diff --git a/experiments/k3/attnres_spike/reproduction.json b/experiments/k3/attnres_spike/reproduction.json new file mode 100644 index 0000000..dc9509e --- /dev/null +++ b/experiments/k3/attnres_spike/reproduction.json @@ -0,0 +1,52 @@ +{ + "artifacts": { + "analyzer": "b9dd7f723e8c5765ecffd9ae3050ff7a95643f95853adaeeaa2c2c323d0ca057", + "manifest": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", + "protocol": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", + "runner": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", + "scoping": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" + }, + "canonical_sha256_without_self": "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515", + "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", + "raw_files": { + "formal-seed-2026073001.json": { + "canonical_sha256": "76b0ccfb55c38baef50c395788ac4b351cbe0d58de70064b50702acb5c93f515", + "file_sha256": "e39e93b7a7fce3c56f5f14f95cfdc04afdce53628affee1202fe62bd1bdb7f71" + }, + "formal-seed-2026073002.json": { + "canonical_sha256": "5352c74eca853b375c0e85933dafd7c5916c39fc59052e742ca14ffd6d68bc78", + "file_sha256": "1c6f6c731030ec0adb2a8e7a4d586e0c4005cc3319568a7ac83c08c2a4b8eaf8" + }, + "formal-seed-2026073003.json": { + "canonical_sha256": "7e764c07e90b78c4cd0acc2e99600225f16428cbb25d5188a0a5a8fe797f8766", + "file_sha256": "115f8245577ece6dfaaa8ada68445c186e6523a7f3b26efcc3eb4c0c4ce82406" + }, + "replay-seed-2026073001.json": { + "canonical_sha256": "c9ebccba0acf10b05fea65beade50e588fd9a83b1807f7ebba8ec1ab836b26fb", + "file_sha256": "4f91c7a2698662ea69af9aa3425e144107834fd9b6b717fb82b84f153b867dc5" + } + }, + "replay_gate": { + "checks": { + "architecture": true, + "artifacts": true, + "batch_size": true, + "depth": true, + "diagnostics": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "round05_equivalence": true, + "seed": true, + "steps": true, + "target_bytes_seen": true, + "training_history": true + }, + "frozen_compare_sha256": "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87", + "passed": true + }, + "schema_version": 1 +} diff --git a/experiments/k3/attnres_spike/results/raw/formal-seed-2026073001.json b/experiments/k3/attnres_spike/results/raw/formal-seed-2026073001.json new file mode 100644 index 0000000..698c27d --- /dev/null +++ b/experiments/k3/attnres_spike/results/raw/formal-seed-2026073001.json @@ -0,0 +1,86460 @@ +{ + "architecture": "block", + "artifacts": { + "protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", + "runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", + "scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" + }, + "batch_size": 32, + "canonical_sha256_without_self": "76b0ccfb55c38baef50c395788ac4b351cbe0d58de70064b50702acb5c93f515", + "depth": 32, + "diagnostics": [ + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "initialization_negative_control": { + "comparisons": { + "detached_learned": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + }, + "uniform_value_backward": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + } + }, + "passed": true + }, + "loss_scale_gate": { + "checks": { + "attention_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "mlp_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_attention_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_mlp_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_attention_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_mlp_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 7.959341947457508, + "forward": { + "activation_sha256": { + "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" + }, + "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", + "loss_nats": 5.516995429992676, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0021613112185150385, + 0.0025601957459002733, + 0.0025323182344436646, + 0.0027253651060163975, + 0.002904455177485943, + 0.0029208383057266474, + 0.00299624796025455, + 0.0030937574338167906, + 0.0033804215490818024, + 0.0034370317589491606, + 0.0038969130255281925, + 0.004255065228790045, + 0.004119740333408117, + 0.003970506135374308, + 0.004385598469525576, + 0.0041999719105660915, + 0.005232699681073427, + 0.004664790350943804, + 0.004898969549685717, + 0.005131533369421959, + 0.005069608334451914, + 0.0051318020559847355, + 0.0058372304774820805, + 0.0059690922498703, + 0.00554921617731452, + 0.0058550527319312096, + 0.006255472078919411, + 0.006502739619463682, + 0.006122888531535864, + 0.007071305997669697, + 0.006241494789719582 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962477027642308, + "first_to_last_ratio": 3.7699040072151497, + "imbalance_abs_log_ratio": 1.3270495388586014, + "last_quartile_mean": 4.234186599205714e-05, + "mean": 8.617327489446325e-05, + "normalized": [ + 2.5428430303860874, + 2.3222064178412967, + 2.1011513502014654, + 1.8911470095267005, + 1.7004447966680725, + 1.5353700031893112, + 1.4229335846259188, + 1.3028620492588128, + 1.2049362956920768, + 1.1705197150696323, + 1.0773612761873523, + 1.0133200843661747, + 0.9906112963967714, + 0.9272530070793641, + 0.8550207003106389, + 0.8185600560490035, + 0.7703816559177782, + 0.7173918665339755, + 0.6760157258502337, + 0.6514535263006792, + 0.6184138115448249, + 0.6053606047045522, + 0.5838821473278404, + 0.5697017460729616, + 0.5457761369570343, + 0.5276086337508958, + 0.5104655265174319, + 0.4951752820428687, + 0.47527333481689216, + 0.465789239220051, + 0.45896414036514044, + 0.45180594922816086 + ], + "peak_layer": 1, + "peak_normalized": 2.5428430303860874, + "population_cv": 0.5705341151299013, + "rest_mean": 9.280180475562466e-05, + "spike_contrast": 0.5428688997682254, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0379213644191625e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021912511147093028, + 0.00020011213200632483, + 0.00018106309289578348, + 0.00016296633111778647, + 0.00014653289690613747, + 0.00013230786134954542, + 0.0001226188469445333, + 0.00011227188952034339, + 0.0001038333066389896, + 0.00010086751717608422, + 9.283974941354245e-05, + 8.732111018616706e-05, + 8.536421955795959e-05, + 7.990442827576771e-05, + 7.367993384832516e-05, + 7.053800072753802e-05, + 6.63863102090545e-05, + 6.182000652188435e-05, + 5.825448897667229e-05, + 5.6137883802875876e-05, + 5.329074338078499e-05, + 5.216590579948388e-05, + 5.031503678765148e-05, + 4.909306517220102e-05, + 4.703131708083674e-05, + 4.5465763832908124e-05, + 4.3988486140733585e-05, + 4.2670875700423494e-05, + 4.095585973118432e-05, + 4.013858415419236e-05, + 3.955044303438626e-05, + 3.893359826179221e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442661636683624, + "first_to_last_ratio": 1.843144421770586, + "imbalance_abs_log_ratio": 0.6114730379455638, + "last_quartile_mean": 0.0001163373926829081, + "mean": 0.00015221274247778638, + "normalized": [ + 1.7741446897870732, + 1.648461275666186, + 1.5301464224684427, + 1.4217121808823485, + 1.3265065496562456, + 1.2454420544341354, + 1.1897998831173078, + 1.1336248235930768, + 1.0869089599614004, + 1.0643003125599004, + 1.0227030055526172, + 0.9941765648180964, + 0.9788958472869692, + 0.9511102301475627, + 0.9228060651723192, + 0.9052418824754537, + 0.8846484396889256, + 0.865292475366091, + 0.8497311645646503, + 0.8388691935254817, + 0.8256303537004397, + 0.8176325372729673, + 0.8076852907347736, + 0.8000670639426607, + 0.7892588692968658, + 0.7814415503461742, + 0.7731202151361662, + 0.7664994567889658, + 0.758490072461763, + 0.7529625284402992, + 0.7485174912671357, + 0.7441725498875049 + ], + "peak_layer": 1, + "peak_normalized": 1.7741446897870732, + "population_cv": 0.27364872432853743, + "rest_mean": 0.00015762320577373935, + "spike_contrast": 0.7803181014868814, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299624067964033, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027004742878489196, + 0.0002509168116375804, + 0.0002329077833564952, + 0.00021640271006617695, + 0.00020191119983792305, + 0.00018957215070258826, + 0.0001811027032090351, + 0.00017255214333999902, + 0.00016544139361940324, + 0.00016200006939470768, + 0.00015566842921543866, + 0.0001513263414381072, + 0.00014900042151566595, + 0.0001447710965294391, + 0.00014046284195501357, + 0.0001377893495373428, + 0.00013465476513374597, + 0.00013170854072086513, + 0.00012933991092722863, + 0.0001276865805266425, + 0.00012567146040964872, + 0.00012445409083738923, + 0.00012293999316170812, + 0.00012178040196886286, + 0.00012013525702059269, + 0.00011894536146428436, + 0.00011767874821089208, + 0.00011667098442558199, + 0.00011545185407157987, + 0.00011461049143690616, + 0.00011393390013836324, + 0.00011327254469506443 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19015566632151604, + "first_to_last_ratio": 1.8431444167877922, + "imbalance_abs_log_ratio": 0.6114730352421437, + "last_quartile_mean": 0.10316916275769472, + "mean": 0.13498378219082952, + "normalized": [ + 1.7741447141792424, + 1.6484611693507762, + 1.5301464164964773, + 1.421712099711509, + 1.3265065258282553, + 1.245442086656075, + 1.1897999853143832, + 1.1336247998774232, + 1.0869090145314644, + 1.0643003539468234, + 1.0227030148964142, + 0.9941765578539046, + 0.978895844117059, + 0.9511102293531465, + 0.9228060470726628, + 0.9052419251832852, + 0.8846484748200404, + 0.8652925213936268, + 0.8497311905993837, + 0.8388692019864461, + 0.8256303029992552, + 0.8176324960789627, + 0.8076852727084025, + 0.800067049482784, + 0.789258877083647, + 0.7814415614650243, + 0.7731201949085273, + 0.7664994759042474, + 0.758490077799444, + 0.7529625178867759, + 0.7485174642151933, + 0.7441725362993381 + ], + "peak_layer": 1, + "peak_normalized": 1.7741447141792424, + "population_cv": 0.2736487184871982, + "rest_mean": 0.13978183573042904, + "spike_contrast": 0.7803180757143806, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907429307699204, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23948076367378235, + 0.2225155234336853, + 0.20654495060443878, + 0.1919080764055252, + 0.17905686795711517, + 0.16811448335647583, + 0.16060370206832886, + 0.1530209630727768, + 0.14671508967876434, + 0.14366328716278076, + 0.13804832100868225, + 0.13419771194458008, + 0.1321350634098053, + 0.12838445603847504, + 0.12456385046243668, + 0.12219297885894775, + 0.11941319704055786, + 0.116800457239151, + 0.11469992995262146, + 0.11323373764753342, + 0.11144670099020004, + 0.11036712676286697, + 0.10902441293001175, + 0.10799607634544373, + 0.10653714835643768, + 0.10548193752765656, + 0.10435868799686432, + 0.1034649983048439, + 0.10238385945558548, + 0.10163772851228714, + 0.10103771835565567, + 0.10045122355222702 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883365585963475e-05, + "first_to_last_ratio": 2.2497546817252747, + "imbalance_abs_log_ratio": 0.8108211799277664, + "last_quartile_mean": 3.059607183786284e-05, + "mean": 4.691841348858361e-05, + "normalized": [ + 1.7359303646705315, + 1.6475859007944922, + 1.5536416135247637, + 1.4921074478374423, + 1.436565604934822, + 1.343834173349448, + 1.2787439414688093, + 1.2483315192759759, + 1.191227587726315, + 1.1603086071531024, + 1.0995306735006558, + 1.059977946939621, + 1.0583984121521266, + 1.0176934649371145, + 0.9754909410255406, + 0.9631621800279858, + 0.9242301483750994, + 0.8752327792179452, + 0.8454160845678106, + 0.8232909667825862, + 0.7863453139125789, + 0.77746685672799, + 0.7497386589361914, + 0.7388508716017261, + 0.7093180450413769, + 0.6908354487677191, + 0.6685471901846501, + 0.6592354484095886, + 0.631440334976849, + 0.6235901145414114, + 0.6199506557967245, + 0.6139807028410055 + ], + "peak_layer": 1, + "peak_normalized": 1.7359303646705315, + "population_cv": 0.3220412433027962, + "rest_mean": 4.907019663530971e-05, + "spike_contrast": 0.7193528234378936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.529878449626267e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144709863699973e-05, + 7.730211655143648e-05, + 7.289439963642508e-05, + 7.000731420703232e-05, + 6.740137905580923e-05, + 6.305056740529835e-05, + 5.999663699185476e-05, + 5.856973439222202e-05, + 5.5890508519951254e-05, + 5.4439839004771784e-05, + 5.1588234782684594e-05, + 4.9732483603293076e-05, + 4.965837433701381e-05, + 4.774856279254891e-05, + 4.576848732540384e-05, + 4.519004141911864e-05, + 4.3363412260077894e-05, + 4.106453343410976e-05, + 3.966558142565191e-05, + 3.862750600092113e-05, + 3.6894074582960457e-05, + 3.647751145763323e-05, + 3.517654840834439e-05, + 3.4665710700210184e-05, + 3.328007733216509e-05, + 3.241290323785506e-05, + 3.136717350571416e-05, + 3.0930281354812905e-05, + 2.9626178729813546e-05, + 2.925785884144716e-05, + 2.9087101211189292e-05, + 2.8807000489905477e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001734125216898974, + "first_to_last_ratio": 1.495517223041696, + "imbalance_abs_log_ratio": 0.4024721155990803, + "last_quartile_mean": 0.00011595488103921525, + "mean": 0.00013990532806928968, + "normalized": [ + 1.3768889945483427, + 1.3340940553978193, + 1.2910049686488183, + 1.2527203273458163, + 1.2147536525239127, + 1.1769854337196524, + 1.14744939519443, + 1.1220955988146075, + 1.0944022472959734, + 1.0773022658134643, + 1.0508559212107247, + 1.03206209473486, + 1.0208171898483585, + 1.000561882500425, + 0.9805161605170362, + 0.9663129296065233, + 0.9487424991891081, + 0.931931984738623, + 0.9181974345490227, + 0.9075104531838204, + 0.8943640941785878, + 0.8862664021756573, + 0.8757830774574916, + 0.86790401991425, + 0.8562894022841879, + 0.8478155487572495, + 0.8387227683262297, + 0.8314011142413483, + 0.8224071457687988, + 0.8161875050523302, + 0.8112199678685613, + 0.8064334645939688 + ], + "peak_layer": 1, + "peak_normalized": 1.3768889945483427, + "population_cv": 0.16325957486062354, + "rest_mean": 0.0001431148236776978, + "spike_contrast": 0.8564734849544942, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225740517838858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926341064972803, + 0.00018664686649572104, + 0.00018061847367789596, + 0.00017526224837638438, + 0.0001699505082797259, + 0.00016466653323732316, + 0.00016053428407758474, + 0.00015698715287726372, + 0.00015311270544771105, + 0.00015072032692842185, + 0.00014702034241054207, + 0.00014439098595175892, + 0.00014281776384450495, + 0.00013998393842484802, + 0.00013717943511437625, + 0.00013519232743419707, + 0.00013273413060232997, + 0.00013038225006312132, + 0.00012846071331296116, + 0.00012696554767899215, + 0.0001251263020094484, + 0.00012399339175317436, + 0.0001225267187692225, + 0.00012142439663875848, + 0.00011979944974882528, + 0.00011861391249112785, + 0.00011734178406186402, + 0.00011631744564510882, + 0.00011505914153531194, + 0.00011418898066040128, + 0.00011349399574100971, + 0.00011282433843007311 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675117381557357, + "first_to_last_ratio": 1.2676837465168667, + "imbalance_abs_log_ratio": 0.23719141364257146, + "last_quartile_mean": 0.00011576323686313117, + "mean": 0.00013042020850662084, + "normalized": [ + 1.169988069421413, + 1.156175152925158, + 1.1417665260225645, + 1.1306726331060184, + 1.1183621029407895, + 1.1062694829504214, + 1.0940115056230622, + 1.0844989956600726, + 1.0711591662817939, + 1.0579765491085218, + 1.0443900226505625, + 1.035471661013805, + 1.0260498632741444, + 1.013992390086173, + 1.0062479316494695, + 0.9977001177294099, + 0.9863163473759529, + 0.9769959732660659, + 0.9687585669786032, + 0.9602628595878113, + 0.9503845997255937, + 0.9426104617668664, + 0.9330243108845692, + 0.9259760938667744, + 0.914797625470175, + 0.906289421438531, + 0.8956357003805738, + 0.888470885240542, + 0.8806216558595509, + 0.8756991509238615, + 0.8712858841028995, + 0.8681382926882502 + ], + "peak_layer": 1, + "peak_normalized": 1.169988069421413, + "population_cv": 0.09123006364857297, + "rest_mean": 0.00013202972385678786, + "spike_contrast": 0.9219804606102014, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012172882561571896, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015259008796419948, + 0.00015078860451467335, + 0.00014890942838974297, + 0.0001474625605624169, + 0.00014585701865144074, + 0.00014427989663090557, + 0.00014268120867200196, + 0.00014144058513920754, + 0.00013970080181024969, + 0.0001379815221298486, + 0.00013620956451632082, + 0.00013504642993211746, + 0.0001338176371064037, + 0.00013224509893916547, + 0.00013123506505507976, + 0.0001301202573813498, + 0.00012863558367826045, + 0.00012742001854348928, + 0.00012634569429792464, + 0.00012523768236860633, + 0.00012394935765769333, + 0.00012293545296415687, + 0.00012168522516731173, + 0.00012076599523425102, + 0.00011930809705518186, + 0.00011819845531135798, + 0.00011680899478960782, + 0.00011587455810513347, + 0.00011485085997264832, + 0.00011420886585256085, + 0.00011363328667357564, + 0.00011322277714498341 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030002388848515693, + "first_to_last_ratio": 2.2836575149168885, + "imbalance_abs_log_ratio": 0.8257783308640597, + "last_quartile_mean": 0.0001313786706305109, + "mean": 0.0001947244400071213, + "normalized": [ + 1.8998833386658438, + 1.7835645331795598, + 1.6713073472446418, + 1.5703972875779748, + 1.4780250735507643, + 1.3761669788219328, + 1.3066447563403325, + 1.2401013786965793, + 1.1762106459027857, + 1.150547637187856, + 1.0768735950666006, + 1.0264900078881174, + 1.0140784159549234, + 0.9666871251601796, + 0.9096841069789114, + 0.8754807117769676, + 0.8421876125778792, + 0.8073809902734768, + 0.77911667677541, + 0.7607651063686129, + 0.7403822030704592, + 0.7308855611321243, + 0.7152791523187013, + 0.7043381880055872, + 0.6944157299089508, + 0.686864629891504, + 0.680599049276497, + 0.6764772714226681, + 0.6702412094778135, + 0.6672258960044359, + 0.6635863560708457, + 0.6581114274310631 + ], + "peak_layer": 1, + "peak_normalized": 1.8998833386658438, + "population_cv": 0.3592853832280775, + "rest_mean": 0.00020492727344390005, + "spike_contrast": 0.6813594750078023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962913944851608, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699537192005664, + 0.0003473036049399525, + 0.0003254443872720003, + 0.00030579473241232336, + 0.00028780760476365685, + 0.00026797334430739284, + 0.0002544356684666127, + 0.0002414780465187505, + 0.0002290369593538344, + 0.00022403974435292184, + 0.00020969360775779933, + 0.00019988269195891917, + 0.0001974658516701311, + 0.00018823760910890996, + 0.00017713772831484675, + 0.00017047749133780599, + 0.00016399451124016196, + 0.0001572168112033978, + 0.00015171305858530104, + 0.00014813955931458622, + 0.00014417050988413393, + 0.00014232128160074353, + 0.00013928233238402754, + 0.0001371518592350185, + 0.00013521971413865685, + 0.00013374933041632175, + 0.00013252926873974502, + 0.00013172665785532445, + 0.00013051234418526292, + 0.00012992518895771354, + 0.00012921648158226162, + 0.000128150379168801 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027073738165199757, + 0.002722575329244137, + 0.0026850115973502398, + 0.0026861224323511124, + 0.00271838391199708, + 0.0026645169127732515, + 0.0026859452482312918, + 0.0027359756641089916, + 0.0026767742820084095, + 0.0027451682835817337, + 0.0026999011170119047, + 0.002663193503394723, + 0.002626648638397455, + 0.0026531771291047335, + 0.0026517712976783514, + 0.002676903735846281, + 0.002670086920261383, + 0.0026355870068073273, + 0.0026610493659973145, + 0.0026388936676084995, + 0.0026469926815479994, + 0.00264862016774714, + 0.002641494618728757, + 0.002674198243767023, + 0.002557571977376938, + 0.00252733426168561, + 0.0025468147359788418, + 0.0026856842450797558, + 0.002551089273765683, + 0.0025778315030038357, + 0.0026001501828432083, + 0.0025662274565547705 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858857113926206, + "first_to_last_ratio": 3.757867690304466, + "imbalance_abs_log_ratio": 1.3238516928952826, + "last_quartile_mean": 4.220174423608114e-05, + "mean": 8.573221236929385e-05, + "normalized": [ + 2.5461641896458365, + 2.3032502265274117, + 2.0924715218207024, + 1.8886722264368014, + 1.7031663195050915, + 1.537434754996766, + 1.4223955235407997, + 1.3049501874408618, + 1.2055297161045848, + 1.1692542365736316, + 1.0769571962730058, + 1.0144505847411098, + 0.9913601784485884, + 0.9317001386730132, + 0.857757082878185, + 0.8155634736467603, + 0.7682612945963089, + 0.7217806418748633, + 0.678338533281248, + 0.6511820384364709, + 0.6192882306865346, + 0.6053793146733001, + 0.5848721334223365, + 0.5718144052517429, + 0.5462281883927541, + 0.527515083157495, + 0.5112993585545178, + 0.496920907994256, + 0.47722358366421574, + 0.46552107689700334, + 0.4601234027970099, + 0.45317424906679377 + ], + "peak_layer": 1, + "peak_normalized": 2.5461641896458365, + "population_cv": 0.5686091737198813, + "rest_mean": 9.231269224848668e-05, + "spike_contrast": 0.5437781067692293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0197621021652594e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021828828903380781, + 0.00019746273756027222, + 0.00017939221288543195, + 0.0001619200484128669, + 0.00014601621660403907, + 0.000131807682919316, + 0.00012194511509733275, + 0.00011187626660102978, + 0.00010335272963857278, + 0.00010024275252362713, + 9.232992306351662e-05, + 8.697109296917915e-05, + 8.499150135321543e-05, + 7.987671415321529e-05, + 7.353741239057854e-05, + 6.992006092332304e-05, + 6.586474046343938e-05, + 6.187985127326101e-05, + 5.8155463193543255e-05, + 5.582727681030519e-05, + 5.309295011102222e-05, + 5.1900507969548926e-05, + 5.0142381951445714e-05, + 4.902291402686387e-05, + 4.682935104938224e-05, + 4.522503513726406e-05, + 4.3834825191879645e-05, + 4.260212881490588e-05, + 4.0913433622336015e-05, + 3.991015182691626e-05, + 3.944739728467539e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242323236947414, + "first_to_last_ratio": 1.8345651540958352, + "imbalance_abs_log_ratio": 0.6068074801791611, + "last_quartile_mean": 0.0001157894184871111, + "mean": 0.00015111673860701558, + "normalized": [ + 1.771707915810251, + 1.6366668487510905, + 1.5242909365839221, + 1.4186705636795862, + 1.3264449439106374, + 1.2453857912414694, + 1.1893415085703798, + 1.1330085309323608, + 1.0858463851390703, + 1.062861239127963, + 1.023237806768965, + 0.9942126959777651, + 0.978436257513136, + 0.9531027445378829, + 0.9232414968561398, + 0.904013237249231, + 0.8853704564278231, + 0.8680164030054869, + 0.8515402788738432, + 0.8395817783841374, + 0.8267435189825344, + 0.8185143642831039, + 0.809172703595369, + 0.8007918347797284, + 0.7909896391756025, + 0.7821052393452608, + 0.7753388667989797, + 0.7682453290925442, + 0.7604995795745633, + 0.7549901567959042, + 0.7508572350862155, + 0.7467737131490535 + ], + "peak_layer": 1, + "peak_normalized": 1.771707915810251, + "population_cv": 0.2714170004285523, + "rest_mean": 0.00015645500981783562, + "spike_contrast": 0.7816309251520459, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229007406858728, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677347220014781, + 0.00024732775636948645, + 0.00023034587502479553, + 0.00021438486874103546, + 0.00020044803386554122, + 0.0001881986390799284, + 0.00017972940986510366, + 0.00017121655400842428, + 0.00016408956435043365, + 0.00016061612404882908, + 0.0001546283601783216, + 0.00015024218009784818, + 0.00014785809617023915, + 0.0001440297783119604, + 0.00013951724395155907, + 0.000136611532070674, + 0.0001337942958343774, + 0.00013117180787958205, + 0.0001286819897359237, + 0.00012687486014328897, + 0.00012493478425312787, + 0.00012369122123345733, + 0.00012227953993715346, + 0.00012101305037504062, + 0.00011953177454415709, + 0.00011818919301731512, + 0.00011716668086592108, + 0.00011609472858253866, + 0.00011492421617731452, + 0.00011409165017539635, + 0.00011346709652571008, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837904930114746, + "first_to_last_ratio": 1.8345651581276443, + "imbalance_abs_log_ratio": 0.6068074823768531, + "last_quartile_mean": 0.10268321540206671, + "mean": 0.13401183625683188, + "normalized": [ + 1.7717078885231814, + 1.6366668119471057, + 1.524290957183596, + 1.418670514273678, + 1.326444920903592, + 1.2453858718827995, + 1.1893415500975306, + 1.1330085716119305, + 1.0858463365841577, + 1.062861212873729, + 1.0232377434953237, + 0.9942127340740732, + 0.9784362443707608, + 0.9531027271661117, + 0.9232414917376033, + 0.904013241653558, + 0.885370477646398, + 0.8680164390463408, + 0.8515402681847718, + 0.8395816969955481, + 0.8267435345782925, + 0.8185144279720658, + 0.8091727254708025, + 0.8007918405918669, + 0.7909896328038464, + 0.7821052665924377, + 0.7753388466443417, + 0.7682453528580236, + 0.7604996016741158, + 0.7549901615676585, + 0.7508572336421532, + 0.7467736753526057 + ], + "peak_layer": 1, + "peak_normalized": 1.7717078885231814, + "population_cv": 0.27141699472708714, + "rest_mean": 0.1387458680956452, + "spike_contrast": 0.7816309473985974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844806432724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23742982745170593, + 0.2193327248096466, + 0.2042730301618576, + 0.19011864066123962, + 0.1777593195438385, + 0.16689644753932953, + 0.15938584506511688, + 0.151836559176445, + 0.1455162614583969, + 0.1424359828233719, + 0.13712596893310547, + 0.13323627412319183, + 0.13112203776836395, + 0.12772704660892487, + 0.1237252876162529, + 0.1211484745144844, + 0.11865012347698212, + 0.11632447689771652, + 0.11411647498607635, + 0.11251388490200043, + 0.11079341918230057, + 0.10969062149524689, + 0.10843872278928757, + 0.10731558501720428, + 0.10600197315216064, + 0.10481136292219162, + 0.10390458256006241, + 0.1029539704322815, + 0.10191594809293747, + 0.10117761790752411, + 0.10062375664710999, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873187476230669e-05, + "first_to_last_ratio": 2.248757500145603, + "imbalance_abs_log_ratio": 0.8103778415280068, + "last_quartile_mean": 3.0564378221242805e-05, + "mean": 4.685761763312257e-05, + "normalized": [ + 1.7400006587426948, + 1.6405194725375354, + 1.5551306973978087, + 1.4944078341288562, + 1.4327724019957897, + 1.3414187773842097, + 1.2804088050749736, + 1.2499327782691996, + 1.1927955171655855, + 1.1608931065609767, + 1.0981756972550076, + 1.0577423904261343, + 1.0542786038595535, + 1.0218640087393278, + 0.9775807389332813, + 0.9570972402588515, + 0.9209982861589985, + 0.8815092249082567, + 0.8478232134430647, + 0.8204952153707635, + 0.7871780668180614, + 0.7759097745574741, + 0.7506963524238334, + 0.7421155343193898, + 0.7072927325438048, + 0.6891814936654451, + 0.6695399899996352, + 0.6616343983211914, + 0.6345111343049281, + 0.6212687927046295, + 0.6200231913288072, + 0.6148038704019305 + ], + "peak_layer": 1, + "peak_normalized": 1.7400006587426948, + "population_cv": 0.3217536161653951, + "rest_mean": 4.900405669845727e-05, + "spike_contrast": 0.719671983430414, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.526684668031521e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153228554874659e-05, + 7.687083416385576e-05, + 7.286971958819777e-05, + 7.002439087955281e-05, + 6.71363013680093e-05, + 6.285568815656006e-05, + 5.999690620228648e-05, + 5.8568872191244736e-05, + 5.5891556257847697e-05, + 5.439668530016206e-05, + 5.145789691596292e-05, + 4.9563288484932855e-05, + 4.9400983698433265e-05, + 4.7882112994557247e-05, + 4.580710447044112e-05, + 4.484729652176611e-05, + 4.3155785533599555e-05, + 4.130542220082134e-05, + 3.972697595600039e-05, + 3.844645107164979e-05, + 3.688528886414133e-05, + 3.635728353401646e-05, + 3.517584264045581e-05, + 3.477376594673842e-05, + 3.314205241622403e-05, + 3.229340290999971e-05, + 3.1373048841487616e-05, + 3.10026116494555e-05, + 2.9731680115219206e-05, + 2.9111175535945222e-05, + 2.9052809622953646e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001719075116852764, + "first_to_last_ratio": 1.4894635014227988, + "imbalance_abs_log_ratio": 0.39841598895878805, + "last_quartile_mean": 0.00011541572621354135, + "mean": 0.00013894635094402474, + "normalized": [ + 1.3738985779819257, + 1.3296082759737424, + 1.2883384004275333, + 1.2499940655466777, + 1.2130730176151365, + 1.1755503979222366, + 1.147409009170091, + 1.11990595257071, + 1.0929037170303153, + 1.075598370036652, + 1.0512762916941902, + 1.0312594730951, + 1.0194418974669357, + 1.0018996501730997, + 0.9803460168062212, + 0.9648004557293595, + 0.9493010807852901, + 0.9344788929945884, + 0.9198953859218769, + 0.908001147597105, + 0.8952913731815082, + 0.886953676542665, + 0.8771894463102521, + 0.8683888934041172, + 0.8579399877904575, + 0.8482799643655919, + 0.8408636339472488, + 0.8330870307470335, + 0.82434015220157, + 0.8181187963300758, + 0.8135579418994486, + 0.809009026741244 + ], + "peak_layer": 1, + "peak_normalized": 1.3738985779819257, + "population_cv": 0.16161423472023242, + "rest_mean": 0.0001421073115866252, + "spike_contrast": 0.8576417505420824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187716347398237, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001908981939777732, + 0.0001847442181315273, + 0.00017900991952046752, + 0.00017368211410939693, + 0.00016855206922627985, + 0.00016333843814209104, + 0.00015942829486448318, + 0.00015560684551019222, + 0.0001518549834145233, + 0.00014945046859793365, + 0.00014607100456487387, + 0.0001432897406630218, + 0.00014164773165248334, + 0.00013921030040364712, + 0.000136215501697734, + 0.0001340555027127266, + 0.0001319019211223349, + 0.00012984243221580982, + 0.0001278161071240902, + 0.00012616344611160457, + 0.00012439746933523566, + 0.00012323897681199014, + 0.00012188227265141904, + 0.00012065946793882176, + 0.00011920763063244522, + 0.00011786540562752634, + 0.00011683493357850239, + 0.00011575440294109285, + 0.0001145390560850501, + 0.00011367462138878182, + 0.00011304090730845928, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552221000485588, + "first_to_last_ratio": 1.262697055715921, + "imbalance_abs_log_ratio": 0.2332499537229619, + "last_quartile_mean": 0.00011524712863320019, + "mean": 0.00012953460168318998, + "normalized": [ + 1.1676213631634658, + 1.1525055688027706, + 1.1391580058081985, + 1.1289307982409096, + 1.1166731577821778, + 1.1054001772932092, + 1.094813144753513, + 1.0822854389705017, + 1.0679308603491235, + 1.0564894822224282, + 1.0456838360716472, + 1.0337072702557388, + 1.0234945545465333, + 1.0158819477934218, + 1.0051772952892828, + 0.9951467950359763, + 0.9866643397779161, + 0.9787559418439278, + 0.970448736962688, + 0.9607908684400951, + 0.9509939230158972, + 0.9428061355388626, + 0.934924980051271, + 0.9261034265480621, + 0.9170582603044295, + 0.9058562786860701, + 0.8981778407016658, + 0.8907564927732574, + 0.8831168120837269, + 0.8778216107141017, + 0.8745192645108896, + 0.8703053916682415 + ], + "peak_layer": 1, + "peak_normalized": 1.1676213631634658, + "population_cv": 0.08967236486753827, + "rest_mean": 0.00013110875065817877, + "spike_contrast": 0.9231588022206524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103419721825049, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124736819416285, + 0.00014928934979252517, + 0.000147560378536582, + 0.00014623560127802193, + 0.00014464781270362437, + 0.00014318757166620344, + 0.00014181618462316692, + 0.00014019341324456036, + 0.0001383339986205101, + 0.00013685194426216185, + 0.00013545223919209093, + 0.00013390085950959474, + 0.00013257795944809914, + 0.00013159186346456409, + 0.00013020524056628346, + 0.00012890594371128827, + 0.00012780717224813998, + 0.00012678276107180864, + 0.0001257066905964166, + 0.0001244556624442339, + 0.00012318661902099848, + 0.0001221260172314942, + 0.00012110513489460573, + 0.0001199624384753406, + 0.00011879077646881342, + 0.00011733973224181682, + 0.00011634510883595794, + 0.00011538378748809919, + 0.0001143941844929941, + 0.00011370827269274741, + 0.00011328050459269434, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029741139769612346, + "first_to_last_ratio": 2.278072415841521, + "imbalance_abs_log_ratio": 0.8233296537390045, + "last_quartile_mean": 0.00013055396993877366, + "mean": 0.00019328556300024502, + "normalized": [ + 1.889398576482247, + 1.780233894901915, + 1.6772594716902607, + 1.566224958959574, + 1.4762232610465842, + 1.375571144634086, + 1.3083686481730985, + 1.2364401527267181, + 1.173148897934763, + 1.1508864184295666, + 1.080457040307468, + 1.0318582947152708, + 1.0081957004611959, + 0.9698994715426978, + 0.9066127339785177, + 0.8754550048258108, + 0.8446146862114313, + 0.8115724449181009, + 0.7806677558079268, + 0.7613480996239201, + 0.7403134026279838, + 0.7294052762839086, + 0.7160186976339193, + 0.7062575707090604, + 0.6942777320753531, + 0.6866005528973866, + 0.6829938476675228, + 0.6753403086482458, + 0.6715967314167978, + 0.6686860556999406, + 0.6640286432895989, + 0.6600445236791291 + ], + "peak_layer": 1, + "peak_normalized": 1.889398576482247, + "population_cv": 0.3579154260008552, + "rest_mean": 0.00020340604748559632, + "spike_contrast": 0.681567477924495, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013863494677934796, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003651934675872326, + 0.0003440935106482357, + 0.00032419004128314555, + 0.0003027286729775369, + 0.0002853326441254467, + 0.00026587804313749075, + 0.00025288877077400684, + 0.00023898603103589267, + 0.00022675274522043765, + 0.00022244972933549434, + 0.0002088367473334074, + 0.00019944331143051386, + 0.00019486967357806861, + 0.00018746756541077048, + 0.00017523515271022916, + 0.00016921281348913908, + 0.0001632518251426518, + 0.00015686523693148047, + 0.00015089180669747293, + 0.00014715759607497603, + 0.00014309189282357693, + 0.00014098350948188454, + 0.00013839607709087431, + 0.0001365093921776861, + 0.0001341938623227179, + 0.00013270997442305088, + 0.00013201285037212074, + 0.00013053353177383542, + 0.0001298099523410201, + 0.00012924736074637622, + 0.00012834715016651899, + 0.00012757707736454904 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0040087271481752396, + 0.005379301495850086, + 0.006410556845366955, + 0.0027253651060163975, + 0.004838948138058186, + 0.006175771821290255, + 0.00742313964292407, + 0.0030937574338167906, + 0.005150026176124811, + 0.007019025273621082, + 0.008143599145114422, + 0.004255065228790045, + 0.0066224560141563416, + 0.008129785768687725, + 0.010099578648805618, + 0.0041999719105660915, + 0.00719039561226964, + 0.009030314162373543, + 0.010736005380749702, + 0.005131533369421959, + 0.00787449348717928, + 0.009912566281855106, + 0.012113703414797783, + 0.0059690922498703, + 0.00876442901790142, + 0.010806876234710217, + 0.012353328987956047, + 0.006502739619463682, + 0.009470963850617409, + 0.012123330496251583, + 0.01384514570236206 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962236011546338, + "first_to_last_ratio": 3.7698422272252903, + "imbalance_abs_log_ratio": 1.3270331510418396, + "last_quartile_mean": 4.2341920561739244e-05, + "mean": 8.61727822893954e-05, + "normalized": [ + 2.542734798705971, + 2.3221448837042376, + 2.101145461274622, + 1.8911858524935854, + 1.7004204056817789, + 1.5354142426015858, + 1.4229001770142031, + 1.3028733810328659, + 1.2049412416934622, + 1.1705258152816296, + 1.0773908232649334, + 1.0133220774376726, + 0.990609360100461, + 0.9272534949358001, + 0.8550385909364457, + 0.8185651575045715, + 0.7703972051466339, + 0.7173947009730917, + 0.6760256695667757, + 0.6514523109028573, + 0.618419077603279, + 0.6053620388084041, + 0.5838930841859182, + 0.5697043695032278, + 0.5457802700901755, + 0.5276149005446632, + 0.5104701332743111, + 0.49517988582787786, + 0.47527562953524777, + 0.46579059316028215, + 0.45896435764104826, + 0.45181000957238127 + ], + "peak_layer": 1, + "peak_normalized": 2.542734798705971, + "population_cv": 0.5705202801130907, + "rest_mean": 9.280119640472414e-05, + "spike_contrast": 0.542873885449774, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.03793460666202e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021911453222855926, + 0.00020010568550787866, + 0.00018106155039276928, + 0.00016296874673571438, + 0.00014652995741926134, + 0.00013231091725174338, + 0.0001226152671733871, + 0.00011227222421439365, + 0.00010383313929196447, + 0.00010086746624438092, + 9.284176485380158e-05, + 8.732078276807442e-05, + 8.536356472177431e-05, + 7.99040135461837e-05, + 7.368105434579775e-05, + 7.053803710732609e-05, + 6.638727063545957e-05, + 6.181989738252014e-05, + 5.825501284562051e-05, + 5.613745815935545e-05, + 5.329089253791608e-05, + 5.216573117650114e-05, + 5.0315691623836756e-05, + 4.9093010602518916e-05, + 4.703140439232811e-05, + 4.546604395727627e-05, + 4.398863165988587e-05, + 4.267102849553339e-05, + 4.095582335139625e-05, + 4.013847137684934e-05, + 3.955023566959426e-05, + 3.893372559105046e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442477373057045, + "first_to_last_ratio": 1.8431254562763404, + "imbalance_abs_log_ratio": 0.6114627481432037, + "last_quartile_mean": 0.00011633759004325839, + "mean": 0.00015221248349917005, + "normalized": [ + 1.7740805953138137, + 1.6484457246979003, + 1.5301434809496548, + 1.4217241600875643, + 1.3264958046529274, + 1.2454577490392524, + 1.1897861330408546, + 1.1336265611690741, + 1.086914920172498, + 1.064301358569379, + 1.0227209980571423, + 0.9941753882589759, + 0.9788979908211966, + 0.9511107967643203, + 0.9228143274453928, + 0.9052447611165039, + 0.8846530997418666, + 0.8652930871753776, + 0.8497381552743479, + 0.8388656494639406, + 0.8256334792987466, + 0.8176309647342147, + 0.8076951735882674, + 0.8000661307359944, + 0.7892605467734257, + 0.78144613040264, + 0.7731219607566042, + 0.7664988488799842, + 0.7584935618388989, + 0.7529639051546407, + 0.7485195296370439, + 0.7441790263875565 + ], + "peak_layer": 1, + "peak_normalized": 1.7740805953138137, + "population_cv": 0.2736411578809574, + "rest_mean": 0.00015762286892384981, + "spike_contrast": 0.7803207938393821, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299640220589936, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027003721334040165, + 0.00025091401766985655, + 0.00023290693934541196, + 0.0002164041652576998, + 0.000201909220777452, + 0.0001895742170745507, + 0.00018110030214302242, + 0.00017255211423616856, + 0.00016544201935175806, + 0.00016199995297938585, + 0.00015567090304102749, + 0.00015132590488065034, + 0.0001490004942752421, + 0.00014477093645837158, + 0.00014046386058907956, + 0.00013778955326415598, + 0.0001346552453469485, + 0.00013170840975362808, + 0.00012934075493831187, + 0.00012768582382705063, + 0.00012567172234412283, + 0.00012445363972801715, + 0.00012294128828216344, + 0.00012178005272289738, + 0.00012013530795229599, + 0.00011894585622940212, + 0.00011767881369451061, + 0.00011667069338727742, + 0.00011545218876563013, + 0.00011461050598882139, + 0.00011393401655368507, + 0.00011327333777444437 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.190154030919075, + "first_to_last_ratio": 1.84312545379361, + "imbalance_abs_log_ratio": 0.6114627467961817, + "last_quartile_mean": 0.10316933691501617, + "mean": 0.13498355145566165, + "normalized": [ + 1.774080628234443, + 1.6484456620184311, + 1.5301434020504485, + 1.4217242444591272, + 1.326495766996536, + 1.2454576834375484, + 1.189786122599316, + 1.133626627255238, + 1.0869149569673349, + 1.0643014004698326, + 1.0227209907477024, + 0.9941753870542451, + 0.9788979589698585, + 0.9511107512164827, + 0.9228143032197887, + 0.9052447972745712, + 0.8846531331847195, + 0.8652931173486506, + 0.8497382179112226, + 0.8388656682549475, + 0.8256334805754935, + 0.8176309131099365, + 0.8076951535479161, + 0.8000661540388709, + 0.7892605573866218, + 0.7814461538045975, + 0.7731219580171443, + 0.7664988542586116, + 0.7584935821787124, + 0.7529638601652359, + 0.7485195164462104, + 0.7441789968002049 + ], + "peak_layer": 1, + "peak_normalized": 1.774080628234443, + "population_cv": 0.27364115345546325, + "rest_mean": 0.13978153549962574, + "spike_contrast": 0.7803207857775153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907443761825561, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2394717037677765, + 0.22251304984092712, + 0.2065441906452179, + 0.19190938770771027, + 0.1790551096200943, + 0.16811630129814148, + 0.16060155630111694, + 0.1530209481716156, + 0.14671564102172852, + 0.1436631828546524, + 0.13805051147937775, + 0.13419732451438904, + 0.13213512301445007, + 0.1283843070268631, + 0.1245647519826889, + 0.12219315767288208, + 0.11941362172365189, + 0.11680033802986145, + 0.11470068246126175, + 0.1132330670952797, + 0.11144693940877914, + 0.11036672443151474, + 0.10902556031942368, + 0.10799577087163925, + 0.10653719305992126, + 0.10548237711191177, + 0.1043587476015091, + 0.10346473753452301, + 0.10238415747880936, + 0.10163773596286774, + 0.10103782266378403, + 0.10045192390680313 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883367768750759e-05, + "first_to_last_ratio": 2.249759357551583, + "imbalance_abs_log_ratio": 0.810823258297238, + "last_quartile_mean": 3.059601795030176e-05, + "mean": 4.691846504556452e-05, + "normalized": [ + 1.7359059710084281, + 1.6475789727938472, + 1.5536456441204052, + 1.4921194549590373, + 1.4365657321885223, + 1.3438513058553838, + 1.2787343947807137, + 1.2483299149152907, + 1.1912354282513808, + 1.1603083401306369, + 1.099547144002948, + 1.059972440022979, + 1.0583955432739494, + 1.0176800955770247, + 0.9754963047746087, + 0.9631595708780456, + 0.9242400656751482, + 0.8752301116137658, + 0.8454245377074081, + 0.8232853322610043, + 0.7863494898171388, + 0.7774642190168813, + 0.7497486129019063, + 0.7388483538626245, + 0.709315327139865, + 0.6908343794807076, + 0.6685469983108738, + 0.6592365073812129, + 0.6314341746588822, + 0.6235898169926835, + 0.6199474545603838, + 0.6139783610863121 + ], + "peak_layer": 1, + "peak_normalized": 1.7359059710084281, + "population_cv": 0.3220406003070692, + "rest_mean": 4.9070239684725595e-05, + "spike_contrast": 0.7193541792517962, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.52988819940947e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144604362314567e-05, + 7.730187644483522e-05, + 7.28946688468568e-05, + 7.000795449130237e-05, + 6.740145909134299e-05, + 6.305144052021205e-05, + 5.999625500408001e-05, + 5.8569723478285596e-05, + 5.589093780145049e-05, + 5.4439886298496276e-05, + 5.158906424185261e-05, + 4.973227987647988e-05, + 4.9658294301480055e-05, + 4.774798799189739e-05, + 4.576878927764483e-05, + 4.51899686595425e-05, + 4.3363925215089694e-05, + 4.1064453398576006e-05, + 3.9666021621087566e-05, + 3.86272840842139e-05, + 3.689431105158292e-05, + 3.6477427784120664e-05, + 3.5177054087398574e-05, + 3.466563066467643e-05, + 3.3279986382694915e-05, + 3.2412888685939834e-05, + 3.136719897156581e-05, + 3.093036502832547e-05, + 2.9625922252307646e-05, + 2.9257877031341195e-05, + 2.9086982976878062e-05, + 2.8806922273361124e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017341253987979144, + "first_to_last_ratio": 1.4955148931299893, + "imbalance_abs_log_ratio": 0.4024705576674927, + "last_quartile_mean": 0.00011595507385209203, + "mean": 0.00013990553861731314, + "normalized": [ + 1.3768818258178377, + 1.3340879911950096, + 1.2910054180628048, + 1.2527236427111486, + 1.2147542166907819, + 1.1769889670731195, + 1.1474411155803916, + 1.1220953663134672, + 1.0944075691302455, + 1.0773007485615413, + 1.0508678613623372, + 1.0320596054419082, + 1.0208161736522385, + 1.000559648637946, + 0.9805197815181176, + 0.9663128275349431, + 0.948744191771126, + 0.9319305822463463, + 0.9182019814348666, + 0.9075038868226692, + 0.8943650364968407, + 0.886262676120959, + 0.8757907045344768, + 0.8679002174812178, + 0.856288373659771, + 0.8478178092769747, + 0.8387215581135022, + 0.8313975747674917, + 0.8224084044020752, + 0.8161864847722404, + 0.8112200992010112, + 0.8064376596145924 + ], + "peak_layer": 1, + "peak_normalized": 1.3768818258178377, + "population_cv": 0.16325888721442564, + "rest_mean": 0.0001431150389921472, + "spike_contrast": 0.8564734877369171, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225742365932092, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926333934534341, + 0.00018664629897102714, + 0.00018061880837194622, + 0.0001752629759721458, + 0.00016995084297377616, + 0.0001646672753849998, + 0.00016053336730692536, + 0.00015698735660407692, + 0.00015311368042603135, + 0.00015072034148033708, + 0.00014702223415952176, + 0.00014439085498452187, + 0.0001428178366040811, + 0.00013998383656144142, + 0.00013718014815822244, + 0.00013519251660909504, + 0.00013273456715978682, + 0.00013038225006312132, + 0.00012846154277212918, + 0.00012696482008323073, + 0.00012512662215158343, + 0.0001239930570591241, + 0.00012252797023393214, + 0.000121424047392793, + 0.00011979948612861335, + 0.00011861440725624561, + 0.00011734179133782163, + 0.0001163171255029738, + 0.00011505949078127742, + 0.00011418900976423174, + 0.00011349418491590768, + 0.00011282509512966499 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675183047074825, + "first_to_last_ratio": 1.2676545313577883, + "imbalance_abs_log_ratio": 0.23716836728277432, + "last_quartile_mean": 0.00011576642282307148, + "mean": 0.00013041823922321782, + "normalized": [ + 1.1700698937843759, + 1.1560440994564782, + 1.1417398043557874, + 1.130463200949706, + 1.118403537277814, + 1.1065304334184665, + 1.0939881912841023, + 1.0846815122144253, + 1.0709468270677047, + 1.0579907390333845, + 1.0442838370166418, + 1.035550673160169, + 1.025959021745254, + 1.0140203095307772, + 1.0062047700154308, + 0.9976631870190885, + 0.9861765922450596, + 0.9770016877798172, + 0.9687702939454803, + 0.9602422119882037, + 0.9504995944091904, + 0.9425883202531964, + 0.9330014667281973, + 0.9259385264566372, + 0.9147443798098084, + 0.906202796813466, + 0.8958065504175365, + 0.8885655303249116, + 0.8806194435723184, + 0.8757905905296257, + 0.8713783728577705, + 0.8681336045391749 + ], + "peak_layer": 1, + "peak_normalized": 1.1700698937843759, + "population_cv": 0.09121429133712369, + "rest_mean": 0.0001320278307604086, + "spike_contrast": 0.921975648780332, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001217264449223876, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001525984553154558, + 0.0001507692359155044, + 0.000148903694935143, + 0.00014743302017450333, + 0.00014586022007279098, + 0.00014431175077334046, + 0.00014267601363826543, + 0.00014146225294098258, + 0.000139670999487862, + 0.00013798128929920495, + 0.00013619365927297622, + 0.00013505469541996717, + 0.00013380376913119107, + 0.00013224674330558628, + 0.00013122745440341532, + 0.00013011347618885338, + 0.00012861541472375393, + 0.00012741883983835578, + 0.0001263453159481287, + 0.0001252330985153094, + 0.0001239624834852293, + 0.00012293070903979242, + 0.00012168040848337114, + 0.00012075927224941552, + 0.00011929935135412961, + 0.00011818537313956767, + 0.00011682951299007982, + 0.00011588515189941972, + 0.00011484883725643158, + 0.00011421906674513593, + 0.00011364363308530301, + 0.00011322045611450449 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030004180916876066, + "first_to_last_ratio": 2.2837506327398005, + "imbalance_abs_log_ratio": 0.8258191057718037, + "last_quartile_mean": 0.00013138116082700435, + "mean": 0.00019473525253488333, + "normalized": [ + 1.9000096511818072, + 1.783700741635724, + 1.6708066910504589, + 1.570366884717585, + 1.4783058799917985, + 1.376185321596543, + 1.3064973297630273, + 1.2402700043336092, + 1.1767953849206394, + 1.1505341943958838, + 1.0771542571420398, + 1.026098162637436, + 1.013898063759585, + 0.9669473025647412, + 0.9095398902231254, + 0.8755205777387188, + 0.842301214161826, + 0.8073124727024382, + 0.7791991818813049, + 0.760574607765185, + 0.7402055397447304, + 0.7308143413325056, + 0.7152667122155245, + 0.7043714155678553, + 0.6942467749765324, + 0.6868749151283189, + 0.680655564574882, + 0.676363713518914, + 0.6701061029149729, + 0.6672276319278244, + 0.6636177363814285, + 0.6582317375530352 + ], + "peak_layer": 1, + "peak_normalized": 1.9000096511818072, + "population_cv": 0.35930746026536914, + "rest_mean": 0.00020494150898971216, + "spike_contrast": 0.6812747128050888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962146767880766, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699988592416048, + 0.0003473494143690914, + 0.0003253649629186839, + 0.0003058057918678969, + 0.0002878782688640058, + 0.0002679917961359024, + 0.0002544210874475539, + 0.00024152429250534624, + 0.00022916354646440595, + 0.000224049566895701, + 0.00020975990628357977, + 0.00019981748482678086, + 0.000197441695490852, + 0.0001882987271528691, + 0.00017711948021315038, + 0.00017049472080543637, + 0.00016402573965024203, + 0.00015721219824627042, + 0.00015173754945863038, + 0.00014811068831477314, + 0.00014414411270990968, + 0.0001423153153154999, + 0.0001392876438330859, + 0.00013716594548895955, + 0.00013519432104658335, + 0.00013375876005738974, + 0.00013254763325676322, + 0.0001317118585575372, + 0.00013049328117631376, + 0.00012993274140171707, + 0.00012922976748086512, + 0.00012818092363886535 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858902861509705, + "first_to_last_ratio": 3.757869257587208, + "imbalance_abs_log_ratio": 1.3238521099622325, + "last_quartile_mean": 4.220184837322449e-05, + "mean": 8.57325238712292e-05, + "normalized": [ + 2.5462257183789148, + 2.3032637538343064, + 2.0924289533379263, + 1.8886492391629932, + 1.7031256747388024, + 1.5374176267930122, + 1.4224231993460965, + 1.3049597038569594, + 1.2055618292025094, + 1.169245405308266, + 1.0769812048488787, + 1.0144334047898542, + 0.9913512297333369, + 0.9317090592952362, + 0.8577499774884012, + 0.81555804918997, + 0.7682725064221458, + 0.7217807351250844, + 0.6783324192661732, + 0.6511800118936032, + 0.6192916242842709, + 0.6053728292385717, + 0.5848720876072855, + 0.5718124973482859, + 0.5462302773879167, + 0.5275173250132701, + 0.5112998770983732, + 0.49691833866078555, + 0.47721658788927634, + 0.46551989467389665, + 0.46012635628943105, + 0.45317260249616614 + ], + "peak_layer": 1, + "peak_normalized": 2.5462257183789148, + "population_cv": 0.5686111305901927, + "rest_mean": 9.231303704803361e-05, + "spike_contrast": 0.5437775023084314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.019775271648541e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021829435718245804, + 0.00019746461475733668, + 0.0001793892151908949, + 0.0001619186659809202, + 0.00014601326256524771, + 0.00013180669338908046, + 0.00012194793089292943, + 0.00011187748896190897, + 0.00010335585830034688, + 0.00010024235962191597, + 9.233231685357168e-05, + 8.69699360919185e-05, + 8.499104296788573e-05, + 7.987776916706935e-05, + 7.353707042057067e-05, + 6.991984992055222e-05, + 6.586594099644572e-05, + 6.188008410390466e-05, + 5.8155150327365845e-05, + 5.582730591413565e-05, + 5.309343396220356e-05, + 5.190014053368941e-05, + 5.014256021240726e-05, + 4.90229285787791e-05, + 4.682970029534772e-05, + 4.522539165918715e-05, + 4.383502891869284e-05, + 4.2602063331287354e-05, + 4.0912982512963936e-05, + 3.991019548266195e-05, + 3.944779382436536e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "rest_mean": 0.00015645541295978345, + "spike_contrast": 0.7816308178118705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229037238284946, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837944976985455, + "first_to_last_ratio": 1.8345658800697515, + "imbalance_abs_log_ratio": 0.6068078758989703, + "last_quartile_mean": 0.10268339328467846, + "mean": 0.1340121782850474, + "normalized": [ + 1.7717451751532867, + 1.6366747548098712, + 1.5242756140248972, + 1.4186514377480737, + 1.3264266357207033, + 1.2453739091726768, + 1.1893513017871604, + 1.1330134634090196, + 1.0858604665419038, + 1.0628542749024097, + 1.0232520332438115, + 0.9942070832344971, + 0.9784308561813967, + 0.9531084116998004, + 0.9232390798282929, + 0.9040072094634526, + 0.8853775581660035, + 0.8680181154203164, + 0.8515367049614931, + 0.8395794986042229, + 0.8267439263823189, + 0.8185133396785964, + 0.809173050926764, + 0.8007913534932437, + 0.7909900046670136, + 0.782107217826699, + 0.7753403703776178, + 0.7682422246066585, + 0.7604971603471317, + 0.7549863443945886, + 0.7508596538022533, + 0.7467717694238248 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451751532867, + "population_cv": 0.27141865758333467, + "rest_mean": 0.13874622461972413, + "spike_contrast": 0.7816308398662988, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844832807779312, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23743543028831482, + 0.21933434903621674, + 0.20427149534225464, + 0.19011656939983368, + 0.17775732278823853, + 0.16689527034759521, + 0.15938755869865417, + 0.15183760225772858, + 0.14551852643489838, + 0.14243541657924652, + 0.13712823390960693, + 0.1332358568906784, + 0.1311216503381729, + 0.12772813439369202, + 0.1237252801656723, + 0.12114797532558441, + 0.1186513751745224, + 0.1163249984383583, + 0.11411628872156143, + 0.11251387745141983, + 0.11079375445842743, + 0.10969075560569763, + 0.10843904316425323, + 0.107315793633461, + 0.10600229352712631, + 0.104811891913414, + 0.10390505194664001, + 0.10295381397008896, + 0.1019158810377121, + 0.10117736458778381, + 0.10062433779239655, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873207303215167e-05, + "first_to_last_ratio": 2.248757814140914, + "imbalance_abs_log_ratio": 0.8103779811585758, + "last_quartile_mean": 3.0564462122129044e-05, + "mean": 4.6857780716891284e-05, + "normalized": [ + 1.7400119939228862, + 1.6405409364325143, + 1.555121247708426, + 1.4944054279879166, + 1.4327475398601994, + 1.3414100714975594, + 1.2804101716466747, + 1.2499370459040815, + 1.192804641985012, + 1.1608858830017554, + 1.098191983597808, + 1.057744066133373, + 1.0542717513613309, + 1.021869458338179, + 0.9775749297615711, + 0.9570927445976433, + 0.9210091333285576, + 0.8815125232807631, + 0.847820728512489, + 0.8204970180441978, + 0.7871838673851995, + 0.7759041238100245, + 0.7506938949785397, + 0.7421070509174572, + 0.7072987335050381, + 0.6891811912847667, + 0.6695407652827761, + 0.6616333377893114, + 0.6345027924930273, + 0.6212629426015038, + 0.6200302724102036, + 0.614801730639213 + ], + "peak_layer": 1, + "peak_normalized": 1.7400119939228862, + "population_cv": 0.32175451551289735, + "rest_mean": 4.9004235565748616e-05, + "spike_contrast": 0.7196709452950106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5266924533061685e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153310045599937e-05, + 7.687210745643824e-05, + 7.28695304132998e-05, + 7.002452184678987e-05, + 6.713537004543468e-05, + 6.28554989816621e-05, + 5.9997179050697014e-05, + 5.8569276006892323e-05, + 5.589217835222371e-05, + 5.439653614303097e-05, + 5.1458839152473956e-05, + 4.9563539505470544e-05, + 4.9400834541302174e-05, + 4.788253500009887e-05, + 4.58069916930981e-05, + 4.4847241952084005e-05, + 4.315644400776364e-05, + 4.130572051508352e-05, + 3.972699778387323e-05, + 3.8446669350378215e-05, + 3.688568904181011e-05, + 3.635714529082179e-05, + 3.5175849916413426e-05, + 3.477348946034908e-05, + 3.3142448955914006e-05, + 3.2293501135427505e-05, + 3.13731943606399e-05, + 3.1002669857116416e-05, + 2.9731392714893445e-05, + 2.9111002731951885e-05, + 2.905324254243169e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017190765356644988, + "first_to_last_ratio": 1.4894621368061003, + "imbalance_abs_log_ratio": 0.3984150727783497, + "last_quartile_mean": 0.00011541592721187044, + "mean": 0.00013894661265112518, + "normalized": [ + 1.3739040544627474, + 1.3296102750467884, + 1.2883342981472405, + 1.2499918159027226, + 1.2130627732820631, + 1.175544413473833, + 1.1474083142393565, + 1.1199112790671786, + 1.092909618037032, + 1.0755929927697745, + 1.0512861461262244, + 1.0312571117882559, + 1.0194388253067554, + 1.0019061415077835, + 0.980344379776871, + 0.9647961249933756, + 0.9493075664595542, + 0.9344817410273759, + 0.9198929201794295, + 0.9079992279088673, + 0.8952920956888009, + 0.8869530532606651, + 0.8771902552778059, + 0.8683889858357474, + 0.8579407806493952, + 0.8482823463732029, + 0.8408658204630546, + 0.8330843095877946, + 0.8243382853592501, + 0.818115422616924, + 0.8135611224181157, + 0.8090075029660142 + ], + "peak_layer": 1, + "peak_normalized": 1.3739040544627474, + "population_cv": 0.16161403917150835, + "rest_mean": 0.00014210757001786045, + "spike_contrast": 0.8576421569620596, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187744287075474, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00019089931447524577, + 0.00018474484386388212, + 0.00017900968668982387, + 0.00017368212866131216, + 0.0001685509632807225, + 0.00016333791427314281, + 0.00015942849859129637, + 0.00015560787869617343, + 0.00015185608936008066, + 0.00014945000293664634, + 0.00014607264893129468, + 0.0001432896824553609, + 0.00014164757158141583, + 0.0001392114645568654, + 0.00013621553080156446, + 0.0001340551534667611, + 0.00013190307072363794, + 0.00012984307250007987, + 0.0001278160052606836, + 0.00012616341700777411, + 0.0001243978040292859, + 0.00012323912233114243, + 0.00012188261462142691, + 0.00012065970804542303, + 0.00011920796532649547, + 0.00011786595860030502, + 0.00011683545744745061, + 0.00011575424287002534, + 0.00011453901242930442, + 0.00011367436673026532, + 0.00011304156214464456, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552358334185556, + "first_to_last_ratio": 1.262638843036328, + "imbalance_abs_log_ratio": 0.2332038508028458, + "last_quartile_mean": 0.00011525352965691127, + "mean": 0.00012953498708156985, + "normalized": [ + 1.1676304712442283, + 1.1525836984023852, + 1.1391521450612143, + 1.1288261090295841, + 1.1167600441451297, + 1.1054423860149858, + 1.094731137320412, + 1.082319740349084, + 1.067957453000188, + 1.0563935463552665, + 1.04560972624621, + 1.0337959762176818, + 1.0235535208863231, + 1.0159040893770892, + 1.0050628637064216, + 0.9950792389307415, + 0.9865814183778941, + 0.9786230528286792, + 0.9704418054120744, + 0.9608381133357455, + 0.9508407831226554, + 0.9429749854457838, + 0.9346839821902475, + 0.9262276149685831, + 0.9170542960948607, + 0.9059488475652709, + 0.8984648923618161, + 0.8908037775288885, + 0.8831483920181569, + 0.8778579808382188, + 0.8744051093280881, + 0.8703028022960919 + ], + "peak_layer": 1, + "peak_normalized": 1.1676304712442283, + "population_cv": 0.0896642712448568, + "rest_mean": 0.00013110964479252557, + "spike_contrast": 0.9231344927669946, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103183544240892, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124899800866842, + 0.00014929991448298097, + 0.00014756005839444697, + 0.00014622247545048594, + 0.00014465949789155275, + 0.00014319346519187093, + 0.00014180598373059183, + 0.00014019827358424664, + 0.00013833785487804562, + 0.00013683992438018322, + 0.0001354430423816666, + 0.00013391274842433631, + 0.0001325859921053052, + 0.00013159512309357524, + 0.00013019080506637692, + 0.00012889757636003196, + 0.00012779681128449738, + 0.00012676592450588942, + 0.00012570616672746837, + 0.00012446215259842575, + 0.00012316714855842292, + 0.00012214825255796313, + 0.00012107427755836397, + 0.00011997888213954866, + 0.00011879061639774591, + 0.00011735207226593047, + 0.0001163826382253319, + 0.0001153902558144182, + 0.00011439861555118114, + 0.00011371332220733166, + 0.00011326605454087257, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029742828155576717, + "first_to_last_ratio": 2.2781890121807993, + "imbalance_abs_log_ratio": 0.8233808344454435, + "last_quartile_mean": 0.00013055469935352448, + "mean": 0.00019329176166138495, + "normalized": [ + 1.8891850069388627, + 1.7801603926271052, + 1.6773544463608039, + 1.566500262900107, + 1.476456430986117, + 1.3754925511314788, + 1.3084504582544147, + 1.236424592515421, + 1.172965600373877, + 1.150871569148191, + 1.0802294364746856, + 1.0319209662753672, + 1.0080063247818516, + 0.9698985570693867, + 0.9067931019504507, + 0.8756163462587222, + 0.8444435806568995, + 0.8113939671049883, + 0.7809677247391495, + 0.7613442367276676, + 0.7402582678385915, + 0.7293562882295931, + 0.7162426695419845, + 0.7062419232587185, + 0.6940340549797547, + 0.6866100033591334, + 0.6831688143117768, + 0.6753320518823956, + 0.6715919825156708, + 0.6686410475337363, + 0.6640239865195754, + 0.6600233567535221 + ], + "peak_layer": 1, + "peak_normalized": 1.8891850069388627, + "population_cv": 0.357915029863326, + "rest_mean": 0.00020341356920888993, + "spike_contrast": 0.6815376252628051, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001386340009048581, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036516389809548855, + 0.0003440903383307159, + 0.0003242187958676368, + 0.0003027915954589844, + 0.00028538686456158757, + 0.00026587137836031616, + 0.0002529126941226423, + 0.00023899068764876574, + 0.00022672458726447076, + 0.00022245399304665625, + 0.0002087994507746771, + 0.00019946182146668434, + 0.00019483931828290224, + 0.00018747340072877705, + 0.00017527563613839447, + 0.00016924942610785365, + 0.00016322398732881993, + 0.00015683576930314302, + 0.0001509546273155138, + 0.00014716156874783337, + 0.0001430858246749267, + 0.0001409785618307069, + 0.00013844380737282336, + 0.00013651074550580233, + 0.00013415106514003128, + 0.00013271605712361634, + 0.00013205090363044292, + 0.00013053612201474607, + 0.00012981319741811603, + 0.00012924280599690974, + 0.00012835036613978446, + 0.00012757707736454904 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.02802751213312149, + 0.01413465291261673, + 0.014225807972252369, + 0.01433455292135477, + 0.01440478302538395, + 0.009703479707241058, + 0.00977881159633398, + 0.009858808480203152, + 0.009950309991836548, + 0.00752287870272994, + 0.007571155205368996, + 0.007693085819482803, + 0.007788818795233965, + 0.006353048607707024, + 0.006389216054230928, + 0.0064707002602517605, + 0.00656962301582098, + 0.005553483963012695, + 0.0056249531917274, + 0.005688921548426151, + 0.005768724251538515, + 0.004966245498508215, + 0.005057469941675663, + 0.005098348017781973, + 0.005216081161051989, + 0.0046758754178881645, + 0.004727107938379049, + 0.004777040798217058, + 0.004837613552808762, + 0.0044186655431985855, + 0.004510571248829365, + 0.0045849340967834 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001580166108396952, + "first_to_last_ratio": 1.9713740353643698, + "imbalance_abs_log_ratio": 0.6787307795042399, + "last_quartile_mean": 8.015557068574708e-05, + "mean": 0.00011843875881822896, + "normalized": [ + 0.8918679597819287, + 1.5814819593588525, + 1.5722239987443252, + 1.3246067564285042, + 1.1252650527356378, + 1.512743589128775, + 1.3788112428745087, + 1.2863035124436846, + 1.097095055843533, + 1.468154353173571, + 1.1523254606068964, + 1.0839888571909224, + 1.054518217028451, + 1.146335080549274, + 1.1665208527784754, + 0.8931136212023528, + 0.8203804970547371, + 1.073741223277434, + 0.919654742394771, + 0.8223971330190447, + 0.7711288002244273, + 0.8470775332584999, + 0.8298711629738247, + 0.7662487463744663, + 0.7210382462310316, + 0.7071376059209309, + 0.7189559392079661, + 0.727980518763353, + 0.6111916361855597, + 0.6986829946379066, + 0.6244678178312321, + 0.6046898327751223 + ], + "peak_layer": 2, + "peak_normalized": 1.5814819593588525, + "population_cv": 0.2904715201464359, + "rest_mean": 0.00012310891098523926, + "spike_contrast": 0.7572151875143335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.321993711637333e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010563173418631777, + 0.0001873087603598833, + 0.00018621225899551064, + 0.00015688478015363216, + 0.0001332749961875379, + 0.00017916747310664505, + 0.00016330469225067645, + 0.00015234819147735834, + 0.00012993857671972364, + 0.00017388637934345752, + 0.0001364799973089248, + 0.0001283862948184833, + 0.00012489582877606153, + 0.00013577050413005054, + 0.00013816128193866462, + 0.00010577926877886057, + 9.71648478298448e-05, + 0.00012717257777694613, + 0.00010892276623053476, + 9.74036956904456e-05, + 9.133153798757121e-05, + 0.0001003268116619438, + 9.828891052166e-05, + 9.07535504666157e-05, + 8.539887494407594e-05, + 8.375250035896897e-05, + 8.515224908478558e-05, + 8.622110908618197e-05, + 7.238877878990024e-05, + 8.275114669231698e-05, + 7.396119326585904e-05, + 7.161871326388791e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001664285200604354, + "first_to_last_ratio": 1.9719337364651075, + "imbalance_abs_log_ratio": 0.6790146534176558, + "last_quartile_mean": 8.439863722742302e-05, + "mean": 0.0001245867656507471, + "normalized": [ + 0.8984334461398115, + 1.5927952483854981, + 1.5658595485525297, + 1.3198735865176374, + 1.131213894321221, + 1.5125315092450449, + 1.3796810696180641, + 1.286366052942558, + 1.0960606282761096, + 1.4536884557404346, + 1.1543342746982923, + 1.0797559630746076, + 1.0497530570909623, + 1.1457182987306291, + 1.1551962694199982, + 0.9023721663305979, + 0.8238710779144349, + 1.0632503993742801, + 0.9211624832492573, + 0.8274335806257342, + 0.7739032412019898, + 0.850990856837025, + 0.8281317613098801, + 0.7681943949939335, + 0.7208251570035091, + 0.7089546837555952, + 0.7203162530736348, + 0.7236174125342815, + 0.6145740290254166, + 0.6994991973458914, + 0.6274381906330746, + 0.6042038120380651 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952483854981, + "population_cv": 0.2889665774838781, + "rest_mean": 0.00012946851159169995, + "spike_contrast": 0.7586812913967177, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.822533756960183e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00011193291720701382, + 0.0001984412083402276, + 0.0001950853766174987, + 0.000164438781212084, + 0.00014093428035266697, + 0.00018844140868168324, + 0.00017189000209327787, + 0.00016026418597903103, + 0.00013655464863404632, + 0.00018111034296453, + 0.00014381477376446128, + 0.0001345233031315729, + 0.00013078533811494708, + 0.00014274133718572557, + 0.00014392216689884663, + 0.00011242362961638719, + 0.00010264343291055411, + 0.0001324669283349067, + 0.00011476465442683548, + 0.00010308727360097691, + 9.641810174798593e-05, + 0.00010602219845168293, + 0.00010317425767425448, + 9.570685506332666e-05, + 8.980527491075918e-05, + 8.832637104205787e-05, + 8.974187221610919e-05, + 9.015315299620852e-05, + 7.656779052922502e-05, + 8.714834257261828e-05, + 7.817049481673166e-05, + 7.527579873567447e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.14759047608822584, + "first_to_last_ratio": 1.9719337947467823, + "imbalance_abs_log_ratio": 0.6790146829732503, + "last_quartile_mean": 0.07484555337578058, + "mean": 0.1104847895912826, + "normalized": [ + 0.8984334974418968, + 1.5927952138130326, + 1.565859646322909, + 1.3198735575362357, + 1.1312139236939245, + 1.5125315007911495, + 1.3796810264894532, + 1.286366010451999, + 1.0960606168617568, + 1.4536884922319049, + 1.1543342013148132, + 1.0797558953237962, + 1.0497530983161607, + 1.1457182562234747, + 1.1551962948455465, + 0.9023721940039697, + 0.8238711061937815, + 1.0632504861804335, + 0.9211625151546727, + 0.8274335809310764, + 0.7739032642161865, + 0.8509909070635955, + 0.8281317391228538, + 0.7681943896830757, + 0.7208251689669645, + 0.7089546589522648, + 0.7203162343708721, + 0.7236173971219663, + 0.6145740294939214, + 0.6994991437375198, + 0.6274381319743458, + 0.6042038211744466 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952138130326, + "population_cv": 0.28896658018674404, + "rest_mean": 0.11481397047086998, + "spike_contrast": 0.7586813040631772, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08710721284151077, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.09926323592662811, + 0.1759796440601349, + 0.1730036735534668, + 0.14582595229148865, + 0.12498193234205246, + 0.16711172461509705, + 0.15243376791477203, + 0.14212387800216675, + 0.12109802663326263, + 0.16061046719551086, + 0.1275363713502884, + 0.1192966029047966, + 0.11598175019025803, + 0.12658444046974182, + 0.12763161957263947, + 0.09969840198755264, + 0.0910252258181572, + 0.11747300624847412, + 0.10177444666624069, + 0.09141882508993149, + 0.08550453931093216, + 0.09402155131101608, + 0.09149596095085144, + 0.08487379550933838, + 0.0796402171254158, + 0.0783287063241005, + 0.07958398759365082, + 0.07994871586561203, + 0.06790108233690262, + 0.07728401571512222, + 0.069322369992733, + 0.06675533205270767 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.093040158499207e-05, + "first_to_last_ratio": 0.8860093039525244, + "imbalance_abs_log_ratio": 0.12102782735710801, + "last_quartile_mean": 5.748291960117058e-05, + "mean": 5.7093861016710434e-05, + "normalized": [ + 0.4337136137178416, + 0.9185569667519601, + 0.9444095926066538, + 0.8728787927927439, + 0.7849140301431445, + 1.1296292591692207, + 1.0726593944235638, + 0.9796135302182263, + 0.9386642177229508, + 1.3046353648934683, + 1.0461832118506482, + 1.0032917361864164, + 0.9662905330123907, + 1.0760080966276753, + 1.1828739682986207, + 0.9087955578115051, + 0.9194941496879511, + 1.198292373503509, + 1.0946132296674007, + 1.0673695485587213, + 0.9244260207272295, + 1.0525679458300632, + 1.125653431968099, + 0.9999504891362015, + 0.9629027708994021, + 1.0063234987710272, + 0.9975933861195996, + 1.0791525148711678, + 0.9412468232151083, + 1.085173475571988, + 1.0031556955576197, + 0.9789667796878816 + ], + "peak_layer": 10, + "peak_normalized": 1.3046353648934683, + "population_cv": 0.1441687540611157, + "rest_mean": 5.695535414647397e-05, + "spike_contrast": 1.0155638391297448, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.784179811598733e-05, + "top_five_layers": [ + 10, + 18, + 15, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.4762384782661684e-05, + 5.2443963795667514e-05, + 5.3919990023132414e-05, + 4.9836020480142906e-05, + 4.481377254705876e-05, + 6.449489592341706e-05, + 6.124226638348773e-05, + 5.592991874436848e-05, + 5.3591964388033375e-05, + 7.448667020071298e-05, + 5.973063889541663e-05, + 5.728179894504137e-05, + 5.516925739357248e-05, + 6.143345672171563e-05, + 6.75348419463262e-05, + 5.1886647270293906e-05, + 5.249747118796222e-05, + 6.84151382301934e-05, + 6.249569560168311e-05, + 6.094024865888059e-05, + 5.277905074763112e-05, + 6.009516800986603e-05, + 6.426790059776977e-05, + 5.7091034250333905e-05, + 5.4975836974335834e-05, + 5.74548939766828e-05, + 5.695645813830197e-05, + 6.161298369988799e-05, + 5.373941530706361e-05, + 6.19567435933277e-05, + 5.727403186028823e-05, + 5.58929932594765e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.801741053801379e-05, + "first_to_last_ratio": 0.954195857595535, + "imbalance_abs_log_ratio": 0.046886327142486896, + "last_quartile_mean": 6.08024129178375e-05, + "mean": 6.296396202287724e-05, + "normalized": [ + 0.4796364558374551, + 0.9568353419130547, + 0.9804638447804884, + 0.8949545354119591, + 0.8237377234766136, + 1.1597846916175047, + 1.085649359633547, + 0.9904450076993521, + 0.9595078978373608, + 1.2941748206155441, + 1.0680753760655664, + 1.0231559828609242, + 0.9904019047543832, + 1.096490963646882, + 1.1850351594596897, + 0.9422215946487361, + 0.9418631353047864, + 1.1705024179981143, + 1.0926353878857977, + 1.0382275689133567, + 0.9320833313178482, + 1.0784583323883348, + 1.0985080196909318, + 0.9917906747078287, + 0.9476388143731139, + 0.9855066582268529, + 0.978396290111904, + 1.028407839812513, + 0.8982055143685478, + 1.0282804954549911, + 0.9451124964564701, + 0.9138123627295472 + ], + "peak_layer": 10, + "peak_normalized": 1.2941748206155441, + "population_cv": 0.13431233496292436, + "rest_mean": 6.285090866084729e-05, + "spike_contrast": 1.0115120295379672, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.357445017783903e-05, + "top_five_layers": [ + 10, + 15, + 18, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.019981159013696e-05, + 6.0246144130360335e-05, + 6.173388828756288e-05, + 5.634988337988034e-05, + 5.186579073779285e-05, + 7.302463927771896e-05, + 6.835678505012766e-05, + 6.236234185053036e-05, + 6.041441884008236e-05, + 8.148637425620109e-05, + 6.725025741616264e-05, + 6.442195444833487e-05, + 6.235962791834027e-05, + 6.903941539349034e-05, + 7.461450877599418e-05, + 5.9326004702597857e-05, + 5.930343468207866e-05, + 7.369946979451925e-05, + 6.879665306769311e-05, + 6.537092122016475e-05, + 5.8687659475253895e-05, + 6.790400948375463e-05, + 6.916641723364592e-05, + 6.244707037694752e-05, + 5.966709431959316e-05, + 6.205140380188823e-05, + 6.160370685392991e-05, + 6.475263216998428e-05, + 5.655457789544016e-05, + 6.474461406469345e-05, + 5.950802733423188e-05, + 5.753724690293893e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7888660295284353e-05, + "first_to_last_ratio": 0.41855462748958633, + "imbalance_abs_log_ratio": 0.8709478659353102, + "last_quartile_mean": 4.2739129185065394e-05, + "mean": 3.1986895351110434e-05, + "normalized": [ + 0.2593955501951102, + 0.4997829486650695, + 0.48480994473028793, + 0.4968760363401999, + 0.5474125689219601, + 0.7788406177150305, + 0.7554473604850604, + 0.6514322393717875, + 0.7578135841604356, + 0.9674810188795085, + 0.9032772556875033, + 0.9019220651946442, + 0.8311356963419073, + 0.9587970177975769, + 1.1401394869948454, + 0.8953886492354074, + 1.0262272842074007, + 1.1413916918814533, + 1.174037958462901, + 1.194927715786966, + 1.0267140632187253, + 1.3434898267078448, + 1.3729067467361273, + 1.2011929483563188, + 1.1735959904166378, + 1.3266745689353452, + 1.2739952988364465, + 1.4049535244931144, + 1.2752889026856997, + 1.4680979746684197, + 1.4182292846928506, + 1.3483241791974143 + ], + "peak_layer": 30, + "peak_normalized": 1.4680979746684197, + "population_cv": 0.3184955307507873, + "rest_mean": 3.066251991657299e-05, + "spike_contrast": 1.276428773763581, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.913852269761264e-05, + "top_five_layers": [ + 30, + 31, + 28, + 23, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.297258318634704e-06, + 1.5986504877218977e-05, + 1.5507564967265353e-05, + 1.589352177688852e-05, + 1.7510028555989265e-05, + 2.491269333404489e-05, + 2.416441566310823e-05, + 2.083729486912489e-05, + 2.4240103812189773e-05, + 3.094671410508454e-05, + 2.889303505071439e-05, + 2.8849686714238487e-05, + 2.6585450541460887e-05, + 3.066893987124786e-05, + 3.6469522456172854e-05, + 2.8640703021665104e-05, + 3.282582474639639e-05, + 3.6509576602838933e-05, + 3.755382931558415e-05, + 3.822202779701911e-05, + 3.2841395295690745e-05, + 4.2974068492185324e-05, + 4.391502443468198e-05, + 3.842243313556537e-05, + 3.7539692129939795e-05, + 4.243620060151443e-05, + 4.0751154301688075e-05, + 4.494010136113502e-05, + 4.0792532672639936e-05, + 4.695989628089592e-05, + 4.5364751713350415e-05, + 4.3128704419359565e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00022091567916504573, + "first_to_last_ratio": 1.232986382535498, + "imbalance_abs_log_ratio": 0.20943917994887448, + "last_quartile_mean": 0.00017917122386279516, + "mean": 0.0002198539991695725, + "normalized": [ + 0.4672441326905094, + 1.0299890566632133, + 1.08986441571188, + 1.0084939820045402, + 0.8814237539069011, + 1.2693826047962031, + 1.1886514905905403, + 1.1035827467935222, + 1.0473307057033197, + 1.4274044616309502, + 1.1442998336482333, + 1.0911283607583675, + 1.1094038041384326, + 1.1946873974128753, + 1.2511466100570947, + 0.9923165311956169, + 0.9239703707694655, + 1.2473578196113966, + 1.1181336030529359, + 1.0160745414007064, + 0.9397394354798955, + 0.9905446518023159, + 1.0121913654482901, + 0.9359943959598221, + 0.8839446941554127, + 0.8510229499852975, + 0.8648513533654478, + 0.8786898836618169, + 0.7455559493639876, + 0.8399702482950041, + 0.7392957942269532, + 0.7163130557190521 + ], + "peak_layer": 10, + "peak_normalized": 1.4274044616309502, + "population_cv": 0.18952709251819305, + "rest_mean": 0.0002217885959436427, + "spike_contrast": 0.9441746799407325, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020940717658959329, + "top_five_layers": [ + 10, + 6, + 15, + 18, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010272549116052687, + 0.00022644721320830286, + 0.00023961105034686625, + 0.00022172143508214504, + 0.0001937845372594893, + 0.0002790788421407342, + 0.0002613297838252038, + 0.00024262708029709756, + 0.00023025984410196543, + 0.000313820579322055, + 0.00025157889467664063, + 0.00023988893372006714, + 0.00024390686303377151, + 0.000262656802078709, + 0.00027506958576850593, + 0.0002181647578254342, + 0.00020313858112785965, + 0.00027423660503700376, + 0.0002458261442370713, + 0.00022338805138133466, + 0.00020660547306761146, + 0.00021777520305477083, + 0.00022253431961871684, + 0.00020578211115207523, + 0.0001943387760547921, + 0.00018710079893935472, + 0.0001901410287246108, + 0.00019318348495289683, + 0.00016391345707233995, + 0.00018467081827111542, + 0.00016253713693004102, + 0.00015748428995721042 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014061377383768559, + 0.014165290631353855, + 0.014281988143920898, + 0.014350161887705326, + 0.00966146495193243, + 0.009741888381540775, + 0.009820535778999329, + 0.00991273857653141, + 0.0074927168898284435, + 0.007546746172010899, + 0.00766373984515667, + 0.007762390188872814, + 0.006325984839349985, + 0.006375366821885109, + 0.006445344537496567, + 0.006555322092026472, + 0.005531130358576775, + 0.005611324217170477, + 0.0056777093559503555, + 0.00574630219489336, + 0.004946626722812653, + 0.0050392793491482735, + 0.005084683652967215, + 0.005203457549214363, + 0.004667261149734259, + 0.004717131145298481, + 0.004768412560224533, + 0.004826710093766451, + 0.0044052014127373695, + 0.004504534415900707, + 0.004574235528707504, + 0.004644096828997135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.729895206561196e-05, + "first_to_last_ratio": 1.2469721805352876, + "imbalance_abs_log_ratio": 0.22071835733540474, + "last_quartile_mean": 2.9911615229138988e-05, + "mean": 3.340667609563752e-05, + "normalized": [ + 1.308752778077389, + 1.2456464596610868, + 1.0236956931411572, + 0.9922285647386262, + 1.26903181062375, + 1.1170846373362797, + 1.0287316533628064, + 0.9469237624012169, + 1.1300261751959062, + 1.1132698789554223, + 1.0126240729566007, + 0.9379202574439188, + 1.1130503370514597, + 1.070713488608195, + 0.9897642722064988, + 0.922085688720865, + 1.036831181462568, + 0.9898105546019274, + 0.9395130252451869, + 0.9001967292772246, + 0.9975952535130206, + 0.9404117749145336, + 0.9224862220158919, + 0.8885787590271026, + 0.9461670180110204, + 0.8974372095121397, + 0.8904691498559152, + 0.8684437934704528, + 0.9244147277585869, + 0.9015689206268677, + 0.8740901912628669, + 0.8604359589635168 + ], + "peak_layer": 1, + "peak_normalized": 1.308752778077389, + "population_cv": 0.1164905978879517, + "rest_mean": 3.378375206213807e-05, + "spike_contrast": 0.9285666618330386, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.137046587653458e-05, + "top_five_layers": [ + 1, + 5, + 2, + 9, + 6 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.3721080146497115e-05, + 4.161290780757554e-05, + 3.419827044126578e-05, + 3.314705827506259e-05, + 4.239413465256803e-05, + 3.73180846509058e-05, + 3.4366505133220926e-05, + 3.1633575417799875e-05, + 3.7750418414361775e-05, + 3.719064625329338e-05, + 3.3828404411906376e-05, + 3.1332798243965954e-05, + 3.718331208801828e-05, + 3.5768978705164045e-05, + 3.306473445263691e-05, + 3.080381793552078e-05, + 3.463708344497718e-05, + 3.3066280593629926e-05, + 3.138600732199848e-05, + 3.007258055731654e-05, + 3.332634150865488e-05, + 3.1416031561093405e-05, + 3.0817198421573266e-05, + 2.9684462788281962e-05, + 3.1608295103069395e-05, + 2.9980394174344838e-05, + 2.9747614462394267e-05, + 2.9011820515734144e-05, + 3.0881623388268054e-05, + 3.01184209092753e-05, + 2.9200447897892445e-05, + 2.874430538213346e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.705871965095866e-05, + "first_to_last_ratio": 0.7180297289018083, + "imbalance_abs_log_ratio": 0.33124430563536084, + "last_quartile_mean": 9.33926785364747e-05, + "mean": 8.228966771639534e-05, + "normalized": [ + 0.8685826408392191, + 0.8249006353131848, + 0.7154905339803773, + 0.6884730939760333, + 0.9470487095509537, + 0.8638069620260335, + 0.8256155016985178, + 0.7853664463613442, + 0.9851686390305121, + 0.9692511243547932, + 0.9083804947955202, + 0.8729855457896263, + 1.0422548534804916, + 1.0255492616133721, + 0.9871528463964067, + 0.9398223255303129, + 1.0958375595009802, + 1.0726325633807163, + 1.0495259774381736, + 1.0230838788771643, + 1.1576441025653552, + 1.1142532603054807, + 1.1008567438213033, + 1.056908774400763, + 1.159063578749664, + 1.1192434432436902, + 1.1032241584740294, + 1.1060435702772593, + 1.1786235078649367, + 1.1628365878275773, + 1.141840825041133, + 1.1085318534950752 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235078649367, + "population_cv": 0.1341844256762087, + "rest_mean": 8.049536754023629e-05, + "spike_contrast": 1.1426606459269575, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.197888866765424e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.14753768988885e-05, + 6.788079917896539e-05, + 5.887747829547152e-05, + 5.66542221349664e-05, + 7.793232362018898e-05, + 7.108238787623122e-05, + 6.793962529627606e-05, + 6.462754390668124e-05, + 8.106919995043427e-05, + 7.975935295689851e-05, + 7.475032907677814e-05, + 7.183769048424438e-05, + 8.576680556870997e-05, + 8.4392107964959e-05, + 8.123247971525416e-05, + 7.733766688033938e-05, + 9.017610864248127e-05, + 8.826657722238451e-05, + 8.636514394311234e-05, + 8.418923243880272e-05, + 9.526214853394777e-05, + 9.169153054244816e-05, + 9.0589135652408e-05, + 8.697267185198143e-05, + 9.537895675748587e-05, + 9.210217103827745e-05, + 9.078394941752777e-05, + 9.101595787797123e-05, + 9.698853682493791e-05, + 9.56894364207983e-05, + 9.396170207764953e-05, + 9.122071787714958e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05946834245696664, + "first_to_last_ratio": 0.7180297222365345, + "imbalance_abs_log_ratio": 0.33124431491808765, + "last_quartile_mean": 0.08282156102359295, + "mean": 0.07297530013602227, + "normalized": [ + 0.8685826122359729, + 0.8249005917343738, + 0.71549052911611, + 0.6884731350635278, + 0.9470486699630107, + 0.8638070113577097, + 0.8256154769796886, + 0.7853664269052657, + 0.9851686341386658, + 0.9692511556795396, + 0.9083805454214465, + 0.8729855579949275, + 1.0422548040385637, + 1.0255492365750694, + 0.9871528950273681, + 0.9398223256645052, + 1.0958375053304956, + 1.07263252629438, + 1.0495259690486791, + 1.0230837911446407, + 1.1576441435704576, + 1.1142533045483842, + 1.1008568124470575, + 1.0569088284972943, + 1.1590636022344982, + 1.119243411839159, + 1.1032241423789504, + 1.1060435591240758, + 1.1786235037173105, + 1.1628366076958818, + 1.141840809884862, + 1.1085318743481278 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235037173105, + "population_cv": 0.13418442668798974, + "rest_mean": 0.07138409617322462, + "spike_contrast": 1.1426607032635474, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08156780153512955, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.06338507682085037, + 0.0601973682641983, + 0.05221313610672951, + 0.050241533666849136, + 0.0691111609339714, + 0.06303657591342926, + 0.060249537229537964, + 0.05731235072016716, + 0.07189297676086426, + 0.07073139399290085, + 0.06628934293985367, + 0.06370638310909271, + 0.07605885714292526, + 0.0748397633433342, + 0.07203777879476547, + 0.06858381628990173, + 0.07996907085180283, + 0.07827568054199219, + 0.07658947259187698, + 0.07465984672307968, + 0.0844794288277626, + 0.08131296932697296, + 0.0803353562951088, + 0.07712823897600174, + 0.08458301424980164, + 0.08167712390422821, + 0.08050811290740967, + 0.08071386069059372, + 0.0860104039311409, + 0.08485835045576096, + 0.08332617580890656, + 0.08089544624090195 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.408341275549901e-06, + "first_to_last_ratio": 0.49124437919492936, + "imbalance_abs_log_ratio": 0.7108135577065074, + "last_quartile_mean": 1.5080765479069669e-05, + "mean": 1.1844782648040564e-05, + "normalized": [ + 0.5657990507718705, + 0.5382233871697658, + 0.49942145823009426, + 0.48906984415931193, + 0.8062921756393574, + 0.7122953827323945, + 0.7013508394120195, + 0.6911626993599347, + 0.8940675881372303, + 0.868722127857376, + 0.9735911331661544, + 0.854855860776803, + 0.989483587379368, + 1.0059088486455374, + 1.0303729006301217, + 0.9941025538400318, + 1.1620912657632405, + 1.127012997009919, + 1.113401270263767, + 1.066097770938846, + 1.1826034542812638, + 1.1643547935214302, + 1.2049778184223279, + 1.1791491538213539, + 1.2365172470752577, + 1.219921447624247, + 1.2337761202582145, + 1.2288278254433027, + 1.3039217543779833, + 1.3631114810840155, + 1.2856635723507386, + 1.3138525898567213 + ], + "peak_layer": 30, + "peak_normalized": 1.3631114810840155, + "population_cv": 0.2549101540724733, + "rest_mean": 1.1420299651271543e-05, + "spike_contrast": 1.237882653019464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4136990830593277e-05, + "top_five_layers": [ + 30, + 32, + 29, + 31, + 25 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.701766778860474e-06, + 6.3751390371180605e-06, + 5.915538622502936e-06, + 5.7929260037781205e-06, + 9.550355571263935e-06, + 8.436983989668079e-06, + 8.307348252856173e-06, + 8.186671948351432e-06, + 1.0590036254143342e-05, + 1.0289824786013924e-05, + 1.1531975360412616e-05, + 1.0125581866304856e-05, + 1.1720218026312068e-05, + 1.1914771675947122e-05, + 1.220454305439489e-05, + 1.1774928680097219e-05, + 1.3764718460151926e-05, + 1.334922399109928e-05, + 1.3187996046326589e-05, + 1.2627696378331166e-05, + 1.4007680874783546e-05, + 1.379152945446549e-05, + 1.4272700354922563e-05, + 1.3966765436634887e-05, + 1.4646278032159898e-05, + 1.4449704394792207e-05, + 1.4613809980801307e-05, + 1.4555198504240252e-05, + 1.5444669770658948e-05, + 1.614575921848882e-05, + 1.5228405572997872e-05, + 1.5562298358418047e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6029601637419546e-05, + "first_to_last_ratio": 0.6070213107493427, + "imbalance_abs_log_ratio": 0.49919138022039955, + "last_quartile_mean": 9.230252817360451e-05, + "mean": 7.840498903988191e-05, + "normalized": [ + 0.6598133090619259, + 0.6419677920829977, + 0.6005256373772865, + 0.5913463573822231, + 0.8562376513989928, + 0.8062968559250673, + 0.7923156566726156, + 0.768439227921366, + 0.9790122729268586, + 0.9665730330981341, + 0.9163710065350301, + 0.8881686280668529, + 1.0631190324760253, + 1.0511030519715439, + 1.0188080221730096, + 0.9712812246823179, + 1.1361272291219966, + 1.1138561419734825, + 1.0918134348286874, + 1.0639217496772315, + 1.203414696914147, + 1.1592426077267617, + 1.1442664103435203, + 1.097952968622282, + 1.2042281788778202, + 1.1625445130141736, + 1.1452986211445626, + 1.1473932954411665, + 1.2233942816523278, + 1.205318760675343, + 1.1829469858778916, + 1.146901364356358 + ], + "peak_layer": 29, + "peak_normalized": 1.2233942816523278, + "population_cv": 0.1898597249088972, + "rest_mean": 7.605543874820728e-05, + "spike_contrast": 1.1977126437531995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.109256061492487e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.173265526536852e-05, + 5.033347770222463e-05, + 4.708420601673424e-05, + 4.636450466932729e-05, + 6.713330367347226e-05, + 6.321769615169615e-05, + 6.212150037754327e-05, + 6.024946924299002e-05, + 7.675944652874023e-05, + 7.578414806630462e-05, + 7.184805872384459e-05, + 6.963685154914856e-05, + 8.335383608937263e-05, + 8.241172326961532e-05, + 7.987963181221858e-05, + 7.615329377586022e-05, + 8.907804294722155e-05, + 8.733187860343605e-05, + 8.560362039133906e-05, + 8.341677312273532e-05, + 9.435371612198651e-05, + 9.089040395338088e-05, + 8.971619536168873e-05, + 8.608499047113582e-05, + 9.441749716643244e-05, + 9.114928980125114e-05, + 8.97971258382313e-05, + 8.996135875349864e-05, + 9.592021524440497e-05, + 9.450300422031432e-05, + 9.274894546251744e-05, + 8.992278890218586e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.8511469685763586e-05, + "first_to_last_ratio": 0.5271352332510719, + "imbalance_abs_log_ratio": 0.6402981537748061, + "last_quartile_mean": 9.202850924339145e-05, + "mean": 7.532146378252946e-05, + "normalized": [ + 0.548104082267993, + 0.5416830199005969, + 0.528884702689083, + 0.5247448675895723, + 0.7777813723908718, + 0.7487613756360456, + 0.7496059866168319, + 0.7329069161583202, + 0.9480676391902623, + 0.9399123881532756, + 0.9000936195631334, + 0.883441930780847, + 1.0560589316963704, + 1.0534552095740957, + 1.032095781768693, + 0.9835620691098637, + 1.1601754560133901, + 1.1467788545782247, + 1.1233340534265408, + 1.0967375255670042, + 1.2396083970871608, + 1.1951467072205837, + 1.1828066060767042, + 1.1317734013234224, + 1.244698956870482, + 1.2025862582796845, + 1.1881970083126314, + 1.1882256015362047, + 1.2689460104605979, + 1.2589162611011824, + 1.2307315535498626, + 1.1921774555104672 + ], + "peak_layer": 29, + "peak_normalized": 1.2689460104605979, + "population_cv": 0.23486334953575583, + "rest_mean": 7.254842300123225e-05, + "spike_contrast": 1.244629176846487, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.029588400153444e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 4.1284001781605184e-05, + 4.080035796505399e-05, + 3.9836369978729635e-05, + 3.952455153921619e-05, + 5.858363147126511e-05, + 5.639780283672735e-05, + 5.6461420172126964e-05, + 5.520362174138427e-05, + 7.140984234865755e-05, + 7.079557690303773e-05, + 6.77963689668104e-05, + 6.654213939327747e-05, + 7.954390457598493e-05, + 7.934778841445222e-05, + 7.773896504659206e-05, + 7.408333476632833e-05, + 8.738611359149218e-05, + 8.637706196168438e-05, + 8.461116522084922e-05, + 8.26078758109361e-05, + 9.336911898571998e-05, + 9.002019942272455e-05, + 8.909072494134307e-05, + 8.524682925781235e-05, + 9.375254740007222e-05, + 9.058055729838088e-05, + 8.949673792812973e-05, + 8.949889161158353e-05, + 9.557887096889317e-05, + 9.482341556577012e-05, + 9.270050213672221e-05, + 8.979655103757977e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.61669284151867e-05, + "first_to_last_ratio": 0.8297833624837898, + "imbalance_abs_log_ratio": 0.18659062131596024, + "last_quartile_mean": 0.00011589401856326731, + "mean": 0.00010716335873439675, + "normalized": [ + 0.9380742209455786, + 0.8898820858744699, + 0.7837417114644826, + 0.7583102664636657, + 1.0822070619439073, + 0.9520749081713696, + 0.9107982467806989, + 0.8640017807748708, + 1.0669538303069324, + 1.0266950649381863, + 0.9594029855154438, + 0.8901506143407661, + 1.0795289063552718, + 1.0379165674077444, + 0.9706971366471856, + 0.914979822300863, + 1.052939971267624, + 1.0225707935964177, + 0.9851238030379684, + 0.9636652893153618, + 1.0990044510799908, + 1.0523904893769196, + 1.0422234788441622, + 1.0049019599938585, + 1.0962247910521359, + 1.0599378706264646, + 1.05039930496957, + 1.0583562345651827, + 1.1199788667078585, + 1.1100529546792268, + 1.0850973923636869, + 1.0717171382921347 + ], + "peak_layer": 29, + "peak_normalized": 1.1199788667078585, + "population_cv": 0.09156717959150794, + "rest_mean": 0.00010599351123716958, + "spike_contrast": 1.0706366257223145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011348053521942347, + "top_five_layers": [ + 29, + 30, + 21, + 25, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010052718425868079, + 9.536275319987908e-05, + 8.398839418077841e-05, + 8.12630751170218e-05, + 0.00011597294360399246, + 0.00010202754492638633, + 9.760419925441965e-05, + 9.25893327803351e-05, + 0.00011433835607022047, + 0.00011002409155480564, + 0.00010281284630764276, + 9.539152961224318e-05, + 0.000115685943455901, + 0.0001112266254494898, + 0.0001040231654769741, + 9.805231093196198e-05, + 0.00011283658386673778, + 0.00010958212078548968, + 0.00010556917550275102, + 0.00010326960909878835, + 0.00011777300824178383, + 0.0001127776995417662, + 0.00011168816854478791, + 0.00010768866923172027, + 0.00011747513053705916, + 0.00011358650226611644, + 0.00011256431753281504, + 0.00011341700883349404, + 0.00012002069706795737, + 0.00011895700299646705, + 0.00011628268111962825, + 0.00011484880815260112 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 7.959341947457508, + "forward": { + "activation_sha256": { + "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" + }, + "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", + "loss_nats": 5.516995429992676, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0021613112185150385, + 0.0025601957459002733, + 0.0025323182344436646, + 0.0027253651060163975, + 0.002904455177485943, + 0.0029208383057266474, + 0.00299624796025455, + 0.0030937574338167906, + 0.0033804215490818024, + 0.0034370317589491606, + 0.0038969130255281925, + 0.004255065228790045, + 0.004119740333408117, + 0.003970506135374308, + 0.004385598469525576, + 0.0041999719105660915, + 0.005232699681073427, + 0.004664790350943804, + 0.004898969549685717, + 0.005131533369421959, + 0.005069608334451914, + 0.0051318020559847355, + 0.0058372304774820805, + 0.0059690922498703, + 0.00554921617731452, + 0.0058550527319312096, + 0.006255472078919411, + 0.006502739619463682, + 0.006122888531535864, + 0.007071305997669697, + 0.006241494789719582 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962477027642308, + "first_to_last_ratio": 3.7699040072151497, + "imbalance_abs_log_ratio": 1.3270495388586014, + "last_quartile_mean": 4.234186599205714e-05, + "mean": 8.617327489446325e-05, + "normalized": [ + 2.5428430303860874, + 2.3222064178412967, + 2.1011513502014654, + 1.8911470095267005, + 1.7004447966680725, + 1.5353700031893112, + 1.4229335846259188, + 1.3028620492588128, + 1.2049362956920768, + 1.1705197150696323, + 1.0773612761873523, + 1.0133200843661747, + 0.9906112963967714, + 0.9272530070793641, + 0.8550207003106389, + 0.8185600560490035, + 0.7703816559177782, + 0.7173918665339755, + 0.6760157258502337, + 0.6514535263006792, + 0.6184138115448249, + 0.6053606047045522, + 0.5838821473278404, + 0.5697017460729616, + 0.5457761369570343, + 0.5276086337508958, + 0.5104655265174319, + 0.4951752820428687, + 0.47527333481689216, + 0.465789239220051, + 0.45896414036514044, + 0.45180594922816086 + ], + "peak_layer": 1, + "peak_normalized": 2.5428430303860874, + "population_cv": 0.5705341151299013, + "rest_mean": 9.280180475562466e-05, + "spike_contrast": 0.5428688997682254, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0379213644191625e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021912511147093028, + 0.00020011213200632483, + 0.00018106309289578348, + 0.00016296633111778647, + 0.00014653289690613747, + 0.00013230786134954542, + 0.0001226188469445333, + 0.00011227188952034339, + 0.0001038333066389896, + 0.00010086751717608422, + 9.283974941354245e-05, + 8.732111018616706e-05, + 8.536421955795959e-05, + 7.990442827576771e-05, + 7.367993384832516e-05, + 7.053800072753802e-05, + 6.63863102090545e-05, + 6.182000652188435e-05, + 5.825448897667229e-05, + 5.6137883802875876e-05, + 5.329074338078499e-05, + 5.216590579948388e-05, + 5.031503678765148e-05, + 4.909306517220102e-05, + 4.703131708083674e-05, + 4.5465763832908124e-05, + 4.3988486140733585e-05, + 4.2670875700423494e-05, + 4.095585973118432e-05, + 4.013858415419236e-05, + 3.955044303438626e-05, + 3.893359826179221e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442661636683624, + "first_to_last_ratio": 1.843144421770586, + "imbalance_abs_log_ratio": 0.6114730379455638, + "last_quartile_mean": 0.0001163373926829081, + "mean": 0.00015221274247778638, + "normalized": [ + 1.7741446897870732, + 1.648461275666186, + 1.5301464224684427, + 1.4217121808823485, + 1.3265065496562456, + 1.2454420544341354, + 1.1897998831173078, + 1.1336248235930768, + 1.0869089599614004, + 1.0643003125599004, + 1.0227030055526172, + 0.9941765648180964, + 0.9788958472869692, + 0.9511102301475627, + 0.9228060651723192, + 0.9052418824754537, + 0.8846484396889256, + 0.865292475366091, + 0.8497311645646503, + 0.8388691935254817, + 0.8256303537004397, + 0.8176325372729673, + 0.8076852907347736, + 0.8000670639426607, + 0.7892588692968658, + 0.7814415503461742, + 0.7731202151361662, + 0.7664994567889658, + 0.758490072461763, + 0.7529625284402992, + 0.7485174912671357, + 0.7441725498875049 + ], + "peak_layer": 1, + "peak_normalized": 1.7741446897870732, + "population_cv": 0.27364872432853743, + "rest_mean": 0.00015762320577373935, + "spike_contrast": 0.7803181014868814, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299624067964033, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027004742878489196, + 0.0002509168116375804, + 0.0002329077833564952, + 0.00021640271006617695, + 0.00020191119983792305, + 0.00018957215070258826, + 0.0001811027032090351, + 0.00017255214333999902, + 0.00016544139361940324, + 0.00016200006939470768, + 0.00015566842921543866, + 0.0001513263414381072, + 0.00014900042151566595, + 0.0001447710965294391, + 0.00014046284195501357, + 0.0001377893495373428, + 0.00013465476513374597, + 0.00013170854072086513, + 0.00012933991092722863, + 0.0001276865805266425, + 0.00012567146040964872, + 0.00012445409083738923, + 0.00012293999316170812, + 0.00012178040196886286, + 0.00012013525702059269, + 0.00011894536146428436, + 0.00011767874821089208, + 0.00011667098442558199, + 0.00011545185407157987, + 0.00011461049143690616, + 0.00011393390013836324, + 0.00011327254469506443 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19015566632151604, + "first_to_last_ratio": 1.8431444167877922, + "imbalance_abs_log_ratio": 0.6114730352421437, + "last_quartile_mean": 0.10316916275769472, + "mean": 0.13498378219082952, + "normalized": [ + 1.7741447141792424, + 1.6484611693507762, + 1.5301464164964773, + 1.421712099711509, + 1.3265065258282553, + 1.245442086656075, + 1.1897999853143832, + 1.1336247998774232, + 1.0869090145314644, + 1.0643003539468234, + 1.0227030148964142, + 0.9941765578539046, + 0.978895844117059, + 0.9511102293531465, + 0.9228060470726628, + 0.9052419251832852, + 0.8846484748200404, + 0.8652925213936268, + 0.8497311905993837, + 0.8388692019864461, + 0.8256303029992552, + 0.8176324960789627, + 0.8076852727084025, + 0.800067049482784, + 0.789258877083647, + 0.7814415614650243, + 0.7731201949085273, + 0.7664994759042474, + 0.758490077799444, + 0.7529625178867759, + 0.7485174642151933, + 0.7441725362993381 + ], + "peak_layer": 1, + "peak_normalized": 1.7741447141792424, + "population_cv": 0.2736487184871982, + "rest_mean": 0.13978183573042904, + "spike_contrast": 0.7803180757143806, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907429307699204, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23948076367378235, + 0.2225155234336853, + 0.20654495060443878, + 0.1919080764055252, + 0.17905686795711517, + 0.16811448335647583, + 0.16060370206832886, + 0.1530209630727768, + 0.14671508967876434, + 0.14366328716278076, + 0.13804832100868225, + 0.13419771194458008, + 0.1321350634098053, + 0.12838445603847504, + 0.12456385046243668, + 0.12219297885894775, + 0.11941319704055786, + 0.116800457239151, + 0.11469992995262146, + 0.11323373764753342, + 0.11144670099020004, + 0.11036712676286697, + 0.10902441293001175, + 0.10799607634544373, + 0.10653714835643768, + 0.10548193752765656, + 0.10435868799686432, + 0.1034649983048439, + 0.10238385945558548, + 0.10163772851228714, + 0.10103771835565567, + 0.10045122355222702 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883365585963475e-05, + "first_to_last_ratio": 2.2497546817252747, + "imbalance_abs_log_ratio": 0.8108211799277664, + "last_quartile_mean": 3.059607183786284e-05, + "mean": 4.691841348858361e-05, + "normalized": [ + 1.7359303646705315, + 1.6475859007944922, + 1.5536416135247637, + 1.4921074478374423, + 1.436565604934822, + 1.343834173349448, + 1.2787439414688093, + 1.2483315192759759, + 1.191227587726315, + 1.1603086071531024, + 1.0995306735006558, + 1.059977946939621, + 1.0583984121521266, + 1.0176934649371145, + 0.9754909410255406, + 0.9631621800279858, + 0.9242301483750994, + 0.8752327792179452, + 0.8454160845678106, + 0.8232909667825862, + 0.7863453139125789, + 0.77746685672799, + 0.7497386589361914, + 0.7388508716017261, + 0.7093180450413769, + 0.6908354487677191, + 0.6685471901846501, + 0.6592354484095886, + 0.631440334976849, + 0.6235901145414114, + 0.6199506557967245, + 0.6139807028410055 + ], + "peak_layer": 1, + "peak_normalized": 1.7359303646705315, + "population_cv": 0.3220412433027962, + "rest_mean": 4.907019663530971e-05, + "spike_contrast": 0.7193528234378936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.529878449626267e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144709863699973e-05, + 7.730211655143648e-05, + 7.289439963642508e-05, + 7.000731420703232e-05, + 6.740137905580923e-05, + 6.305056740529835e-05, + 5.999663699185476e-05, + 5.856973439222202e-05, + 5.5890508519951254e-05, + 5.4439839004771784e-05, + 5.1588234782684594e-05, + 4.9732483603293076e-05, + 4.965837433701381e-05, + 4.774856279254891e-05, + 4.576848732540384e-05, + 4.519004141911864e-05, + 4.3363412260077894e-05, + 4.106453343410976e-05, + 3.966558142565191e-05, + 3.862750600092113e-05, + 3.6894074582960457e-05, + 3.647751145763323e-05, + 3.517654840834439e-05, + 3.4665710700210184e-05, + 3.328007733216509e-05, + 3.241290323785506e-05, + 3.136717350571416e-05, + 3.0930281354812905e-05, + 2.9626178729813546e-05, + 2.925785884144716e-05, + 2.9087101211189292e-05, + 2.8807000489905477e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001734125216898974, + "first_to_last_ratio": 1.495517223041696, + "imbalance_abs_log_ratio": 0.4024721155990803, + "last_quartile_mean": 0.00011595488103921525, + "mean": 0.00013990532806928968, + "normalized": [ + 1.3768889945483427, + 1.3340940553978193, + 1.2910049686488183, + 1.2527203273458163, + 1.2147536525239127, + 1.1769854337196524, + 1.14744939519443, + 1.1220955988146075, + 1.0944022472959734, + 1.0773022658134643, + 1.0508559212107247, + 1.03206209473486, + 1.0208171898483585, + 1.000561882500425, + 0.9805161605170362, + 0.9663129296065233, + 0.9487424991891081, + 0.931931984738623, + 0.9181974345490227, + 0.9075104531838204, + 0.8943640941785878, + 0.8862664021756573, + 0.8757830774574916, + 0.86790401991425, + 0.8562894022841879, + 0.8478155487572495, + 0.8387227683262297, + 0.8314011142413483, + 0.8224071457687988, + 0.8161875050523302, + 0.8112199678685613, + 0.8064334645939688 + ], + "peak_layer": 1, + "peak_normalized": 1.3768889945483427, + "population_cv": 0.16325957486062354, + "rest_mean": 0.0001431148236776978, + "spike_contrast": 0.8564734849544942, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225740517838858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926341064972803, + 0.00018664686649572104, + 0.00018061847367789596, + 0.00017526224837638438, + 0.0001699505082797259, + 0.00016466653323732316, + 0.00016053428407758474, + 0.00015698715287726372, + 0.00015311270544771105, + 0.00015072032692842185, + 0.00014702034241054207, + 0.00014439098595175892, + 0.00014281776384450495, + 0.00013998393842484802, + 0.00013717943511437625, + 0.00013519232743419707, + 0.00013273413060232997, + 0.00013038225006312132, + 0.00012846071331296116, + 0.00012696554767899215, + 0.0001251263020094484, + 0.00012399339175317436, + 0.0001225267187692225, + 0.00012142439663875848, + 0.00011979944974882528, + 0.00011861391249112785, + 0.00011734178406186402, + 0.00011631744564510882, + 0.00011505914153531194, + 0.00011418898066040128, + 0.00011349399574100971, + 0.00011282433843007311 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675117381557357, + "first_to_last_ratio": 1.2676837465168667, + "imbalance_abs_log_ratio": 0.23719141364257146, + "last_quartile_mean": 0.00011576323686313117, + "mean": 0.00013042020850662084, + "normalized": [ + 1.169988069421413, + 1.156175152925158, + 1.1417665260225645, + 1.1306726331060184, + 1.1183621029407895, + 1.1062694829504214, + 1.0940115056230622, + 1.0844989956600726, + 1.0711591662817939, + 1.0579765491085218, + 1.0443900226505625, + 1.035471661013805, + 1.0260498632741444, + 1.013992390086173, + 1.0062479316494695, + 0.9977001177294099, + 0.9863163473759529, + 0.9769959732660659, + 0.9687585669786032, + 0.9602628595878113, + 0.9503845997255937, + 0.9426104617668664, + 0.9330243108845692, + 0.9259760938667744, + 0.914797625470175, + 0.906289421438531, + 0.8956357003805738, + 0.888470885240542, + 0.8806216558595509, + 0.8756991509238615, + 0.8712858841028995, + 0.8681382926882502 + ], + "peak_layer": 1, + "peak_normalized": 1.169988069421413, + "population_cv": 0.09123006364857297, + "rest_mean": 0.00013202972385678786, + "spike_contrast": 0.9219804606102014, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012172882561571896, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015259008796419948, + 0.00015078860451467335, + 0.00014890942838974297, + 0.0001474625605624169, + 0.00014585701865144074, + 0.00014427989663090557, + 0.00014268120867200196, + 0.00014144058513920754, + 0.00013970080181024969, + 0.0001379815221298486, + 0.00013620956451632082, + 0.00013504642993211746, + 0.0001338176371064037, + 0.00013224509893916547, + 0.00013123506505507976, + 0.0001301202573813498, + 0.00012863558367826045, + 0.00012742001854348928, + 0.00012634569429792464, + 0.00012523768236860633, + 0.00012394935765769333, + 0.00012293545296415687, + 0.00012168522516731173, + 0.00012076599523425102, + 0.00011930809705518186, + 0.00011819845531135798, + 0.00011680899478960782, + 0.00011587455810513347, + 0.00011485085997264832, + 0.00011420886585256085, + 0.00011363328667357564, + 0.00011322277714498341 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030002388848515693, + "first_to_last_ratio": 2.2836575149168885, + "imbalance_abs_log_ratio": 0.8257783308640597, + "last_quartile_mean": 0.0001313786706305109, + "mean": 0.0001947244400071213, + "normalized": [ + 1.8998833386658438, + 1.7835645331795598, + 1.6713073472446418, + 1.5703972875779748, + 1.4780250735507643, + 1.3761669788219328, + 1.3066447563403325, + 1.2401013786965793, + 1.1762106459027857, + 1.150547637187856, + 1.0768735950666006, + 1.0264900078881174, + 1.0140784159549234, + 0.9666871251601796, + 0.9096841069789114, + 0.8754807117769676, + 0.8421876125778792, + 0.8073809902734768, + 0.77911667677541, + 0.7607651063686129, + 0.7403822030704592, + 0.7308855611321243, + 0.7152791523187013, + 0.7043381880055872, + 0.6944157299089508, + 0.686864629891504, + 0.680599049276497, + 0.6764772714226681, + 0.6702412094778135, + 0.6672258960044359, + 0.6635863560708457, + 0.6581114274310631 + ], + "peak_layer": 1, + "peak_normalized": 1.8998833386658438, + "population_cv": 0.3592853832280775, + "rest_mean": 0.00020492727344390005, + "spike_contrast": 0.6813594750078023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962913944851608, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699537192005664, + 0.0003473036049399525, + 0.0003254443872720003, + 0.00030579473241232336, + 0.00028780760476365685, + 0.00026797334430739284, + 0.0002544356684666127, + 0.0002414780465187505, + 0.0002290369593538344, + 0.00022403974435292184, + 0.00020969360775779933, + 0.00019988269195891917, + 0.0001974658516701311, + 0.00018823760910890996, + 0.00017713772831484675, + 0.00017047749133780599, + 0.00016399451124016196, + 0.0001572168112033978, + 0.00015171305858530104, + 0.00014813955931458622, + 0.00014417050988413393, + 0.00014232128160074353, + 0.00013928233238402754, + 0.0001371518592350185, + 0.00013521971413865685, + 0.00013374933041632175, + 0.00013252926873974502, + 0.00013172665785532445, + 0.00013051234418526292, + 0.00012992518895771354, + 0.00012921648158226162, + 0.000128150379168801 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027073738165199757, + 0.002722575329244137, + 0.0026850115973502398, + 0.0026861224323511124, + 0.00271838391199708, + 0.0026645169127732515, + 0.0026859452482312918, + 0.0027359756641089916, + 0.0026767742820084095, + 0.0027451682835817337, + 0.0026999011170119047, + 0.002663193503394723, + 0.002626648638397455, + 0.0026531771291047335, + 0.0026517712976783514, + 0.002676903735846281, + 0.002670086920261383, + 0.0026355870068073273, + 0.0026610493659973145, + 0.0026388936676084995, + 0.0026469926815479994, + 0.00264862016774714, + 0.002641494618728757, + 0.002674198243767023, + 0.002557571977376938, + 0.00252733426168561, + 0.0025468147359788418, + 0.0026856842450797558, + 0.002551089273765683, + 0.0025778315030038357, + 0.0026001501828432083, + 0.0025662274565547705 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858857113926206, + "first_to_last_ratio": 3.757867690304466, + "imbalance_abs_log_ratio": 1.3238516928952826, + "last_quartile_mean": 4.220174423608114e-05, + "mean": 8.573221236929385e-05, + "normalized": [ + 2.5461641896458365, + 2.3032502265274117, + 2.0924715218207024, + 1.8886722264368014, + 1.7031663195050915, + 1.537434754996766, + 1.4223955235407997, + 1.3049501874408618, + 1.2055297161045848, + 1.1692542365736316, + 1.0769571962730058, + 1.0144505847411098, + 0.9913601784485884, + 0.9317001386730132, + 0.857757082878185, + 0.8155634736467603, + 0.7682612945963089, + 0.7217806418748633, + 0.678338533281248, + 0.6511820384364709, + 0.6192882306865346, + 0.6053793146733001, + 0.5848721334223365, + 0.5718144052517429, + 0.5462281883927541, + 0.527515083157495, + 0.5112993585545178, + 0.496920907994256, + 0.47722358366421574, + 0.46552107689700334, + 0.4601234027970099, + 0.45317424906679377 + ], + "peak_layer": 1, + "peak_normalized": 2.5461641896458365, + "population_cv": 0.5686091737198813, + "rest_mean": 9.231269224848668e-05, + "spike_contrast": 0.5437781067692293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0197621021652594e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021828828903380781, + 0.00019746273756027222, + 0.00017939221288543195, + 0.0001619200484128669, + 0.00014601621660403907, + 0.000131807682919316, + 0.00012194511509733275, + 0.00011187626660102978, + 0.00010335272963857278, + 0.00010024275252362713, + 9.232992306351662e-05, + 8.697109296917915e-05, + 8.499150135321543e-05, + 7.987671415321529e-05, + 7.353741239057854e-05, + 6.992006092332304e-05, + 6.586474046343938e-05, + 6.187985127326101e-05, + 5.8155463193543255e-05, + 5.582727681030519e-05, + 5.309295011102222e-05, + 5.1900507969548926e-05, + 5.0142381951445714e-05, + 4.902291402686387e-05, + 4.682935104938224e-05, + 4.522503513726406e-05, + 4.3834825191879645e-05, + 4.260212881490588e-05, + 4.0913433622336015e-05, + 3.991015182691626e-05, + 3.944739728467539e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242323236947414, + "first_to_last_ratio": 1.8345651540958352, + "imbalance_abs_log_ratio": 0.6068074801791611, + "last_quartile_mean": 0.0001157894184871111, + "mean": 0.00015111673860701558, + "normalized": [ + 1.771707915810251, + 1.6366668487510905, + 1.5242909365839221, + 1.4186705636795862, + 1.3264449439106374, + 1.2453857912414694, + 1.1893415085703798, + 1.1330085309323608, + 1.0858463851390703, + 1.062861239127963, + 1.023237806768965, + 0.9942126959777651, + 0.978436257513136, + 0.9531027445378829, + 0.9232414968561398, + 0.904013237249231, + 0.8853704564278231, + 0.8680164030054869, + 0.8515402788738432, + 0.8395817783841374, + 0.8267435189825344, + 0.8185143642831039, + 0.809172703595369, + 0.8007918347797284, + 0.7909896391756025, + 0.7821052393452608, + 0.7753388667989797, + 0.7682453290925442, + 0.7604995795745633, + 0.7549901567959042, + 0.7508572350862155, + 0.7467737131490535 + ], + "peak_layer": 1, + "peak_normalized": 1.771707915810251, + "population_cv": 0.2714170004285523, + "rest_mean": 0.00015645500981783562, + "spike_contrast": 0.7816309251520459, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229007406858728, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677347220014781, + 0.00024732775636948645, + 0.00023034587502479553, + 0.00021438486874103546, + 0.00020044803386554122, + 0.0001881986390799284, + 0.00017972940986510366, + 0.00017121655400842428, + 0.00016408956435043365, + 0.00016061612404882908, + 0.0001546283601783216, + 0.00015024218009784818, + 0.00014785809617023915, + 0.0001440297783119604, + 0.00013951724395155907, + 0.000136611532070674, + 0.0001337942958343774, + 0.00013117180787958205, + 0.0001286819897359237, + 0.00012687486014328897, + 0.00012493478425312787, + 0.00012369122123345733, + 0.00012227953993715346, + 0.00012101305037504062, + 0.00011953177454415709, + 0.00011818919301731512, + 0.00011716668086592108, + 0.00011609472858253866, + 0.00011492421617731452, + 0.00011409165017539635, + 0.00011346709652571008, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837904930114746, + "first_to_last_ratio": 1.8345651581276443, + "imbalance_abs_log_ratio": 0.6068074823768531, + "last_quartile_mean": 0.10268321540206671, + "mean": 0.13401183625683188, + "normalized": [ + 1.7717078885231814, + 1.6366668119471057, + 1.524290957183596, + 1.418670514273678, + 1.326444920903592, + 1.2453858718827995, + 1.1893415500975306, + 1.1330085716119305, + 1.0858463365841577, + 1.062861212873729, + 1.0232377434953237, + 0.9942127340740732, + 0.9784362443707608, + 0.9531027271661117, + 0.9232414917376033, + 0.904013241653558, + 0.885370477646398, + 0.8680164390463408, + 0.8515402681847718, + 0.8395816969955481, + 0.8267435345782925, + 0.8185144279720658, + 0.8091727254708025, + 0.8007918405918669, + 0.7909896328038464, + 0.7821052665924377, + 0.7753388466443417, + 0.7682453528580236, + 0.7604996016741158, + 0.7549901615676585, + 0.7508572336421532, + 0.7467736753526057 + ], + "peak_layer": 1, + "peak_normalized": 1.7717078885231814, + "population_cv": 0.27141699472708714, + "rest_mean": 0.1387458680956452, + "spike_contrast": 0.7816309473985974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844806432724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23742982745170593, + 0.2193327248096466, + 0.2042730301618576, + 0.19011864066123962, + 0.1777593195438385, + 0.16689644753932953, + 0.15938584506511688, + 0.151836559176445, + 0.1455162614583969, + 0.1424359828233719, + 0.13712596893310547, + 0.13323627412319183, + 0.13112203776836395, + 0.12772704660892487, + 0.1237252876162529, + 0.1211484745144844, + 0.11865012347698212, + 0.11632447689771652, + 0.11411647498607635, + 0.11251388490200043, + 0.11079341918230057, + 0.10969062149524689, + 0.10843872278928757, + 0.10731558501720428, + 0.10600197315216064, + 0.10481136292219162, + 0.10390458256006241, + 0.1029539704322815, + 0.10191594809293747, + 0.10117761790752411, + 0.10062375664710999, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873187476230669e-05, + "first_to_last_ratio": 2.248757500145603, + "imbalance_abs_log_ratio": 0.8103778415280068, + "last_quartile_mean": 3.0564378221242805e-05, + "mean": 4.685761763312257e-05, + "normalized": [ + 1.7400006587426948, + 1.6405194725375354, + 1.5551306973978087, + 1.4944078341288562, + 1.4327724019957897, + 1.3414187773842097, + 1.2804088050749736, + 1.2499327782691996, + 1.1927955171655855, + 1.1608931065609767, + 1.0981756972550076, + 1.0577423904261343, + 1.0542786038595535, + 1.0218640087393278, + 0.9775807389332813, + 0.9570972402588515, + 0.9209982861589985, + 0.8815092249082567, + 0.8478232134430647, + 0.8204952153707635, + 0.7871780668180614, + 0.7759097745574741, + 0.7506963524238334, + 0.7421155343193898, + 0.7072927325438048, + 0.6891814936654451, + 0.6695399899996352, + 0.6616343983211914, + 0.6345111343049281, + 0.6212687927046295, + 0.6200231913288072, + 0.6148038704019305 + ], + "peak_layer": 1, + "peak_normalized": 1.7400006587426948, + "population_cv": 0.3217536161653951, + "rest_mean": 4.900405669845727e-05, + "spike_contrast": 0.719671983430414, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.526684668031521e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153228554874659e-05, + 7.687083416385576e-05, + 7.286971958819777e-05, + 7.002439087955281e-05, + 6.71363013680093e-05, + 6.285568815656006e-05, + 5.999690620228648e-05, + 5.8568872191244736e-05, + 5.5891556257847697e-05, + 5.439668530016206e-05, + 5.145789691596292e-05, + 4.9563288484932855e-05, + 4.9400983698433265e-05, + 4.7882112994557247e-05, + 4.580710447044112e-05, + 4.484729652176611e-05, + 4.3155785533599555e-05, + 4.130542220082134e-05, + 3.972697595600039e-05, + 3.844645107164979e-05, + 3.688528886414133e-05, + 3.635728353401646e-05, + 3.517584264045581e-05, + 3.477376594673842e-05, + 3.314205241622403e-05, + 3.229340290999971e-05, + 3.1373048841487616e-05, + 3.10026116494555e-05, + 2.9731680115219206e-05, + 2.9111175535945222e-05, + 2.9052809622953646e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001719075116852764, + "first_to_last_ratio": 1.4894635014227988, + "imbalance_abs_log_ratio": 0.39841598895878805, + "last_quartile_mean": 0.00011541572621354135, + "mean": 0.00013894635094402474, + "normalized": [ + 1.3738985779819257, + 1.3296082759737424, + 1.2883384004275333, + 1.2499940655466777, + 1.2130730176151365, + 1.1755503979222366, + 1.147409009170091, + 1.11990595257071, + 1.0929037170303153, + 1.075598370036652, + 1.0512762916941902, + 1.0312594730951, + 1.0194418974669357, + 1.0018996501730997, + 0.9803460168062212, + 0.9648004557293595, + 0.9493010807852901, + 0.9344788929945884, + 0.9198953859218769, + 0.908001147597105, + 0.8952913731815082, + 0.886953676542665, + 0.8771894463102521, + 0.8683888934041172, + 0.8579399877904575, + 0.8482799643655919, + 0.8408636339472488, + 0.8330870307470335, + 0.82434015220157, + 0.8181187963300758, + 0.8135579418994486, + 0.809009026741244 + ], + "peak_layer": 1, + "peak_normalized": 1.3738985779819257, + "population_cv": 0.16161423472023242, + "rest_mean": 0.0001421073115866252, + "spike_contrast": 0.8576417505420824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187716347398237, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001908981939777732, + 0.0001847442181315273, + 0.00017900991952046752, + 0.00017368211410939693, + 0.00016855206922627985, + 0.00016333843814209104, + 0.00015942829486448318, + 0.00015560684551019222, + 0.0001518549834145233, + 0.00014945046859793365, + 0.00014607100456487387, + 0.0001432897406630218, + 0.00014164773165248334, + 0.00013921030040364712, + 0.000136215501697734, + 0.0001340555027127266, + 0.0001319019211223349, + 0.00012984243221580982, + 0.0001278161071240902, + 0.00012616344611160457, + 0.00012439746933523566, + 0.00012323897681199014, + 0.00012188227265141904, + 0.00012065946793882176, + 0.00011920763063244522, + 0.00011786540562752634, + 0.00011683493357850239, + 0.00011575440294109285, + 0.0001145390560850501, + 0.00011367462138878182, + 0.00011304090730845928, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552221000485588, + "first_to_last_ratio": 1.262697055715921, + "imbalance_abs_log_ratio": 0.2332499537229619, + "last_quartile_mean": 0.00011524712863320019, + "mean": 0.00012953460168318998, + "normalized": [ + 1.1676213631634658, + 1.1525055688027706, + 1.1391580058081985, + 1.1289307982409096, + 1.1166731577821778, + 1.1054001772932092, + 1.094813144753513, + 1.0822854389705017, + 1.0679308603491235, + 1.0564894822224282, + 1.0456838360716472, + 1.0337072702557388, + 1.0234945545465333, + 1.0158819477934218, + 1.0051772952892828, + 0.9951467950359763, + 0.9866643397779161, + 0.9787559418439278, + 0.970448736962688, + 0.9607908684400951, + 0.9509939230158972, + 0.9428061355388626, + 0.934924980051271, + 0.9261034265480621, + 0.9170582603044295, + 0.9058562786860701, + 0.8981778407016658, + 0.8907564927732574, + 0.8831168120837269, + 0.8778216107141017, + 0.8745192645108896, + 0.8703053916682415 + ], + "peak_layer": 1, + "peak_normalized": 1.1676213631634658, + "population_cv": 0.08967236486753827, + "rest_mean": 0.00013110875065817877, + "spike_contrast": 0.9231588022206524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103419721825049, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124736819416285, + 0.00014928934979252517, + 0.000147560378536582, + 0.00014623560127802193, + 0.00014464781270362437, + 0.00014318757166620344, + 0.00014181618462316692, + 0.00014019341324456036, + 0.0001383339986205101, + 0.00013685194426216185, + 0.00013545223919209093, + 0.00013390085950959474, + 0.00013257795944809914, + 0.00013159186346456409, + 0.00013020524056628346, + 0.00012890594371128827, + 0.00012780717224813998, + 0.00012678276107180864, + 0.0001257066905964166, + 0.0001244556624442339, + 0.00012318661902099848, + 0.0001221260172314942, + 0.00012110513489460573, + 0.0001199624384753406, + 0.00011879077646881342, + 0.00011733973224181682, + 0.00011634510883595794, + 0.00011538378748809919, + 0.0001143941844929941, + 0.00011370827269274741, + 0.00011328050459269434, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029741139769612346, + "first_to_last_ratio": 2.278072415841521, + "imbalance_abs_log_ratio": 0.8233296537390045, + "last_quartile_mean": 0.00013055396993877366, + "mean": 0.00019328556300024502, + "normalized": [ + 1.889398576482247, + 1.780233894901915, + 1.6772594716902607, + 1.566224958959574, + 1.4762232610465842, + 1.375571144634086, + 1.3083686481730985, + 1.2364401527267181, + 1.173148897934763, + 1.1508864184295666, + 1.080457040307468, + 1.0318582947152708, + 1.0081957004611959, + 0.9698994715426978, + 0.9066127339785177, + 0.8754550048258108, + 0.8446146862114313, + 0.8115724449181009, + 0.7806677558079268, + 0.7613480996239201, + 0.7403134026279838, + 0.7294052762839086, + 0.7160186976339193, + 0.7062575707090604, + 0.6942777320753531, + 0.6866005528973866, + 0.6829938476675228, + 0.6753403086482458, + 0.6715967314167978, + 0.6686860556999406, + 0.6640286432895989, + 0.6600445236791291 + ], + "peak_layer": 1, + "peak_normalized": 1.889398576482247, + "population_cv": 0.3579154260008552, + "rest_mean": 0.00020340604748559632, + "spike_contrast": 0.681567477924495, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013863494677934796, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003651934675872326, + 0.0003440935106482357, + 0.00032419004128314555, + 0.0003027286729775369, + 0.0002853326441254467, + 0.00026587804313749075, + 0.00025288877077400684, + 0.00023898603103589267, + 0.00022675274522043765, + 0.00022244972933549434, + 0.0002088367473334074, + 0.00019944331143051386, + 0.00019486967357806861, + 0.00018746756541077048, + 0.00017523515271022916, + 0.00016921281348913908, + 0.0001632518251426518, + 0.00015686523693148047, + 0.00015089180669747293, + 0.00014715759607497603, + 0.00014309189282357693, + 0.00014098350948188454, + 0.00013839607709087431, + 0.0001365093921776861, + 0.0001341938623227179, + 0.00013270997442305088, + 0.00013201285037212074, + 0.00013053353177383542, + 0.0001298099523410201, + 0.00012924736074637622, + 0.00012834715016651899, + 0.00012757707736454904 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0040087271481752396, + 0.005379301495850086, + 0.006410556845366955, + 0.0027253651060163975, + 0.004838948138058186, + 0.006175771821290255, + 0.00742313964292407, + 0.0030937574338167906, + 0.005150026176124811, + 0.007019025273621082, + 0.008143599145114422, + 0.004255065228790045, + 0.0066224560141563416, + 0.008129785768687725, + 0.010099578648805618, + 0.0041999719105660915, + 0.00719039561226964, + 0.009030314162373543, + 0.010736005380749702, + 0.005131533369421959, + 0.00787449348717928, + 0.009912566281855106, + 0.012113703414797783, + 0.0059690922498703, + 0.00876442901790142, + 0.010806876234710217, + 0.012353328987956047, + 0.006502739619463682, + 0.009470963850617409, + 0.012123330496251583, + 0.01384514570236206 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962236011546338, + "first_to_last_ratio": 3.7698422272252903, + "imbalance_abs_log_ratio": 1.3270331510418396, + "last_quartile_mean": 4.2341920561739244e-05, + "mean": 8.61727822893954e-05, + "normalized": [ + 2.542734798705971, + 2.3221448837042376, + 2.101145461274622, + 1.8911858524935854, + 1.7004204056817789, + 1.5354142426015858, + 1.4229001770142031, + 1.3028733810328659, + 1.2049412416934622, + 1.1705258152816296, + 1.0773908232649334, + 1.0133220774376726, + 0.990609360100461, + 0.9272534949358001, + 0.8550385909364457, + 0.8185651575045715, + 0.7703972051466339, + 0.7173947009730917, + 0.6760256695667757, + 0.6514523109028573, + 0.618419077603279, + 0.6053620388084041, + 0.5838930841859182, + 0.5697043695032278, + 0.5457802700901755, + 0.5276149005446632, + 0.5104701332743111, + 0.49517988582787786, + 0.47527562953524777, + 0.46579059316028215, + 0.45896435764104826, + 0.45181000957238127 + ], + "peak_layer": 1, + "peak_normalized": 2.542734798705971, + "population_cv": 0.5705202801130907, + "rest_mean": 9.280119640472414e-05, + "spike_contrast": 0.542873885449774, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.03793460666202e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021911453222855926, + 0.00020010568550787866, + 0.00018106155039276928, + 0.00016296874673571438, + 0.00014652995741926134, + 0.00013231091725174338, + 0.0001226152671733871, + 0.00011227222421439365, + 0.00010383313929196447, + 0.00010086746624438092, + 9.284176485380158e-05, + 8.732078276807442e-05, + 8.536356472177431e-05, + 7.99040135461837e-05, + 7.368105434579775e-05, + 7.053803710732609e-05, + 6.638727063545957e-05, + 6.181989738252014e-05, + 5.825501284562051e-05, + 5.613745815935545e-05, + 5.329089253791608e-05, + 5.216573117650114e-05, + 5.0315691623836756e-05, + 4.9093010602518916e-05, + 4.703140439232811e-05, + 4.546604395727627e-05, + 4.398863165988587e-05, + 4.267102849553339e-05, + 4.095582335139625e-05, + 4.013847137684934e-05, + 3.955023566959426e-05, + 3.893372559105046e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442477373057045, + "first_to_last_ratio": 1.8431254562763404, + "imbalance_abs_log_ratio": 0.6114627481432037, + "last_quartile_mean": 0.00011633759004325839, + "mean": 0.00015221248349917005, + "normalized": [ + 1.7740805953138137, + 1.6484457246979003, + 1.5301434809496548, + 1.4217241600875643, + 1.3264958046529274, + 1.2454577490392524, + 1.1897861330408546, + 1.1336265611690741, + 1.086914920172498, + 1.064301358569379, + 1.0227209980571423, + 0.9941753882589759, + 0.9788979908211966, + 0.9511107967643203, + 0.9228143274453928, + 0.9052447611165039, + 0.8846530997418666, + 0.8652930871753776, + 0.8497381552743479, + 0.8388656494639406, + 0.8256334792987466, + 0.8176309647342147, + 0.8076951735882674, + 0.8000661307359944, + 0.7892605467734257, + 0.78144613040264, + 0.7731219607566042, + 0.7664988488799842, + 0.7584935618388989, + 0.7529639051546407, + 0.7485195296370439, + 0.7441790263875565 + ], + "peak_layer": 1, + "peak_normalized": 1.7740805953138137, + "population_cv": 0.2736411578809574, + "rest_mean": 0.00015762286892384981, + "spike_contrast": 0.7803207938393821, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299640220589936, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027003721334040165, + 0.00025091401766985655, + 0.00023290693934541196, + 0.0002164041652576998, + 0.000201909220777452, + 0.0001895742170745507, + 0.00018110030214302242, + 0.00017255211423616856, + 0.00016544201935175806, + 0.00016199995297938585, + 0.00015567090304102749, + 0.00015132590488065034, + 0.0001490004942752421, + 0.00014477093645837158, + 0.00014046386058907956, + 0.00013778955326415598, + 0.0001346552453469485, + 0.00013170840975362808, + 0.00012934075493831187, + 0.00012768582382705063, + 0.00012567172234412283, + 0.00012445363972801715, + 0.00012294128828216344, + 0.00012178005272289738, + 0.00012013530795229599, + 0.00011894585622940212, + 0.00011767881369451061, + 0.00011667069338727742, + 0.00011545218876563013, + 0.00011461050598882139, + 0.00011393401655368507, + 0.00011327333777444437 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.190154030919075, + "first_to_last_ratio": 1.84312545379361, + "imbalance_abs_log_ratio": 0.6114627467961817, + "last_quartile_mean": 0.10316933691501617, + "mean": 0.13498355145566165, + "normalized": [ + 1.774080628234443, + 1.6484456620184311, + 1.5301434020504485, + 1.4217242444591272, + 1.326495766996536, + 1.2454576834375484, + 1.189786122599316, + 1.133626627255238, + 1.0869149569673349, + 1.0643014004698326, + 1.0227209907477024, + 0.9941753870542451, + 0.9788979589698585, + 0.9511107512164827, + 0.9228143032197887, + 0.9052447972745712, + 0.8846531331847195, + 0.8652931173486506, + 0.8497382179112226, + 0.8388656682549475, + 0.8256334805754935, + 0.8176309131099365, + 0.8076951535479161, + 0.8000661540388709, + 0.7892605573866218, + 0.7814461538045975, + 0.7731219580171443, + 0.7664988542586116, + 0.7584935821787124, + 0.7529638601652359, + 0.7485195164462104, + 0.7441789968002049 + ], + "peak_layer": 1, + "peak_normalized": 1.774080628234443, + "population_cv": 0.27364115345546325, + "rest_mean": 0.13978153549962574, + "spike_contrast": 0.7803207857775153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907443761825561, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2394717037677765, + 0.22251304984092712, + 0.2065441906452179, + 0.19190938770771027, + 0.1790551096200943, + 0.16811630129814148, + 0.16060155630111694, + 0.1530209481716156, + 0.14671564102172852, + 0.1436631828546524, + 0.13805051147937775, + 0.13419732451438904, + 0.13213512301445007, + 0.1283843070268631, + 0.1245647519826889, + 0.12219315767288208, + 0.11941362172365189, + 0.11680033802986145, + 0.11470068246126175, + 0.1132330670952797, + 0.11144693940877914, + 0.11036672443151474, + 0.10902556031942368, + 0.10799577087163925, + 0.10653719305992126, + 0.10548237711191177, + 0.1043587476015091, + 0.10346473753452301, + 0.10238415747880936, + 0.10163773596286774, + 0.10103782266378403, + 0.10045192390680313 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883367768750759e-05, + "first_to_last_ratio": 2.249759357551583, + "imbalance_abs_log_ratio": 0.810823258297238, + "last_quartile_mean": 3.059601795030176e-05, + "mean": 4.691846504556452e-05, + "normalized": [ + 1.7359059710084281, + 1.6475789727938472, + 1.5536456441204052, + 1.4921194549590373, + 1.4365657321885223, + 1.3438513058553838, + 1.2787343947807137, + 1.2483299149152907, + 1.1912354282513808, + 1.1603083401306369, + 1.099547144002948, + 1.059972440022979, + 1.0583955432739494, + 1.0176800955770247, + 0.9754963047746087, + 0.9631595708780456, + 0.9242400656751482, + 0.8752301116137658, + 0.8454245377074081, + 0.8232853322610043, + 0.7863494898171388, + 0.7774642190168813, + 0.7497486129019063, + 0.7388483538626245, + 0.709315327139865, + 0.6908343794807076, + 0.6685469983108738, + 0.6592365073812129, + 0.6314341746588822, + 0.6235898169926835, + 0.6199474545603838, + 0.6139783610863121 + ], + "peak_layer": 1, + "peak_normalized": 1.7359059710084281, + "population_cv": 0.3220406003070692, + "rest_mean": 4.9070239684725595e-05, + "spike_contrast": 0.7193541792517962, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.52988819940947e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144604362314567e-05, + 7.730187644483522e-05, + 7.28946688468568e-05, + 7.000795449130237e-05, + 6.740145909134299e-05, + 6.305144052021205e-05, + 5.999625500408001e-05, + 5.8569723478285596e-05, + 5.589093780145049e-05, + 5.4439886298496276e-05, + 5.158906424185261e-05, + 4.973227987647988e-05, + 4.9658294301480055e-05, + 4.774798799189739e-05, + 4.576878927764483e-05, + 4.51899686595425e-05, + 4.3363925215089694e-05, + 4.1064453398576006e-05, + 3.9666021621087566e-05, + 3.86272840842139e-05, + 3.689431105158292e-05, + 3.6477427784120664e-05, + 3.5177054087398574e-05, + 3.466563066467643e-05, + 3.3279986382694915e-05, + 3.2412888685939834e-05, + 3.136719897156581e-05, + 3.093036502832547e-05, + 2.9625922252307646e-05, + 2.9257877031341195e-05, + 2.9086982976878062e-05, + 2.8806922273361124e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017341253987979144, + "first_to_last_ratio": 1.4955148931299893, + "imbalance_abs_log_ratio": 0.4024705576674927, + "last_quartile_mean": 0.00011595507385209203, + "mean": 0.00013990553861731314, + "normalized": [ + 1.3768818258178377, + 1.3340879911950096, + 1.2910054180628048, + 1.2527236427111486, + 1.2147542166907819, + 1.1769889670731195, + 1.1474411155803916, + 1.1220953663134672, + 1.0944075691302455, + 1.0773007485615413, + 1.0508678613623372, + 1.0320596054419082, + 1.0208161736522385, + 1.000559648637946, + 0.9805197815181176, + 0.9663128275349431, + 0.948744191771126, + 0.9319305822463463, + 0.9182019814348666, + 0.9075038868226692, + 0.8943650364968407, + 0.886262676120959, + 0.8757907045344768, + 0.8679002174812178, + 0.856288373659771, + 0.8478178092769747, + 0.8387215581135022, + 0.8313975747674917, + 0.8224084044020752, + 0.8161864847722404, + 0.8112200992010112, + 0.8064376596145924 + ], + "peak_layer": 1, + "peak_normalized": 1.3768818258178377, + "population_cv": 0.16325888721442564, + "rest_mean": 0.0001431150389921472, + "spike_contrast": 0.8564734877369171, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225742365932092, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926333934534341, + 0.00018664629897102714, + 0.00018061880837194622, + 0.0001752629759721458, + 0.00016995084297377616, + 0.0001646672753849998, + 0.00016053336730692536, + 0.00015698735660407692, + 0.00015311368042603135, + 0.00015072034148033708, + 0.00014702223415952176, + 0.00014439085498452187, + 0.0001428178366040811, + 0.00013998383656144142, + 0.00013718014815822244, + 0.00013519251660909504, + 0.00013273456715978682, + 0.00013038225006312132, + 0.00012846154277212918, + 0.00012696482008323073, + 0.00012512662215158343, + 0.0001239930570591241, + 0.00012252797023393214, + 0.000121424047392793, + 0.00011979948612861335, + 0.00011861440725624561, + 0.00011734179133782163, + 0.0001163171255029738, + 0.00011505949078127742, + 0.00011418900976423174, + 0.00011349418491590768, + 0.00011282509512966499 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675183047074825, + "first_to_last_ratio": 1.2676545313577883, + "imbalance_abs_log_ratio": 0.23716836728277432, + "last_quartile_mean": 0.00011576642282307148, + "mean": 0.00013041823922321782, + "normalized": [ + 1.1700698937843759, + 1.1560440994564782, + 1.1417398043557874, + 1.130463200949706, + 1.118403537277814, + 1.1065304334184665, + 1.0939881912841023, + 1.0846815122144253, + 1.0709468270677047, + 1.0579907390333845, + 1.0442838370166418, + 1.035550673160169, + 1.025959021745254, + 1.0140203095307772, + 1.0062047700154308, + 0.9976631870190885, + 0.9861765922450596, + 0.9770016877798172, + 0.9687702939454803, + 0.9602422119882037, + 0.9504995944091904, + 0.9425883202531964, + 0.9330014667281973, + 0.9259385264566372, + 0.9147443798098084, + 0.906202796813466, + 0.8958065504175365, + 0.8885655303249116, + 0.8806194435723184, + 0.8757905905296257, + 0.8713783728577705, + 0.8681336045391749 + ], + "peak_layer": 1, + "peak_normalized": 1.1700698937843759, + "population_cv": 0.09121429133712369, + "rest_mean": 0.0001320278307604086, + "spike_contrast": 0.921975648780332, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001217264449223876, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001525984553154558, + 0.0001507692359155044, + 0.000148903694935143, + 0.00014743302017450333, + 0.00014586022007279098, + 0.00014431175077334046, + 0.00014267601363826543, + 0.00014146225294098258, + 0.000139670999487862, + 0.00013798128929920495, + 0.00013619365927297622, + 0.00013505469541996717, + 0.00013380376913119107, + 0.00013224674330558628, + 0.00013122745440341532, + 0.00013011347618885338, + 0.00012861541472375393, + 0.00012741883983835578, + 0.0001263453159481287, + 0.0001252330985153094, + 0.0001239624834852293, + 0.00012293070903979242, + 0.00012168040848337114, + 0.00012075927224941552, + 0.00011929935135412961, + 0.00011818537313956767, + 0.00011682951299007982, + 0.00011588515189941972, + 0.00011484883725643158, + 0.00011421906674513593, + 0.00011364363308530301, + 0.00011322045611450449 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030004180916876066, + "first_to_last_ratio": 2.2837506327398005, + "imbalance_abs_log_ratio": 0.8258191057718037, + "last_quartile_mean": 0.00013138116082700435, + "mean": 0.00019473525253488333, + "normalized": [ + 1.9000096511818072, + 1.783700741635724, + 1.6708066910504589, + 1.570366884717585, + 1.4783058799917985, + 1.376185321596543, + 1.3064973297630273, + 1.2402700043336092, + 1.1767953849206394, + 1.1505341943958838, + 1.0771542571420398, + 1.026098162637436, + 1.013898063759585, + 0.9669473025647412, + 0.9095398902231254, + 0.8755205777387188, + 0.842301214161826, + 0.8073124727024382, + 0.7791991818813049, + 0.760574607765185, + 0.7402055397447304, + 0.7308143413325056, + 0.7152667122155245, + 0.7043714155678553, + 0.6942467749765324, + 0.6868749151283189, + 0.680655564574882, + 0.676363713518914, + 0.6701061029149729, + 0.6672276319278244, + 0.6636177363814285, + 0.6582317375530352 + ], + "peak_layer": 1, + "peak_normalized": 1.9000096511818072, + "population_cv": 0.35930746026536914, + "rest_mean": 0.00020494150898971216, + "spike_contrast": 0.6812747128050888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962146767880766, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699988592416048, + 0.0003473494143690914, + 0.0003253649629186839, + 0.0003058057918678969, + 0.0002878782688640058, + 0.0002679917961359024, + 0.0002544210874475539, + 0.00024152429250534624, + 0.00022916354646440595, + 0.000224049566895701, + 0.00020975990628357977, + 0.00019981748482678086, + 0.000197441695490852, + 0.0001882987271528691, + 0.00017711948021315038, + 0.00017049472080543637, + 0.00016402573965024203, + 0.00015721219824627042, + 0.00015173754945863038, + 0.00014811068831477314, + 0.00014414411270990968, + 0.0001423153153154999, + 0.0001392876438330859, + 0.00013716594548895955, + 0.00013519432104658335, + 0.00013375876005738974, + 0.00013254763325676322, + 0.0001317118585575372, + 0.00013049328117631376, + 0.00012993274140171707, + 0.00012922976748086512, + 0.00012818092363886535 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858902861509705, + "first_to_last_ratio": 3.757869257587208, + "imbalance_abs_log_ratio": 1.3238521099622325, + "last_quartile_mean": 4.220184837322449e-05, + "mean": 8.57325238712292e-05, + "normalized": [ + 2.5462257183789148, + 2.3032637538343064, + 2.0924289533379263, + 1.8886492391629932, + 1.7031256747388024, + 1.5374176267930122, + 1.4224231993460965, + 1.3049597038569594, + 1.2055618292025094, + 1.169245405308266, + 1.0769812048488787, + 1.0144334047898542, + 0.9913512297333369, + 0.9317090592952362, + 0.8577499774884012, + 0.81555804918997, + 0.7682725064221458, + 0.7217807351250844, + 0.6783324192661732, + 0.6511800118936032, + 0.6192916242842709, + 0.6053728292385717, + 0.5848720876072855, + 0.5718124973482859, + 0.5462302773879167, + 0.5275173250132701, + 0.5112998770983732, + 0.49691833866078555, + 0.47721658788927634, + 0.46551989467389665, + 0.46012635628943105, + 0.45317260249616614 + ], + "peak_layer": 1, + "peak_normalized": 2.5462257183789148, + "population_cv": 0.5686111305901927, + "rest_mean": 9.231303704803361e-05, + "spike_contrast": 0.5437775023084314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.019775271648541e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021829435718245804, + 0.00019746461475733668, + 0.0001793892151908949, + 0.0001619186659809202, + 0.00014601326256524771, + 0.00013180669338908046, + 0.00012194793089292943, + 0.00011187748896190897, + 0.00010335585830034688, + 0.00010024235962191597, + 9.233231685357168e-05, + 8.69699360919185e-05, + 8.499104296788573e-05, + 7.987776916706935e-05, + 7.353707042057067e-05, + 6.991984992055222e-05, + 6.586594099644572e-05, + 6.188008410390466e-05, + 5.8155150327365845e-05, + 5.582730591413565e-05, + 5.309343396220356e-05, + 5.190014053368941e-05, + 5.014256021240726e-05, + 4.90229285787791e-05, + 4.682970029534772e-05, + 4.522539165918715e-05, + 4.383502891869284e-05, + 4.2602063331287354e-05, + 4.0912982512963936e-05, + 3.991019548266195e-05, + 3.944779382436536e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "rest_mean": 0.00015645541295978345, + "spike_contrast": 0.7816308178118705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229037238284946, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837944976985455, + "first_to_last_ratio": 1.8345658800697515, + "imbalance_abs_log_ratio": 0.6068078758989703, + "last_quartile_mean": 0.10268339328467846, + "mean": 0.1340121782850474, + "normalized": [ + 1.7717451751532867, + 1.6366747548098712, + 1.5242756140248972, + 1.4186514377480737, + 1.3264266357207033, + 1.2453739091726768, + 1.1893513017871604, + 1.1330134634090196, + 1.0858604665419038, + 1.0628542749024097, + 1.0232520332438115, + 0.9942070832344971, + 0.9784308561813967, + 0.9531084116998004, + 0.9232390798282929, + 0.9040072094634526, + 0.8853775581660035, + 0.8680181154203164, + 0.8515367049614931, + 0.8395794986042229, + 0.8267439263823189, + 0.8185133396785964, + 0.809173050926764, + 0.8007913534932437, + 0.7909900046670136, + 0.782107217826699, + 0.7753403703776178, + 0.7682422246066585, + 0.7604971603471317, + 0.7549863443945886, + 0.7508596538022533, + 0.7467717694238248 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451751532867, + "population_cv": 0.27141865758333467, + "rest_mean": 0.13874622461972413, + "spike_contrast": 0.7816308398662988, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844832807779312, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23743543028831482, + 0.21933434903621674, + 0.20427149534225464, + 0.19011656939983368, + 0.17775732278823853, + 0.16689527034759521, + 0.15938755869865417, + 0.15183760225772858, + 0.14551852643489838, + 0.14243541657924652, + 0.13712823390960693, + 0.1332358568906784, + 0.1311216503381729, + 0.12772813439369202, + 0.1237252801656723, + 0.12114797532558441, + 0.1186513751745224, + 0.1163249984383583, + 0.11411628872156143, + 0.11251387745141983, + 0.11079375445842743, + 0.10969075560569763, + 0.10843904316425323, + 0.107315793633461, + 0.10600229352712631, + 0.104811891913414, + 0.10390505194664001, + 0.10295381397008896, + 0.1019158810377121, + 0.10117736458778381, + 0.10062433779239655, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873207303215167e-05, + "first_to_last_ratio": 2.248757814140914, + "imbalance_abs_log_ratio": 0.8103779811585758, + "last_quartile_mean": 3.0564462122129044e-05, + "mean": 4.6857780716891284e-05, + "normalized": [ + 1.7400119939228862, + 1.6405409364325143, + 1.555121247708426, + 1.4944054279879166, + 1.4327475398601994, + 1.3414100714975594, + 1.2804101716466747, + 1.2499370459040815, + 1.192804641985012, + 1.1608858830017554, + 1.098191983597808, + 1.057744066133373, + 1.0542717513613309, + 1.021869458338179, + 0.9775749297615711, + 0.9570927445976433, + 0.9210091333285576, + 0.8815125232807631, + 0.847820728512489, + 0.8204970180441978, + 0.7871838673851995, + 0.7759041238100245, + 0.7506938949785397, + 0.7421070509174572, + 0.7072987335050381, + 0.6891811912847667, + 0.6695407652827761, + 0.6616333377893114, + 0.6345027924930273, + 0.6212629426015038, + 0.6200302724102036, + 0.614801730639213 + ], + "peak_layer": 1, + "peak_normalized": 1.7400119939228862, + "population_cv": 0.32175451551289735, + "rest_mean": 4.9004235565748616e-05, + "spike_contrast": 0.7196709452950106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5266924533061685e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153310045599937e-05, + 7.687210745643824e-05, + 7.28695304132998e-05, + 7.002452184678987e-05, + 6.713537004543468e-05, + 6.28554989816621e-05, + 5.9997179050697014e-05, + 5.8569276006892323e-05, + 5.589217835222371e-05, + 5.439653614303097e-05, + 5.1458839152473956e-05, + 4.9563539505470544e-05, + 4.9400834541302174e-05, + 4.788253500009887e-05, + 4.58069916930981e-05, + 4.4847241952084005e-05, + 4.315644400776364e-05, + 4.130572051508352e-05, + 3.972699778387323e-05, + 3.8446669350378215e-05, + 3.688568904181011e-05, + 3.635714529082179e-05, + 3.5175849916413426e-05, + 3.477348946034908e-05, + 3.3142448955914006e-05, + 3.2293501135427505e-05, + 3.13731943606399e-05, + 3.1002669857116416e-05, + 2.9731392714893445e-05, + 2.9111002731951885e-05, + 2.905324254243169e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017190765356644988, + "first_to_last_ratio": 1.4894621368061003, + "imbalance_abs_log_ratio": 0.3984150727783497, + "last_quartile_mean": 0.00011541592721187044, + "mean": 0.00013894661265112518, + "normalized": [ + 1.3739040544627474, + 1.3296102750467884, + 1.2883342981472405, + 1.2499918159027226, + 1.2130627732820631, + 1.175544413473833, + 1.1474083142393565, + 1.1199112790671786, + 1.092909618037032, + 1.0755929927697745, + 1.0512861461262244, + 1.0312571117882559, + 1.0194388253067554, + 1.0019061415077835, + 0.980344379776871, + 0.9647961249933756, + 0.9493075664595542, + 0.9344817410273759, + 0.9198929201794295, + 0.9079992279088673, + 0.8952920956888009, + 0.8869530532606651, + 0.8771902552778059, + 0.8683889858357474, + 0.8579407806493952, + 0.8482823463732029, + 0.8408658204630546, + 0.8330843095877946, + 0.8243382853592501, + 0.818115422616924, + 0.8135611224181157, + 0.8090075029660142 + ], + "peak_layer": 1, + "peak_normalized": 1.3739040544627474, + "population_cv": 0.16161403917150835, + "rest_mean": 0.00014210757001786045, + "spike_contrast": 0.8576421569620596, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187744287075474, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00019089931447524577, + 0.00018474484386388212, + 0.00017900968668982387, + 0.00017368212866131216, + 0.0001685509632807225, + 0.00016333791427314281, + 0.00015942849859129637, + 0.00015560787869617343, + 0.00015185608936008066, + 0.00014945000293664634, + 0.00014607264893129468, + 0.0001432896824553609, + 0.00014164757158141583, + 0.0001392114645568654, + 0.00013621553080156446, + 0.0001340551534667611, + 0.00013190307072363794, + 0.00012984307250007987, + 0.0001278160052606836, + 0.00012616341700777411, + 0.0001243978040292859, + 0.00012323912233114243, + 0.00012188261462142691, + 0.00012065970804542303, + 0.00011920796532649547, + 0.00011786595860030502, + 0.00011683545744745061, + 0.00011575424287002534, + 0.00011453901242930442, + 0.00011367436673026532, + 0.00011304156214464456, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552358334185556, + "first_to_last_ratio": 1.262638843036328, + "imbalance_abs_log_ratio": 0.2332038508028458, + "last_quartile_mean": 0.00011525352965691127, + "mean": 0.00012953498708156985, + "normalized": [ + 1.1676304712442283, + 1.1525836984023852, + 1.1391521450612143, + 1.1288261090295841, + 1.1167600441451297, + 1.1054423860149858, + 1.094731137320412, + 1.082319740349084, + 1.067957453000188, + 1.0563935463552665, + 1.04560972624621, + 1.0337959762176818, + 1.0235535208863231, + 1.0159040893770892, + 1.0050628637064216, + 0.9950792389307415, + 0.9865814183778941, + 0.9786230528286792, + 0.9704418054120744, + 0.9608381133357455, + 0.9508407831226554, + 0.9429749854457838, + 0.9346839821902475, + 0.9262276149685831, + 0.9170542960948607, + 0.9059488475652709, + 0.8984648923618161, + 0.8908037775288885, + 0.8831483920181569, + 0.8778579808382188, + 0.8744051093280881, + 0.8703028022960919 + ], + "peak_layer": 1, + "peak_normalized": 1.1676304712442283, + "population_cv": 0.0896642712448568, + "rest_mean": 0.00013110964479252557, + "spike_contrast": 0.9231344927669946, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103183544240892, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124899800866842, + 0.00014929991448298097, + 0.00014756005839444697, + 0.00014622247545048594, + 0.00014465949789155275, + 0.00014319346519187093, + 0.00014180598373059183, + 0.00014019827358424664, + 0.00013833785487804562, + 0.00013683992438018322, + 0.0001354430423816666, + 0.00013391274842433631, + 0.0001325859921053052, + 0.00013159512309357524, + 0.00013019080506637692, + 0.00012889757636003196, + 0.00012779681128449738, + 0.00012676592450588942, + 0.00012570616672746837, + 0.00012446215259842575, + 0.00012316714855842292, + 0.00012214825255796313, + 0.00012107427755836397, + 0.00011997888213954866, + 0.00011879061639774591, + 0.00011735207226593047, + 0.0001163826382253319, + 0.0001153902558144182, + 0.00011439861555118114, + 0.00011371332220733166, + 0.00011326605454087257, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029742828155576717, + "first_to_last_ratio": 2.2781890121807993, + "imbalance_abs_log_ratio": 0.8233808344454435, + "last_quartile_mean": 0.00013055469935352448, + "mean": 0.00019329176166138495, + "normalized": [ + 1.8891850069388627, + 1.7801603926271052, + 1.6773544463608039, + 1.566500262900107, + 1.476456430986117, + 1.3754925511314788, + 1.3084504582544147, + 1.236424592515421, + 1.172965600373877, + 1.150871569148191, + 1.0802294364746856, + 1.0319209662753672, + 1.0080063247818516, + 0.9698985570693867, + 0.9067931019504507, + 0.8756163462587222, + 0.8444435806568995, + 0.8113939671049883, + 0.7809677247391495, + 0.7613442367276676, + 0.7402582678385915, + 0.7293562882295931, + 0.7162426695419845, + 0.7062419232587185, + 0.6940340549797547, + 0.6866100033591334, + 0.6831688143117768, + 0.6753320518823956, + 0.6715919825156708, + 0.6686410475337363, + 0.6640239865195754, + 0.6600233567535221 + ], + "peak_layer": 1, + "peak_normalized": 1.8891850069388627, + "population_cv": 0.357915029863326, + "rest_mean": 0.00020341356920888993, + "spike_contrast": 0.6815376252628051, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001386340009048581, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036516389809548855, + 0.0003440903383307159, + 0.0003242187958676368, + 0.0003027915954589844, + 0.00028538686456158757, + 0.00026587137836031616, + 0.0002529126941226423, + 0.00023899068764876574, + 0.00022672458726447076, + 0.00022245399304665625, + 0.0002087994507746771, + 0.00019946182146668434, + 0.00019483931828290224, + 0.00018747340072877705, + 0.00017527563613839447, + 0.00016924942610785365, + 0.00016322398732881993, + 0.00015683576930314302, + 0.0001509546273155138, + 0.00014716156874783337, + 0.0001430858246749267, + 0.0001409785618307069, + 0.00013844380737282336, + 0.00013651074550580233, + 0.00013415106514003128, + 0.00013271605712361634, + 0.00013205090363044292, + 0.00013053612201474607, + 0.00012981319741811603, + 0.00012924280599690974, + 0.00012835036613978446, + 0.00012757707736454904 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.02802751213312149, + 0.01413465291261673, + 0.014225807972252369, + 0.01433455292135477, + 0.01440478302538395, + 0.009703479707241058, + 0.00977881159633398, + 0.009858808480203152, + 0.009950309991836548, + 0.00752287870272994, + 0.007571155205368996, + 0.007693085819482803, + 0.007788818795233965, + 0.006353048607707024, + 0.006389216054230928, + 0.0064707002602517605, + 0.00656962301582098, + 0.005553483963012695, + 0.0056249531917274, + 0.005688921548426151, + 0.005768724251538515, + 0.004966245498508215, + 0.005057469941675663, + 0.005098348017781973, + 0.005216081161051989, + 0.0046758754178881645, + 0.004727107938379049, + 0.004777040798217058, + 0.004837613552808762, + 0.0044186655431985855, + 0.004510571248829365, + 0.0045849340967834 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001580166108396952, + "first_to_last_ratio": 1.9713740353643698, + "imbalance_abs_log_ratio": 0.6787307795042399, + "last_quartile_mean": 8.015557068574708e-05, + "mean": 0.00011843875881822896, + "normalized": [ + 0.8918679597819287, + 1.5814819593588525, + 1.5722239987443252, + 1.3246067564285042, + 1.1252650527356378, + 1.512743589128775, + 1.3788112428745087, + 1.2863035124436846, + 1.097095055843533, + 1.468154353173571, + 1.1523254606068964, + 1.0839888571909224, + 1.054518217028451, + 1.146335080549274, + 1.1665208527784754, + 0.8931136212023528, + 0.8203804970547371, + 1.073741223277434, + 0.919654742394771, + 0.8223971330190447, + 0.7711288002244273, + 0.8470775332584999, + 0.8298711629738247, + 0.7662487463744663, + 0.7210382462310316, + 0.7071376059209309, + 0.7189559392079661, + 0.727980518763353, + 0.6111916361855597, + 0.6986829946379066, + 0.6244678178312321, + 0.6046898327751223 + ], + "peak_layer": 2, + "peak_normalized": 1.5814819593588525, + "population_cv": 0.2904715201464359, + "rest_mean": 0.00012310891098523926, + "spike_contrast": 0.7572151875143335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.321993711637333e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010563173418631777, + 0.0001873087603598833, + 0.00018621225899551064, + 0.00015688478015363216, + 0.0001332749961875379, + 0.00017916747310664505, + 0.00016330469225067645, + 0.00015234819147735834, + 0.00012993857671972364, + 0.00017388637934345752, + 0.0001364799973089248, + 0.0001283862948184833, + 0.00012489582877606153, + 0.00013577050413005054, + 0.00013816128193866462, + 0.00010577926877886057, + 9.71648478298448e-05, + 0.00012717257777694613, + 0.00010892276623053476, + 9.74036956904456e-05, + 9.133153798757121e-05, + 0.0001003268116619438, + 9.828891052166e-05, + 9.07535504666157e-05, + 8.539887494407594e-05, + 8.375250035896897e-05, + 8.515224908478558e-05, + 8.622110908618197e-05, + 7.238877878990024e-05, + 8.275114669231698e-05, + 7.396119326585904e-05, + 7.161871326388791e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001664285200604354, + "first_to_last_ratio": 1.9719337364651075, + "imbalance_abs_log_ratio": 0.6790146534176558, + "last_quartile_mean": 8.439863722742302e-05, + "mean": 0.0001245867656507471, + "normalized": [ + 0.8984334461398115, + 1.5927952483854981, + 1.5658595485525297, + 1.3198735865176374, + 1.131213894321221, + 1.5125315092450449, + 1.3796810696180641, + 1.286366052942558, + 1.0960606282761096, + 1.4536884557404346, + 1.1543342746982923, + 1.0797559630746076, + 1.0497530570909623, + 1.1457182987306291, + 1.1551962694199982, + 0.9023721663305979, + 0.8238710779144349, + 1.0632503993742801, + 0.9211624832492573, + 0.8274335806257342, + 0.7739032412019898, + 0.850990856837025, + 0.8281317613098801, + 0.7681943949939335, + 0.7208251570035091, + 0.7089546837555952, + 0.7203162530736348, + 0.7236174125342815, + 0.6145740290254166, + 0.6994991973458914, + 0.6274381906330746, + 0.6042038120380651 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952483854981, + "population_cv": 0.2889665774838781, + "rest_mean": 0.00012946851159169995, + "spike_contrast": 0.7586812913967177, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.822533756960183e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00011193291720701382, + 0.0001984412083402276, + 0.0001950853766174987, + 0.000164438781212084, + 0.00014093428035266697, + 0.00018844140868168324, + 0.00017189000209327787, + 0.00016026418597903103, + 0.00013655464863404632, + 0.00018111034296453, + 0.00014381477376446128, + 0.0001345233031315729, + 0.00013078533811494708, + 0.00014274133718572557, + 0.00014392216689884663, + 0.00011242362961638719, + 0.00010264343291055411, + 0.0001324669283349067, + 0.00011476465442683548, + 0.00010308727360097691, + 9.641810174798593e-05, + 0.00010602219845168293, + 0.00010317425767425448, + 9.570685506332666e-05, + 8.980527491075918e-05, + 8.832637104205787e-05, + 8.974187221610919e-05, + 9.015315299620852e-05, + 7.656779052922502e-05, + 8.714834257261828e-05, + 7.817049481673166e-05, + 7.527579873567447e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.14759047608822584, + "first_to_last_ratio": 1.9719337947467823, + "imbalance_abs_log_ratio": 0.6790146829732503, + "last_quartile_mean": 0.07484555337578058, + "mean": 0.1104847895912826, + "normalized": [ + 0.8984334974418968, + 1.5927952138130326, + 1.565859646322909, + 1.3198735575362357, + 1.1312139236939245, + 1.5125315007911495, + 1.3796810264894532, + 1.286366010451999, + 1.0960606168617568, + 1.4536884922319049, + 1.1543342013148132, + 1.0797558953237962, + 1.0497530983161607, + 1.1457182562234747, + 1.1551962948455465, + 0.9023721940039697, + 0.8238711061937815, + 1.0632504861804335, + 0.9211625151546727, + 0.8274335809310764, + 0.7739032642161865, + 0.8509909070635955, + 0.8281317391228538, + 0.7681943896830757, + 0.7208251689669645, + 0.7089546589522648, + 0.7203162343708721, + 0.7236173971219663, + 0.6145740294939214, + 0.6994991437375198, + 0.6274381319743458, + 0.6042038211744466 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952138130326, + "population_cv": 0.28896658018674404, + "rest_mean": 0.11481397047086998, + "spike_contrast": 0.7586813040631772, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08710721284151077, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.09926323592662811, + 0.1759796440601349, + 0.1730036735534668, + 0.14582595229148865, + 0.12498193234205246, + 0.16711172461509705, + 0.15243376791477203, + 0.14212387800216675, + 0.12109802663326263, + 0.16061046719551086, + 0.1275363713502884, + 0.1192966029047966, + 0.11598175019025803, + 0.12658444046974182, + 0.12763161957263947, + 0.09969840198755264, + 0.0910252258181572, + 0.11747300624847412, + 0.10177444666624069, + 0.09141882508993149, + 0.08550453931093216, + 0.09402155131101608, + 0.09149596095085144, + 0.08487379550933838, + 0.0796402171254158, + 0.0783287063241005, + 0.07958398759365082, + 0.07994871586561203, + 0.06790108233690262, + 0.07728401571512222, + 0.069322369992733, + 0.06675533205270767 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.093040158499207e-05, + "first_to_last_ratio": 0.8860093039525244, + "imbalance_abs_log_ratio": 0.12102782735710801, + "last_quartile_mean": 5.748291960117058e-05, + "mean": 5.7093861016710434e-05, + "normalized": [ + 0.4337136137178416, + 0.9185569667519601, + 0.9444095926066538, + 0.8728787927927439, + 0.7849140301431445, + 1.1296292591692207, + 1.0726593944235638, + 0.9796135302182263, + 0.9386642177229508, + 1.3046353648934683, + 1.0461832118506482, + 1.0032917361864164, + 0.9662905330123907, + 1.0760080966276753, + 1.1828739682986207, + 0.9087955578115051, + 0.9194941496879511, + 1.198292373503509, + 1.0946132296674007, + 1.0673695485587213, + 0.9244260207272295, + 1.0525679458300632, + 1.125653431968099, + 0.9999504891362015, + 0.9629027708994021, + 1.0063234987710272, + 0.9975933861195996, + 1.0791525148711678, + 0.9412468232151083, + 1.085173475571988, + 1.0031556955576197, + 0.9789667796878816 + ], + "peak_layer": 10, + "peak_normalized": 1.3046353648934683, + "population_cv": 0.1441687540611157, + "rest_mean": 5.695535414647397e-05, + "spike_contrast": 1.0155638391297448, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.784179811598733e-05, + "top_five_layers": [ + 10, + 18, + 15, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.4762384782661684e-05, + 5.2443963795667514e-05, + 5.3919990023132414e-05, + 4.9836020480142906e-05, + 4.481377254705876e-05, + 6.449489592341706e-05, + 6.124226638348773e-05, + 5.592991874436848e-05, + 5.3591964388033375e-05, + 7.448667020071298e-05, + 5.973063889541663e-05, + 5.728179894504137e-05, + 5.516925739357248e-05, + 6.143345672171563e-05, + 6.75348419463262e-05, + 5.1886647270293906e-05, + 5.249747118796222e-05, + 6.84151382301934e-05, + 6.249569560168311e-05, + 6.094024865888059e-05, + 5.277905074763112e-05, + 6.009516800986603e-05, + 6.426790059776977e-05, + 5.7091034250333905e-05, + 5.4975836974335834e-05, + 5.74548939766828e-05, + 5.695645813830197e-05, + 6.161298369988799e-05, + 5.373941530706361e-05, + 6.19567435933277e-05, + 5.727403186028823e-05, + 5.58929932594765e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.801741053801379e-05, + "first_to_last_ratio": 0.954195857595535, + "imbalance_abs_log_ratio": 0.046886327142486896, + "last_quartile_mean": 6.08024129178375e-05, + "mean": 6.296396202287724e-05, + "normalized": [ + 0.4796364558374551, + 0.9568353419130547, + 0.9804638447804884, + 0.8949545354119591, + 0.8237377234766136, + 1.1597846916175047, + 1.085649359633547, + 0.9904450076993521, + 0.9595078978373608, + 1.2941748206155441, + 1.0680753760655664, + 1.0231559828609242, + 0.9904019047543832, + 1.096490963646882, + 1.1850351594596897, + 0.9422215946487361, + 0.9418631353047864, + 1.1705024179981143, + 1.0926353878857977, + 1.0382275689133567, + 0.9320833313178482, + 1.0784583323883348, + 1.0985080196909318, + 0.9917906747078287, + 0.9476388143731139, + 0.9855066582268529, + 0.978396290111904, + 1.028407839812513, + 0.8982055143685478, + 1.0282804954549911, + 0.9451124964564701, + 0.9138123627295472 + ], + "peak_layer": 10, + "peak_normalized": 1.2941748206155441, + "population_cv": 0.13431233496292436, + "rest_mean": 6.285090866084729e-05, + "spike_contrast": 1.0115120295379672, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.357445017783903e-05, + "top_five_layers": [ + 10, + 15, + 18, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.019981159013696e-05, + 6.0246144130360335e-05, + 6.173388828756288e-05, + 5.634988337988034e-05, + 5.186579073779285e-05, + 7.302463927771896e-05, + 6.835678505012766e-05, + 6.236234185053036e-05, + 6.041441884008236e-05, + 8.148637425620109e-05, + 6.725025741616264e-05, + 6.442195444833487e-05, + 6.235962791834027e-05, + 6.903941539349034e-05, + 7.461450877599418e-05, + 5.9326004702597857e-05, + 5.930343468207866e-05, + 7.369946979451925e-05, + 6.879665306769311e-05, + 6.537092122016475e-05, + 5.8687659475253895e-05, + 6.790400948375463e-05, + 6.916641723364592e-05, + 6.244707037694752e-05, + 5.966709431959316e-05, + 6.205140380188823e-05, + 6.160370685392991e-05, + 6.475263216998428e-05, + 5.655457789544016e-05, + 6.474461406469345e-05, + 5.950802733423188e-05, + 5.753724690293893e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7888660295284353e-05, + "first_to_last_ratio": 0.41855462748958633, + "imbalance_abs_log_ratio": 0.8709478659353102, + "last_quartile_mean": 4.2739129185065394e-05, + "mean": 3.1986895351110434e-05, + "normalized": [ + 0.2593955501951102, + 0.4997829486650695, + 0.48480994473028793, + 0.4968760363401999, + 0.5474125689219601, + 0.7788406177150305, + 0.7554473604850604, + 0.6514322393717875, + 0.7578135841604356, + 0.9674810188795085, + 0.9032772556875033, + 0.9019220651946442, + 0.8311356963419073, + 0.9587970177975769, + 1.1401394869948454, + 0.8953886492354074, + 1.0262272842074007, + 1.1413916918814533, + 1.174037958462901, + 1.194927715786966, + 1.0267140632187253, + 1.3434898267078448, + 1.3729067467361273, + 1.2011929483563188, + 1.1735959904166378, + 1.3266745689353452, + 1.2739952988364465, + 1.4049535244931144, + 1.2752889026856997, + 1.4680979746684197, + 1.4182292846928506, + 1.3483241791974143 + ], + "peak_layer": 30, + "peak_normalized": 1.4680979746684197, + "population_cv": 0.3184955307507873, + "rest_mean": 3.066251991657299e-05, + "spike_contrast": 1.276428773763581, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.913852269761264e-05, + "top_five_layers": [ + 30, + 31, + 28, + 23, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.297258318634704e-06, + 1.5986504877218977e-05, + 1.5507564967265353e-05, + 1.589352177688852e-05, + 1.7510028555989265e-05, + 2.491269333404489e-05, + 2.416441566310823e-05, + 2.083729486912489e-05, + 2.4240103812189773e-05, + 3.094671410508454e-05, + 2.889303505071439e-05, + 2.8849686714238487e-05, + 2.6585450541460887e-05, + 3.066893987124786e-05, + 3.6469522456172854e-05, + 2.8640703021665104e-05, + 3.282582474639639e-05, + 3.6509576602838933e-05, + 3.755382931558415e-05, + 3.822202779701911e-05, + 3.2841395295690745e-05, + 4.2974068492185324e-05, + 4.391502443468198e-05, + 3.842243313556537e-05, + 3.7539692129939795e-05, + 4.243620060151443e-05, + 4.0751154301688075e-05, + 4.494010136113502e-05, + 4.0792532672639936e-05, + 4.695989628089592e-05, + 4.5364751713350415e-05, + 4.3128704419359565e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00022091567916504573, + "first_to_last_ratio": 1.232986382535498, + "imbalance_abs_log_ratio": 0.20943917994887448, + "last_quartile_mean": 0.00017917122386279516, + "mean": 0.0002198539991695725, + "normalized": [ + 0.4672441326905094, + 1.0299890566632133, + 1.08986441571188, + 1.0084939820045402, + 0.8814237539069011, + 1.2693826047962031, + 1.1886514905905403, + 1.1035827467935222, + 1.0473307057033197, + 1.4274044616309502, + 1.1442998336482333, + 1.0911283607583675, + 1.1094038041384326, + 1.1946873974128753, + 1.2511466100570947, + 0.9923165311956169, + 0.9239703707694655, + 1.2473578196113966, + 1.1181336030529359, + 1.0160745414007064, + 0.9397394354798955, + 0.9905446518023159, + 1.0121913654482901, + 0.9359943959598221, + 0.8839446941554127, + 0.8510229499852975, + 0.8648513533654478, + 0.8786898836618169, + 0.7455559493639876, + 0.8399702482950041, + 0.7392957942269532, + 0.7163130557190521 + ], + "peak_layer": 10, + "peak_normalized": 1.4274044616309502, + "population_cv": 0.18952709251819305, + "rest_mean": 0.0002217885959436427, + "spike_contrast": 0.9441746799407325, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020940717658959329, + "top_five_layers": [ + 10, + 6, + 15, + 18, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010272549116052687, + 0.00022644721320830286, + 0.00023961105034686625, + 0.00022172143508214504, + 0.0001937845372594893, + 0.0002790788421407342, + 0.0002613297838252038, + 0.00024262708029709756, + 0.00023025984410196543, + 0.000313820579322055, + 0.00025157889467664063, + 0.00023988893372006714, + 0.00024390686303377151, + 0.000262656802078709, + 0.00027506958576850593, + 0.0002181647578254342, + 0.00020313858112785965, + 0.00027423660503700376, + 0.0002458261442370713, + 0.00022338805138133466, + 0.00020660547306761146, + 0.00021777520305477083, + 0.00022253431961871684, + 0.00020578211115207523, + 0.0001943387760547921, + 0.00018710079893935472, + 0.0001901410287246108, + 0.00019318348495289683, + 0.00016391345707233995, + 0.00018467081827111542, + 0.00016253713693004102, + 0.00015748428995721042 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014061377383768559, + 0.014165290631353855, + 0.014281988143920898, + 0.014350161887705326, + 0.00966146495193243, + 0.009741888381540775, + 0.009820535778999329, + 0.00991273857653141, + 0.0074927168898284435, + 0.007546746172010899, + 0.00766373984515667, + 0.007762390188872814, + 0.006325984839349985, + 0.006375366821885109, + 0.006445344537496567, + 0.006555322092026472, + 0.005531130358576775, + 0.005611324217170477, + 0.0056777093559503555, + 0.00574630219489336, + 0.004946626722812653, + 0.0050392793491482735, + 0.005084683652967215, + 0.005203457549214363, + 0.004667261149734259, + 0.004717131145298481, + 0.004768412560224533, + 0.004826710093766451, + 0.0044052014127373695, + 0.004504534415900707, + 0.004574235528707504, + 0.004644096828997135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.729895206561196e-05, + "first_to_last_ratio": 1.2469721805352876, + "imbalance_abs_log_ratio": 0.22071835733540474, + "last_quartile_mean": 2.9911615229138988e-05, + "mean": 3.340667609563752e-05, + "normalized": [ + 1.308752778077389, + 1.2456464596610868, + 1.0236956931411572, + 0.9922285647386262, + 1.26903181062375, + 1.1170846373362797, + 1.0287316533628064, + 0.9469237624012169, + 1.1300261751959062, + 1.1132698789554223, + 1.0126240729566007, + 0.9379202574439188, + 1.1130503370514597, + 1.070713488608195, + 0.9897642722064988, + 0.922085688720865, + 1.036831181462568, + 0.9898105546019274, + 0.9395130252451869, + 0.9001967292772246, + 0.9975952535130206, + 0.9404117749145336, + 0.9224862220158919, + 0.8885787590271026, + 0.9461670180110204, + 0.8974372095121397, + 0.8904691498559152, + 0.8684437934704528, + 0.9244147277585869, + 0.9015689206268677, + 0.8740901912628669, + 0.8604359589635168 + ], + "peak_layer": 1, + "peak_normalized": 1.308752778077389, + "population_cv": 0.1164905978879517, + "rest_mean": 3.378375206213807e-05, + "spike_contrast": 0.9285666618330386, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.137046587653458e-05, + "top_five_layers": [ + 1, + 5, + 2, + 9, + 6 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.3721080146497115e-05, + 4.161290780757554e-05, + 3.419827044126578e-05, + 3.314705827506259e-05, + 4.239413465256803e-05, + 3.73180846509058e-05, + 3.4366505133220926e-05, + 3.1633575417799875e-05, + 3.7750418414361775e-05, + 3.719064625329338e-05, + 3.3828404411906376e-05, + 3.1332798243965954e-05, + 3.718331208801828e-05, + 3.5768978705164045e-05, + 3.306473445263691e-05, + 3.080381793552078e-05, + 3.463708344497718e-05, + 3.3066280593629926e-05, + 3.138600732199848e-05, + 3.007258055731654e-05, + 3.332634150865488e-05, + 3.1416031561093405e-05, + 3.0817198421573266e-05, + 2.9684462788281962e-05, + 3.1608295103069395e-05, + 2.9980394174344838e-05, + 2.9747614462394267e-05, + 2.9011820515734144e-05, + 3.0881623388268054e-05, + 3.01184209092753e-05, + 2.9200447897892445e-05, + 2.874430538213346e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.705871965095866e-05, + "first_to_last_ratio": 0.7180297289018083, + "imbalance_abs_log_ratio": 0.33124430563536084, + "last_quartile_mean": 9.33926785364747e-05, + "mean": 8.228966771639534e-05, + "normalized": [ + 0.8685826408392191, + 0.8249006353131848, + 0.7154905339803773, + 0.6884730939760333, + 0.9470487095509537, + 0.8638069620260335, + 0.8256155016985178, + 0.7853664463613442, + 0.9851686390305121, + 0.9692511243547932, + 0.9083804947955202, + 0.8729855457896263, + 1.0422548534804916, + 1.0255492616133721, + 0.9871528463964067, + 0.9398223255303129, + 1.0958375595009802, + 1.0726325633807163, + 1.0495259774381736, + 1.0230838788771643, + 1.1576441025653552, + 1.1142532603054807, + 1.1008567438213033, + 1.056908774400763, + 1.159063578749664, + 1.1192434432436902, + 1.1032241584740294, + 1.1060435702772593, + 1.1786235078649367, + 1.1628365878275773, + 1.141840825041133, + 1.1085318534950752 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235078649367, + "population_cv": 0.1341844256762087, + "rest_mean": 8.049536754023629e-05, + "spike_contrast": 1.1426606459269575, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.197888866765424e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.14753768988885e-05, + 6.788079917896539e-05, + 5.887747829547152e-05, + 5.66542221349664e-05, + 7.793232362018898e-05, + 7.108238787623122e-05, + 6.793962529627606e-05, + 6.462754390668124e-05, + 8.106919995043427e-05, + 7.975935295689851e-05, + 7.475032907677814e-05, + 7.183769048424438e-05, + 8.576680556870997e-05, + 8.4392107964959e-05, + 8.123247971525416e-05, + 7.733766688033938e-05, + 9.017610864248127e-05, + 8.826657722238451e-05, + 8.636514394311234e-05, + 8.418923243880272e-05, + 9.526214853394777e-05, + 9.169153054244816e-05, + 9.0589135652408e-05, + 8.697267185198143e-05, + 9.537895675748587e-05, + 9.210217103827745e-05, + 9.078394941752777e-05, + 9.101595787797123e-05, + 9.698853682493791e-05, + 9.56894364207983e-05, + 9.396170207764953e-05, + 9.122071787714958e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05946834245696664, + "first_to_last_ratio": 0.7180297222365345, + "imbalance_abs_log_ratio": 0.33124431491808765, + "last_quartile_mean": 0.08282156102359295, + "mean": 0.07297530013602227, + "normalized": [ + 0.8685826122359729, + 0.8249005917343738, + 0.71549052911611, + 0.6884731350635278, + 0.9470486699630107, + 0.8638070113577097, + 0.8256154769796886, + 0.7853664269052657, + 0.9851686341386658, + 0.9692511556795396, + 0.9083805454214465, + 0.8729855579949275, + 1.0422548040385637, + 1.0255492365750694, + 0.9871528950273681, + 0.9398223256645052, + 1.0958375053304956, + 1.07263252629438, + 1.0495259690486791, + 1.0230837911446407, + 1.1576441435704576, + 1.1142533045483842, + 1.1008568124470575, + 1.0569088284972943, + 1.1590636022344982, + 1.119243411839159, + 1.1032241423789504, + 1.1060435591240758, + 1.1786235037173105, + 1.1628366076958818, + 1.141840809884862, + 1.1085318743481278 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235037173105, + "population_cv": 0.13418442668798974, + "rest_mean": 0.07138409617322462, + "spike_contrast": 1.1426607032635474, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08156780153512955, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.06338507682085037, + 0.0601973682641983, + 0.05221313610672951, + 0.050241533666849136, + 0.0691111609339714, + 0.06303657591342926, + 0.060249537229537964, + 0.05731235072016716, + 0.07189297676086426, + 0.07073139399290085, + 0.06628934293985367, + 0.06370638310909271, + 0.07605885714292526, + 0.0748397633433342, + 0.07203777879476547, + 0.06858381628990173, + 0.07996907085180283, + 0.07827568054199219, + 0.07658947259187698, + 0.07465984672307968, + 0.0844794288277626, + 0.08131296932697296, + 0.0803353562951088, + 0.07712823897600174, + 0.08458301424980164, + 0.08167712390422821, + 0.08050811290740967, + 0.08071386069059372, + 0.0860104039311409, + 0.08485835045576096, + 0.08332617580890656, + 0.08089544624090195 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.408341275549901e-06, + "first_to_last_ratio": 0.49124437919492936, + "imbalance_abs_log_ratio": 0.7108135577065074, + "last_quartile_mean": 1.5080765479069669e-05, + "mean": 1.1844782648040564e-05, + "normalized": [ + 0.5657990507718705, + 0.5382233871697658, + 0.49942145823009426, + 0.48906984415931193, + 0.8062921756393574, + 0.7122953827323945, + 0.7013508394120195, + 0.6911626993599347, + 0.8940675881372303, + 0.868722127857376, + 0.9735911331661544, + 0.854855860776803, + 0.989483587379368, + 1.0059088486455374, + 1.0303729006301217, + 0.9941025538400318, + 1.1620912657632405, + 1.127012997009919, + 1.113401270263767, + 1.066097770938846, + 1.1826034542812638, + 1.1643547935214302, + 1.2049778184223279, + 1.1791491538213539, + 1.2365172470752577, + 1.219921447624247, + 1.2337761202582145, + 1.2288278254433027, + 1.3039217543779833, + 1.3631114810840155, + 1.2856635723507386, + 1.3138525898567213 + ], + "peak_layer": 30, + "peak_normalized": 1.3631114810840155, + "population_cv": 0.2549101540724733, + "rest_mean": 1.1420299651271543e-05, + "spike_contrast": 1.237882653019464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4136990830593277e-05, + "top_five_layers": [ + 30, + 32, + 29, + 31, + 25 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.701766778860474e-06, + 6.3751390371180605e-06, + 5.915538622502936e-06, + 5.7929260037781205e-06, + 9.550355571263935e-06, + 8.436983989668079e-06, + 8.307348252856173e-06, + 8.186671948351432e-06, + 1.0590036254143342e-05, + 1.0289824786013924e-05, + 1.1531975360412616e-05, + 1.0125581866304856e-05, + 1.1720218026312068e-05, + 1.1914771675947122e-05, + 1.220454305439489e-05, + 1.1774928680097219e-05, + 1.3764718460151926e-05, + 1.334922399109928e-05, + 1.3187996046326589e-05, + 1.2627696378331166e-05, + 1.4007680874783546e-05, + 1.379152945446549e-05, + 1.4272700354922563e-05, + 1.3966765436634887e-05, + 1.4646278032159898e-05, + 1.4449704394792207e-05, + 1.4613809980801307e-05, + 1.4555198504240252e-05, + 1.5444669770658948e-05, + 1.614575921848882e-05, + 1.5228405572997872e-05, + 1.5562298358418047e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6029601637419546e-05, + "first_to_last_ratio": 0.6070213107493427, + "imbalance_abs_log_ratio": 0.49919138022039955, + "last_quartile_mean": 9.230252817360451e-05, + "mean": 7.840498903988191e-05, + "normalized": [ + 0.6598133090619259, + 0.6419677920829977, + 0.6005256373772865, + 0.5913463573822231, + 0.8562376513989928, + 0.8062968559250673, + 0.7923156566726156, + 0.768439227921366, + 0.9790122729268586, + 0.9665730330981341, + 0.9163710065350301, + 0.8881686280668529, + 1.0631190324760253, + 1.0511030519715439, + 1.0188080221730096, + 0.9712812246823179, + 1.1361272291219966, + 1.1138561419734825, + 1.0918134348286874, + 1.0639217496772315, + 1.203414696914147, + 1.1592426077267617, + 1.1442664103435203, + 1.097952968622282, + 1.2042281788778202, + 1.1625445130141736, + 1.1452986211445626, + 1.1473932954411665, + 1.2233942816523278, + 1.205318760675343, + 1.1829469858778916, + 1.146901364356358 + ], + "peak_layer": 29, + "peak_normalized": 1.2233942816523278, + "population_cv": 0.1898597249088972, + "rest_mean": 7.605543874820728e-05, + "spike_contrast": 1.1977126437531995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.109256061492487e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.173265526536852e-05, + 5.033347770222463e-05, + 4.708420601673424e-05, + 4.636450466932729e-05, + 6.713330367347226e-05, + 6.321769615169615e-05, + 6.212150037754327e-05, + 6.024946924299002e-05, + 7.675944652874023e-05, + 7.578414806630462e-05, + 7.184805872384459e-05, + 6.963685154914856e-05, + 8.335383608937263e-05, + 8.241172326961532e-05, + 7.987963181221858e-05, + 7.615329377586022e-05, + 8.907804294722155e-05, + 8.733187860343605e-05, + 8.560362039133906e-05, + 8.341677312273532e-05, + 9.435371612198651e-05, + 9.089040395338088e-05, + 8.971619536168873e-05, + 8.608499047113582e-05, + 9.441749716643244e-05, + 9.114928980125114e-05, + 8.97971258382313e-05, + 8.996135875349864e-05, + 9.592021524440497e-05, + 9.450300422031432e-05, + 9.274894546251744e-05, + 8.992278890218586e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.8511469685763586e-05, + "first_to_last_ratio": 0.5271352332510719, + "imbalance_abs_log_ratio": 0.6402981537748061, + "last_quartile_mean": 9.202850924339145e-05, + "mean": 7.532146378252946e-05, + "normalized": [ + 0.548104082267993, + 0.5416830199005969, + 0.528884702689083, + 0.5247448675895723, + 0.7777813723908718, + 0.7487613756360456, + 0.7496059866168319, + 0.7329069161583202, + 0.9480676391902623, + 0.9399123881532756, + 0.9000936195631334, + 0.883441930780847, + 1.0560589316963704, + 1.0534552095740957, + 1.032095781768693, + 0.9835620691098637, + 1.1601754560133901, + 1.1467788545782247, + 1.1233340534265408, + 1.0967375255670042, + 1.2396083970871608, + 1.1951467072205837, + 1.1828066060767042, + 1.1317734013234224, + 1.244698956870482, + 1.2025862582796845, + 1.1881970083126314, + 1.1882256015362047, + 1.2689460104605979, + 1.2589162611011824, + 1.2307315535498626, + 1.1921774555104672 + ], + "peak_layer": 29, + "peak_normalized": 1.2689460104605979, + "population_cv": 0.23486334953575583, + "rest_mean": 7.254842300123225e-05, + "spike_contrast": 1.244629176846487, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.029588400153444e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 4.1284001781605184e-05, + 4.080035796505399e-05, + 3.9836369978729635e-05, + 3.952455153921619e-05, + 5.858363147126511e-05, + 5.639780283672735e-05, + 5.6461420172126964e-05, + 5.520362174138427e-05, + 7.140984234865755e-05, + 7.079557690303773e-05, + 6.77963689668104e-05, + 6.654213939327747e-05, + 7.954390457598493e-05, + 7.934778841445222e-05, + 7.773896504659206e-05, + 7.408333476632833e-05, + 8.738611359149218e-05, + 8.637706196168438e-05, + 8.461116522084922e-05, + 8.26078758109361e-05, + 9.336911898571998e-05, + 9.002019942272455e-05, + 8.909072494134307e-05, + 8.524682925781235e-05, + 9.375254740007222e-05, + 9.058055729838088e-05, + 8.949673792812973e-05, + 8.949889161158353e-05, + 9.557887096889317e-05, + 9.482341556577012e-05, + 9.270050213672221e-05, + 8.979655103757977e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.61669284151867e-05, + "first_to_last_ratio": 0.8297833624837898, + "imbalance_abs_log_ratio": 0.18659062131596024, + "last_quartile_mean": 0.00011589401856326731, + "mean": 0.00010716335873439675, + "normalized": [ + 0.9380742209455786, + 0.8898820858744699, + 0.7837417114644826, + 0.7583102664636657, + 1.0822070619439073, + 0.9520749081713696, + 0.9107982467806989, + 0.8640017807748708, + 1.0669538303069324, + 1.0266950649381863, + 0.9594029855154438, + 0.8901506143407661, + 1.0795289063552718, + 1.0379165674077444, + 0.9706971366471856, + 0.914979822300863, + 1.052939971267624, + 1.0225707935964177, + 0.9851238030379684, + 0.9636652893153618, + 1.0990044510799908, + 1.0523904893769196, + 1.0422234788441622, + 1.0049019599938585, + 1.0962247910521359, + 1.0599378706264646, + 1.05039930496957, + 1.0583562345651827, + 1.1199788667078585, + 1.1100529546792268, + 1.0850973923636869, + 1.0717171382921347 + ], + "peak_layer": 29, + "peak_normalized": 1.1199788667078585, + "population_cv": 0.09156717959150794, + "rest_mean": 0.00010599351123716958, + "spike_contrast": 1.0706366257223145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011348053521942347, + "top_five_layers": [ + 29, + 30, + 21, + 25, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010052718425868079, + 9.536275319987908e-05, + 8.398839418077841e-05, + 8.12630751170218e-05, + 0.00011597294360399246, + 0.00010202754492638633, + 9.760419925441965e-05, + 9.25893327803351e-05, + 0.00011433835607022047, + 0.00011002409155480564, + 0.00010281284630764276, + 9.539152961224318e-05, + 0.000115685943455901, + 0.0001112266254494898, + 0.0001040231654769741, + 9.805231093196198e-05, + 0.00011283658386673778, + 0.00010958212078548968, + 0.00010556917550275102, + 0.00010326960909878835, + 0.00011777300824178383, + 0.0001127776995417662, + 0.00011168816854478791, + 0.00010768866923172027, + 0.00011747513053705916, + 0.00011358650226611644, + 0.00011256431753281504, + 0.00011341700883349404, + 0.00012002069706795737, + 0.00011895700299646705, + 0.00011628268111962825, + 0.00011484880815260112 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 7.959341947457508, + "forward": { + "activation_sha256": { + "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" + }, + "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", + "loss_nats": 5.516995429992676, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0021613112185150385, + 0.0025601957459002733, + 0.0025323182344436646, + 0.0027253651060163975, + 0.002904455177485943, + 0.0029208383057266474, + 0.00299624796025455, + 0.0030937574338167906, + 0.0033804215490818024, + 0.0034370317589491606, + 0.0038969130255281925, + 0.004255065228790045, + 0.004119740333408117, + 0.003970506135374308, + 0.004385598469525576, + 0.0041999719105660915, + 0.005232699681073427, + 0.004664790350943804, + 0.004898969549685717, + 0.005131533369421959, + 0.005069608334451914, + 0.0051318020559847355, + 0.0058372304774820805, + 0.0059690922498703, + 0.00554921617731452, + 0.0058550527319312096, + 0.006255472078919411, + 0.006502739619463682, + 0.006122888531535864, + 0.007071305997669697, + 0.006241494789719582 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962477027642308, + "first_to_last_ratio": 3.7699040072151497, + "imbalance_abs_log_ratio": 1.3270495388586014, + "last_quartile_mean": 4.234186599205714e-05, + "mean": 8.617327489446325e-05, + "normalized": [ + 2.5428430303860874, + 2.3222064178412967, + 2.1011513502014654, + 1.8911470095267005, + 1.7004447966680725, + 1.5353700031893112, + 1.4229335846259188, + 1.3028620492588128, + 1.2049362956920768, + 1.1705197150696323, + 1.0773612761873523, + 1.0133200843661747, + 0.9906112963967714, + 0.9272530070793641, + 0.8550207003106389, + 0.8185600560490035, + 0.7703816559177782, + 0.7173918665339755, + 0.6760157258502337, + 0.6514535263006792, + 0.6184138115448249, + 0.6053606047045522, + 0.5838821473278404, + 0.5697017460729616, + 0.5457761369570343, + 0.5276086337508958, + 0.5104655265174319, + 0.4951752820428687, + 0.47527333481689216, + 0.465789239220051, + 0.45896414036514044, + 0.45180594922816086 + ], + "peak_layer": 1, + "peak_normalized": 2.5428430303860874, + "population_cv": 0.5705341151299013, + "rest_mean": 9.280180475562466e-05, + "spike_contrast": 0.5428688997682254, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0379213644191625e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021912511147093028, + 0.00020011213200632483, + 0.00018106309289578348, + 0.00016296633111778647, + 0.00014653289690613747, + 0.00013230786134954542, + 0.0001226188469445333, + 0.00011227188952034339, + 0.0001038333066389896, + 0.00010086751717608422, + 9.283974941354245e-05, + 8.732111018616706e-05, + 8.536421955795959e-05, + 7.990442827576771e-05, + 7.367993384832516e-05, + 7.053800072753802e-05, + 6.63863102090545e-05, + 6.182000652188435e-05, + 5.825448897667229e-05, + 5.6137883802875876e-05, + 5.329074338078499e-05, + 5.216590579948388e-05, + 5.031503678765148e-05, + 4.909306517220102e-05, + 4.703131708083674e-05, + 4.5465763832908124e-05, + 4.3988486140733585e-05, + 4.2670875700423494e-05, + 4.095585973118432e-05, + 4.013858415419236e-05, + 3.955044303438626e-05, + 3.893359826179221e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442661636683624, + "first_to_last_ratio": 1.843144421770586, + "imbalance_abs_log_ratio": 0.6114730379455638, + "last_quartile_mean": 0.0001163373926829081, + "mean": 0.00015221274247778638, + "normalized": [ + 1.7741446897870732, + 1.648461275666186, + 1.5301464224684427, + 1.4217121808823485, + 1.3265065496562456, + 1.2454420544341354, + 1.1897998831173078, + 1.1336248235930768, + 1.0869089599614004, + 1.0643003125599004, + 1.0227030055526172, + 0.9941765648180964, + 0.9788958472869692, + 0.9511102301475627, + 0.9228060651723192, + 0.9052418824754537, + 0.8846484396889256, + 0.865292475366091, + 0.8497311645646503, + 0.8388691935254817, + 0.8256303537004397, + 0.8176325372729673, + 0.8076852907347736, + 0.8000670639426607, + 0.7892588692968658, + 0.7814415503461742, + 0.7731202151361662, + 0.7664994567889658, + 0.758490072461763, + 0.7529625284402992, + 0.7485174912671357, + 0.7441725498875049 + ], + "peak_layer": 1, + "peak_normalized": 1.7741446897870732, + "population_cv": 0.27364872432853743, + "rest_mean": 0.00015762320577373935, + "spike_contrast": 0.7803181014868814, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299624067964033, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027004742878489196, + 0.0002509168116375804, + 0.0002329077833564952, + 0.00021640271006617695, + 0.00020191119983792305, + 0.00018957215070258826, + 0.0001811027032090351, + 0.00017255214333999902, + 0.00016544139361940324, + 0.00016200006939470768, + 0.00015566842921543866, + 0.0001513263414381072, + 0.00014900042151566595, + 0.0001447710965294391, + 0.00014046284195501357, + 0.0001377893495373428, + 0.00013465476513374597, + 0.00013170854072086513, + 0.00012933991092722863, + 0.0001276865805266425, + 0.00012567146040964872, + 0.00012445409083738923, + 0.00012293999316170812, + 0.00012178040196886286, + 0.00012013525702059269, + 0.00011894536146428436, + 0.00011767874821089208, + 0.00011667098442558199, + 0.00011545185407157987, + 0.00011461049143690616, + 0.00011393390013836324, + 0.00011327254469506443 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19015566632151604, + "first_to_last_ratio": 1.8431444167877922, + "imbalance_abs_log_ratio": 0.6114730352421437, + "last_quartile_mean": 0.10316916275769472, + "mean": 0.13498378219082952, + "normalized": [ + 1.7741447141792424, + 1.6484611693507762, + 1.5301464164964773, + 1.421712099711509, + 1.3265065258282553, + 1.245442086656075, + 1.1897999853143832, + 1.1336247998774232, + 1.0869090145314644, + 1.0643003539468234, + 1.0227030148964142, + 0.9941765578539046, + 0.978895844117059, + 0.9511102293531465, + 0.9228060470726628, + 0.9052419251832852, + 0.8846484748200404, + 0.8652925213936268, + 0.8497311905993837, + 0.8388692019864461, + 0.8256303029992552, + 0.8176324960789627, + 0.8076852727084025, + 0.800067049482784, + 0.789258877083647, + 0.7814415614650243, + 0.7731201949085273, + 0.7664994759042474, + 0.758490077799444, + 0.7529625178867759, + 0.7485174642151933, + 0.7441725362993381 + ], + "peak_layer": 1, + "peak_normalized": 1.7741447141792424, + "population_cv": 0.2736487184871982, + "rest_mean": 0.13978183573042904, + "spike_contrast": 0.7803180757143806, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907429307699204, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23948076367378235, + 0.2225155234336853, + 0.20654495060443878, + 0.1919080764055252, + 0.17905686795711517, + 0.16811448335647583, + 0.16060370206832886, + 0.1530209630727768, + 0.14671508967876434, + 0.14366328716278076, + 0.13804832100868225, + 0.13419771194458008, + 0.1321350634098053, + 0.12838445603847504, + 0.12456385046243668, + 0.12219297885894775, + 0.11941319704055786, + 0.116800457239151, + 0.11469992995262146, + 0.11323373764753342, + 0.11144670099020004, + 0.11036712676286697, + 0.10902441293001175, + 0.10799607634544373, + 0.10653714835643768, + 0.10548193752765656, + 0.10435868799686432, + 0.1034649983048439, + 0.10238385945558548, + 0.10163772851228714, + 0.10103771835565567, + 0.10045122355222702 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883365585963475e-05, + "first_to_last_ratio": 2.2497546817252747, + "imbalance_abs_log_ratio": 0.8108211799277664, + "last_quartile_mean": 3.059607183786284e-05, + "mean": 4.691841348858361e-05, + "normalized": [ + 1.7359303646705315, + 1.6475859007944922, + 1.5536416135247637, + 1.4921074478374423, + 1.436565604934822, + 1.343834173349448, + 1.2787439414688093, + 1.2483315192759759, + 1.191227587726315, + 1.1603086071531024, + 1.0995306735006558, + 1.059977946939621, + 1.0583984121521266, + 1.0176934649371145, + 0.9754909410255406, + 0.9631621800279858, + 0.9242301483750994, + 0.8752327792179452, + 0.8454160845678106, + 0.8232909667825862, + 0.7863453139125789, + 0.77746685672799, + 0.7497386589361914, + 0.7388508716017261, + 0.7093180450413769, + 0.6908354487677191, + 0.6685471901846501, + 0.6592354484095886, + 0.631440334976849, + 0.6235901145414114, + 0.6199506557967245, + 0.6139807028410055 + ], + "peak_layer": 1, + "peak_normalized": 1.7359303646705315, + "population_cv": 0.3220412433027962, + "rest_mean": 4.907019663530971e-05, + "spike_contrast": 0.7193528234378936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.529878449626267e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144709863699973e-05, + 7.730211655143648e-05, + 7.289439963642508e-05, + 7.000731420703232e-05, + 6.740137905580923e-05, + 6.305056740529835e-05, + 5.999663699185476e-05, + 5.856973439222202e-05, + 5.5890508519951254e-05, + 5.4439839004771784e-05, + 5.1588234782684594e-05, + 4.9732483603293076e-05, + 4.965837433701381e-05, + 4.774856279254891e-05, + 4.576848732540384e-05, + 4.519004141911864e-05, + 4.3363412260077894e-05, + 4.106453343410976e-05, + 3.966558142565191e-05, + 3.862750600092113e-05, + 3.6894074582960457e-05, + 3.647751145763323e-05, + 3.517654840834439e-05, + 3.4665710700210184e-05, + 3.328007733216509e-05, + 3.241290323785506e-05, + 3.136717350571416e-05, + 3.0930281354812905e-05, + 2.9626178729813546e-05, + 2.925785884144716e-05, + 2.9087101211189292e-05, + 2.8807000489905477e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001734125216898974, + "first_to_last_ratio": 1.495517223041696, + "imbalance_abs_log_ratio": 0.4024721155990803, + "last_quartile_mean": 0.00011595488103921525, + "mean": 0.00013990532806928968, + "normalized": [ + 1.3768889945483427, + 1.3340940553978193, + 1.2910049686488183, + 1.2527203273458163, + 1.2147536525239127, + 1.1769854337196524, + 1.14744939519443, + 1.1220955988146075, + 1.0944022472959734, + 1.0773022658134643, + 1.0508559212107247, + 1.03206209473486, + 1.0208171898483585, + 1.000561882500425, + 0.9805161605170362, + 0.9663129296065233, + 0.9487424991891081, + 0.931931984738623, + 0.9181974345490227, + 0.9075104531838204, + 0.8943640941785878, + 0.8862664021756573, + 0.8757830774574916, + 0.86790401991425, + 0.8562894022841879, + 0.8478155487572495, + 0.8387227683262297, + 0.8314011142413483, + 0.8224071457687988, + 0.8161875050523302, + 0.8112199678685613, + 0.8064334645939688 + ], + "peak_layer": 1, + "peak_normalized": 1.3768889945483427, + "population_cv": 0.16325957486062354, + "rest_mean": 0.0001431148236776978, + "spike_contrast": 0.8564734849544942, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225740517838858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926341064972803, + 0.00018664686649572104, + 0.00018061847367789596, + 0.00017526224837638438, + 0.0001699505082797259, + 0.00016466653323732316, + 0.00016053428407758474, + 0.00015698715287726372, + 0.00015311270544771105, + 0.00015072032692842185, + 0.00014702034241054207, + 0.00014439098595175892, + 0.00014281776384450495, + 0.00013998393842484802, + 0.00013717943511437625, + 0.00013519232743419707, + 0.00013273413060232997, + 0.00013038225006312132, + 0.00012846071331296116, + 0.00012696554767899215, + 0.0001251263020094484, + 0.00012399339175317436, + 0.0001225267187692225, + 0.00012142439663875848, + 0.00011979944974882528, + 0.00011861391249112785, + 0.00011734178406186402, + 0.00011631744564510882, + 0.00011505914153531194, + 0.00011418898066040128, + 0.00011349399574100971, + 0.00011282433843007311 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675117381557357, + "first_to_last_ratio": 1.2676837465168667, + "imbalance_abs_log_ratio": 0.23719141364257146, + "last_quartile_mean": 0.00011576323686313117, + "mean": 0.00013042020850662084, + "normalized": [ + 1.169988069421413, + 1.156175152925158, + 1.1417665260225645, + 1.1306726331060184, + 1.1183621029407895, + 1.1062694829504214, + 1.0940115056230622, + 1.0844989956600726, + 1.0711591662817939, + 1.0579765491085218, + 1.0443900226505625, + 1.035471661013805, + 1.0260498632741444, + 1.013992390086173, + 1.0062479316494695, + 0.9977001177294099, + 0.9863163473759529, + 0.9769959732660659, + 0.9687585669786032, + 0.9602628595878113, + 0.9503845997255937, + 0.9426104617668664, + 0.9330243108845692, + 0.9259760938667744, + 0.914797625470175, + 0.906289421438531, + 0.8956357003805738, + 0.888470885240542, + 0.8806216558595509, + 0.8756991509238615, + 0.8712858841028995, + 0.8681382926882502 + ], + "peak_layer": 1, + "peak_normalized": 1.169988069421413, + "population_cv": 0.09123006364857297, + "rest_mean": 0.00013202972385678786, + "spike_contrast": 0.9219804606102014, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012172882561571896, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015259008796419948, + 0.00015078860451467335, + 0.00014890942838974297, + 0.0001474625605624169, + 0.00014585701865144074, + 0.00014427989663090557, + 0.00014268120867200196, + 0.00014144058513920754, + 0.00013970080181024969, + 0.0001379815221298486, + 0.00013620956451632082, + 0.00013504642993211746, + 0.0001338176371064037, + 0.00013224509893916547, + 0.00013123506505507976, + 0.0001301202573813498, + 0.00012863558367826045, + 0.00012742001854348928, + 0.00012634569429792464, + 0.00012523768236860633, + 0.00012394935765769333, + 0.00012293545296415687, + 0.00012168522516731173, + 0.00012076599523425102, + 0.00011930809705518186, + 0.00011819845531135798, + 0.00011680899478960782, + 0.00011587455810513347, + 0.00011485085997264832, + 0.00011420886585256085, + 0.00011363328667357564, + 0.00011322277714498341 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030002388848515693, + "first_to_last_ratio": 2.2836575149168885, + "imbalance_abs_log_ratio": 0.8257783308640597, + "last_quartile_mean": 0.0001313786706305109, + "mean": 0.0001947244400071213, + "normalized": [ + 1.8998833386658438, + 1.7835645331795598, + 1.6713073472446418, + 1.5703972875779748, + 1.4780250735507643, + 1.3761669788219328, + 1.3066447563403325, + 1.2401013786965793, + 1.1762106459027857, + 1.150547637187856, + 1.0768735950666006, + 1.0264900078881174, + 1.0140784159549234, + 0.9666871251601796, + 0.9096841069789114, + 0.8754807117769676, + 0.8421876125778792, + 0.8073809902734768, + 0.77911667677541, + 0.7607651063686129, + 0.7403822030704592, + 0.7308855611321243, + 0.7152791523187013, + 0.7043381880055872, + 0.6944157299089508, + 0.686864629891504, + 0.680599049276497, + 0.6764772714226681, + 0.6702412094778135, + 0.6672258960044359, + 0.6635863560708457, + 0.6581114274310631 + ], + "peak_layer": 1, + "peak_normalized": 1.8998833386658438, + "population_cv": 0.3592853832280775, + "rest_mean": 0.00020492727344390005, + "spike_contrast": 0.6813594750078023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962913944851608, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699537192005664, + 0.0003473036049399525, + 0.0003254443872720003, + 0.00030579473241232336, + 0.00028780760476365685, + 0.00026797334430739284, + 0.0002544356684666127, + 0.0002414780465187505, + 0.0002290369593538344, + 0.00022403974435292184, + 0.00020969360775779933, + 0.00019988269195891917, + 0.0001974658516701311, + 0.00018823760910890996, + 0.00017713772831484675, + 0.00017047749133780599, + 0.00016399451124016196, + 0.0001572168112033978, + 0.00015171305858530104, + 0.00014813955931458622, + 0.00014417050988413393, + 0.00014232128160074353, + 0.00013928233238402754, + 0.0001371518592350185, + 0.00013521971413865685, + 0.00013374933041632175, + 0.00013252926873974502, + 0.00013172665785532445, + 0.00013051234418526292, + 0.00012992518895771354, + 0.00012921648158226162, + 0.000128150379168801 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027073738165199757, + 0.002722575329244137, + 0.0026850115973502398, + 0.0026861224323511124, + 0.00271838391199708, + 0.0026645169127732515, + 0.0026859452482312918, + 0.0027359756641089916, + 0.0026767742820084095, + 0.0027451682835817337, + 0.0026999011170119047, + 0.002663193503394723, + 0.002626648638397455, + 0.0026531771291047335, + 0.0026517712976783514, + 0.002676903735846281, + 0.002670086920261383, + 0.0026355870068073273, + 0.0026610493659973145, + 0.0026388936676084995, + 0.0026469926815479994, + 0.00264862016774714, + 0.002641494618728757, + 0.002674198243767023, + 0.002557571977376938, + 0.00252733426168561, + 0.0025468147359788418, + 0.0026856842450797558, + 0.002551089273765683, + 0.0025778315030038357, + 0.0026001501828432083, + 0.0025662274565547705 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858857113926206, + "first_to_last_ratio": 3.757867690304466, + "imbalance_abs_log_ratio": 1.3238516928952826, + "last_quartile_mean": 4.220174423608114e-05, + "mean": 8.573221236929385e-05, + "normalized": [ + 2.5461641896458365, + 2.3032502265274117, + 2.0924715218207024, + 1.8886722264368014, + 1.7031663195050915, + 1.537434754996766, + 1.4223955235407997, + 1.3049501874408618, + 1.2055297161045848, + 1.1692542365736316, + 1.0769571962730058, + 1.0144505847411098, + 0.9913601784485884, + 0.9317001386730132, + 0.857757082878185, + 0.8155634736467603, + 0.7682612945963089, + 0.7217806418748633, + 0.678338533281248, + 0.6511820384364709, + 0.6192882306865346, + 0.6053793146733001, + 0.5848721334223365, + 0.5718144052517429, + 0.5462281883927541, + 0.527515083157495, + 0.5112993585545178, + 0.496920907994256, + 0.47722358366421574, + 0.46552107689700334, + 0.4601234027970099, + 0.45317424906679377 + ], + "peak_layer": 1, + "peak_normalized": 2.5461641896458365, + "population_cv": 0.5686091737198813, + "rest_mean": 9.231269224848668e-05, + "spike_contrast": 0.5437781067692293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0197621021652594e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021828828903380781, + 0.00019746273756027222, + 0.00017939221288543195, + 0.0001619200484128669, + 0.00014601621660403907, + 0.000131807682919316, + 0.00012194511509733275, + 0.00011187626660102978, + 0.00010335272963857278, + 0.00010024275252362713, + 9.232992306351662e-05, + 8.697109296917915e-05, + 8.499150135321543e-05, + 7.987671415321529e-05, + 7.353741239057854e-05, + 6.992006092332304e-05, + 6.586474046343938e-05, + 6.187985127326101e-05, + 5.8155463193543255e-05, + 5.582727681030519e-05, + 5.309295011102222e-05, + 5.1900507969548926e-05, + 5.0142381951445714e-05, + 4.902291402686387e-05, + 4.682935104938224e-05, + 4.522503513726406e-05, + 4.3834825191879645e-05, + 4.260212881490588e-05, + 4.0913433622336015e-05, + 3.991015182691626e-05, + 3.944739728467539e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242323236947414, + "first_to_last_ratio": 1.8345651540958352, + "imbalance_abs_log_ratio": 0.6068074801791611, + "last_quartile_mean": 0.0001157894184871111, + "mean": 0.00015111673860701558, + "normalized": [ + 1.771707915810251, + 1.6366668487510905, + 1.5242909365839221, + 1.4186705636795862, + 1.3264449439106374, + 1.2453857912414694, + 1.1893415085703798, + 1.1330085309323608, + 1.0858463851390703, + 1.062861239127963, + 1.023237806768965, + 0.9942126959777651, + 0.978436257513136, + 0.9531027445378829, + 0.9232414968561398, + 0.904013237249231, + 0.8853704564278231, + 0.8680164030054869, + 0.8515402788738432, + 0.8395817783841374, + 0.8267435189825344, + 0.8185143642831039, + 0.809172703595369, + 0.8007918347797284, + 0.7909896391756025, + 0.7821052393452608, + 0.7753388667989797, + 0.7682453290925442, + 0.7604995795745633, + 0.7549901567959042, + 0.7508572350862155, + 0.7467737131490535 + ], + "peak_layer": 1, + "peak_normalized": 1.771707915810251, + "population_cv": 0.2714170004285523, + "rest_mean": 0.00015645500981783562, + "spike_contrast": 0.7816309251520459, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229007406858728, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677347220014781, + 0.00024732775636948645, + 0.00023034587502479553, + 0.00021438486874103546, + 0.00020044803386554122, + 0.0001881986390799284, + 0.00017972940986510366, + 0.00017121655400842428, + 0.00016408956435043365, + 0.00016061612404882908, + 0.0001546283601783216, + 0.00015024218009784818, + 0.00014785809617023915, + 0.0001440297783119604, + 0.00013951724395155907, + 0.000136611532070674, + 0.0001337942958343774, + 0.00013117180787958205, + 0.0001286819897359237, + 0.00012687486014328897, + 0.00012493478425312787, + 0.00012369122123345733, + 0.00012227953993715346, + 0.00012101305037504062, + 0.00011953177454415709, + 0.00011818919301731512, + 0.00011716668086592108, + 0.00011609472858253866, + 0.00011492421617731452, + 0.00011409165017539635, + 0.00011346709652571008, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837904930114746, + "first_to_last_ratio": 1.8345651581276443, + "imbalance_abs_log_ratio": 0.6068074823768531, + "last_quartile_mean": 0.10268321540206671, + "mean": 0.13401183625683188, + "normalized": [ + 1.7717078885231814, + 1.6366668119471057, + 1.524290957183596, + 1.418670514273678, + 1.326444920903592, + 1.2453858718827995, + 1.1893415500975306, + 1.1330085716119305, + 1.0858463365841577, + 1.062861212873729, + 1.0232377434953237, + 0.9942127340740732, + 0.9784362443707608, + 0.9531027271661117, + 0.9232414917376033, + 0.904013241653558, + 0.885370477646398, + 0.8680164390463408, + 0.8515402681847718, + 0.8395816969955481, + 0.8267435345782925, + 0.8185144279720658, + 0.8091727254708025, + 0.8007918405918669, + 0.7909896328038464, + 0.7821052665924377, + 0.7753388466443417, + 0.7682453528580236, + 0.7604996016741158, + 0.7549901615676585, + 0.7508572336421532, + 0.7467736753526057 + ], + "peak_layer": 1, + "peak_normalized": 1.7717078885231814, + "population_cv": 0.27141699472708714, + "rest_mean": 0.1387458680956452, + "spike_contrast": 0.7816309473985974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844806432724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23742982745170593, + 0.2193327248096466, + 0.2042730301618576, + 0.19011864066123962, + 0.1777593195438385, + 0.16689644753932953, + 0.15938584506511688, + 0.151836559176445, + 0.1455162614583969, + 0.1424359828233719, + 0.13712596893310547, + 0.13323627412319183, + 0.13112203776836395, + 0.12772704660892487, + 0.1237252876162529, + 0.1211484745144844, + 0.11865012347698212, + 0.11632447689771652, + 0.11411647498607635, + 0.11251388490200043, + 0.11079341918230057, + 0.10969062149524689, + 0.10843872278928757, + 0.10731558501720428, + 0.10600197315216064, + 0.10481136292219162, + 0.10390458256006241, + 0.1029539704322815, + 0.10191594809293747, + 0.10117761790752411, + 0.10062375664710999, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873187476230669e-05, + "first_to_last_ratio": 2.248757500145603, + "imbalance_abs_log_ratio": 0.8103778415280068, + "last_quartile_mean": 3.0564378221242805e-05, + "mean": 4.685761763312257e-05, + "normalized": [ + 1.7400006587426948, + 1.6405194725375354, + 1.5551306973978087, + 1.4944078341288562, + 1.4327724019957897, + 1.3414187773842097, + 1.2804088050749736, + 1.2499327782691996, + 1.1927955171655855, + 1.1608931065609767, + 1.0981756972550076, + 1.0577423904261343, + 1.0542786038595535, + 1.0218640087393278, + 0.9775807389332813, + 0.9570972402588515, + 0.9209982861589985, + 0.8815092249082567, + 0.8478232134430647, + 0.8204952153707635, + 0.7871780668180614, + 0.7759097745574741, + 0.7506963524238334, + 0.7421155343193898, + 0.7072927325438048, + 0.6891814936654451, + 0.6695399899996352, + 0.6616343983211914, + 0.6345111343049281, + 0.6212687927046295, + 0.6200231913288072, + 0.6148038704019305 + ], + "peak_layer": 1, + "peak_normalized": 1.7400006587426948, + "population_cv": 0.3217536161653951, + "rest_mean": 4.900405669845727e-05, + "spike_contrast": 0.719671983430414, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.526684668031521e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153228554874659e-05, + 7.687083416385576e-05, + 7.286971958819777e-05, + 7.002439087955281e-05, + 6.71363013680093e-05, + 6.285568815656006e-05, + 5.999690620228648e-05, + 5.8568872191244736e-05, + 5.5891556257847697e-05, + 5.439668530016206e-05, + 5.145789691596292e-05, + 4.9563288484932855e-05, + 4.9400983698433265e-05, + 4.7882112994557247e-05, + 4.580710447044112e-05, + 4.484729652176611e-05, + 4.3155785533599555e-05, + 4.130542220082134e-05, + 3.972697595600039e-05, + 3.844645107164979e-05, + 3.688528886414133e-05, + 3.635728353401646e-05, + 3.517584264045581e-05, + 3.477376594673842e-05, + 3.314205241622403e-05, + 3.229340290999971e-05, + 3.1373048841487616e-05, + 3.10026116494555e-05, + 2.9731680115219206e-05, + 2.9111175535945222e-05, + 2.9052809622953646e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001719075116852764, + "first_to_last_ratio": 1.4894635014227988, + "imbalance_abs_log_ratio": 0.39841598895878805, + "last_quartile_mean": 0.00011541572621354135, + "mean": 0.00013894635094402474, + "normalized": [ + 1.3738985779819257, + 1.3296082759737424, + 1.2883384004275333, + 1.2499940655466777, + 1.2130730176151365, + 1.1755503979222366, + 1.147409009170091, + 1.11990595257071, + 1.0929037170303153, + 1.075598370036652, + 1.0512762916941902, + 1.0312594730951, + 1.0194418974669357, + 1.0018996501730997, + 0.9803460168062212, + 0.9648004557293595, + 0.9493010807852901, + 0.9344788929945884, + 0.9198953859218769, + 0.908001147597105, + 0.8952913731815082, + 0.886953676542665, + 0.8771894463102521, + 0.8683888934041172, + 0.8579399877904575, + 0.8482799643655919, + 0.8408636339472488, + 0.8330870307470335, + 0.82434015220157, + 0.8181187963300758, + 0.8135579418994486, + 0.809009026741244 + ], + "peak_layer": 1, + "peak_normalized": 1.3738985779819257, + "population_cv": 0.16161423472023242, + "rest_mean": 0.0001421073115866252, + "spike_contrast": 0.8576417505420824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187716347398237, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001908981939777732, + 0.0001847442181315273, + 0.00017900991952046752, + 0.00017368211410939693, + 0.00016855206922627985, + 0.00016333843814209104, + 0.00015942829486448318, + 0.00015560684551019222, + 0.0001518549834145233, + 0.00014945046859793365, + 0.00014607100456487387, + 0.0001432897406630218, + 0.00014164773165248334, + 0.00013921030040364712, + 0.000136215501697734, + 0.0001340555027127266, + 0.0001319019211223349, + 0.00012984243221580982, + 0.0001278161071240902, + 0.00012616344611160457, + 0.00012439746933523566, + 0.00012323897681199014, + 0.00012188227265141904, + 0.00012065946793882176, + 0.00011920763063244522, + 0.00011786540562752634, + 0.00011683493357850239, + 0.00011575440294109285, + 0.0001145390560850501, + 0.00011367462138878182, + 0.00011304090730845928, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552221000485588, + "first_to_last_ratio": 1.262697055715921, + "imbalance_abs_log_ratio": 0.2332499537229619, + "last_quartile_mean": 0.00011524712863320019, + "mean": 0.00012953460168318998, + "normalized": [ + 1.1676213631634658, + 1.1525055688027706, + 1.1391580058081985, + 1.1289307982409096, + 1.1166731577821778, + 1.1054001772932092, + 1.094813144753513, + 1.0822854389705017, + 1.0679308603491235, + 1.0564894822224282, + 1.0456838360716472, + 1.0337072702557388, + 1.0234945545465333, + 1.0158819477934218, + 1.0051772952892828, + 0.9951467950359763, + 0.9866643397779161, + 0.9787559418439278, + 0.970448736962688, + 0.9607908684400951, + 0.9509939230158972, + 0.9428061355388626, + 0.934924980051271, + 0.9261034265480621, + 0.9170582603044295, + 0.9058562786860701, + 0.8981778407016658, + 0.8907564927732574, + 0.8831168120837269, + 0.8778216107141017, + 0.8745192645108896, + 0.8703053916682415 + ], + "peak_layer": 1, + "peak_normalized": 1.1676213631634658, + "population_cv": 0.08967236486753827, + "rest_mean": 0.00013110875065817877, + "spike_contrast": 0.9231588022206524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103419721825049, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124736819416285, + 0.00014928934979252517, + 0.000147560378536582, + 0.00014623560127802193, + 0.00014464781270362437, + 0.00014318757166620344, + 0.00014181618462316692, + 0.00014019341324456036, + 0.0001383339986205101, + 0.00013685194426216185, + 0.00013545223919209093, + 0.00013390085950959474, + 0.00013257795944809914, + 0.00013159186346456409, + 0.00013020524056628346, + 0.00012890594371128827, + 0.00012780717224813998, + 0.00012678276107180864, + 0.0001257066905964166, + 0.0001244556624442339, + 0.00012318661902099848, + 0.0001221260172314942, + 0.00012110513489460573, + 0.0001199624384753406, + 0.00011879077646881342, + 0.00011733973224181682, + 0.00011634510883595794, + 0.00011538378748809919, + 0.0001143941844929941, + 0.00011370827269274741, + 0.00011328050459269434, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029741139769612346, + "first_to_last_ratio": 2.278072415841521, + "imbalance_abs_log_ratio": 0.8233296537390045, + "last_quartile_mean": 0.00013055396993877366, + "mean": 0.00019328556300024502, + "normalized": [ + 1.889398576482247, + 1.780233894901915, + 1.6772594716902607, + 1.566224958959574, + 1.4762232610465842, + 1.375571144634086, + 1.3083686481730985, + 1.2364401527267181, + 1.173148897934763, + 1.1508864184295666, + 1.080457040307468, + 1.0318582947152708, + 1.0081957004611959, + 0.9698994715426978, + 0.9066127339785177, + 0.8754550048258108, + 0.8446146862114313, + 0.8115724449181009, + 0.7806677558079268, + 0.7613480996239201, + 0.7403134026279838, + 0.7294052762839086, + 0.7160186976339193, + 0.7062575707090604, + 0.6942777320753531, + 0.6866005528973866, + 0.6829938476675228, + 0.6753403086482458, + 0.6715967314167978, + 0.6686860556999406, + 0.6640286432895989, + 0.6600445236791291 + ], + "peak_layer": 1, + "peak_normalized": 1.889398576482247, + "population_cv": 0.3579154260008552, + "rest_mean": 0.00020340604748559632, + "spike_contrast": 0.681567477924495, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013863494677934796, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003651934675872326, + 0.0003440935106482357, + 0.00032419004128314555, + 0.0003027286729775369, + 0.0002853326441254467, + 0.00026587804313749075, + 0.00025288877077400684, + 0.00023898603103589267, + 0.00022675274522043765, + 0.00022244972933549434, + 0.0002088367473334074, + 0.00019944331143051386, + 0.00019486967357806861, + 0.00018746756541077048, + 0.00017523515271022916, + 0.00016921281348913908, + 0.0001632518251426518, + 0.00015686523693148047, + 0.00015089180669747293, + 0.00014715759607497603, + 0.00014309189282357693, + 0.00014098350948188454, + 0.00013839607709087431, + 0.0001365093921776861, + 0.0001341938623227179, + 0.00013270997442305088, + 0.00013201285037212074, + 0.00013053353177383542, + 0.0001298099523410201, + 0.00012924736074637622, + 0.00012834715016651899, + 0.00012757707736454904 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0040087271481752396, + 0.005379301495850086, + 0.006410556845366955, + 0.0027253651060163975, + 0.004838948138058186, + 0.006175771821290255, + 0.00742313964292407, + 0.0030937574338167906, + 0.005150026176124811, + 0.007019025273621082, + 0.008143599145114422, + 0.004255065228790045, + 0.0066224560141563416, + 0.008129785768687725, + 0.010099578648805618, + 0.0041999719105660915, + 0.00719039561226964, + 0.009030314162373543, + 0.010736005380749702, + 0.005131533369421959, + 0.00787449348717928, + 0.009912566281855106, + 0.012113703414797783, + 0.0059690922498703, + 0.00876442901790142, + 0.010806876234710217, + 0.012353328987956047, + 0.006502739619463682, + 0.009470963850617409, + 0.012123330496251583, + 0.01384514570236206 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962236011546338, + "first_to_last_ratio": 3.7698422272252903, + "imbalance_abs_log_ratio": 1.3270331510418396, + "last_quartile_mean": 4.2341920561739244e-05, + "mean": 8.61727822893954e-05, + "normalized": [ + 2.542734798705971, + 2.3221448837042376, + 2.101145461274622, + 1.8911858524935854, + 1.7004204056817789, + 1.5354142426015858, + 1.4229001770142031, + 1.3028733810328659, + 1.2049412416934622, + 1.1705258152816296, + 1.0773908232649334, + 1.0133220774376726, + 0.990609360100461, + 0.9272534949358001, + 0.8550385909364457, + 0.8185651575045715, + 0.7703972051466339, + 0.7173947009730917, + 0.6760256695667757, + 0.6514523109028573, + 0.618419077603279, + 0.6053620388084041, + 0.5838930841859182, + 0.5697043695032278, + 0.5457802700901755, + 0.5276149005446632, + 0.5104701332743111, + 0.49517988582787786, + 0.47527562953524777, + 0.46579059316028215, + 0.45896435764104826, + 0.45181000957238127 + ], + "peak_layer": 1, + "peak_normalized": 2.542734798705971, + "population_cv": 0.5705202801130907, + "rest_mean": 9.280119640472414e-05, + "spike_contrast": 0.542873885449774, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.03793460666202e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021911453222855926, + 0.00020010568550787866, + 0.00018106155039276928, + 0.00016296874673571438, + 0.00014652995741926134, + 0.00013231091725174338, + 0.0001226152671733871, + 0.00011227222421439365, + 0.00010383313929196447, + 0.00010086746624438092, + 9.284176485380158e-05, + 8.732078276807442e-05, + 8.536356472177431e-05, + 7.99040135461837e-05, + 7.368105434579775e-05, + 7.053803710732609e-05, + 6.638727063545957e-05, + 6.181989738252014e-05, + 5.825501284562051e-05, + 5.613745815935545e-05, + 5.329089253791608e-05, + 5.216573117650114e-05, + 5.0315691623836756e-05, + 4.9093010602518916e-05, + 4.703140439232811e-05, + 4.546604395727627e-05, + 4.398863165988587e-05, + 4.267102849553339e-05, + 4.095582335139625e-05, + 4.013847137684934e-05, + 3.955023566959426e-05, + 3.893372559105046e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442477373057045, + "first_to_last_ratio": 1.8431254562763404, + "imbalance_abs_log_ratio": 0.6114627481432037, + "last_quartile_mean": 0.00011633759004325839, + "mean": 0.00015221248349917005, + "normalized": [ + 1.7740805953138137, + 1.6484457246979003, + 1.5301434809496548, + 1.4217241600875643, + 1.3264958046529274, + 1.2454577490392524, + 1.1897861330408546, + 1.1336265611690741, + 1.086914920172498, + 1.064301358569379, + 1.0227209980571423, + 0.9941753882589759, + 0.9788979908211966, + 0.9511107967643203, + 0.9228143274453928, + 0.9052447611165039, + 0.8846530997418666, + 0.8652930871753776, + 0.8497381552743479, + 0.8388656494639406, + 0.8256334792987466, + 0.8176309647342147, + 0.8076951735882674, + 0.8000661307359944, + 0.7892605467734257, + 0.78144613040264, + 0.7731219607566042, + 0.7664988488799842, + 0.7584935618388989, + 0.7529639051546407, + 0.7485195296370439, + 0.7441790263875565 + ], + "peak_layer": 1, + "peak_normalized": 1.7740805953138137, + "population_cv": 0.2736411578809574, + "rest_mean": 0.00015762286892384981, + "spike_contrast": 0.7803207938393821, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299640220589936, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027003721334040165, + 0.00025091401766985655, + 0.00023290693934541196, + 0.0002164041652576998, + 0.000201909220777452, + 0.0001895742170745507, + 0.00018110030214302242, + 0.00017255211423616856, + 0.00016544201935175806, + 0.00016199995297938585, + 0.00015567090304102749, + 0.00015132590488065034, + 0.0001490004942752421, + 0.00014477093645837158, + 0.00014046386058907956, + 0.00013778955326415598, + 0.0001346552453469485, + 0.00013170840975362808, + 0.00012934075493831187, + 0.00012768582382705063, + 0.00012567172234412283, + 0.00012445363972801715, + 0.00012294128828216344, + 0.00012178005272289738, + 0.00012013530795229599, + 0.00011894585622940212, + 0.00011767881369451061, + 0.00011667069338727742, + 0.00011545218876563013, + 0.00011461050598882139, + 0.00011393401655368507, + 0.00011327333777444437 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.190154030919075, + "first_to_last_ratio": 1.84312545379361, + "imbalance_abs_log_ratio": 0.6114627467961817, + "last_quartile_mean": 0.10316933691501617, + "mean": 0.13498355145566165, + "normalized": [ + 1.774080628234443, + 1.6484456620184311, + 1.5301434020504485, + 1.4217242444591272, + 1.326495766996536, + 1.2454576834375484, + 1.189786122599316, + 1.133626627255238, + 1.0869149569673349, + 1.0643014004698326, + 1.0227209907477024, + 0.9941753870542451, + 0.9788979589698585, + 0.9511107512164827, + 0.9228143032197887, + 0.9052447972745712, + 0.8846531331847195, + 0.8652931173486506, + 0.8497382179112226, + 0.8388656682549475, + 0.8256334805754935, + 0.8176309131099365, + 0.8076951535479161, + 0.8000661540388709, + 0.7892605573866218, + 0.7814461538045975, + 0.7731219580171443, + 0.7664988542586116, + 0.7584935821787124, + 0.7529638601652359, + 0.7485195164462104, + 0.7441789968002049 + ], + "peak_layer": 1, + "peak_normalized": 1.774080628234443, + "population_cv": 0.27364115345546325, + "rest_mean": 0.13978153549962574, + "spike_contrast": 0.7803207857775153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907443761825561, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2394717037677765, + 0.22251304984092712, + 0.2065441906452179, + 0.19190938770771027, + 0.1790551096200943, + 0.16811630129814148, + 0.16060155630111694, + 0.1530209481716156, + 0.14671564102172852, + 0.1436631828546524, + 0.13805051147937775, + 0.13419732451438904, + 0.13213512301445007, + 0.1283843070268631, + 0.1245647519826889, + 0.12219315767288208, + 0.11941362172365189, + 0.11680033802986145, + 0.11470068246126175, + 0.1132330670952797, + 0.11144693940877914, + 0.11036672443151474, + 0.10902556031942368, + 0.10799577087163925, + 0.10653719305992126, + 0.10548237711191177, + 0.1043587476015091, + 0.10346473753452301, + 0.10238415747880936, + 0.10163773596286774, + 0.10103782266378403, + 0.10045192390680313 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883367768750759e-05, + "first_to_last_ratio": 2.249759357551583, + "imbalance_abs_log_ratio": 0.810823258297238, + "last_quartile_mean": 3.059601795030176e-05, + "mean": 4.691846504556452e-05, + "normalized": [ + 1.7359059710084281, + 1.6475789727938472, + 1.5536456441204052, + 1.4921194549590373, + 1.4365657321885223, + 1.3438513058553838, + 1.2787343947807137, + 1.2483299149152907, + 1.1912354282513808, + 1.1603083401306369, + 1.099547144002948, + 1.059972440022979, + 1.0583955432739494, + 1.0176800955770247, + 0.9754963047746087, + 0.9631595708780456, + 0.9242400656751482, + 0.8752301116137658, + 0.8454245377074081, + 0.8232853322610043, + 0.7863494898171388, + 0.7774642190168813, + 0.7497486129019063, + 0.7388483538626245, + 0.709315327139865, + 0.6908343794807076, + 0.6685469983108738, + 0.6592365073812129, + 0.6314341746588822, + 0.6235898169926835, + 0.6199474545603838, + 0.6139783610863121 + ], + "peak_layer": 1, + "peak_normalized": 1.7359059710084281, + "population_cv": 0.3220406003070692, + "rest_mean": 4.9070239684725595e-05, + "spike_contrast": 0.7193541792517962, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.52988819940947e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144604362314567e-05, + 7.730187644483522e-05, + 7.28946688468568e-05, + 7.000795449130237e-05, + 6.740145909134299e-05, + 6.305144052021205e-05, + 5.999625500408001e-05, + 5.8569723478285596e-05, + 5.589093780145049e-05, + 5.4439886298496276e-05, + 5.158906424185261e-05, + 4.973227987647988e-05, + 4.9658294301480055e-05, + 4.774798799189739e-05, + 4.576878927764483e-05, + 4.51899686595425e-05, + 4.3363925215089694e-05, + 4.1064453398576006e-05, + 3.9666021621087566e-05, + 3.86272840842139e-05, + 3.689431105158292e-05, + 3.6477427784120664e-05, + 3.5177054087398574e-05, + 3.466563066467643e-05, + 3.3279986382694915e-05, + 3.2412888685939834e-05, + 3.136719897156581e-05, + 3.093036502832547e-05, + 2.9625922252307646e-05, + 2.9257877031341195e-05, + 2.9086982976878062e-05, + 2.8806922273361124e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017341253987979144, + "first_to_last_ratio": 1.4955148931299893, + "imbalance_abs_log_ratio": 0.4024705576674927, + "last_quartile_mean": 0.00011595507385209203, + "mean": 0.00013990553861731314, + "normalized": [ + 1.3768818258178377, + 1.3340879911950096, + 1.2910054180628048, + 1.2527236427111486, + 1.2147542166907819, + 1.1769889670731195, + 1.1474411155803916, + 1.1220953663134672, + 1.0944075691302455, + 1.0773007485615413, + 1.0508678613623372, + 1.0320596054419082, + 1.0208161736522385, + 1.000559648637946, + 0.9805197815181176, + 0.9663128275349431, + 0.948744191771126, + 0.9319305822463463, + 0.9182019814348666, + 0.9075038868226692, + 0.8943650364968407, + 0.886262676120959, + 0.8757907045344768, + 0.8679002174812178, + 0.856288373659771, + 0.8478178092769747, + 0.8387215581135022, + 0.8313975747674917, + 0.8224084044020752, + 0.8161864847722404, + 0.8112200992010112, + 0.8064376596145924 + ], + "peak_layer": 1, + "peak_normalized": 1.3768818258178377, + "population_cv": 0.16325888721442564, + "rest_mean": 0.0001431150389921472, + "spike_contrast": 0.8564734877369171, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225742365932092, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926333934534341, + 0.00018664629897102714, + 0.00018061880837194622, + 0.0001752629759721458, + 0.00016995084297377616, + 0.0001646672753849998, + 0.00016053336730692536, + 0.00015698735660407692, + 0.00015311368042603135, + 0.00015072034148033708, + 0.00014702223415952176, + 0.00014439085498452187, + 0.0001428178366040811, + 0.00013998383656144142, + 0.00013718014815822244, + 0.00013519251660909504, + 0.00013273456715978682, + 0.00013038225006312132, + 0.00012846154277212918, + 0.00012696482008323073, + 0.00012512662215158343, + 0.0001239930570591241, + 0.00012252797023393214, + 0.000121424047392793, + 0.00011979948612861335, + 0.00011861440725624561, + 0.00011734179133782163, + 0.0001163171255029738, + 0.00011505949078127742, + 0.00011418900976423174, + 0.00011349418491590768, + 0.00011282509512966499 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675183047074825, + "first_to_last_ratio": 1.2676545313577883, + "imbalance_abs_log_ratio": 0.23716836728277432, + "last_quartile_mean": 0.00011576642282307148, + "mean": 0.00013041823922321782, + "normalized": [ + 1.1700698937843759, + 1.1560440994564782, + 1.1417398043557874, + 1.130463200949706, + 1.118403537277814, + 1.1065304334184665, + 1.0939881912841023, + 1.0846815122144253, + 1.0709468270677047, + 1.0579907390333845, + 1.0442838370166418, + 1.035550673160169, + 1.025959021745254, + 1.0140203095307772, + 1.0062047700154308, + 0.9976631870190885, + 0.9861765922450596, + 0.9770016877798172, + 0.9687702939454803, + 0.9602422119882037, + 0.9504995944091904, + 0.9425883202531964, + 0.9330014667281973, + 0.9259385264566372, + 0.9147443798098084, + 0.906202796813466, + 0.8958065504175365, + 0.8885655303249116, + 0.8806194435723184, + 0.8757905905296257, + 0.8713783728577705, + 0.8681336045391749 + ], + "peak_layer": 1, + "peak_normalized": 1.1700698937843759, + "population_cv": 0.09121429133712369, + "rest_mean": 0.0001320278307604086, + "spike_contrast": 0.921975648780332, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001217264449223876, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001525984553154558, + 0.0001507692359155044, + 0.000148903694935143, + 0.00014743302017450333, + 0.00014586022007279098, + 0.00014431175077334046, + 0.00014267601363826543, + 0.00014146225294098258, + 0.000139670999487862, + 0.00013798128929920495, + 0.00013619365927297622, + 0.00013505469541996717, + 0.00013380376913119107, + 0.00013224674330558628, + 0.00013122745440341532, + 0.00013011347618885338, + 0.00012861541472375393, + 0.00012741883983835578, + 0.0001263453159481287, + 0.0001252330985153094, + 0.0001239624834852293, + 0.00012293070903979242, + 0.00012168040848337114, + 0.00012075927224941552, + 0.00011929935135412961, + 0.00011818537313956767, + 0.00011682951299007982, + 0.00011588515189941972, + 0.00011484883725643158, + 0.00011421906674513593, + 0.00011364363308530301, + 0.00011322045611450449 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030004180916876066, + "first_to_last_ratio": 2.2837506327398005, + "imbalance_abs_log_ratio": 0.8258191057718037, + "last_quartile_mean": 0.00013138116082700435, + "mean": 0.00019473525253488333, + "normalized": [ + 1.9000096511818072, + 1.783700741635724, + 1.6708066910504589, + 1.570366884717585, + 1.4783058799917985, + 1.376185321596543, + 1.3064973297630273, + 1.2402700043336092, + 1.1767953849206394, + 1.1505341943958838, + 1.0771542571420398, + 1.026098162637436, + 1.013898063759585, + 0.9669473025647412, + 0.9095398902231254, + 0.8755205777387188, + 0.842301214161826, + 0.8073124727024382, + 0.7791991818813049, + 0.760574607765185, + 0.7402055397447304, + 0.7308143413325056, + 0.7152667122155245, + 0.7043714155678553, + 0.6942467749765324, + 0.6868749151283189, + 0.680655564574882, + 0.676363713518914, + 0.6701061029149729, + 0.6672276319278244, + 0.6636177363814285, + 0.6582317375530352 + ], + "peak_layer": 1, + "peak_normalized": 1.9000096511818072, + "population_cv": 0.35930746026536914, + "rest_mean": 0.00020494150898971216, + "spike_contrast": 0.6812747128050888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962146767880766, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699988592416048, + 0.0003473494143690914, + 0.0003253649629186839, + 0.0003058057918678969, + 0.0002878782688640058, + 0.0002679917961359024, + 0.0002544210874475539, + 0.00024152429250534624, + 0.00022916354646440595, + 0.000224049566895701, + 0.00020975990628357977, + 0.00019981748482678086, + 0.000197441695490852, + 0.0001882987271528691, + 0.00017711948021315038, + 0.00017049472080543637, + 0.00016402573965024203, + 0.00015721219824627042, + 0.00015173754945863038, + 0.00014811068831477314, + 0.00014414411270990968, + 0.0001423153153154999, + 0.0001392876438330859, + 0.00013716594548895955, + 0.00013519432104658335, + 0.00013375876005738974, + 0.00013254763325676322, + 0.0001317118585575372, + 0.00013049328117631376, + 0.00012993274140171707, + 0.00012922976748086512, + 0.00012818092363886535 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858902861509705, + "first_to_last_ratio": 3.757869257587208, + "imbalance_abs_log_ratio": 1.3238521099622325, + "last_quartile_mean": 4.220184837322449e-05, + "mean": 8.57325238712292e-05, + "normalized": [ + 2.5462257183789148, + 2.3032637538343064, + 2.0924289533379263, + 1.8886492391629932, + 1.7031256747388024, + 1.5374176267930122, + 1.4224231993460965, + 1.3049597038569594, + 1.2055618292025094, + 1.169245405308266, + 1.0769812048488787, + 1.0144334047898542, + 0.9913512297333369, + 0.9317090592952362, + 0.8577499774884012, + 0.81555804918997, + 0.7682725064221458, + 0.7217807351250844, + 0.6783324192661732, + 0.6511800118936032, + 0.6192916242842709, + 0.6053728292385717, + 0.5848720876072855, + 0.5718124973482859, + 0.5462302773879167, + 0.5275173250132701, + 0.5112998770983732, + 0.49691833866078555, + 0.47721658788927634, + 0.46551989467389665, + 0.46012635628943105, + 0.45317260249616614 + ], + "peak_layer": 1, + "peak_normalized": 2.5462257183789148, + "population_cv": 0.5686111305901927, + "rest_mean": 9.231303704803361e-05, + "spike_contrast": 0.5437775023084314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.019775271648541e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021829435718245804, + 0.00019746461475733668, + 0.0001793892151908949, + 0.0001619186659809202, + 0.00014601326256524771, + 0.00013180669338908046, + 0.00012194793089292943, + 0.00011187748896190897, + 0.00010335585830034688, + 0.00010024235962191597, + 9.233231685357168e-05, + 8.69699360919185e-05, + 8.499104296788573e-05, + 7.987776916706935e-05, + 7.353707042057067e-05, + 6.991984992055222e-05, + 6.586594099644572e-05, + 6.188008410390466e-05, + 5.8155150327365845e-05, + 5.582730591413565e-05, + 5.309343396220356e-05, + 5.190014053368941e-05, + 5.014256021240726e-05, + 4.90229285787791e-05, + 4.682970029534772e-05, + 4.522539165918715e-05, + 4.383502891869284e-05, + 4.2602063331287354e-05, + 4.0912982512963936e-05, + 3.991019548266195e-05, + 3.944779382436536e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "rest_mean": 0.00015645541295978345, + "spike_contrast": 0.7816308178118705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229037238284946, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837944976985455, + "first_to_last_ratio": 1.8345658800697515, + "imbalance_abs_log_ratio": 0.6068078758989703, + "last_quartile_mean": 0.10268339328467846, + "mean": 0.1340121782850474, + "normalized": [ + 1.7717451751532867, + 1.6366747548098712, + 1.5242756140248972, + 1.4186514377480737, + 1.3264266357207033, + 1.2453739091726768, + 1.1893513017871604, + 1.1330134634090196, + 1.0858604665419038, + 1.0628542749024097, + 1.0232520332438115, + 0.9942070832344971, + 0.9784308561813967, + 0.9531084116998004, + 0.9232390798282929, + 0.9040072094634526, + 0.8853775581660035, + 0.8680181154203164, + 0.8515367049614931, + 0.8395794986042229, + 0.8267439263823189, + 0.8185133396785964, + 0.809173050926764, + 0.8007913534932437, + 0.7909900046670136, + 0.782107217826699, + 0.7753403703776178, + 0.7682422246066585, + 0.7604971603471317, + 0.7549863443945886, + 0.7508596538022533, + 0.7467717694238248 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451751532867, + "population_cv": 0.27141865758333467, + "rest_mean": 0.13874622461972413, + "spike_contrast": 0.7816308398662988, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844832807779312, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23743543028831482, + 0.21933434903621674, + 0.20427149534225464, + 0.19011656939983368, + 0.17775732278823853, + 0.16689527034759521, + 0.15938755869865417, + 0.15183760225772858, + 0.14551852643489838, + 0.14243541657924652, + 0.13712823390960693, + 0.1332358568906784, + 0.1311216503381729, + 0.12772813439369202, + 0.1237252801656723, + 0.12114797532558441, + 0.1186513751745224, + 0.1163249984383583, + 0.11411628872156143, + 0.11251387745141983, + 0.11079375445842743, + 0.10969075560569763, + 0.10843904316425323, + 0.107315793633461, + 0.10600229352712631, + 0.104811891913414, + 0.10390505194664001, + 0.10295381397008896, + 0.1019158810377121, + 0.10117736458778381, + 0.10062433779239655, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873207303215167e-05, + "first_to_last_ratio": 2.248757814140914, + "imbalance_abs_log_ratio": 0.8103779811585758, + "last_quartile_mean": 3.0564462122129044e-05, + "mean": 4.6857780716891284e-05, + "normalized": [ + 1.7400119939228862, + 1.6405409364325143, + 1.555121247708426, + 1.4944054279879166, + 1.4327475398601994, + 1.3414100714975594, + 1.2804101716466747, + 1.2499370459040815, + 1.192804641985012, + 1.1608858830017554, + 1.098191983597808, + 1.057744066133373, + 1.0542717513613309, + 1.021869458338179, + 0.9775749297615711, + 0.9570927445976433, + 0.9210091333285576, + 0.8815125232807631, + 0.847820728512489, + 0.8204970180441978, + 0.7871838673851995, + 0.7759041238100245, + 0.7506938949785397, + 0.7421070509174572, + 0.7072987335050381, + 0.6891811912847667, + 0.6695407652827761, + 0.6616333377893114, + 0.6345027924930273, + 0.6212629426015038, + 0.6200302724102036, + 0.614801730639213 + ], + "peak_layer": 1, + "peak_normalized": 1.7400119939228862, + "population_cv": 0.32175451551289735, + "rest_mean": 4.9004235565748616e-05, + "spike_contrast": 0.7196709452950106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5266924533061685e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153310045599937e-05, + 7.687210745643824e-05, + 7.28695304132998e-05, + 7.002452184678987e-05, + 6.713537004543468e-05, + 6.28554989816621e-05, + 5.9997179050697014e-05, + 5.8569276006892323e-05, + 5.589217835222371e-05, + 5.439653614303097e-05, + 5.1458839152473956e-05, + 4.9563539505470544e-05, + 4.9400834541302174e-05, + 4.788253500009887e-05, + 4.58069916930981e-05, + 4.4847241952084005e-05, + 4.315644400776364e-05, + 4.130572051508352e-05, + 3.972699778387323e-05, + 3.8446669350378215e-05, + 3.688568904181011e-05, + 3.635714529082179e-05, + 3.5175849916413426e-05, + 3.477348946034908e-05, + 3.3142448955914006e-05, + 3.2293501135427505e-05, + 3.13731943606399e-05, + 3.1002669857116416e-05, + 2.9731392714893445e-05, + 2.9111002731951885e-05, + 2.905324254243169e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017190765356644988, + "first_to_last_ratio": 1.4894621368061003, + "imbalance_abs_log_ratio": 0.3984150727783497, + "last_quartile_mean": 0.00011541592721187044, + "mean": 0.00013894661265112518, + "normalized": [ + 1.3739040544627474, + 1.3296102750467884, + 1.2883342981472405, + 1.2499918159027226, + 1.2130627732820631, + 1.175544413473833, + 1.1474083142393565, + 1.1199112790671786, + 1.092909618037032, + 1.0755929927697745, + 1.0512861461262244, + 1.0312571117882559, + 1.0194388253067554, + 1.0019061415077835, + 0.980344379776871, + 0.9647961249933756, + 0.9493075664595542, + 0.9344817410273759, + 0.9198929201794295, + 0.9079992279088673, + 0.8952920956888009, + 0.8869530532606651, + 0.8771902552778059, + 0.8683889858357474, + 0.8579407806493952, + 0.8482823463732029, + 0.8408658204630546, + 0.8330843095877946, + 0.8243382853592501, + 0.818115422616924, + 0.8135611224181157, + 0.8090075029660142 + ], + "peak_layer": 1, + "peak_normalized": 1.3739040544627474, + "population_cv": 0.16161403917150835, + "rest_mean": 0.00014210757001786045, + "spike_contrast": 0.8576421569620596, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187744287075474, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00019089931447524577, + 0.00018474484386388212, + 0.00017900968668982387, + 0.00017368212866131216, + 0.0001685509632807225, + 0.00016333791427314281, + 0.00015942849859129637, + 0.00015560787869617343, + 0.00015185608936008066, + 0.00014945000293664634, + 0.00014607264893129468, + 0.0001432896824553609, + 0.00014164757158141583, + 0.0001392114645568654, + 0.00013621553080156446, + 0.0001340551534667611, + 0.00013190307072363794, + 0.00012984307250007987, + 0.0001278160052606836, + 0.00012616341700777411, + 0.0001243978040292859, + 0.00012323912233114243, + 0.00012188261462142691, + 0.00012065970804542303, + 0.00011920796532649547, + 0.00011786595860030502, + 0.00011683545744745061, + 0.00011575424287002534, + 0.00011453901242930442, + 0.00011367436673026532, + 0.00011304156214464456, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552358334185556, + "first_to_last_ratio": 1.262638843036328, + "imbalance_abs_log_ratio": 0.2332038508028458, + "last_quartile_mean": 0.00011525352965691127, + "mean": 0.00012953498708156985, + "normalized": [ + 1.1676304712442283, + 1.1525836984023852, + 1.1391521450612143, + 1.1288261090295841, + 1.1167600441451297, + 1.1054423860149858, + 1.094731137320412, + 1.082319740349084, + 1.067957453000188, + 1.0563935463552665, + 1.04560972624621, + 1.0337959762176818, + 1.0235535208863231, + 1.0159040893770892, + 1.0050628637064216, + 0.9950792389307415, + 0.9865814183778941, + 0.9786230528286792, + 0.9704418054120744, + 0.9608381133357455, + 0.9508407831226554, + 0.9429749854457838, + 0.9346839821902475, + 0.9262276149685831, + 0.9170542960948607, + 0.9059488475652709, + 0.8984648923618161, + 0.8908037775288885, + 0.8831483920181569, + 0.8778579808382188, + 0.8744051093280881, + 0.8703028022960919 + ], + "peak_layer": 1, + "peak_normalized": 1.1676304712442283, + "population_cv": 0.0896642712448568, + "rest_mean": 0.00013110964479252557, + "spike_contrast": 0.9231344927669946, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103183544240892, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124899800866842, + 0.00014929991448298097, + 0.00014756005839444697, + 0.00014622247545048594, + 0.00014465949789155275, + 0.00014319346519187093, + 0.00014180598373059183, + 0.00014019827358424664, + 0.00013833785487804562, + 0.00013683992438018322, + 0.0001354430423816666, + 0.00013391274842433631, + 0.0001325859921053052, + 0.00013159512309357524, + 0.00013019080506637692, + 0.00012889757636003196, + 0.00012779681128449738, + 0.00012676592450588942, + 0.00012570616672746837, + 0.00012446215259842575, + 0.00012316714855842292, + 0.00012214825255796313, + 0.00012107427755836397, + 0.00011997888213954866, + 0.00011879061639774591, + 0.00011735207226593047, + 0.0001163826382253319, + 0.0001153902558144182, + 0.00011439861555118114, + 0.00011371332220733166, + 0.00011326605454087257, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029742828155576717, + "first_to_last_ratio": 2.2781890121807993, + "imbalance_abs_log_ratio": 0.8233808344454435, + "last_quartile_mean": 0.00013055469935352448, + "mean": 0.00019329176166138495, + "normalized": [ + 1.8891850069388627, + 1.7801603926271052, + 1.6773544463608039, + 1.566500262900107, + 1.476456430986117, + 1.3754925511314788, + 1.3084504582544147, + 1.236424592515421, + 1.172965600373877, + 1.150871569148191, + 1.0802294364746856, + 1.0319209662753672, + 1.0080063247818516, + 0.9698985570693867, + 0.9067931019504507, + 0.8756163462587222, + 0.8444435806568995, + 0.8113939671049883, + 0.7809677247391495, + 0.7613442367276676, + 0.7402582678385915, + 0.7293562882295931, + 0.7162426695419845, + 0.7062419232587185, + 0.6940340549797547, + 0.6866100033591334, + 0.6831688143117768, + 0.6753320518823956, + 0.6715919825156708, + 0.6686410475337363, + 0.6640239865195754, + 0.6600233567535221 + ], + "peak_layer": 1, + "peak_normalized": 1.8891850069388627, + "population_cv": 0.357915029863326, + "rest_mean": 0.00020341356920888993, + "spike_contrast": 0.6815376252628051, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001386340009048581, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036516389809548855, + 0.0003440903383307159, + 0.0003242187958676368, + 0.0003027915954589844, + 0.00028538686456158757, + 0.00026587137836031616, + 0.0002529126941226423, + 0.00023899068764876574, + 0.00022672458726447076, + 0.00022245399304665625, + 0.0002087994507746771, + 0.00019946182146668434, + 0.00019483931828290224, + 0.00018747340072877705, + 0.00017527563613839447, + 0.00016924942610785365, + 0.00016322398732881993, + 0.00015683576930314302, + 0.0001509546273155138, + 0.00014716156874783337, + 0.0001430858246749267, + 0.0001409785618307069, + 0.00013844380737282336, + 0.00013651074550580233, + 0.00013415106514003128, + 0.00013271605712361634, + 0.00013205090363044292, + 0.00013053612201474607, + 0.00012981319741811603, + 0.00012924280599690974, + 0.00012835036613978446, + 0.00012757707736454904 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.02802751213312149, + 0.01413465291261673, + 0.014225807972252369, + 0.01433455292135477, + 0.01440478302538395, + 0.009703479707241058, + 0.00977881159633398, + 0.009858808480203152, + 0.009950309991836548, + 0.00752287870272994, + 0.007571155205368996, + 0.007693085819482803, + 0.007788818795233965, + 0.006353048607707024, + 0.006389216054230928, + 0.0064707002602517605, + 0.00656962301582098, + 0.005553483963012695, + 0.0056249531917274, + 0.005688921548426151, + 0.005768724251538515, + 0.004966245498508215, + 0.005057469941675663, + 0.005098348017781973, + 0.005216081161051989, + 0.0046758754178881645, + 0.004727107938379049, + 0.004777040798217058, + 0.004837613552808762, + 0.0044186655431985855, + 0.004510571248829365, + 0.0045849340967834 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001580166108396952, + "first_to_last_ratio": 1.9713740353643698, + "imbalance_abs_log_ratio": 0.6787307795042399, + "last_quartile_mean": 8.015557068574708e-05, + "mean": 0.00011843875881822896, + "normalized": [ + 0.8918679597819287, + 1.5814819593588525, + 1.5722239987443252, + 1.3246067564285042, + 1.1252650527356378, + 1.512743589128775, + 1.3788112428745087, + 1.2863035124436846, + 1.097095055843533, + 1.468154353173571, + 1.1523254606068964, + 1.0839888571909224, + 1.054518217028451, + 1.146335080549274, + 1.1665208527784754, + 0.8931136212023528, + 0.8203804970547371, + 1.073741223277434, + 0.919654742394771, + 0.8223971330190447, + 0.7711288002244273, + 0.8470775332584999, + 0.8298711629738247, + 0.7662487463744663, + 0.7210382462310316, + 0.7071376059209309, + 0.7189559392079661, + 0.727980518763353, + 0.6111916361855597, + 0.6986829946379066, + 0.6244678178312321, + 0.6046898327751223 + ], + "peak_layer": 2, + "peak_normalized": 1.5814819593588525, + "population_cv": 0.2904715201464359, + "rest_mean": 0.00012310891098523926, + "spike_contrast": 0.7572151875143335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.321993711637333e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010563173418631777, + 0.0001873087603598833, + 0.00018621225899551064, + 0.00015688478015363216, + 0.0001332749961875379, + 0.00017916747310664505, + 0.00016330469225067645, + 0.00015234819147735834, + 0.00012993857671972364, + 0.00017388637934345752, + 0.0001364799973089248, + 0.0001283862948184833, + 0.00012489582877606153, + 0.00013577050413005054, + 0.00013816128193866462, + 0.00010577926877886057, + 9.71648478298448e-05, + 0.00012717257777694613, + 0.00010892276623053476, + 9.74036956904456e-05, + 9.133153798757121e-05, + 0.0001003268116619438, + 9.828891052166e-05, + 9.07535504666157e-05, + 8.539887494407594e-05, + 8.375250035896897e-05, + 8.515224908478558e-05, + 8.622110908618197e-05, + 7.238877878990024e-05, + 8.275114669231698e-05, + 7.396119326585904e-05, + 7.161871326388791e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001664285200604354, + "first_to_last_ratio": 1.9719337364651075, + "imbalance_abs_log_ratio": 0.6790146534176558, + "last_quartile_mean": 8.439863722742302e-05, + "mean": 0.0001245867656507471, + "normalized": [ + 0.8984334461398115, + 1.5927952483854981, + 1.5658595485525297, + 1.3198735865176374, + 1.131213894321221, + 1.5125315092450449, + 1.3796810696180641, + 1.286366052942558, + 1.0960606282761096, + 1.4536884557404346, + 1.1543342746982923, + 1.0797559630746076, + 1.0497530570909623, + 1.1457182987306291, + 1.1551962694199982, + 0.9023721663305979, + 0.8238710779144349, + 1.0632503993742801, + 0.9211624832492573, + 0.8274335806257342, + 0.7739032412019898, + 0.850990856837025, + 0.8281317613098801, + 0.7681943949939335, + 0.7208251570035091, + 0.7089546837555952, + 0.7203162530736348, + 0.7236174125342815, + 0.6145740290254166, + 0.6994991973458914, + 0.6274381906330746, + 0.6042038120380651 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952483854981, + "population_cv": 0.2889665774838781, + "rest_mean": 0.00012946851159169995, + "spike_contrast": 0.7586812913967177, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.822533756960183e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00011193291720701382, + 0.0001984412083402276, + 0.0001950853766174987, + 0.000164438781212084, + 0.00014093428035266697, + 0.00018844140868168324, + 0.00017189000209327787, + 0.00016026418597903103, + 0.00013655464863404632, + 0.00018111034296453, + 0.00014381477376446128, + 0.0001345233031315729, + 0.00013078533811494708, + 0.00014274133718572557, + 0.00014392216689884663, + 0.00011242362961638719, + 0.00010264343291055411, + 0.0001324669283349067, + 0.00011476465442683548, + 0.00010308727360097691, + 9.641810174798593e-05, + 0.00010602219845168293, + 0.00010317425767425448, + 9.570685506332666e-05, + 8.980527491075918e-05, + 8.832637104205787e-05, + 8.974187221610919e-05, + 9.015315299620852e-05, + 7.656779052922502e-05, + 8.714834257261828e-05, + 7.817049481673166e-05, + 7.527579873567447e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.14759047608822584, + "first_to_last_ratio": 1.9719337947467823, + "imbalance_abs_log_ratio": 0.6790146829732503, + "last_quartile_mean": 0.07484555337578058, + "mean": 0.1104847895912826, + "normalized": [ + 0.8984334974418968, + 1.5927952138130326, + 1.565859646322909, + 1.3198735575362357, + 1.1312139236939245, + 1.5125315007911495, + 1.3796810264894532, + 1.286366010451999, + 1.0960606168617568, + 1.4536884922319049, + 1.1543342013148132, + 1.0797558953237962, + 1.0497530983161607, + 1.1457182562234747, + 1.1551962948455465, + 0.9023721940039697, + 0.8238711061937815, + 1.0632504861804335, + 0.9211625151546727, + 0.8274335809310764, + 0.7739032642161865, + 0.8509909070635955, + 0.8281317391228538, + 0.7681943896830757, + 0.7208251689669645, + 0.7089546589522648, + 0.7203162343708721, + 0.7236173971219663, + 0.6145740294939214, + 0.6994991437375198, + 0.6274381319743458, + 0.6042038211744466 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952138130326, + "population_cv": 0.28896658018674404, + "rest_mean": 0.11481397047086998, + "spike_contrast": 0.7586813040631772, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08710721284151077, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.09926323592662811, + 0.1759796440601349, + 0.1730036735534668, + 0.14582595229148865, + 0.12498193234205246, + 0.16711172461509705, + 0.15243376791477203, + 0.14212387800216675, + 0.12109802663326263, + 0.16061046719551086, + 0.1275363713502884, + 0.1192966029047966, + 0.11598175019025803, + 0.12658444046974182, + 0.12763161957263947, + 0.09969840198755264, + 0.0910252258181572, + 0.11747300624847412, + 0.10177444666624069, + 0.09141882508993149, + 0.08550453931093216, + 0.09402155131101608, + 0.09149596095085144, + 0.08487379550933838, + 0.0796402171254158, + 0.0783287063241005, + 0.07958398759365082, + 0.07994871586561203, + 0.06790108233690262, + 0.07728401571512222, + 0.069322369992733, + 0.06675533205270767 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.093040158499207e-05, + "first_to_last_ratio": 0.8860093039525244, + "imbalance_abs_log_ratio": 0.12102782735710801, + "last_quartile_mean": 5.748291960117058e-05, + "mean": 5.7093861016710434e-05, + "normalized": [ + 0.4337136137178416, + 0.9185569667519601, + 0.9444095926066538, + 0.8728787927927439, + 0.7849140301431445, + 1.1296292591692207, + 1.0726593944235638, + 0.9796135302182263, + 0.9386642177229508, + 1.3046353648934683, + 1.0461832118506482, + 1.0032917361864164, + 0.9662905330123907, + 1.0760080966276753, + 1.1828739682986207, + 0.9087955578115051, + 0.9194941496879511, + 1.198292373503509, + 1.0946132296674007, + 1.0673695485587213, + 0.9244260207272295, + 1.0525679458300632, + 1.125653431968099, + 0.9999504891362015, + 0.9629027708994021, + 1.0063234987710272, + 0.9975933861195996, + 1.0791525148711678, + 0.9412468232151083, + 1.085173475571988, + 1.0031556955576197, + 0.9789667796878816 + ], + "peak_layer": 10, + "peak_normalized": 1.3046353648934683, + "population_cv": 0.1441687540611157, + "rest_mean": 5.695535414647397e-05, + "spike_contrast": 1.0155638391297448, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.784179811598733e-05, + "top_five_layers": [ + 10, + 18, + 15, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.4762384782661684e-05, + 5.2443963795667514e-05, + 5.3919990023132414e-05, + 4.9836020480142906e-05, + 4.481377254705876e-05, + 6.449489592341706e-05, + 6.124226638348773e-05, + 5.592991874436848e-05, + 5.3591964388033375e-05, + 7.448667020071298e-05, + 5.973063889541663e-05, + 5.728179894504137e-05, + 5.516925739357248e-05, + 6.143345672171563e-05, + 6.75348419463262e-05, + 5.1886647270293906e-05, + 5.249747118796222e-05, + 6.84151382301934e-05, + 6.249569560168311e-05, + 6.094024865888059e-05, + 5.277905074763112e-05, + 6.009516800986603e-05, + 6.426790059776977e-05, + 5.7091034250333905e-05, + 5.4975836974335834e-05, + 5.74548939766828e-05, + 5.695645813830197e-05, + 6.161298369988799e-05, + 5.373941530706361e-05, + 6.19567435933277e-05, + 5.727403186028823e-05, + 5.58929932594765e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.801741053801379e-05, + "first_to_last_ratio": 0.954195857595535, + "imbalance_abs_log_ratio": 0.046886327142486896, + "last_quartile_mean": 6.08024129178375e-05, + "mean": 6.296396202287724e-05, + "normalized": [ + 0.4796364558374551, + 0.9568353419130547, + 0.9804638447804884, + 0.8949545354119591, + 0.8237377234766136, + 1.1597846916175047, + 1.085649359633547, + 0.9904450076993521, + 0.9595078978373608, + 1.2941748206155441, + 1.0680753760655664, + 1.0231559828609242, + 0.9904019047543832, + 1.096490963646882, + 1.1850351594596897, + 0.9422215946487361, + 0.9418631353047864, + 1.1705024179981143, + 1.0926353878857977, + 1.0382275689133567, + 0.9320833313178482, + 1.0784583323883348, + 1.0985080196909318, + 0.9917906747078287, + 0.9476388143731139, + 0.9855066582268529, + 0.978396290111904, + 1.028407839812513, + 0.8982055143685478, + 1.0282804954549911, + 0.9451124964564701, + 0.9138123627295472 + ], + "peak_layer": 10, + "peak_normalized": 1.2941748206155441, + "population_cv": 0.13431233496292436, + "rest_mean": 6.285090866084729e-05, + "spike_contrast": 1.0115120295379672, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.357445017783903e-05, + "top_five_layers": [ + 10, + 15, + 18, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.019981159013696e-05, + 6.0246144130360335e-05, + 6.173388828756288e-05, + 5.634988337988034e-05, + 5.186579073779285e-05, + 7.302463927771896e-05, + 6.835678505012766e-05, + 6.236234185053036e-05, + 6.041441884008236e-05, + 8.148637425620109e-05, + 6.725025741616264e-05, + 6.442195444833487e-05, + 6.235962791834027e-05, + 6.903941539349034e-05, + 7.461450877599418e-05, + 5.9326004702597857e-05, + 5.930343468207866e-05, + 7.369946979451925e-05, + 6.879665306769311e-05, + 6.537092122016475e-05, + 5.8687659475253895e-05, + 6.790400948375463e-05, + 6.916641723364592e-05, + 6.244707037694752e-05, + 5.966709431959316e-05, + 6.205140380188823e-05, + 6.160370685392991e-05, + 6.475263216998428e-05, + 5.655457789544016e-05, + 6.474461406469345e-05, + 5.950802733423188e-05, + 5.753724690293893e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7888660295284353e-05, + "first_to_last_ratio": 0.41855462748958633, + "imbalance_abs_log_ratio": 0.8709478659353102, + "last_quartile_mean": 4.2739129185065394e-05, + "mean": 3.1986895351110434e-05, + "normalized": [ + 0.2593955501951102, + 0.4997829486650695, + 0.48480994473028793, + 0.4968760363401999, + 0.5474125689219601, + 0.7788406177150305, + 0.7554473604850604, + 0.6514322393717875, + 0.7578135841604356, + 0.9674810188795085, + 0.9032772556875033, + 0.9019220651946442, + 0.8311356963419073, + 0.9587970177975769, + 1.1401394869948454, + 0.8953886492354074, + 1.0262272842074007, + 1.1413916918814533, + 1.174037958462901, + 1.194927715786966, + 1.0267140632187253, + 1.3434898267078448, + 1.3729067467361273, + 1.2011929483563188, + 1.1735959904166378, + 1.3266745689353452, + 1.2739952988364465, + 1.4049535244931144, + 1.2752889026856997, + 1.4680979746684197, + 1.4182292846928506, + 1.3483241791974143 + ], + "peak_layer": 30, + "peak_normalized": 1.4680979746684197, + "population_cv": 0.3184955307507873, + "rest_mean": 3.066251991657299e-05, + "spike_contrast": 1.276428773763581, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.913852269761264e-05, + "top_five_layers": [ + 30, + 31, + 28, + 23, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.297258318634704e-06, + 1.5986504877218977e-05, + 1.5507564967265353e-05, + 1.589352177688852e-05, + 1.7510028555989265e-05, + 2.491269333404489e-05, + 2.416441566310823e-05, + 2.083729486912489e-05, + 2.4240103812189773e-05, + 3.094671410508454e-05, + 2.889303505071439e-05, + 2.8849686714238487e-05, + 2.6585450541460887e-05, + 3.066893987124786e-05, + 3.6469522456172854e-05, + 2.8640703021665104e-05, + 3.282582474639639e-05, + 3.6509576602838933e-05, + 3.755382931558415e-05, + 3.822202779701911e-05, + 3.2841395295690745e-05, + 4.2974068492185324e-05, + 4.391502443468198e-05, + 3.842243313556537e-05, + 3.7539692129939795e-05, + 4.243620060151443e-05, + 4.0751154301688075e-05, + 4.494010136113502e-05, + 4.0792532672639936e-05, + 4.695989628089592e-05, + 4.5364751713350415e-05, + 4.3128704419359565e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00022091567916504573, + "first_to_last_ratio": 1.232986382535498, + "imbalance_abs_log_ratio": 0.20943917994887448, + "last_quartile_mean": 0.00017917122386279516, + "mean": 0.0002198539991695725, + "normalized": [ + 0.4672441326905094, + 1.0299890566632133, + 1.08986441571188, + 1.0084939820045402, + 0.8814237539069011, + 1.2693826047962031, + 1.1886514905905403, + 1.1035827467935222, + 1.0473307057033197, + 1.4274044616309502, + 1.1442998336482333, + 1.0911283607583675, + 1.1094038041384326, + 1.1946873974128753, + 1.2511466100570947, + 0.9923165311956169, + 0.9239703707694655, + 1.2473578196113966, + 1.1181336030529359, + 1.0160745414007064, + 0.9397394354798955, + 0.9905446518023159, + 1.0121913654482901, + 0.9359943959598221, + 0.8839446941554127, + 0.8510229499852975, + 0.8648513533654478, + 0.8786898836618169, + 0.7455559493639876, + 0.8399702482950041, + 0.7392957942269532, + 0.7163130557190521 + ], + "peak_layer": 10, + "peak_normalized": 1.4274044616309502, + "population_cv": 0.18952709251819305, + "rest_mean": 0.0002217885959436427, + "spike_contrast": 0.9441746799407325, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020940717658959329, + "top_five_layers": [ + 10, + 6, + 15, + 18, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010272549116052687, + 0.00022644721320830286, + 0.00023961105034686625, + 0.00022172143508214504, + 0.0001937845372594893, + 0.0002790788421407342, + 0.0002613297838252038, + 0.00024262708029709756, + 0.00023025984410196543, + 0.000313820579322055, + 0.00025157889467664063, + 0.00023988893372006714, + 0.00024390686303377151, + 0.000262656802078709, + 0.00027506958576850593, + 0.0002181647578254342, + 0.00020313858112785965, + 0.00027423660503700376, + 0.0002458261442370713, + 0.00022338805138133466, + 0.00020660547306761146, + 0.00021777520305477083, + 0.00022253431961871684, + 0.00020578211115207523, + 0.0001943387760547921, + 0.00018710079893935472, + 0.0001901410287246108, + 0.00019318348495289683, + 0.00016391345707233995, + 0.00018467081827111542, + 0.00016253713693004102, + 0.00015748428995721042 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014061377383768559, + 0.014165290631353855, + 0.014281988143920898, + 0.014350161887705326, + 0.00966146495193243, + 0.009741888381540775, + 0.009820535778999329, + 0.00991273857653141, + 0.0074927168898284435, + 0.007546746172010899, + 0.00766373984515667, + 0.007762390188872814, + 0.006325984839349985, + 0.006375366821885109, + 0.006445344537496567, + 0.006555322092026472, + 0.005531130358576775, + 0.005611324217170477, + 0.0056777093559503555, + 0.00574630219489336, + 0.004946626722812653, + 0.0050392793491482735, + 0.005084683652967215, + 0.005203457549214363, + 0.004667261149734259, + 0.004717131145298481, + 0.004768412560224533, + 0.004826710093766451, + 0.0044052014127373695, + 0.004504534415900707, + 0.004574235528707504, + 0.004644096828997135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.729895206561196e-05, + "first_to_last_ratio": 1.2469721805352876, + "imbalance_abs_log_ratio": 0.22071835733540474, + "last_quartile_mean": 2.9911615229138988e-05, + "mean": 3.340667609563752e-05, + "normalized": [ + 1.308752778077389, + 1.2456464596610868, + 1.0236956931411572, + 0.9922285647386262, + 1.26903181062375, + 1.1170846373362797, + 1.0287316533628064, + 0.9469237624012169, + 1.1300261751959062, + 1.1132698789554223, + 1.0126240729566007, + 0.9379202574439188, + 1.1130503370514597, + 1.070713488608195, + 0.9897642722064988, + 0.922085688720865, + 1.036831181462568, + 0.9898105546019274, + 0.9395130252451869, + 0.9001967292772246, + 0.9975952535130206, + 0.9404117749145336, + 0.9224862220158919, + 0.8885787590271026, + 0.9461670180110204, + 0.8974372095121397, + 0.8904691498559152, + 0.8684437934704528, + 0.9244147277585869, + 0.9015689206268677, + 0.8740901912628669, + 0.8604359589635168 + ], + "peak_layer": 1, + "peak_normalized": 1.308752778077389, + "population_cv": 0.1164905978879517, + "rest_mean": 3.378375206213807e-05, + "spike_contrast": 0.9285666618330386, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.137046587653458e-05, + "top_five_layers": [ + 1, + 5, + 2, + 9, + 6 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.3721080146497115e-05, + 4.161290780757554e-05, + 3.419827044126578e-05, + 3.314705827506259e-05, + 4.239413465256803e-05, + 3.73180846509058e-05, + 3.4366505133220926e-05, + 3.1633575417799875e-05, + 3.7750418414361775e-05, + 3.719064625329338e-05, + 3.3828404411906376e-05, + 3.1332798243965954e-05, + 3.718331208801828e-05, + 3.5768978705164045e-05, + 3.306473445263691e-05, + 3.080381793552078e-05, + 3.463708344497718e-05, + 3.3066280593629926e-05, + 3.138600732199848e-05, + 3.007258055731654e-05, + 3.332634150865488e-05, + 3.1416031561093405e-05, + 3.0817198421573266e-05, + 2.9684462788281962e-05, + 3.1608295103069395e-05, + 2.9980394174344838e-05, + 2.9747614462394267e-05, + 2.9011820515734144e-05, + 3.0881623388268054e-05, + 3.01184209092753e-05, + 2.9200447897892445e-05, + 2.874430538213346e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.705871965095866e-05, + "first_to_last_ratio": 0.7180297289018083, + "imbalance_abs_log_ratio": 0.33124430563536084, + "last_quartile_mean": 9.33926785364747e-05, + "mean": 8.228966771639534e-05, + "normalized": [ + 0.8685826408392191, + 0.8249006353131848, + 0.7154905339803773, + 0.6884730939760333, + 0.9470487095509537, + 0.8638069620260335, + 0.8256155016985178, + 0.7853664463613442, + 0.9851686390305121, + 0.9692511243547932, + 0.9083804947955202, + 0.8729855457896263, + 1.0422548534804916, + 1.0255492616133721, + 0.9871528463964067, + 0.9398223255303129, + 1.0958375595009802, + 1.0726325633807163, + 1.0495259774381736, + 1.0230838788771643, + 1.1576441025653552, + 1.1142532603054807, + 1.1008567438213033, + 1.056908774400763, + 1.159063578749664, + 1.1192434432436902, + 1.1032241584740294, + 1.1060435702772593, + 1.1786235078649367, + 1.1628365878275773, + 1.141840825041133, + 1.1085318534950752 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235078649367, + "population_cv": 0.1341844256762087, + "rest_mean": 8.049536754023629e-05, + "spike_contrast": 1.1426606459269575, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.197888866765424e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.14753768988885e-05, + 6.788079917896539e-05, + 5.887747829547152e-05, + 5.66542221349664e-05, + 7.793232362018898e-05, + 7.108238787623122e-05, + 6.793962529627606e-05, + 6.462754390668124e-05, + 8.106919995043427e-05, + 7.975935295689851e-05, + 7.475032907677814e-05, + 7.183769048424438e-05, + 8.576680556870997e-05, + 8.4392107964959e-05, + 8.123247971525416e-05, + 7.733766688033938e-05, + 9.017610864248127e-05, + 8.826657722238451e-05, + 8.636514394311234e-05, + 8.418923243880272e-05, + 9.526214853394777e-05, + 9.169153054244816e-05, + 9.0589135652408e-05, + 8.697267185198143e-05, + 9.537895675748587e-05, + 9.210217103827745e-05, + 9.078394941752777e-05, + 9.101595787797123e-05, + 9.698853682493791e-05, + 9.56894364207983e-05, + 9.396170207764953e-05, + 9.122071787714958e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05946834245696664, + "first_to_last_ratio": 0.7180297222365345, + "imbalance_abs_log_ratio": 0.33124431491808765, + "last_quartile_mean": 0.08282156102359295, + "mean": 0.07297530013602227, + "normalized": [ + 0.8685826122359729, + 0.8249005917343738, + 0.71549052911611, + 0.6884731350635278, + 0.9470486699630107, + 0.8638070113577097, + 0.8256154769796886, + 0.7853664269052657, + 0.9851686341386658, + 0.9692511556795396, + 0.9083805454214465, + 0.8729855579949275, + 1.0422548040385637, + 1.0255492365750694, + 0.9871528950273681, + 0.9398223256645052, + 1.0958375053304956, + 1.07263252629438, + 1.0495259690486791, + 1.0230837911446407, + 1.1576441435704576, + 1.1142533045483842, + 1.1008568124470575, + 1.0569088284972943, + 1.1590636022344982, + 1.119243411839159, + 1.1032241423789504, + 1.1060435591240758, + 1.1786235037173105, + 1.1628366076958818, + 1.141840809884862, + 1.1085318743481278 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235037173105, + "population_cv": 0.13418442668798974, + "rest_mean": 0.07138409617322462, + "spike_contrast": 1.1426607032635474, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08156780153512955, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.06338507682085037, + 0.0601973682641983, + 0.05221313610672951, + 0.050241533666849136, + 0.0691111609339714, + 0.06303657591342926, + 0.060249537229537964, + 0.05731235072016716, + 0.07189297676086426, + 0.07073139399290085, + 0.06628934293985367, + 0.06370638310909271, + 0.07605885714292526, + 0.0748397633433342, + 0.07203777879476547, + 0.06858381628990173, + 0.07996907085180283, + 0.07827568054199219, + 0.07658947259187698, + 0.07465984672307968, + 0.0844794288277626, + 0.08131296932697296, + 0.0803353562951088, + 0.07712823897600174, + 0.08458301424980164, + 0.08167712390422821, + 0.08050811290740967, + 0.08071386069059372, + 0.0860104039311409, + 0.08485835045576096, + 0.08332617580890656, + 0.08089544624090195 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.408341275549901e-06, + "first_to_last_ratio": 0.49124437919492936, + "imbalance_abs_log_ratio": 0.7108135577065074, + "last_quartile_mean": 1.5080765479069669e-05, + "mean": 1.1844782648040564e-05, + "normalized": [ + 0.5657990507718705, + 0.5382233871697658, + 0.49942145823009426, + 0.48906984415931193, + 0.8062921756393574, + 0.7122953827323945, + 0.7013508394120195, + 0.6911626993599347, + 0.8940675881372303, + 0.868722127857376, + 0.9735911331661544, + 0.854855860776803, + 0.989483587379368, + 1.0059088486455374, + 1.0303729006301217, + 0.9941025538400318, + 1.1620912657632405, + 1.127012997009919, + 1.113401270263767, + 1.066097770938846, + 1.1826034542812638, + 1.1643547935214302, + 1.2049778184223279, + 1.1791491538213539, + 1.2365172470752577, + 1.219921447624247, + 1.2337761202582145, + 1.2288278254433027, + 1.3039217543779833, + 1.3631114810840155, + 1.2856635723507386, + 1.3138525898567213 + ], + "peak_layer": 30, + "peak_normalized": 1.3631114810840155, + "population_cv": 0.2549101540724733, + "rest_mean": 1.1420299651271543e-05, + "spike_contrast": 1.237882653019464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4136990830593277e-05, + "top_five_layers": [ + 30, + 32, + 29, + 31, + 25 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.701766778860474e-06, + 6.3751390371180605e-06, + 5.915538622502936e-06, + 5.7929260037781205e-06, + 9.550355571263935e-06, + 8.436983989668079e-06, + 8.307348252856173e-06, + 8.186671948351432e-06, + 1.0590036254143342e-05, + 1.0289824786013924e-05, + 1.1531975360412616e-05, + 1.0125581866304856e-05, + 1.1720218026312068e-05, + 1.1914771675947122e-05, + 1.220454305439489e-05, + 1.1774928680097219e-05, + 1.3764718460151926e-05, + 1.334922399109928e-05, + 1.3187996046326589e-05, + 1.2627696378331166e-05, + 1.4007680874783546e-05, + 1.379152945446549e-05, + 1.4272700354922563e-05, + 1.3966765436634887e-05, + 1.4646278032159898e-05, + 1.4449704394792207e-05, + 1.4613809980801307e-05, + 1.4555198504240252e-05, + 1.5444669770658948e-05, + 1.614575921848882e-05, + 1.5228405572997872e-05, + 1.5562298358418047e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6029601637419546e-05, + "first_to_last_ratio": 0.6070213107493427, + "imbalance_abs_log_ratio": 0.49919138022039955, + "last_quartile_mean": 9.230252817360451e-05, + "mean": 7.840498903988191e-05, + "normalized": [ + 0.6598133090619259, + 0.6419677920829977, + 0.6005256373772865, + 0.5913463573822231, + 0.8562376513989928, + 0.8062968559250673, + 0.7923156566726156, + 0.768439227921366, + 0.9790122729268586, + 0.9665730330981341, + 0.9163710065350301, + 0.8881686280668529, + 1.0631190324760253, + 1.0511030519715439, + 1.0188080221730096, + 0.9712812246823179, + 1.1361272291219966, + 1.1138561419734825, + 1.0918134348286874, + 1.0639217496772315, + 1.203414696914147, + 1.1592426077267617, + 1.1442664103435203, + 1.097952968622282, + 1.2042281788778202, + 1.1625445130141736, + 1.1452986211445626, + 1.1473932954411665, + 1.2233942816523278, + 1.205318760675343, + 1.1829469858778916, + 1.146901364356358 + ], + "peak_layer": 29, + "peak_normalized": 1.2233942816523278, + "population_cv": 0.1898597249088972, + "rest_mean": 7.605543874820728e-05, + "spike_contrast": 1.1977126437531995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.109256061492487e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.173265526536852e-05, + 5.033347770222463e-05, + 4.708420601673424e-05, + 4.636450466932729e-05, + 6.713330367347226e-05, + 6.321769615169615e-05, + 6.212150037754327e-05, + 6.024946924299002e-05, + 7.675944652874023e-05, + 7.578414806630462e-05, + 7.184805872384459e-05, + 6.963685154914856e-05, + 8.335383608937263e-05, + 8.241172326961532e-05, + 7.987963181221858e-05, + 7.615329377586022e-05, + 8.907804294722155e-05, + 8.733187860343605e-05, + 8.560362039133906e-05, + 8.341677312273532e-05, + 9.435371612198651e-05, + 9.089040395338088e-05, + 8.971619536168873e-05, + 8.608499047113582e-05, + 9.441749716643244e-05, + 9.114928980125114e-05, + 8.97971258382313e-05, + 8.996135875349864e-05, + 9.592021524440497e-05, + 9.450300422031432e-05, + 9.274894546251744e-05, + 8.992278890218586e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.8511469685763586e-05, + "first_to_last_ratio": 0.5271352332510719, + "imbalance_abs_log_ratio": 0.6402981537748061, + "last_quartile_mean": 9.202850924339145e-05, + "mean": 7.532146378252946e-05, + "normalized": [ + 0.548104082267993, + 0.5416830199005969, + 0.528884702689083, + 0.5247448675895723, + 0.7777813723908718, + 0.7487613756360456, + 0.7496059866168319, + 0.7329069161583202, + 0.9480676391902623, + 0.9399123881532756, + 0.9000936195631334, + 0.883441930780847, + 1.0560589316963704, + 1.0534552095740957, + 1.032095781768693, + 0.9835620691098637, + 1.1601754560133901, + 1.1467788545782247, + 1.1233340534265408, + 1.0967375255670042, + 1.2396083970871608, + 1.1951467072205837, + 1.1828066060767042, + 1.1317734013234224, + 1.244698956870482, + 1.2025862582796845, + 1.1881970083126314, + 1.1882256015362047, + 1.2689460104605979, + 1.2589162611011824, + 1.2307315535498626, + 1.1921774555104672 + ], + "peak_layer": 29, + "peak_normalized": 1.2689460104605979, + "population_cv": 0.23486334953575583, + "rest_mean": 7.254842300123225e-05, + "spike_contrast": 1.244629176846487, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.029588400153444e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 4.1284001781605184e-05, + 4.080035796505399e-05, + 3.9836369978729635e-05, + 3.952455153921619e-05, + 5.858363147126511e-05, + 5.639780283672735e-05, + 5.6461420172126964e-05, + 5.520362174138427e-05, + 7.140984234865755e-05, + 7.079557690303773e-05, + 6.77963689668104e-05, + 6.654213939327747e-05, + 7.954390457598493e-05, + 7.934778841445222e-05, + 7.773896504659206e-05, + 7.408333476632833e-05, + 8.738611359149218e-05, + 8.637706196168438e-05, + 8.461116522084922e-05, + 8.26078758109361e-05, + 9.336911898571998e-05, + 9.002019942272455e-05, + 8.909072494134307e-05, + 8.524682925781235e-05, + 9.375254740007222e-05, + 9.058055729838088e-05, + 8.949673792812973e-05, + 8.949889161158353e-05, + 9.557887096889317e-05, + 9.482341556577012e-05, + 9.270050213672221e-05, + 8.979655103757977e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.61669284151867e-05, + "first_to_last_ratio": 0.8297833624837898, + "imbalance_abs_log_ratio": 0.18659062131596024, + "last_quartile_mean": 0.00011589401856326731, + "mean": 0.00010716335873439675, + "normalized": [ + 0.9380742209455786, + 0.8898820858744699, + 0.7837417114644826, + 0.7583102664636657, + 1.0822070619439073, + 0.9520749081713696, + 0.9107982467806989, + 0.8640017807748708, + 1.0669538303069324, + 1.0266950649381863, + 0.9594029855154438, + 0.8901506143407661, + 1.0795289063552718, + 1.0379165674077444, + 0.9706971366471856, + 0.914979822300863, + 1.052939971267624, + 1.0225707935964177, + 0.9851238030379684, + 0.9636652893153618, + 1.0990044510799908, + 1.0523904893769196, + 1.0422234788441622, + 1.0049019599938585, + 1.0962247910521359, + 1.0599378706264646, + 1.05039930496957, + 1.0583562345651827, + 1.1199788667078585, + 1.1100529546792268, + 1.0850973923636869, + 1.0717171382921347 + ], + "peak_layer": 29, + "peak_normalized": 1.1199788667078585, + "population_cv": 0.09156717959150794, + "rest_mean": 0.00010599351123716958, + "spike_contrast": 1.0706366257223145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011348053521942347, + "top_five_layers": [ + 29, + 30, + 21, + 25, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010052718425868079, + 9.536275319987908e-05, + 8.398839418077841e-05, + 8.12630751170218e-05, + 0.00011597294360399246, + 0.00010202754492638633, + 9.760419925441965e-05, + 9.25893327803351e-05, + 0.00011433835607022047, + 0.00011002409155480564, + 0.00010281284630764276, + 9.539152961224318e-05, + 0.000115685943455901, + 0.0001112266254494898, + 0.0001040231654769741, + 9.805231093196198e-05, + 0.00011283658386673778, + 0.00010958212078548968, + 0.00010556917550275102, + 0.00010326960909878835, + 0.00011777300824178383, + 0.0001127776995417662, + 0.00011168816854478791, + 0.00010768866923172027, + 0.00011747513053705916, + 0.00011358650226611644, + 0.00011256431753281504, + 0.00011341700883349404, + 0.00012002069706795737, + 0.00011895700299646705, + 0.00011628268111962825, + 0.00011484880815260112 + ] + } + } + } + } + } + }, + "step": 0 + }, + { + "modes": { + "learned": { + "bits_per_byte": 5.150682269327325, + "forward": { + "activation_sha256": { + "attention_branch_output": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", + "mlp_branch_output": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", + "post_attention_state": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", + "post_mlp_state": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", + "pre_attention_input": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", + "pre_mlp_input": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f" + }, + "logits_sha256": "bdadcb0e52fa595647848016703b410d7bcf45d1b7a624e8a3f61c3e1956509d", + "loss_nats": 3.570180892944336, + "mixer_summary_sha256": "023101d1bfcf41624a0444a7f14bed4c44082764bd9047e90be0fb0a456542ad" + }, + "loss_nats": 3.570180892944336, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6930912733078003, + "group": 1, + "latest_source_mass": 0.49764734506607056, + "layer": 1, + "max_source_mass": 0.5023525953292847, + "mixer_index": 1, + "normalized_entropy": 0.9999193428845807, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5023525953292847, + "median": 0.5031890273094177, + "p05": 0.4936259984970093, + "p95": 0.5090169310569763 + }, + { + "label": "current_group_partial", + "mean": 0.49764734506607056, + "median": 0.4968109726905823, + "p05": 0.4909830391407013, + "p95": 0.5063740015029907 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.692903995513916, + "group": 1, + "latest_source_mass": 0.48966121673583984, + "layer": 2, + "max_source_mass": 0.5103387832641602, + "mixer_index": 2, + "normalized_entropy": 0.9996491581400753, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5103387832641602, + "median": 0.5101123452186584, + "p05": 0.5045775175094604, + "p95": 0.5169917345046997 + }, + { + "label": "current_group_partial", + "mean": 0.48966121673583984, + "median": 0.48988765478134155, + "p05": 0.4830082654953003, + "p95": 0.49542248249053955 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6928629875183105, + "group": 1, + "latest_source_mass": 0.4946908950805664, + "layer": 2, + "max_source_mass": 0.5053091049194336, + "mixer_index": 3, + "normalized_entropy": 0.9995899961081784, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5053091049194336, + "median": 0.5084525346755981, + "p05": 0.4841057062149048, + "p95": 0.5181366801261902 + }, + { + "label": "current_group_partial", + "mean": 0.4946908950805664, + "median": 0.49154743552207947, + "p05": 0.4818633496761322, + "p95": 0.5158942937850952 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6927823424339294, + "group": 1, + "latest_source_mass": 0.4873165488243103, + "layer": 3, + "max_source_mass": 0.5126834511756897, + "mixer_index": 4, + "normalized_entropy": 0.9994736498448697, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5126834511756897, + "median": 0.5128336548805237, + "p05": 0.5050104260444641, + "p95": 0.5203934907913208 + }, + { + "label": "current_group_partial", + "mean": 0.4873165488243103, + "median": 0.4871663451194763, + "p05": 0.4796065092086792, + "p95": 0.4949895739555359 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6928155422210693, + "group": 1, + "latest_source_mass": 0.49704670906066895, + "layer": 3, + "max_source_mass": 0.502953290939331, + "mixer_index": 5, + "normalized_entropy": 0.999521547013135, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.502953290939331, + "median": 0.5012187957763672, + "p05": 0.4864824414253235, + "p95": 0.5226666927337646 + }, + { + "label": "current_group_partial", + "mean": 0.49704670906066895, + "median": 0.4987812042236328, + "p05": 0.4773333668708801, + "p95": 0.5135174989700317 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.692751407623291, + "group": 1, + "latest_source_mass": 0.48708435893058777, + "layer": 4, + "max_source_mass": 0.5129156112670898, + "mixer_index": 6, + "normalized_entropy": 0.9994290203469708, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5129156112670898, + "median": 0.512997567653656, + "p05": 0.504144549369812, + "p95": 0.5216691493988037 + }, + { + "label": "current_group_partial", + "mean": 0.48708435893058777, + "median": 0.4870024621486664, + "p05": 0.4783308207988739, + "p95": 0.4958554208278656 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6925709247589111, + "group": 1, + "latest_source_mass": 0.48873060941696167, + "layer": 4, + "max_source_mass": 0.5112694501876831, + "mixer_index": 7, + "normalized_entropy": 0.9991686386135645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5112694501876831, + "median": 0.5127402544021606, + "p05": 0.4926953613758087, + "p95": 0.5298716425895691 + }, + { + "label": "current_group_partial", + "mean": 0.48873060941696167, + "median": 0.48725977540016174, + "p05": 0.4701283872127533, + "p95": 0.5073046088218689 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6929247379302979, + "group": 2, + "latest_source_mass": 0.49048691987991333, + "layer": 5, + "max_source_mass": 0.5095131397247314, + "mixer_index": 8, + "normalized_entropy": 0.9996790831213254, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5095131397247314, + "median": 0.5093363523483276, + "p05": 0.502471923828125, + "p95": 0.5177270770072937 + }, + { + "label": "completed_group_1", + "mean": 0.49048691987991333, + "median": 0.49066367745399475, + "p05": 0.4822728931903839, + "p95": 0.4975281059741974 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0983518362045288, + "group": 2, + "latest_source_mass": 0.33183109760284424, + "layer": 5, + "max_source_mass": 0.33983534574508667, + "mixer_index": 9, + "normalized_entropy": 0.999762925950977, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33983534574508667, + "median": 0.3411724865436554, + "p05": 0.32688987255096436, + "p95": 0.3497551679611206 + }, + { + "label": "completed_group_1", + "mean": 0.32833361625671387, + "median": 0.328115850687027, + "p05": 0.31867384910583496, + "p95": 0.33767229318618774 + }, + { + "label": "current_group_partial", + "mean": 0.33183109760284424, + "median": 0.331381618976593, + "p05": 0.3268507122993469, + "p95": 0.33714187145233154 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.098447322845459, + "group": 2, + "latest_source_mass": 0.3288182318210602, + "layer": 6, + "max_source_mass": 0.340813547372818, + "mixer_index": 10, + "normalized_entropy": 0.9998498416371705, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.340813547372818, + "median": 0.340879887342453, + "p05": 0.33481645584106445, + "p95": 0.34668514132499695 + }, + { + "label": "completed_group_1", + "mean": 0.3303682208061218, + "median": 0.33054590225219727, + "p05": 0.325665682554245, + "p95": 0.33449190855026245 + }, + { + "label": "current_group_partial", + "mean": 0.3288182318210602, + "median": 0.3287218511104584, + "p05": 0.3254890441894531, + "p95": 0.33243927359580994 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0980432033538818, + "group": 2, + "latest_source_mass": 0.32141047716140747, + "layer": 6, + "max_source_mass": 0.34726741909980774, + "mixer_index": 11, + "normalized_entropy": 0.9994819962236925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34726741909980774, + "median": 0.34668511152267456, + "p05": 0.3404209613800049, + "p95": 0.35565274953842163 + }, + { + "label": "completed_group_1", + "mean": 0.3313220739364624, + "median": 0.3316044509410858, + "p05": 0.3264642655849457, + "p95": 0.33527836203575134 + }, + { + "label": "current_group_partial", + "mean": 0.32141047716140747, + "median": 0.321965754032135, + "p05": 0.3149893283843994, + "p95": 0.326220840215683 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0985240936279297, + "group": 2, + "latest_source_mass": 0.32844245433807373, + "layer": 7, + "max_source_mass": 0.3373557925224304, + "mixer_index": 12, + "normalized_entropy": 0.9999197214148341, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3373557925224304, + "median": 0.33718767762184143, + "p05": 0.33250534534454346, + "p95": 0.34282490611076355 + }, + { + "label": "completed_group_1", + "mean": 0.33420175313949585, + "median": 0.3341767191886902, + "p05": 0.3307935893535614, + "p95": 0.3377305567264557 + }, + { + "label": "current_group_partial", + "mean": 0.32844245433807373, + "median": 0.3287346363067627, + "p05": 0.3250963091850281, + "p95": 0.33099210262298584 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.098268985748291, + "group": 2, + "latest_source_mass": 0.32483887672424316, + "layer": 7, + "max_source_mass": 0.3445510268211365, + "mixer_index": 13, + "normalized_entropy": 0.9996875122157655, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3445510268211365, + "median": 0.3445873260498047, + "p05": 0.3388661742210388, + "p95": 0.3502572774887085 + }, + { + "label": "completed_group_1", + "mean": 0.330610066652298, + "median": 0.330571711063385, + "p05": 0.3262380361557007, + "p95": 0.3352304697036743 + }, + { + "label": "current_group_partial", + "mean": 0.32483887672424316, + "median": 0.32478615641593933, + "p05": 0.3213251233100891, + "p95": 0.3284890055656433 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0985302925109863, + "group": 2, + "latest_source_mass": 0.3295760750770569, + "layer": 8, + "max_source_mass": 0.3371829390525818, + "mixer_index": 14, + "normalized_entropy": 0.9999253638813536, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3371829390525818, + "median": 0.33710986375808716, + "p05": 0.330502450466156, + "p95": 0.34345686435699463 + }, + { + "label": "completed_group_1", + "mean": 0.3332410156726837, + "median": 0.33316272497177124, + "p05": 0.32938438653945923, + "p95": 0.33757662773132324 + }, + { + "label": "current_group_partial", + "mean": 0.3295760750770569, + "median": 0.3295734226703644, + "p05": 0.3265466094017029, + "p95": 0.3326798677444458 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0983161926269531, + "group": 2, + "latest_source_mass": 0.32498204708099365, + "layer": 8, + "max_source_mass": 0.3423102796077728, + "mixer_index": 15, + "normalized_entropy": 0.9997304817684903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3423102796077728, + "median": 0.34171032905578613, + "p05": 0.3346346318721771, + "p95": 0.35120290517807007 + }, + { + "label": "completed_group_1", + "mean": 0.3327076733112335, + "median": 0.3330804705619812, + "p05": 0.32750606536865234, + "p95": 0.33743682503700256 + }, + { + "label": "current_group_partial", + "mean": 0.32498204708099365, + "median": 0.3252435028553009, + "p05": 0.31977349519729614, + "p95": 0.3294060230255127 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.098489761352539, + "group": 3, + "latest_source_mass": 0.327616423368454, + "layer": 9, + "max_source_mass": 0.33793729543685913, + "mixer_index": 16, + "normalized_entropy": 0.9998884708310343, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33793729543685913, + "median": 0.3376197814941406, + "p05": 0.3321118652820587, + "p95": 0.34503501653671265 + }, + { + "label": "completed_group_1", + "mean": 0.3344462513923645, + "median": 0.3347727060317993, + "p05": 0.33071762323379517, + "p95": 0.33719509840011597 + }, + { + "label": "completed_group_2", + "mean": 0.327616423368454, + "median": 0.3275512754917145, + "p05": 0.32320931553840637, + "p95": 0.33203738927841187 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3861088752746582, + "group": 3, + "latest_source_mass": 0.2466907799243927, + "layer": 9, + "max_source_mass": 0.2561652958393097, + "mixer_index": 17, + "normalized_entropy": 0.9998662002454641, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2561652958393097, + "median": 0.25643306970596313, + "p05": 0.25058528780937195, + "p95": 0.2609494626522064 + }, + { + "label": "completed_group_1", + "mean": 0.2513129711151123, + "median": 0.2511752247810364, + "p05": 0.24789966642856598, + "p95": 0.25490516424179077 + }, + { + "label": "completed_group_2", + "mean": 0.2458309531211853, + "median": 0.2456333041191101, + "p05": 0.2419084906578064, + "p95": 0.2499997466802597 + }, + { + "label": "current_group_partial", + "mean": 0.2466907799243927, + "median": 0.246699720621109, + "p05": 0.24314038455486298, + "p95": 0.25063008069992065 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3861844539642334, + "group": 3, + "latest_source_mass": 0.24695077538490295, + "layer": 10, + "max_source_mass": 0.2536832094192505, + "mixer_index": 18, + "normalized_entropy": 0.9999207187457876, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2536832094192505, + "median": 0.253880113363266, + "p05": 0.24814625084400177, + "p95": 0.2593393921852112 + }, + { + "label": "completed_group_1", + "mean": 0.25214672088623047, + "median": 0.2522420883178711, + "p05": 0.24960598349571228, + "p95": 0.2543295919895172 + }, + { + "label": "completed_group_2", + "mean": 0.24721932411193848, + "median": 0.24704834818840027, + "p05": 0.24384759366512299, + "p95": 0.25051552057266235 + }, + { + "label": "current_group_partial", + "mean": 0.24695077538490295, + "median": 0.24677348136901855, + "p05": 0.24500520527362823, + "p95": 0.2492748200893402 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3860987424850464, + "group": 3, + "latest_source_mass": 0.24684447050094604, + "layer": 10, + "max_source_mass": 0.25705933570861816, + "mixer_index": 19, + "normalized_entropy": 0.9998588909828025, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25705933570861816, + "median": 0.2568470239639282, + "p05": 0.2513251304626465, + "p95": 0.26331475377082825 + }, + { + "label": "completed_group_1", + "mean": 0.24983704090118408, + "median": 0.24969333410263062, + "p05": 0.2465076744556427, + "p95": 0.25355273485183716 + }, + { + "label": "completed_group_2", + "mean": 0.2462591528892517, + "median": 0.24622508883476257, + "p05": 0.24337750673294067, + "p95": 0.24924828112125397 + }, + { + "label": "current_group_partial", + "mean": 0.24684447050094604, + "median": 0.24683824181556702, + "p05": 0.24403956532478333, + "p95": 0.24949496984481812 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862221240997314, + "group": 3, + "latest_source_mass": 0.24620777368545532, + "layer": 11, + "max_source_mass": 0.25285202264785767, + "mixer_index": 20, + "normalized_entropy": 0.9999478920046239, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2514326572418213, + "median": 0.25176799297332764, + "p05": 0.24684108793735504, + "p95": 0.2554701566696167 + }, + { + "label": "completed_group_1", + "mean": 0.25285202264785767, + "median": 0.25268346071243286, + "p05": 0.2510001063346863, + "p95": 0.2550324499607086 + }, + { + "label": "completed_group_2", + "mean": 0.24950754642486572, + "median": 0.2494041621685028, + "p05": 0.2475442737340927, + "p95": 0.2515066862106323 + }, + { + "label": "current_group_partial", + "mean": 0.24620777368545532, + "median": 0.24642038345336914, + "p05": 0.2445429414510727, + "p95": 0.2473805695772171 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.386016845703125, + "group": 3, + "latest_source_mass": 0.24290117621421814, + "layer": 11, + "max_source_mass": 0.25482746958732605, + "mixer_index": 21, + "normalized_entropy": 0.999799814942231, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25482746958732605, + "median": 0.2545710802078247, + "p05": 0.2466055005788803, + "p95": 0.2635689973831177 + }, + { + "label": "completed_group_1", + "mean": 0.2536851167678833, + "median": 0.2534823715686798, + "p05": 0.2508774995803833, + "p95": 0.2568957209587097 + }, + { + "label": "completed_group_2", + "mean": 0.24858617782592773, + "median": 0.24861112236976624, + "p05": 0.24390698969364166, + "p95": 0.252436101436615 + }, + { + "label": "current_group_partial", + "mean": 0.24290117621421814, + "median": 0.24232372641563416, + "p05": 0.23918122053146362, + "p95": 0.24772994220256805 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862277269363403, + "group": 3, + "latest_source_mass": 0.24656552076339722, + "layer": 12, + "max_source_mass": 0.2530350983142853, + "mixer_index": 22, + "normalized_entropy": 0.9999519335969191, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25171470642089844, + "median": 0.2513781189918518, + "p05": 0.24833296239376068, + "p95": 0.25546184182167053 + }, + { + "label": "completed_group_1", + "mean": 0.2530350983142853, + "median": 0.25303736329078674, + "p05": 0.25148919224739075, + "p95": 0.2544969618320465 + }, + { + "label": "completed_group_2", + "mean": 0.24868467450141907, + "median": 0.24892255663871765, + "p05": 0.2467103898525238, + "p95": 0.25010940432548523 + }, + { + "label": "current_group_partial", + "mean": 0.24656552076339722, + "median": 0.24677836894989014, + "p05": 0.24485518038272858, + "p95": 0.2477675825357437 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3859888315200806, + "group": 3, + "latest_source_mass": 0.24197682738304138, + "layer": 12, + "max_source_mass": 0.2546530067920685, + "mixer_index": 23, + "normalized_entropy": 0.9997796069807546, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25458407402038574, + "median": 0.2539653182029724, + "p05": 0.24643409252166748, + "p95": 0.26325899362564087 + }, + { + "label": "completed_group_1", + "mean": 0.2546530067920685, + "median": 0.2548711895942688, + "p05": 0.25055474042892456, + "p95": 0.25881263613700867 + }, + { + "label": "completed_group_2", + "mean": 0.248786062002182, + "median": 0.24910664558410645, + "p05": 0.2454056739807129, + "p95": 0.2517758011817932 + }, + { + "label": "current_group_partial", + "mean": 0.24197682738304138, + "median": 0.2416902482509613, + "p05": 0.23907747864723206, + "p95": 0.2456972599029541 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862541913986206, + "group": 4, + "latest_source_mass": 0.24767187237739563, + "layer": 13, + "max_source_mass": 0.2521768808364868, + "mixer_index": 24, + "normalized_entropy": 0.999971023671165, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2501963973045349, + "median": 0.250087708234787, + "p05": 0.24644337594509125, + "p95": 0.25403982400894165 + }, + { + "label": "completed_group_1", + "mean": 0.2521768808364868, + "median": 0.2521744668483734, + "p05": 0.25072887539863586, + "p95": 0.25345146656036377 + }, + { + "label": "completed_group_2", + "mean": 0.24995486438274384, + "median": 0.2498224377632141, + "p05": 0.2482302337884903, + "p95": 0.2518054246902466 + }, + { + "label": "completed_group_3", + "mean": 0.24767187237739563, + "median": 0.2475992739200592, + "p05": 0.24603278934955597, + "p95": 0.2497682422399521 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6090729236602783, + "group": 4, + "latest_source_mass": 0.19462624192237854, + "layer": 13, + "max_source_mass": 0.20631682872772217, + "mixer_index": 25, + "normalized_entropy": 0.9997732197241024, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20360878109931946, + "median": 0.20399323105812073, + "p05": 0.19613273441791534, + "p95": 0.21039927005767822 + }, + { + "label": "completed_group_1", + "mean": 0.20631682872772217, + "median": 0.20582494139671326, + "p05": 0.20368342101573944, + "p95": 0.2099049985408783 + }, + { + "label": "completed_group_2", + "mean": 0.19986985623836517, + "median": 0.19948531687259674, + "p05": 0.19628161191940308, + "p95": 0.20406028628349304 + }, + { + "label": "completed_group_3", + "mean": 0.19557830691337585, + "median": 0.19561558961868286, + "p05": 0.1910029649734497, + "p95": 0.20039188861846924 + }, + { + "label": "current_group_partial", + "mean": 0.19462624192237854, + "median": 0.1954103410243988, + "p05": 0.19068367779254913, + "p95": 0.19781020283699036 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093083620071411, + "group": 4, + "latest_source_mass": 0.1976698487997055, + "layer": 14, + "max_source_mass": 0.20362424850463867, + "mixer_index": 26, + "normalized_entropy": 0.9999195057939432, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2017374038696289, + "median": 0.2026747167110443, + "p05": 0.19666710495948792, + "p95": 0.20696133375167847 + }, + { + "label": "completed_group_1", + "mean": 0.20362424850463867, + "median": 0.20354315638542175, + "p05": 0.20209211111068726, + "p95": 0.20522044599056244 + }, + { + "label": "completed_group_2", + "mean": 0.20011675357818604, + "median": 0.1997961699962616, + "p05": 0.19781088829040527, + "p95": 0.2023434340953827 + }, + { + "label": "completed_group_3", + "mean": 0.1968517303466797, + "median": 0.19661611318588257, + "p05": 0.19452345371246338, + "p95": 0.19933368265628815 + }, + { + "label": "current_group_partial", + "mean": 0.1976698487997055, + "median": 0.19780653715133667, + "p05": 0.19629782438278198, + "p95": 0.19887655973434448 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.608644723892212, + "group": 4, + "latest_source_mass": 0.18768835067749023, + "layer": 14, + "max_source_mass": 0.20831988751888275, + "mixer_index": 27, + "normalized_entropy": 0.9995071642492324, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20431089401245117, + "median": 0.20455676317214966, + "p05": 0.19753149151802063, + "p95": 0.21038773655891418 + }, + { + "label": "completed_group_1", + "mean": 0.20831988751888275, + "median": 0.20877230167388916, + "p05": 0.2030324637889862, + "p95": 0.2128065973520279 + }, + { + "label": "completed_group_2", + "mean": 0.20422273874282837, + "median": 0.2047901451587677, + "p05": 0.20030567049980164, + "p95": 0.20715001225471497 + }, + { + "label": "completed_group_3", + "mean": 0.19545812904834747, + "median": 0.19552549719810486, + "p05": 0.1923498660326004, + "p95": 0.19825047254562378 + }, + { + "label": "current_group_partial", + "mean": 0.18768835067749023, + "median": 0.18748146295547485, + "p05": 0.18511706590652466, + "p95": 0.19059684872627258 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.609345555305481, + "group": 4, + "latest_source_mass": 0.19806164503097534, + "layer": 15, + "max_source_mass": 0.20386312901973724, + "mixer_index": 28, + "normalized_entropy": 0.9999426152895332, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2002599537372589, + "median": 0.20108085870742798, + "p05": 0.19542044401168823, + "p95": 0.20405235886573792 + }, + { + "label": "completed_group_1", + "mean": 0.20386312901973724, + "median": 0.20386025309562683, + "p05": 0.20271247625350952, + "p95": 0.20498213171958923 + }, + { + "label": "completed_group_2", + "mean": 0.2001732587814331, + "median": 0.19999440014362335, + "p05": 0.1985137164592743, + "p95": 0.2019573152065277 + }, + { + "label": "completed_group_3", + "mean": 0.1976420283317566, + "median": 0.19760513305664062, + "p05": 0.1957571804523468, + "p95": 0.20000392198562622 + }, + { + "label": "current_group_partial", + "mean": 0.19806164503097534, + "median": 0.19796663522720337, + "p05": 0.1969158947467804, + "p95": 0.19929495453834534 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6086196899414062, + "group": 4, + "latest_source_mass": 0.18893863260746002, + "layer": 15, + "max_source_mass": 0.21025606989860535, + "mixer_index": 29, + "normalized_entropy": 0.9994916097810468, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20515945553779602, + "median": 0.2049580067396164, + "p05": 0.19868221879005432, + "p95": 0.21147878468036652 + }, + { + "label": "completed_group_1", + "mean": 0.21025606989860535, + "median": 0.2101147472858429, + "p05": 0.20689907670021057, + "p95": 0.21399712562561035 + }, + { + "label": "completed_group_2", + "mean": 0.20179016888141632, + "median": 0.20176216959953308, + "p05": 0.199407696723938, + "p95": 0.20402206480503082 + }, + { + "label": "completed_group_3", + "mean": 0.1938556730747223, + "median": 0.19379490613937378, + "p05": 0.19004228711128235, + "p95": 0.1977127194404602 + }, + { + "label": "current_group_partial", + "mean": 0.18893863260746002, + "median": 0.1887960582971573, + "p05": 0.18639500439167023, + "p95": 0.19145210087299347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.609370470046997, + "group": 4, + "latest_source_mass": 0.1981920301914215, + "layer": 16, + "max_source_mass": 0.20340535044670105, + "mixer_index": 30, + "normalized_entropy": 0.9999580956888227, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2002836912870407, + "median": 0.20032534003257751, + "p05": 0.19668522477149963, + "p95": 0.20378869771957397 + }, + { + "label": "completed_group_1", + "mean": 0.20340535044670105, + "median": 0.20332223176956177, + "p05": 0.2024117112159729, + "p95": 0.20457619428634644 + }, + { + "label": "completed_group_2", + "mean": 0.20010054111480713, + "median": 0.1998724788427353, + "p05": 0.19848622381687164, + "p95": 0.20170091092586517 + }, + { + "label": "completed_group_3", + "mean": 0.1980184018611908, + "median": 0.19809064269065857, + "p05": 0.19669371843338013, + "p95": 0.19908645749092102 + }, + { + "label": "current_group_partial", + "mean": 0.1981920301914215, + "median": 0.19810855388641357, + "p05": 0.1971922218799591, + "p95": 0.19958281517028809 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6083989143371582, + "group": 4, + "latest_source_mass": 0.18735264241695404, + "layer": 16, + "max_source_mass": 0.21095217764377594, + "mixer_index": 31, + "normalized_entropy": 0.9993544341854289, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20363345742225647, + "median": 0.20384372770786285, + "p05": 0.19405215978622437, + "p95": 0.21326100826263428 + }, + { + "label": "completed_group_1", + "mean": 0.21095217764377594, + "median": 0.21146853268146515, + "p05": 0.20567592978477478, + "p95": 0.21522513031959534 + }, + { + "label": "completed_group_2", + "mean": 0.20392385125160217, + "median": 0.20381325483322144, + "p05": 0.20115600526332855, + "p95": 0.2067122757434845 + }, + { + "label": "completed_group_3", + "mean": 0.19413790106773376, + "median": 0.19391843676567078, + "p05": 0.18873284757137299, + "p95": 0.1996975690126419 + }, + { + "label": "current_group_partial", + "mean": 0.18735264241695404, + "median": 0.18673613667488098, + "p05": 0.1829182654619217, + "p95": 0.1927962601184845 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093392372131348, + "group": 5, + "latest_source_mass": 0.19563516974449158, + "layer": 17, + "max_source_mass": 0.20331618189811707, + "mixer_index": 32, + "normalized_entropy": 0.9999386896380388, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20084109902381897, + "median": 0.20074692368507385, + "p05": 0.198189377784729, + "p95": 0.20358796417713165 + }, + { + "label": "completed_group_1", + "mean": 0.20331618189811707, + "median": 0.20337940752506256, + "p05": 0.20178094506263733, + "p95": 0.20489434897899628 + }, + { + "label": "completed_group_2", + "mean": 0.20119993388652802, + "median": 0.20122915506362915, + "p05": 0.20026232302188873, + "p95": 0.2019881308078766 + }, + { + "label": "completed_group_3", + "mean": 0.19900763034820557, + "median": 0.198960542678833, + "p05": 0.19746547937393188, + "p95": 0.20062875747680664 + }, + { + "label": "completed_group_4", + "mean": 0.19563516974449158, + "median": 0.19558461010456085, + "p05": 0.193980410695076, + "p95": 0.19730384647846222 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7908520698547363, + "group": 5, + "latest_source_mass": 0.1625179648399353, + "layer": 17, + "max_source_mass": 0.17730244994163513, + "mixer_index": 33, + "normalized_entropy": 0.9994935707672249, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17160072922706604, + "median": 0.1718936264514923, + "p05": 0.16487310826778412, + "p95": 0.17849747836589813 + }, + { + "label": "completed_group_1", + "mean": 0.17730244994163513, + "median": 0.1778835654258728, + "p05": 0.17289136350154877, + "p95": 0.18126235902309418 + }, + { + "label": "completed_group_2", + "mean": 0.1690303534269333, + "median": 0.16928547620773315, + "p05": 0.16646793484687805, + "p95": 0.17125293612480164 + }, + { + "label": "completed_group_3", + "mean": 0.16099892556667328, + "median": 0.16061408817768097, + "p05": 0.15715192258358002, + "p95": 0.1656636893749237 + }, + { + "label": "completed_group_4", + "mean": 0.15854957699775696, + "median": 0.15825147926807404, + "p05": 0.15517033636569977, + "p95": 0.1620505452156067 + }, + { + "label": "current_group_partial", + "mean": 0.1625179648399353, + "median": 0.16315963864326477, + "p05": 0.15939058363437653, + "p95": 0.16489268839359283 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7916775941848755, + "group": 5, + "latest_source_mass": 0.16463731229305267, + "layer": 18, + "max_source_mass": 0.16981256008148193, + "mixer_index": 34, + "normalized_entropy": 0.9999543046683522, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16706335544586182, + "median": 0.16679415106773376, + "p05": 0.16531850397586823, + "p95": 0.16906341910362244 + }, + { + "label": "completed_group_1", + "mean": 0.16981256008148193, + "median": 0.1699288785457611, + "p05": 0.1682884395122528, + "p95": 0.17106440663337708 + }, + { + "label": "completed_group_2", + "mean": 0.16858771443367004, + "median": 0.16860879957675934, + "p05": 0.16782470047473907, + "p95": 0.16934865713119507 + }, + { + "label": "completed_group_3", + "mean": 0.16527186334133148, + "median": 0.16520482301712036, + "p05": 0.1643052101135254, + "p95": 0.16638660430908203 + }, + { + "label": "completed_group_4", + "mean": 0.16462719440460205, + "median": 0.16459274291992188, + "p05": 0.1638425886631012, + "p95": 0.16539013385772705 + }, + { + "label": "current_group_partial", + "mean": 0.16463731229305267, + "median": 0.16451048851013184, + "p05": 0.1639435589313507, + "p95": 0.16552197933197021 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7898297309875488, + "group": 5, + "latest_source_mass": 0.153368279337883, + "layer": 18, + "max_source_mass": 0.18101517856121063, + "mixer_index": 35, + "normalized_entropy": 0.9989229925815112, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17284835875034332, + "median": 0.17269954085350037, + "p05": 0.1658562421798706, + "p95": 0.18014436960220337 + }, + { + "label": "completed_group_1", + "mean": 0.18101517856121063, + "median": 0.1814284771680832, + "p05": 0.17416825890541077, + "p95": 0.18707150220870972 + }, + { + "label": "completed_group_2", + "mean": 0.1735592633485794, + "median": 0.17370939254760742, + "p05": 0.17075185477733612, + "p95": 0.17625606060028076 + }, + { + "label": "completed_group_3", + "mean": 0.16231875121593475, + "median": 0.1621963381767273, + "p05": 0.15821577608585358, + "p95": 0.16650474071502686 + }, + { + "label": "completed_group_4", + "mean": 0.15689018368721008, + "median": 0.15674838423728943, + "p05": 0.15215492248535156, + "p95": 0.1616627275943756 + }, + { + "label": "current_group_partial", + "mean": 0.153368279337883, + "median": 0.1528519243001938, + "p05": 0.15056025981903076, + "p95": 0.15731072425842285 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917213439941406, + "group": 5, + "latest_source_mass": 0.16463561356067657, + "layer": 19, + "max_source_mass": 0.16885727643966675, + "mixer_index": 36, + "normalized_entropy": 0.9999787219018127, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1661592274904251, + "median": 0.1661478579044342, + "p05": 0.16439414024353027, + "p95": 0.1678619235754013 + }, + { + "label": "completed_group_1", + "mean": 0.16885727643966675, + "median": 0.16877895593643188, + "p05": 0.16819745302200317, + "p95": 0.1697411835193634 + }, + { + "label": "completed_group_2", + "mean": 0.16764025390148163, + "median": 0.16765299439430237, + "p05": 0.1669837087392807, + "p95": 0.1682416945695877 + }, + { + "label": "completed_group_3", + "mean": 0.16661542654037476, + "median": 0.1665019690990448, + "p05": 0.1658560335636139, + "p95": 0.16755254566669464 + }, + { + "label": "completed_group_4", + "mean": 0.16609220206737518, + "median": 0.16612619161605835, + "p05": 0.16555683314800262, + "p95": 0.16653065383434296 + }, + { + "label": "current_group_partial", + "mean": 0.16463561356067657, + "median": 0.16466060280799866, + "p05": 0.16384081542491913, + "p95": 0.16551899909973145 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7890949249267578, + "group": 5, + "latest_source_mass": 0.15094393491744995, + "layer": 19, + "max_source_mass": 0.18413972854614258, + "mixer_index": 37, + "normalized_entropy": 0.9985128895105295, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17241930961608887, + "median": 0.17162901163101196, + "p05": 0.1626318097114563, + "p95": 0.18299198150634766 + }, + { + "label": "completed_group_1", + "mean": 0.18413972854614258, + "median": 0.1848709136247635, + "p05": 0.17655718326568604, + "p95": 0.19053134322166443 + }, + { + "label": "completed_group_2", + "mean": 0.17484498023986816, + "median": 0.17485198378562927, + "p05": 0.1718076914548874, + "p95": 0.17815767228603363 + }, + { + "label": "completed_group_3", + "mean": 0.1621202528476715, + "median": 0.16197144985198975, + "p05": 0.15698590874671936, + "p95": 0.16731801629066467 + }, + { + "label": "completed_group_4", + "mean": 0.15553176403045654, + "median": 0.15512266755104065, + "p05": 0.1502593755722046, + "p95": 0.16059254109859467 + }, + { + "label": "current_group_partial", + "mean": 0.15094393491744995, + "median": 0.15083211660385132, + "p05": 0.1470862179994583, + "p95": 0.1549530029296875 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917208671569824, + "group": 5, + "latest_source_mass": 0.16669265925884247, + "layer": 20, + "max_source_mass": 0.168935626745224, + "mixer_index": 38, + "normalized_entropy": 0.9999784557739275, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16663745045661926, + "median": 0.16669239103794098, + "p05": 0.16406413912773132, + "p95": 0.16939817368984222 + }, + { + "label": "completed_group_1", + "mean": 0.168935626745224, + "median": 0.16887718439102173, + "p05": 0.16814815998077393, + "p95": 0.1698606014251709 + }, + { + "label": "completed_group_2", + "mean": 0.16597259044647217, + "median": 0.16582076251506805, + "p05": 0.16482511162757874, + "p95": 0.1670236438512802 + }, + { + "label": "completed_group_3", + "mean": 0.16514870524406433, + "median": 0.165022611618042, + "p05": 0.16397784650325775, + "p95": 0.1662973016500473 + }, + { + "label": "completed_group_4", + "mean": 0.16661295294761658, + "median": 0.16667577624320984, + "p05": 0.16593852639198303, + "p95": 0.16715332865715027 + }, + { + "label": "current_group_partial", + "mean": 0.16669265925884247, + "median": 0.1666145920753479, + "p05": 0.16592761874198914, + "p95": 0.1674128621816635 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.789101004600525, + "group": 5, + "latest_source_mass": 0.14993149042129517, + "layer": 20, + "max_source_mass": 0.18365857005119324, + "mixer_index": 39, + "normalized_entropy": 0.9985162826410648, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17229264974594116, + "median": 0.1720556616783142, + "p05": 0.16247707605361938, + "p95": 0.18216519057750702 + }, + { + "label": "completed_group_1", + "mean": 0.18365857005119324, + "median": 0.18395906686782837, + "p05": 0.17734748125076294, + "p95": 0.18972161412239075 + }, + { + "label": "completed_group_2", + "mean": 0.1747075319290161, + "median": 0.1746184229850769, + "p05": 0.17210862040519714, + "p95": 0.1774953156709671 + }, + { + "label": "completed_group_3", + "mean": 0.1640234887599945, + "median": 0.1635349988937378, + "p05": 0.15958356857299805, + "p95": 0.16904467344284058 + }, + { + "label": "completed_group_4", + "mean": 0.15538626909255981, + "median": 0.15498605370521545, + "p05": 0.14995241165161133, + "p95": 0.16101743280887604 + }, + { + "label": "current_group_partial", + "mean": 0.14993149042129517, + "median": 0.15014703571796417, + "p05": 0.14564283192157745, + "p95": 0.1527552753686905 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917447090148926, + "group": 6, + "latest_source_mass": 0.1661863625049591, + "layer": 21, + "max_source_mass": 0.16768339276313782, + "mixer_index": 40, + "normalized_entropy": 0.9999917621681839, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1658121943473816, + "median": 0.1657177209854126, + "p05": 0.16437731683254242, + "p95": 0.16778677701950073 + }, + { + "label": "completed_group_1", + "mean": 0.16768339276313782, + "median": 0.16760629415512085, + "p05": 0.1666821986436844, + "p95": 0.16875657439231873 + }, + { + "label": "completed_group_2", + "mean": 0.16629494726657867, + "median": 0.16622786223888397, + "p05": 0.1655496209859848, + "p95": 0.16718709468841553 + }, + { + "label": "completed_group_3", + "mean": 0.1669958382844925, + "median": 0.16695073246955872, + "p05": 0.1664678305387497, + "p95": 0.16760030388832092 + }, + { + "label": "completed_group_4", + "mean": 0.16702723503112793, + "median": 0.16713355481624603, + "p05": 0.16618871688842773, + "p95": 0.16781742870807648 + }, + { + "label": "completed_group_5", + "mean": 0.1661863625049591, + "median": 0.16608929634094238, + "p05": 0.16534176468849182, + "p95": 0.16705378890037537 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9436898231506348, + "group": 6, + "latest_source_mass": 0.13651785254478455, + "layer": 21, + "max_source_mass": 0.15856821835041046, + "mixer_index": 41, + "normalized_entropy": 0.9988589781980651, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14778707921504974, + "median": 0.14752492308616638, + "p05": 0.13982102274894714, + "p95": 0.1569063663482666 + }, + { + "label": "completed_group_1", + "mean": 0.15856821835041046, + "median": 0.1585446149110794, + "p05": 0.15396656095981598, + "p95": 0.16390106081962585 + }, + { + "label": "completed_group_2", + "mean": 0.1500256508588791, + "median": 0.14993107318878174, + "p05": 0.14748509228229523, + "p95": 0.15272916853427887 + }, + { + "label": "completed_group_3", + "mean": 0.1408001333475113, + "median": 0.14052921533584595, + "p05": 0.13647492229938507, + "p95": 0.14521925151348114 + }, + { + "label": "completed_group_4", + "mean": 0.13533657789230347, + "median": 0.13547882437705994, + "p05": 0.13068656623363495, + "p95": 0.13946673274040222 + }, + { + "label": "completed_group_5", + "mean": 0.130964457988739, + "median": 0.13209164142608643, + "p05": 0.12633010745048523, + "p95": 0.13486123085021973 + }, + { + "label": "current_group_partial", + "mean": 0.13651785254478455, + "median": 0.1365797072649002, + "p05": 0.13271403312683105, + "p95": 0.14000433683395386 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458942413330078, + "group": 6, + "latest_source_mass": 0.14246684312820435, + "layer": 22, + "max_source_mass": 0.14393000304698944, + "mixer_index": 42, + "normalized_entropy": 0.9999918250478763, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.142832413315773, + "median": 0.14290162920951843, + "p05": 0.14148348569869995, + "p95": 0.1440550833940506 + }, + { + "label": "completed_group_1", + "mean": 0.1429906189441681, + "median": 0.14299973845481873, + "p05": 0.14231431484222412, + "p95": 0.14357265830039978 + }, + { + "label": "completed_group_2", + "mean": 0.14157448709011078, + "median": 0.14162302017211914, + "p05": 0.1408008486032486, + "p95": 0.1421196460723877 + }, + { + "label": "completed_group_3", + "mean": 0.14285019040107727, + "median": 0.14287817478179932, + "p05": 0.1423148512840271, + "p95": 0.14345195889472961 + }, + { + "label": "completed_group_4", + "mean": 0.14393000304698944, + "median": 0.14395871758460999, + "p05": 0.14325706660747528, + "p95": 0.14449745416641235 + }, + { + "label": "completed_group_5", + "mean": 0.14335544407367706, + "median": 0.14326418936252594, + "p05": 0.14292818307876587, + "p95": 0.14392802119255066 + }, + { + "label": "current_group_partial", + "mean": 0.14246684312820435, + "median": 0.14247724413871765, + "p05": 0.14211606979370117, + "p95": 0.14281725883483887 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9425597190856934, + "group": 6, + "latest_source_mass": 0.12947803735733032, + "layer": 22, + "max_source_mass": 0.1615946888923645, + "mixer_index": 43, + "normalized_entropy": 0.9982782195923864, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1483290195465088, + "median": 0.14828777313232422, + "p05": 0.13995377719402313, + "p95": 0.15759113430976868 + }, + { + "label": "completed_group_1", + "mean": 0.1615946888923645, + "median": 0.16178560256958008, + "p05": 0.15419340133666992, + "p95": 0.16922280192375183 + }, + { + "label": "completed_group_2", + "mean": 0.15280696749687195, + "median": 0.1528012454509735, + "p05": 0.15010300278663635, + "p95": 0.1558753252029419 + }, + { + "label": "completed_group_3", + "mean": 0.14239415526390076, + "median": 0.14164817333221436, + "p05": 0.1380041539669037, + "p95": 0.14742238819599152 + }, + { + "label": "completed_group_4", + "mean": 0.13462956249713898, + "median": 0.1342291682958603, + "p05": 0.12952053546905518, + "p95": 0.14025796949863434 + }, + { + "label": "completed_group_5", + "mean": 0.1307675838470459, + "median": 0.13075318932533264, + "p05": 0.12602169811725616, + "p95": 0.13501127064228058 + }, + { + "label": "current_group_partial", + "mean": 0.12947803735733032, + "median": 0.1295393407344818, + "p05": 0.1264311969280243, + "p95": 0.13163118064403534 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458987712860107, + "group": 6, + "latest_source_mass": 0.14346569776535034, + "layer": 23, + "max_source_mass": 0.14346569776535034, + "mixer_index": 44, + "normalized_entropy": 0.9999941529832156, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1426449418067932, + "median": 0.14265665411949158, + "p05": 0.14160317182540894, + "p95": 0.1436314880847931 + }, + { + "label": "completed_group_1", + "mean": 0.14291533827781677, + "median": 0.14293473958969116, + "p05": 0.1422625482082367, + "p95": 0.14348529279232025 + }, + { + "label": "completed_group_2", + "mean": 0.1416938751935959, + "median": 0.14168253540992737, + "p05": 0.14088502526283264, + "p95": 0.14245855808258057 + }, + { + "label": "completed_group_3", + "mean": 0.14274153113365173, + "median": 0.14274577796459198, + "p05": 0.14230182766914368, + "p95": 0.14324283599853516 + }, + { + "label": "completed_group_4", + "mean": 0.14325544238090515, + "median": 0.1432678997516632, + "p05": 0.14277629554271698, + "p95": 0.14376230537891388 + }, + { + "label": "completed_group_5", + "mean": 0.1432831883430481, + "median": 0.14325812458992004, + "p05": 0.14287351071834564, + "p95": 0.1437322348356247 + }, + { + "label": "current_group_partial", + "mean": 0.14346569776535034, + "median": 0.1435171663761139, + "p05": 0.142959862947464, + "p95": 0.1437678039073944 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.941068410873413, + "group": 6, + "latest_source_mass": 0.1254161298274994, + "layer": 23, + "max_source_mass": 0.16456490755081177, + "mixer_index": 45, + "normalized_entropy": 0.9975118387741332, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14912807941436768, + "median": 0.14865416288375854, + "p05": 0.13610364496707916, + "p95": 0.16207551956176758 + }, + { + "label": "completed_group_1", + "mean": 0.16456490755081177, + "median": 0.16586321592330933, + "p05": 0.15482422709465027, + "p95": 0.17167139053344727 + }, + { + "label": "completed_group_2", + "mean": 0.15460123121738434, + "median": 0.1543833166360855, + "p05": 0.15155401825904846, + "p95": 0.15836837887763977 + }, + { + "label": "completed_group_3", + "mean": 0.1425517499446869, + "median": 0.14188045263290405, + "p05": 0.13683314621448517, + "p95": 0.1494884341955185 + }, + { + "label": "completed_group_4", + "mean": 0.13539987802505493, + "median": 0.13422535359859467, + "p05": 0.12956365942955017, + "p95": 0.14296601712703705 + }, + { + "label": "completed_group_5", + "mean": 0.1283380389213562, + "median": 0.12796106934547424, + "p05": 0.12316946685314178, + "p95": 0.13433398306369781 + }, + { + "label": "current_group_partial", + "mean": 0.1254161298274994, + "median": 0.12523823976516724, + "p05": 0.12156305462121964, + "p95": 0.12899920344352722 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458861351013184, + "group": 6, + "latest_source_mass": 0.14452049136161804, + "layer": 24, + "max_source_mass": 0.14452049136161804, + "mixer_index": 46, + "normalized_entropy": 0.9999876592688483, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14264336228370667, + "median": 0.1426892876625061, + "p05": 0.14129234850406647, + "p95": 0.14382319152355194 + }, + { + "label": "completed_group_1", + "mean": 0.1430070698261261, + "median": 0.14298146963119507, + "p05": 0.14245536923408508, + "p95": 0.1435846984386444 + }, + { + "label": "completed_group_2", + "mean": 0.14314037561416626, + "median": 0.14310884475708008, + "p05": 0.14277565479278564, + "p95": 0.14360389113426208 + }, + { + "label": "completed_group_3", + "mean": 0.14221979677677155, + "median": 0.14216911792755127, + "p05": 0.14183637499809265, + "p95": 0.14285607635974884 + }, + { + "label": "completed_group_4", + "mean": 0.14132915437221527, + "median": 0.1413242518901825, + "p05": 0.140913724899292, + "p95": 0.14174436032772064 + }, + { + "label": "completed_group_5", + "mean": 0.1431397795677185, + "median": 0.14310240745544434, + "p05": 0.14276552200317383, + "p95": 0.14377698302268982 + }, + { + "label": "current_group_partial", + "mean": 0.14452049136161804, + "median": 0.1444905698299408, + "p05": 0.14409083127975464, + "p95": 0.14507406949996948 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9415628910064697, + "group": 6, + "latest_source_mass": 0.1248510330915451, + "layer": 24, + "max_source_mass": 0.16342055797576904, + "mixer_index": 47, + "normalized_entropy": 0.9977659512948458, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14895881712436676, + "median": 0.14996516704559326, + "p05": 0.13763123750686646, + "p95": 0.15872029960155487 + }, + { + "label": "completed_group_1", + "mean": 0.16342055797576904, + "median": 0.1639186143875122, + "p05": 0.15447522699832916, + "p95": 0.17102839052677155 + }, + { + "label": "completed_group_2", + "mean": 0.15406624972820282, + "median": 0.1540084183216095, + "p05": 0.1514483094215393, + "p95": 0.1571783721446991 + }, + { + "label": "completed_group_3", + "mean": 0.14293310046195984, + "median": 0.14186778664588928, + "p05": 0.13832569122314453, + "p95": 0.1491597443819046 + }, + { + "label": "completed_group_4", + "mean": 0.1363803744316101, + "median": 0.13557225465774536, + "p05": 0.13139937818050385, + "p95": 0.14291447401046753 + }, + { + "label": "completed_group_5", + "mean": 0.12938985228538513, + "median": 0.12914851307868958, + "p05": 0.12471500039100647, + "p95": 0.13491052389144897 + }, + { + "label": "current_group_partial", + "mean": 0.1248510330915451, + "median": 0.12450477480888367, + "p05": 0.120968759059906, + "p95": 0.12821169197559357 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458723068237305, + "group": 7, + "latest_source_mass": 0.14437910914421082, + "layer": 25, + "max_source_mass": 0.14437910914421082, + "mixer_index": 48, + "normalized_entropy": 0.999980552939918, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14191991090774536, + "median": 0.14168427884578705, + "p05": 0.14056351780891418, + "p95": 0.14397425949573517 + }, + { + "label": "completed_group_1", + "mean": 0.1418565809726715, + "median": 0.1417136788368225, + "p05": 0.14059004187583923, + "p95": 0.14328429102897644 + }, + { + "label": "completed_group_2", + "mean": 0.14164230227470398, + "median": 0.14142388105392456, + "p05": 0.14061963558197021, + "p95": 0.14320334792137146 + }, + { + "label": "completed_group_3", + "mean": 0.14289972186088562, + "median": 0.14285793900489807, + "p05": 0.1423202008008957, + "p95": 0.14347271621227264 + }, + { + "label": "completed_group_4", + "mean": 0.14397987723350525, + "median": 0.1439221203327179, + "p05": 0.14329849183559418, + "p95": 0.14462903141975403 + }, + { + "label": "completed_group_5", + "mean": 0.14332249760627747, + "median": 0.14335353672504425, + "p05": 0.14246267080307007, + "p95": 0.14414797723293304 + }, + { + "label": "completed_group_6", + "mean": 0.14437910914421082, + "median": 0.14459103345870972, + "p05": 0.14338219165802002, + "p95": 0.14524540305137634 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.073223114013672, + "group": 7, + "latest_source_mass": 0.11086538434028625, + "layer": 25, + "max_source_mass": 0.14827430248260498, + "mixer_index": 49, + "normalized_entropy": 0.9970095684146328, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13280925154685974, + "median": 0.13328123092651367, + "p05": 0.11957645416259766, + "p95": 0.1447097212076187 + }, + { + "label": "completed_group_1", + "mean": 0.14827430248260498, + "median": 0.14952056109905243, + "p05": 0.13636183738708496, + "p95": 0.15699218213558197 + }, + { + "label": "completed_group_2", + "mean": 0.13796664774417877, + "median": 0.1381954699754715, + "p05": 0.13450786471366882, + "p95": 0.1413417011499405 + }, + { + "label": "completed_group_3", + "mean": 0.12756124138832092, + "median": 0.12679177522659302, + "p05": 0.12232367694377899, + "p95": 0.13387270271778107 + }, + { + "label": "completed_group_4", + "mean": 0.12019774317741394, + "median": 0.11888951063156128, + "p05": 0.11402077227830887, + "p95": 0.12811852991580963 + }, + { + "label": "completed_group_5", + "mean": 0.11310221254825592, + "median": 0.11254298686981201, + "p05": 0.10767743736505508, + "p95": 0.12006338685750961 + }, + { + "label": "completed_group_6", + "mean": 0.10922321677207947, + "median": 0.10910944640636444, + "p05": 0.10472994297742844, + "p95": 0.11491495370864868 + }, + { + "label": "current_group_partial", + "mean": 0.11086538434028625, + "median": 0.11154524236917496, + "p05": 0.10694204270839691, + "p95": 0.11396989226341248 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079413890838623, + "group": 7, + "latest_source_mass": 0.1263430416584015, + "layer": 26, + "max_source_mass": 0.1263430416584015, + "mixer_index": 50, + "normalized_entropy": 0.9999867027561686, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12475188821554184, + "median": 0.12463723123073578, + "p05": 0.12364111095666885, + "p95": 0.1259622424840927 + }, + { + "label": "completed_group_1", + "mean": 0.12460128962993622, + "median": 0.12450964748859406, + "p05": 0.12383783608675003, + "p95": 0.125508114695549 + }, + { + "label": "completed_group_2", + "mean": 0.12424702197313309, + "median": 0.12409250438213348, + "p05": 0.1235705241560936, + "p95": 0.12524327635765076 + }, + { + "label": "completed_group_3", + "mean": 0.12420950829982758, + "median": 0.12421596050262451, + "p05": 0.12374235689640045, + "p95": 0.12457412481307983 + }, + { + "label": "completed_group_4", + "mean": 0.12484367191791534, + "median": 0.12490126490592957, + "p05": 0.12441762536764145, + "p95": 0.1252550482749939 + }, + { + "label": "completed_group_5", + "mean": 0.12469653785228729, + "median": 0.12477756291627884, + "p05": 0.12382431328296661, + "p95": 0.12524525821208954 + }, + { + "label": "completed_group_6", + "mean": 0.12630701065063477, + "median": 0.12644168734550476, + "p05": 0.12546542286872864, + "p95": 0.12699897587299347 + }, + { + "label": "current_group_partial", + "mean": 0.1263430416584015, + "median": 0.12645362317562103, + "p05": 0.12567421793937683, + "p95": 0.1267925351858139 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.072925567626953, + "group": 7, + "latest_source_mass": 0.10599855333566666, + "layer": 26, + "max_source_mass": 0.14843900501728058, + "mixer_index": 51, + "normalized_entropy": 0.9968664788491151, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13228744268417358, + "median": 0.13281425833702087, + "p05": 0.12057555466890335, + "p95": 0.14273115992546082 + }, + { + "label": "completed_group_1", + "mean": 0.14843900501728058, + "median": 0.14951951801776886, + "p05": 0.13698264956474304, + "p95": 0.1572267711162567 + }, + { + "label": "completed_group_2", + "mean": 0.13848665356636047, + "median": 0.1387266218662262, + "p05": 0.13503478467464447, + "p95": 0.1420348733663559 + }, + { + "label": "completed_group_3", + "mean": 0.12843601405620575, + "median": 0.12770286202430725, + "p05": 0.12394566833972931, + "p95": 0.13404949009418488 + }, + { + "label": "completed_group_4", + "mean": 0.12169650197029114, + "median": 0.12072620540857315, + "p05": 0.11642728000879288, + "p95": 0.12845680117607117 + }, + { + "label": "completed_group_5", + "mean": 0.11442162096500397, + "median": 0.11409753561019897, + "p05": 0.10971812158823013, + "p95": 0.12062239646911621 + }, + { + "label": "completed_group_6", + "mean": 0.11023421585559845, + "median": 0.11024647206068039, + "p05": 0.10620385408401489, + "p95": 0.11548472940921783 + }, + { + "label": "current_group_partial", + "mean": 0.10599855333566666, + "median": 0.10623474419116974, + "p05": 0.1023266613483429, + "p95": 0.10840564221143723 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0793263912200928, + "group": 7, + "latest_source_mass": 0.12782885134220123, + "layer": 27, + "max_source_mass": 0.12782885134220123, + "mixer_index": 52, + "normalized_entropy": 0.9999446243342576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12420415878295898, + "median": 0.12419730424880981, + "p05": 0.12226121127605438, + "p95": 0.12590435147285461 + }, + { + "label": "completed_group_1", + "mean": 0.12273682653903961, + "median": 0.12260904908180237, + "p05": 0.12171939015388489, + "p95": 0.12380002439022064 + }, + { + "label": "completed_group_2", + "mean": 0.12360995262861252, + "median": 0.12356993556022644, + "p05": 0.12293750047683716, + "p95": 0.12423694133758545 + }, + { + "label": "completed_group_3", + "mean": 0.1238352432847023, + "median": 0.12381016463041306, + "p05": 0.12339816987514496, + "p95": 0.12445184588432312 + }, + { + "label": "completed_group_4", + "mean": 0.1241140365600586, + "median": 0.12404628843069077, + "p05": 0.12358136475086212, + "p95": 0.1247347816824913 + }, + { + "label": "completed_group_5", + "mean": 0.12607695162296295, + "median": 0.1261017620563507, + "p05": 0.12524889409542084, + "p95": 0.12690351903438568 + }, + { + "label": "completed_group_6", + "mean": 0.127593994140625, + "median": 0.12773489952087402, + "p05": 0.1267755776643753, + "p95": 0.1284082531929016 + }, + { + "label": "current_group_partial", + "mean": 0.12782885134220123, + "median": 0.1278689205646515, + "p05": 0.12723052501678467, + "p95": 0.12838256359100342 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.073427200317383, + "group": 7, + "latest_source_mass": 0.10513068735599518, + "layer": 27, + "max_source_mass": 0.1465209424495697, + "mixer_index": 53, + "normalized_entropy": 0.9971077131807252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13135746121406555, + "median": 0.13279247283935547, + "p05": 0.11876361072063446, + "p95": 0.14146237075328827 + }, + { + "label": "completed_group_1", + "mean": 0.1465209424495697, + "median": 0.1469956636428833, + "p05": 0.13593894243240356, + "p95": 0.1559162735939026 + }, + { + "label": "completed_group_2", + "mean": 0.13772746920585632, + "median": 0.137643963098526, + "p05": 0.13452906906604767, + "p95": 0.14171628654003143 + }, + { + "label": "completed_group_3", + "mean": 0.12966199219226837, + "median": 0.1287793666124344, + "p05": 0.12537093460559845, + "p95": 0.13508695363998413 + }, + { + "label": "completed_group_4", + "mean": 0.12215200066566467, + "median": 0.12143252789974213, + "p05": 0.11671329289674759, + "p95": 0.129044771194458 + }, + { + "label": "completed_group_5", + "mean": 0.11654195189476013, + "median": 0.11616532504558563, + "p05": 0.11162350326776505, + "p95": 0.12263412773609161 + }, + { + "label": "completed_group_6", + "mean": 0.11090746521949768, + "median": 0.11034701764583588, + "p05": 0.10647446662187576, + "p95": 0.11587852239608765 + }, + { + "label": "current_group_partial", + "mean": 0.10513068735599518, + "median": 0.10488355159759521, + "p05": 0.10140107572078705, + "p95": 0.10828588157892227 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079422950744629, + "group": 7, + "latest_source_mass": 0.12521177530288696, + "layer": 28, + "max_source_mass": 0.12591606378555298, + "mixer_index": 54, + "normalized_entropy": 0.9999910596499906, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12477636337280273, + "median": 0.12485313415527344, + "p05": 0.12292245030403137, + "p95": 0.12664391100406647 + }, + { + "label": "completed_group_1", + "mean": 0.12493627518415451, + "median": 0.12481105327606201, + "p05": 0.12430030107498169, + "p95": 0.12584592401981354 + }, + { + "label": "completed_group_2", + "mean": 0.1242394968867302, + "median": 0.12431945651769638, + "p05": 0.12357322871685028, + "p95": 0.12491215020418167 + }, + { + "label": "completed_group_3", + "mean": 0.12470392882823944, + "median": 0.12473497539758682, + "p05": 0.12409180402755737, + "p95": 0.12546151876449585 + }, + { + "label": "completed_group_4", + "mean": 0.12560302019119263, + "median": 0.12554584443569183, + "p05": 0.12500521540641785, + "p95": 0.12630382180213928 + }, + { + "label": "completed_group_5", + "mean": 0.12461308389902115, + "median": 0.12459056079387665, + "p05": 0.12410678714513779, + "p95": 0.12512892484664917 + }, + { + "label": "completed_group_6", + "mean": 0.12591606378555298, + "median": 0.12592178583145142, + "p05": 0.12557773292064667, + "p95": 0.12638571858406067 + }, + { + "label": "current_group_partial", + "mean": 0.12521177530288696, + "median": 0.12524773180484772, + "p05": 0.12447575479745865, + "p95": 0.1259688436985016 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0725789070129395, + "group": 7, + "latest_source_mass": 0.10425455868244171, + "layer": 28, + "max_source_mass": 0.14827941358089447, + "mixer_index": 55, + "normalized_entropy": 0.9966997703328787, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13186156749725342, + "median": 0.13315711915493011, + "p05": 0.11918869614601135, + "p95": 0.14256390929222107 + }, + { + "label": "completed_group_1", + "mean": 0.14827941358089447, + "median": 0.14931634068489075, + "p05": 0.13603198528289795, + "p95": 0.15790212154388428 + }, + { + "label": "completed_group_2", + "mean": 0.13866659998893738, + "median": 0.13880926370620728, + "p05": 0.1348804384469986, + "p95": 0.14264559745788574 + }, + { + "label": "completed_group_3", + "mean": 0.12988534569740295, + "median": 0.12922212481498718, + "p05": 0.12565752863883972, + "p95": 0.13522912561893463 + }, + { + "label": "completed_group_4", + "mean": 0.12183371186256409, + "median": 0.12081405520439148, + "p05": 0.1163119301199913, + "p95": 0.12904007732868195 + }, + { + "label": "completed_group_5", + "mean": 0.11524046957492828, + "median": 0.1147729679942131, + "p05": 0.11042807996273041, + "p95": 0.12173305451869965 + }, + { + "label": "completed_group_6", + "mean": 0.1099783331155777, + "median": 0.10993096977472305, + "p05": 0.10556004196405411, + "p95": 0.11569196730852127 + }, + { + "label": "current_group_partial", + "mean": 0.10425455868244171, + "median": 0.10401678085327148, + "p05": 0.10061369091272354, + "p95": 0.10750719904899597 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0793380737304688, + "group": 8, + "latest_source_mass": 0.12683480978012085, + "layer": 29, + "max_source_mass": 0.12747105956077576, + "mixer_index": 56, + "normalized_entropy": 0.9999502424341857, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12417487800121307, + "median": 0.1240256130695343, + "p05": 0.12294038385152817, + "p95": 0.1256686896085739 + }, + { + "label": "completed_group_1", + "mean": 0.1225123256444931, + "median": 0.12230424582958221, + "p05": 0.12168098986148834, + "p95": 0.12367479503154755 + }, + { + "label": "completed_group_2", + "mean": 0.12277349829673767, + "median": 0.1226874440908432, + "p05": 0.12208077311515808, + "p95": 0.12351377308368683 + }, + { + "label": "completed_group_3", + "mean": 0.1243998259305954, + "median": 0.12447299808263779, + "p05": 0.12367013841867447, + "p95": 0.12505075335502625 + }, + { + "label": "completed_group_4", + "mean": 0.12567630410194397, + "median": 0.12574473023414612, + "p05": 0.12491010874509811, + "p95": 0.1263245940208435 + }, + { + "label": "completed_group_5", + "mean": 0.12615728378295898, + "median": 0.12627410888671875, + "p05": 0.1253039538860321, + "p95": 0.1267564296722412 + }, + { + "label": "completed_group_6", + "mean": 0.12747105956077576, + "median": 0.12754422426223755, + "p05": 0.1268254816532135, + "p95": 0.1280263066291809 + }, + { + "label": "completed_group_7", + "mean": 0.12683480978012085, + "median": 0.12676092982292175, + "p05": 0.1260336935520172, + "p95": 0.12750548124313354 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1906280517578125, + "group": 8, + "latest_source_mass": 0.09856453537940979, + "layer": 29, + "max_source_mass": 0.13291704654693604, + "mixer_index": 57, + "normalized_entropy": 0.9969977918295433, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11854428797960281, + "median": 0.11933885514736176, + "p05": 0.10912582278251648, + "p95": 0.12757083773612976 + }, + { + "label": "completed_group_1", + "mean": 0.13291704654693604, + "median": 0.13307878375053406, + "p05": 0.12339349836111069, + "p95": 0.14226418733596802 + }, + { + "label": "completed_group_2", + "mean": 0.12378417700529099, + "median": 0.1235300600528717, + "p05": 0.12091188877820969, + "p95": 0.1279762238264084 + }, + { + "label": "completed_group_3", + "mean": 0.11745279282331467, + "median": 0.11693152785301208, + "p05": 0.11398620158433914, + "p95": 0.12128151953220367 + }, + { + "label": "completed_group_4", + "mean": 0.11140252649784088, + "median": 0.11079759150743484, + "p05": 0.10681547224521637, + "p95": 0.11677644401788712 + }, + { + "label": "completed_group_5", + "mean": 0.10409101843833923, + "median": 0.1040785163640976, + "p05": 0.09952308982610703, + "p95": 0.10878291726112366 + }, + { + "label": "completed_group_6", + "mean": 0.09838292002677917, + "median": 0.09840121120214462, + "p05": 0.09399936348199844, + "p95": 0.10255295783281326 + }, + { + "label": "completed_group_7", + "mean": 0.09486069530248642, + "median": 0.09467358142137527, + "p05": 0.09144656360149384, + "p95": 0.09743223339319229 + }, + { + "label": "current_group_partial", + "mean": 0.09856453537940979, + "median": 0.09877555072307587, + "p05": 0.09461960196495056, + "p95": 0.10164139419794083 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.197185516357422, + "group": 8, + "latest_source_mass": 0.11096570640802383, + "layer": 30, + "max_source_mass": 0.11249847710132599, + "mixer_index": 58, + "normalized_entropy": 0.999982222582434, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11084984987974167, + "median": 0.11077362298965454, + "p05": 0.108895443379879, + "p95": 0.11311701685190201 + }, + { + "label": "completed_group_1", + "mean": 0.11021924018859863, + "median": 0.11018644273281097, + "p05": 0.10941747575998306, + "p95": 0.11104057729244232 + }, + { + "label": "completed_group_2", + "mean": 0.11018813401460648, + "median": 0.11026144027709961, + "p05": 0.10942888259887695, + "p95": 0.1108226552605629 + }, + { + "label": "completed_group_3", + "mean": 0.11046157777309418, + "median": 0.11042240262031555, + "p05": 0.10973355174064636, + "p95": 0.11123604327440262 + }, + { + "label": "completed_group_4", + "mean": 0.11131839454174042, + "median": 0.11132219433784485, + "p05": 0.1105581521987915, + "p95": 0.11199404299259186 + }, + { + "label": "completed_group_5", + "mean": 0.11167056858539581, + "median": 0.1118055060505867, + "p05": 0.11075222492218018, + "p95": 0.1122528463602066 + }, + { + "label": "completed_group_6", + "mean": 0.11249847710132599, + "median": 0.11257164180278778, + "p05": 0.11187897622585297, + "p95": 0.11291356384754181 + }, + { + "label": "completed_group_7", + "mean": 0.1118280440568924, + "median": 0.11168843507766724, + "p05": 0.11146453768014908, + "p95": 0.11255130171775818 + }, + { + "label": "current_group_partial", + "mean": 0.11096570640802383, + "median": 0.11086957156658173, + "p05": 0.11044884473085403, + "p95": 0.11181740462779999 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1899101734161377, + "group": 8, + "latest_source_mass": 0.09411308169364929, + "layer": 30, + "max_source_mass": 0.13352498412132263, + "mixer_index": 59, + "normalized_entropy": 0.9966710713162742, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1190595030784607, + "median": 0.11995811760425568, + "p05": 0.10872812569141388, + "p95": 0.1278010904788971 + }, + { + "label": "completed_group_1", + "mean": 0.13352498412132263, + "median": 0.13314831256866455, + "p05": 0.12346142530441284, + "p95": 0.14397428929805756 + }, + { + "label": "completed_group_2", + "mean": 0.1239522397518158, + "median": 0.12362635135650635, + "p05": 0.12061451375484467, + "p95": 0.12857040762901306 + }, + { + "label": "completed_group_3", + "mean": 0.11779220402240753, + "median": 0.11750936508178711, + "p05": 0.11429227143526077, + "p95": 0.12138796597719193 + }, + { + "label": "completed_group_4", + "mean": 0.11234382539987564, + "median": 0.11192750930786133, + "p05": 0.1077679693698883, + "p95": 0.11798053979873657 + }, + { + "label": "completed_group_5", + "mean": 0.10513705015182495, + "median": 0.1050291359424591, + "p05": 0.10075211524963379, + "p95": 0.11021918058395386 + }, + { + "label": "completed_group_6", + "mean": 0.09944089502096176, + "median": 0.09912091493606567, + "p05": 0.09499282389879227, + "p95": 0.10404862463474274 + }, + { + "label": "completed_group_7", + "mean": 0.09463620185852051, + "median": 0.09429345279932022, + "p05": 0.090972401201725, + "p95": 0.0975509062409401 + }, + { + "label": "current_group_partial", + "mean": 0.09411308169364929, + "median": 0.09450676292181015, + "p05": 0.09012747555971146, + "p95": 0.0978383868932724 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.197190284729004, + "group": 8, + "latest_source_mass": 0.11225657165050507, + "layer": 31, + "max_source_mass": 0.11242865771055222, + "mixer_index": 60, + "normalized_entropy": 0.9999843927618645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11067462712526321, + "median": 0.11063650250434875, + "p05": 0.10940495133399963, + "p95": 0.11191637068986893 + }, + { + "label": "completed_group_1", + "mean": 0.11021950840950012, + "median": 0.11016013473272324, + "p05": 0.10925821959972382, + "p95": 0.11120235174894333 + }, + { + "label": "completed_group_2", + "mean": 0.11038416624069214, + "median": 0.1102733239531517, + "p05": 0.10960154980421066, + "p95": 0.11125709116458893 + }, + { + "label": "completed_group_3", + "mean": 0.11103622615337372, + "median": 0.11098648607730865, + "p05": 0.11065587401390076, + "p95": 0.11149194091558456 + }, + { + "label": "completed_group_4", + "mean": 0.11016052961349487, + "median": 0.11016929149627686, + "p05": 0.10976231843233109, + "p95": 0.11053308844566345 + }, + { + "label": "completed_group_5", + "mean": 0.11144963651895523, + "median": 0.11151091754436493, + "p05": 0.11074185371398926, + "p95": 0.11193440854549408 + }, + { + "label": "completed_group_6", + "mean": 0.11242865771055222, + "median": 0.11247988045215607, + "p05": 0.11164618283510208, + "p95": 0.11297892034053802 + }, + { + "label": "completed_group_7", + "mean": 0.11139006912708282, + "median": 0.11143708229064941, + "p05": 0.11081651598215103, + "p95": 0.1119559034705162 + }, + { + "label": "current_group_partial", + "mean": 0.11225657165050507, + "median": 0.11227837204933167, + "p05": 0.1117941215634346, + "p95": 0.11280770599842072 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1896841526031494, + "group": 8, + "latest_source_mass": 0.09325435757637024, + "layer": 31, + "max_source_mass": 0.13428261876106262, + "mixer_index": 61, + "normalized_entropy": 0.9965682048112662, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11968795955181122, + "median": 0.1202392578125, + "p05": 0.11006385833024979, + "p95": 0.12905627489089966 + }, + { + "label": "completed_group_1", + "mean": 0.13428261876106262, + "median": 0.13435032963752747, + "p05": 0.12363448739051819, + "p95": 0.14427264034748077 + }, + { + "label": "completed_group_2", + "mean": 0.12373493611812592, + "median": 0.12346962094306946, + "p05": 0.12061072885990143, + "p95": 0.1282179057598114 + }, + { + "label": "completed_group_3", + "mean": 0.1179073229432106, + "median": 0.11775766313076019, + "p05": 0.11464057117700577, + "p95": 0.12117566168308258 + }, + { + "label": "completed_group_4", + "mean": 0.11132999509572983, + "median": 0.11077313870191574, + "p05": 0.10667262226343155, + "p95": 0.1171092763543129 + }, + { + "label": "completed_group_5", + "mean": 0.10504785180091858, + "median": 0.1049019992351532, + "p05": 0.10071787983179092, + "p95": 0.10987957566976547 + }, + { + "label": "completed_group_6", + "mean": 0.09994535893201828, + "median": 0.09992853552103043, + "p05": 0.09573008865118027, + "p95": 0.1044701635837555 + }, + { + "label": "completed_group_7", + "mean": 0.09480959922075272, + "median": 0.09453058242797852, + "p05": 0.09107209742069244, + "p95": 0.09781570732593536 + }, + { + "label": "current_group_partial", + "mean": 0.09325435757637024, + "median": 0.09382589161396027, + "p05": 0.08936256170272827, + "p95": 0.09653929620981216 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1971938610076904, + "group": 8, + "latest_source_mass": 0.10965746641159058, + "layer": 32, + "max_source_mass": 0.11217275261878967, + "mixer_index": 62, + "normalized_entropy": 0.9999860203964374, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11116788536310196, + "median": 0.11119179427623749, + "p05": 0.11027558147907257, + "p95": 0.11194087564945221 + }, + { + "label": "completed_group_1", + "mean": 0.1117495447397232, + "median": 0.11171998083591461, + "p05": 0.11123906075954437, + "p95": 0.1124686524271965 + }, + { + "label": "completed_group_2", + "mean": 0.11143069714307785, + "median": 0.11137667298316956, + "p05": 0.11108753085136414, + "p95": 0.1119472086429596 + }, + { + "label": "completed_group_3", + "mean": 0.11117202788591385, + "median": 0.11118572950363159, + "p05": 0.11080538481473923, + "p95": 0.11158231645822525 + }, + { + "label": "completed_group_4", + "mean": 0.11217275261878967, + "median": 0.11215600371360779, + "p05": 0.1119328960776329, + "p95": 0.11246690899133682 + }, + { + "label": "completed_group_5", + "mean": 0.11135637760162354, + "median": 0.11132287979125977, + "p05": 0.11110222339630127, + "p95": 0.1116970106959343 + }, + { + "label": "completed_group_6", + "mean": 0.11160033196210861, + "median": 0.11164811253547668, + "p05": 0.11120022088289261, + "p95": 0.111964151263237 + }, + { + "label": "completed_group_7", + "mean": 0.10969290882349014, + "median": 0.10973770171403885, + "p05": 0.10930771380662918, + "p95": 0.10996787250041962 + }, + { + "label": "current_group_partial", + "mean": 0.10965746641159058, + "median": 0.10969755053520203, + "p05": 0.10922328382730484, + "p95": 0.1099214255809784 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.188610792160034, + "group": 8, + "latest_source_mass": 0.09157620370388031, + "layer": 32, + "max_source_mass": 0.1360917091369629, + "mixer_index": 63, + "normalized_entropy": 0.9960796974214496, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11987313628196716, + "median": 0.12083137035369873, + "p05": 0.10883265733718872, + "p95": 0.12953230738639832 + }, + { + "label": "completed_group_1", + "mean": 0.1360917091369629, + "median": 0.13644108176231384, + "p05": 0.12369051575660706, + "p95": 0.14713600277900696 + }, + { + "label": "completed_group_2", + "mean": 0.12462760508060455, + "median": 0.12453563511371613, + "p05": 0.12107435613870621, + "p95": 0.12926824390888214 + }, + { + "label": "completed_group_3", + "mean": 0.11755925416946411, + "median": 0.11717041581869125, + "p05": 0.11395573616027832, + "p95": 0.12139421701431274 + }, + { + "label": "completed_group_4", + "mean": 0.11237476021051407, + "median": 0.11172428727149963, + "p05": 0.10730767995119095, + "p95": 0.11879656463861465 + }, + { + "label": "completed_group_5", + "mean": 0.10438287258148193, + "median": 0.10406170785427094, + "p05": 0.09989674389362335, + "p95": 0.10985209792852402 + }, + { + "label": "completed_group_6", + "mean": 0.09920359402894974, + "median": 0.09911085665225983, + "p05": 0.0946069061756134, + "p95": 0.10452912747859955 + }, + { + "label": "completed_group_7", + "mean": 0.09431086480617523, + "median": 0.09389802813529968, + "p05": 0.09041668474674225, + "p95": 0.09788760542869568 + }, + { + "label": "current_group_partial", + "mean": 0.09157620370388031, + "median": 0.09197241812944412, + "p05": 0.0875491350889206, + "p95": 0.09489993005990982 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1930694580078125, + "group": null, + "latest_source_mass": 0.11946354806423187, + "layer": null, + "max_source_mass": 0.12094368040561676, + "mixer_index": null, + "normalized_entropy": 0.9981089236979843, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10071488469839096, + "median": 0.09996568411588669, + "p05": 0.09106086939573288, + "p95": 0.1130409762263298 + }, + { + "label": "completed_group_1", + "mean": 0.09411877393722534, + "median": 0.09266924858093262, + "p05": 0.08850140869617462, + "p95": 0.10204548388719559 + }, + { + "label": "completed_group_2", + "mean": 0.10181896388530731, + "median": 0.10170605033636093, + "p05": 0.09935513883829117, + "p95": 0.10529708862304688 + }, + { + "label": "completed_group_3", + "mean": 0.10989432036876678, + "median": 0.11005622148513794, + "p05": 0.1059446707367897, + "p95": 0.11359670758247375 + }, + { + "label": "completed_group_4", + "mean": 0.1145806610584259, + "median": 0.11529573798179626, + "p05": 0.10914603620767593, + "p95": 0.11935751885175705 + }, + { + "label": "completed_group_5", + "mean": 0.11861918121576309, + "median": 0.11936421692371368, + "p05": 0.11333125829696655, + "p95": 0.12279194593429565 + }, + { + "label": "completed_group_6", + "mean": 0.11984598636627197, + "median": 0.12033014744520187, + "p05": 0.11644290387630463, + "p95": 0.12263244390487671 + }, + { + "label": "completed_group_7", + "mean": 0.12094368040561676, + "median": 0.12095880508422852, + "p05": 0.118732750415802, + "p95": 0.12314030528068542 + }, + { + "label": "completed_group_8", + "mean": 0.11946354806423187, + "median": 0.118886798620224, + "p05": 0.11734164506196976, + "p95": 0.12209592759609222 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", + "optimizer_state_before": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.002919268561527133, + 0.0026904908008873463, + 0.0028405722696334124, + 0.0029144894797354937, + 0.0030634324066340923, + 0.0032732728868722916, + 0.003848863998427987, + 0.003392934100702405, + 0.0037359180860221386, + 0.004302032291889191, + 0.004344160668551922, + 0.004835603758692741, + 0.005531937349587679, + 0.005483300890773535, + 0.005397035740315914, + 0.005739198997616768, + 0.005659298039972782, + 0.007230010814964771, + 0.006938732694834471, + 0.007343437522649765, + 0.007882392033934593, + 0.00830139685422182, + 0.008992243558168411, + 0.009393857792019844, + 0.012831038795411587, + 0.00905293133109808, + 0.012243949808180332, + 0.012258118949830532, + 0.013431218452751637, + 0.01483316533267498, + 0.015388047322630882, + 0.014634119346737862 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.570841959619429e-05, + "first_to_last_ratio": 3.472594525431938, + "imbalance_abs_log_ratio": 1.2449020164225795, + "last_quartile_mean": 4.523539814726973e-06, + "mean": 8.683279858701098e-06, + "normalized": [ + 2.229894338065868, + 2.1043034837063286, + 1.976946278509285, + 1.8612974320657731, + 1.7187480804487953, + 1.6201790212455152, + 1.5254325865776683, + 1.4355362848487128, + 1.3017373037261373, + 1.2100830304331005, + 1.133220479031311, + 1.0535486714736453, + 0.9690659040803394, + 0.9036010498699347, + 0.8438110062145913, + 0.7926660684312774, + 0.749751881449241, + 0.7082035160351038, + 0.6752208628773162, + 0.6475764877057508, + 0.6244377002240272, + 0.6007972561165787, + 0.5813749906205735, + 0.5649799471303525, + 0.5550363681005585, + 0.5437815898914894, + 0.5335487688172357, + 0.5248093961192007, + 0.5123845047471113, + 0.5051729354463279, + 0.4992577444849533, + 0.49359503150589623 + ], + "peak_layer": 1, + "peak_normalized": 2.229894338065868, + "population_cv": 0.5276859177486317, + "rest_mean": 9.350082985259144e-06, + "spike_contrast": 0.543582659458801, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.082542975287651e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.936279659275897e-05, + 1.8272256056661718e-05, + 1.7166377801913768e-05, + 1.6162166502908804e-05, + 1.49243705891422e-05, + 1.4068467862671241e-05, + 1.3245758054836188e-05, + 1.2465163308661431e-05, + 1.1303349310765043e-05, + 1.050748960551573e-05, + 9.840070561040193e-06, + 9.148257959168404e-06, + 8.414670446654782e-06, + 7.846220796636771e-06, + 7.3270471148134675e-06, + 6.8829413066850975e-06, + 6.510305411211448e-06, + 6.149529326648917e-06, + 5.863131718797376e-06, + 5.623087872663746e-06, + 5.42216730536893e-06, + 5.216890713199973e-06, + 5.048241746408166e-06, + 4.905878995487001e-06, + 4.819536115974188e-06, + 4.7218077270372305e-06, + 4.632953277905472e-06, + 4.557066858978942e-06, + 4.4491780499811284e-06, + 4.38655797552201e-06, + 4.335194716986734e-06, + 4.286023795430083e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.106548153184121e-05, + "first_to_last_ratio": 3.396664379322294, + "imbalance_abs_log_ratio": 1.2227938851577476, + "last_quartile_mean": 1.7978073401536676e-05, + "mean": 3.421872185072061e-05, + "normalized": [ + 2.125690241222297, + 2.035753111677548, + 1.9424653444151043, + 1.8443604241196392, + 1.713002582679694, + 1.6251047960941019, + 1.5375111780105508, + 1.4526202049778152, + 1.3158411261187606, + 1.226794175835664, + 1.149073116741286, + 1.0695301297827082, + 0.9835511244361359, + 0.9163244633971447, + 0.8552224227935168, + 0.8029396208596036, + 0.7578907347648619, + 0.7156154592837152, + 0.6817576032295484, + 0.6538669675872317, + 0.6299332335154186, + 0.605919603383038, + 0.5863501185278558, + 0.569785799336057, + 0.5595323652477712, + 0.5483698809054829, + 0.5380511691309848, + 0.5293393622233508, + 0.5166837305455455, + 0.5094821346031305, + 0.5036034749610113, + 0.498034299593426 + ], + "peak_layer": 1, + "peak_normalized": 2.125690241222297, + "population_cv": 0.5121606369233174, + "rest_mean": 3.681488218447797e-05, + "spike_contrast": 0.5486763735168884, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0199456048430876e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.273840310517699e-05, + 6.966086948523298e-05, + 6.646868132520467e-05, + 6.311165634542704e-05, + 5.8616758906282485e-05, + 5.560900899581611e-05, + 5.261166734271683e-05, + 4.970680674887262e-05, + 4.502640149439685e-05, + 4.197932867100462e-05, + 3.931981336791068e-05, + 3.659795402199961e-05, + 3.365586235304363e-05, + 3.1355451937997714e-05, + 2.9264618206070736e-05, + 2.747556754911784e-05, + 2.593405224615708e-05, + 2.448744635330513e-05, + 2.3328873794525862e-05, + 2.2374491891241632e-05, + 2.1555510102189146e-05, + 2.073379437206313e-05, + 2.006415161304176e-05, + 1.9497341781971045e-05, + 1.9146482372889295e-05, + 1.8764516426017508e-05, + 1.8411423297948204e-05, + 1.8113316400558688e-05, + 1.76802568603307e-05, + 1.7433827451895922e-05, + 1.723266723274719e-05, + 1.70420971699059e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.054153479635715485, + "first_to_last_ratio": 3.3966644557946295, + "imbalance_abs_log_ratio": 1.2227939076716985, + "last_quartile_mean": 0.01594313490204513, + "mean": 0.030345504521392286, + "normalized": [ + 2.1256903113360446, + 2.035753146214189, + 1.9424653011478115, + 1.8443605007189683, + 1.71300252819031, + 1.6251048166450035, + 1.5375111878366567, + 1.452620175683203, + 1.3158411334809852, + 1.2267941193182312, + 1.1490731743165223, + 1.0695301881432524, + 0.9835511318647784, + 0.9163244786384582, + 0.8552223856485347, + 0.8029396127492687, + 0.7578907435068827, + 0.7156154476500971, + 0.6817576088078939, + 0.6538669486644281, + 0.6299332311205004, + 0.6059195951826442, + 0.586350084864764, + 0.569785800788618, + 0.5595323690576035, + 0.5483698809409665, + 0.5380511395546987, + 0.5293393594226915, + 0.5166837108984478, + 0.5094821244722924, + 0.5036034882338023, + 0.4980342749014515 + ], + "peak_layer": 1, + "peak_normalized": 2.1256903113360446, + "population_cv": 0.5121606472523333, + "rest_mean": 0.0326478054953946, + "spike_contrast": 0.5486763655311122, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017913079261779784, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06450514495372772, + 0.06177595630288124, + 0.05894508957862854, + 0.05596804991364479, + 0.05198192596435547, + 0.049314625561237335, + 0.04665655270218849, + 0.04408049210906029, + 0.039929863065481186, + 0.03722768649458885, + 0.034869205206632614, + 0.032455433160066605, + 0.029846355319023132, + 0.027806328609585762, + 0.025952154770493507, + 0.024365607649087906, + 0.02299857698380947, + 0.021715711802244186, + 0.02068827860057354, + 0.01984192244708538, + 0.019115641713142395, + 0.01838693581521511, + 0.017793089151382446, + 0.01729043759405613, + 0.016979292035102844, + 0.01664056070148945, + 0.01632743328809738, + 0.01606306992471218, + 0.015679027885198593, + 0.015460492111742496, + 0.015282101929187775, + 0.015113101340830326 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.484954840540013e-06, + "first_to_last_ratio": 4.317118296735341, + "imbalance_abs_log_ratio": 1.462588118730986, + "last_quartile_mean": 1.2705129819323702e-06, + "mean": 2.775344690064685e-06, + "normalized": [ + 2.642311887415898, + 2.416230245995366, + 2.21143682159549, + 2.0333492354809493, + 1.8430856012839139, + 1.6939904160227164, + 1.5446558421616008, + 1.4254585070372157, + 1.2827451334693212, + 1.1755524566426843, + 1.0923958043799127, + 1.0016640021807137, + 0.9226630606403423, + 0.8576190807297013, + 0.7936412075441582, + 0.7370849544400642, + 0.6937134258189417, + 0.6483319177623661, + 0.6136592809430983, + 0.5846488972705621, + 0.5617626261670844, + 0.5391223387293338, + 0.5197976899046192, + 0.5027943403920482, + 0.49023336166728243, + 0.4787835081603097, + 0.4696409445710295, + 0.46085508801940983, + 0.450168541694312, + 0.44355065941556593, + 0.43718887869853934, + 0.4318642437654502 + ], + "peak_layer": 1, + "peak_normalized": 2.642311887415898, + "population_cv": 0.6359571845984164, + "rest_mean": 3.020632700807053e-06, + "spike_contrast": 0.48029322852403633, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.450789432055899e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.333326266234508e-06, + 6.705871783196926e-06, + 6.1374994402285665e-06, + 5.643245003739139e-06, + 5.1151978368579876e-06, + 4.701407306129113e-06, + 4.286952389520593e-06, + 3.95613869841327e-06, + 3.5600598948803963e-06, + 3.2625632684357697e-06, + 3.031774895134731e-06, + 2.779962869681185e-06, + 2.5607080260670045e-06, + 2.380188561801333e-06, + 2.202627911174204e-06, + 2.0456648144318024e-06, + 1.9252938727731816e-06, + 1.7993445453612367e-06, + 1.7031160268743406e-06, + 1.622602212592028e-06, + 1.5590849216096103e-06, + 1.4962503200877109e-06, + 1.4426177585846744e-06, + 1.3954276028016466e-06, + 1.3605665571958525e-06, + 1.3287892670632573e-06, + 1.3034155017521698e-06, + 1.2790317214239622e-06, + 1.2493728718254715e-06, + 1.2310059673836804e-06, + 1.2133498330513248e-06, + 1.1985721357632428e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.000926375942072e-05, + "first_to_last_ratio": 2.861076547442411, + "imbalance_abs_log_ratio": 1.0511979691949909, + "last_quartile_mean": 1.7479177131463075e-05, + "mean": 3.001758557275025e-05, + "normalized": [ + 1.966517821002492, + 1.8844494706669952, + 1.8016591622079403, + 1.7146543004927615, + 1.602713342111759, + 1.525842314618408, + 1.4518503935852287, + 1.3803041859872354, + 1.2718526716709437, + 1.1969100977048808, + 1.1317671023781757, + 1.06503487809827, + 0.9973441232158428, + 0.941363281859895, + 0.8896494105618908, + 0.8445592656875696, + 0.8090600018046681, + 0.7713325693634966, + 0.7401533589124044, + 0.7139326567185207, + 0.692297727813069, + 0.6685034690624881, + 0.6485427267474059, + 0.6313224439068734, + 0.6206897714046083, + 0.6084823527417841, + 0.5970360381608709, + 0.5872159169829965, + 0.572597446631392, + 0.564250570870669, + 0.5573663965692749, + 0.5507447304591896 + ], + "peak_layer": 1, + "peak_normalized": 1.966517821002492, + "population_cv": 0.43784911874358545, + "rest_mean": 3.1950544494234406e-05, + "spike_contrast": 0.6128098192589182, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.957960739673581e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.9030116972280666e-05, + 5.656662324327044e-05, + 5.408145807450637e-05, + 5.146978219272569e-05, + 4.810958489542827e-05, + 4.580210224958137e-05, + 4.3581043428275734e-05, + 4.143339901929721e-05, + 3.817794640781358e-05, + 3.592835128074512e-05, + 3.397291584406048e-05, + 3.196977559127845e-05, + 2.993786256411113e-05, + 2.8257452868274413e-05, + 2.670512731128838e-05, + 2.5351630029035732e-05, + 2.4286027837661095e-05, + 2.3153541405918077e-05, + 2.2217616788111627e-05, + 2.1430534616229124e-05, + 2.0781106286449358e-05, + 2.006686008826364e-05, + 1.9467686797725037e-05, + 1.8950775483972393e-05, + 1.8631608327268623e-05, + 1.8265171092934906e-05, + 1.7921580365509726e-05, + 1.7626804037718102e-05, + 1.7187992852996103e-05, + 1.6937439795583487e-05, + 1.673079350439366e-05, + 1.6532027075299993e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.89006590921781e-05, + "first_to_last_ratio": 2.1665975586085744, + "imbalance_abs_log_ratio": 0.7731579916979808, + "last_quartile_mean": 1.7954723034563358e-05, + "mean": 2.587040432899812e-05, + "normalized": [ + 1.7831448102947671, + 1.6974226005892297, + 1.620036533641332, + 1.533717800359919, + 1.4465980773830347, + 1.3745334732258696, + 1.316859965180907, + 1.257080202982369, + 1.1876570648434144, + 1.1203272435244702, + 1.0718495078032313, + 1.0193313865605265, + 0.978296344153428, + 0.937610047259807, + 0.9066038289594328, + 0.877320807240572, + 0.8649781691504772, + 0.8401000985755385, + 0.8145599740177561, + 0.792555397726043, + 0.7795027531450014, + 0.7587311609912605, + 0.7416782788097599, + 0.7272996278499536, + 0.7222997702968516, + 0.7135070943498951, + 0.7058159198574021, + 0.6988176661381593, + 0.6867067750666979, + 0.6814848031835403, + 0.675082580859005, + 0.668490235980348 + ], + "peak_layer": 1, + "peak_normalized": 1.7831448102947671, + "population_cv": 0.3297379118959793, + "rest_mean": 2.708773947098827e-05, + "spike_contrast": 0.7123811340152144, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9296794562251307e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.6130677219480276e-05, + 4.391300899442285e-05, + 4.191100015304983e-05, + 3.967789962189272e-05, + 3.742407716345042e-05, + 3.555973671609536e-05, + 3.406769974390045e-05, + 3.252117312513292e-05, + 3.072516847169027e-05, + 2.8983318770769984e-05, + 2.7729180146707222e-05, + 2.63705151155591e-05, + 2.530892197682988e-05, + 2.4256351025542244e-05, + 2.3454207621398382e-05, + 2.269664400955662e-05, + 2.2377334971679375e-05, + 2.173372922698036e-05, + 2.1072995878057554e-05, + 2.050372859230265e-05, + 2.0166051399428397e-05, + 1.9628681911854073e-05, + 1.9187516954843886e-05, + 1.8815535440808162e-05, + 1.8686187104322016e-05, + 1.8458717022440396e-05, + 1.8259743228554726e-05, + 1.8078695575241e-05, + 1.776538192643784e-05, + 1.7630287402425893e-05, + 1.7464659322286025e-05, + 1.729411269479897e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011786452796513913, + "first_to_last_ratio": 5.272419649365271, + "imbalance_abs_log_ratio": 1.6624893936843081, + "last_quartile_mean": 2.2354921611622558e-05, + "mean": 5.981176059322024e-05, + "normalized": [ + 2.342046121512161, + 2.2441163908348334, + 2.142933559820905, + 2.032574241487723, + 1.8909432158278896, + 1.7811012647720825, + 1.7046454997674345, + 1.6263691566595195, + 1.4513535263674326, + 1.3484633711734058, + 1.2565665103260477, + 1.1592512217173825, + 1.0440274588303153, + 0.9501757209203151, + 0.8639054268917737, + 0.7923577171085253, + 0.7123005746652016, + 0.6458221031841243, + 0.5963034371094506, + 0.5569297557861542, + 0.5166112374905155, + 0.4756772431215453, + 0.44740588281370386, + 0.42808243396944073, + 0.41207290926584317, + 0.3977052623291704, + 0.38540407372369656, + 0.3748332006962577, + 0.3630559483389979, + 0.3565727390637327, + 0.35157439090519904, + 0.34881840351921967 + ], + "peak_layer": 1, + "peak_normalized": 2.342046121512161, + "population_cv": 0.6458102290367006, + "rest_mean": 6.583757459776718e-05, + "spike_contrast": 0.41423708475420845, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.7272364968666805e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014008190191816539, + 0.00013422455231193453, + 0.00012817262904718518, + 0.00012157184391980991, + 0.00011310064292047173, + 0.00010653080244082958, + 0.00010195784852840006, + 9.727600263431668e-05, + 8.680800965521485e-05, + 8.065396832535043e-05, + 7.515745528507978e-05, + 6.933685654075816e-05, + 6.244512042030692e-05, + 5.683168274117634e-05, + 5.167170456843451e-05, + 4.739231007988565e-05, + 4.2603951442288235e-05, + 3.862775702145882e-05, + 3.566595842130482e-05, + 3.331094922032207e-05, + 3.089942765654996e-05, + 2.8451093385228887e-05, + 2.6760133550851606e-05, + 2.56043640547432e-05, + 2.464680619596038e-05, + 2.3787451937096193e-05, + 2.3051696189213544e-05, + 2.241943366243504e-05, + 2.171501546399668e-05, + 2.1327243302948773e-05, + 2.1028283299528994e-05, + 2.086344284180086e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.005079356953501701, + 0.0052208867855370045, + 0.005689430516213179, + 0.005868426524102688, + 0.00590060418471694, + 0.0059102908708155155, + 0.0058776624500751495, + 0.006663382984697819, + 0.006277414970099926, + 0.006413500756025314, + 0.006856496445834637, + 0.006499431096017361, + 0.006718362681567669, + 0.0070298356004059315, + 0.006969383917748928, + 0.0068367342464625835, + 0.007033169735223055, + 0.007157569285482168, + 0.0075136395171284676, + 0.007708695251494646, + 0.007887008599936962, + 0.008205465972423553, + 0.00814912747591734, + 0.008287571370601654, + 0.008026733994483948, + 0.008120592683553696, + 0.008479002863168716, + 0.00907627958804369, + 0.00841403380036354, + 0.008773921057581902, + 0.008821850642561913, + 0.009164483286440372 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5008465311439068e-05, + "first_to_last_ratio": 3.3668968034019833, + "imbalance_abs_log_ratio": 1.213991490288974, + "last_quartile_mean": 4.457655279566097e-06, + "mean": 8.299250268350988e-06, + "normalized": [ + 2.2568151540450523, + 2.1152935419859507, + 1.9767562187382288, + 1.86698957878165, + 1.7143188557206628, + 1.6128346254213421, + 1.5135326124803135, + 1.410756892623042, + 1.2736883716277778, + 1.1902867485409472, + 1.1070374521817832, + 1.0322313304507316, + 0.9480382659054776, + 0.8848529200942086, + 0.8311995014573598, + 0.7835740569083032, + 0.742040914352895, + 0.7071972712748903, + 0.6782348645232779, + 0.6514608668278806, + 0.6287747596064291, + 0.6086256047198574, + 0.5913261103101772, + 0.5772101349211559, + 0.569144818247716, + 0.5582752103529628, + 0.549190400479895, + 0.5409965375823161, + 0.5285413070813697, + 0.5224600186388942, + 0.5164682084504763, + 0.5118468456669756 + ], + "peak_layer": 1, + "peak_normalized": 2.2568151540450523, + "population_cv": 0.524209711679407, + "rest_mean": 8.921669009227858e-06, + "spike_contrast": 0.5535050742757021, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.9381890676158944e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8729873772826977e-05, + 1.7555350495968014e-05, + 1.640559457882773e-05, + 1.5494613762712106e-05, + 1.422756122337887e-05, + 1.338531819783384e-05, + 1.2561185940285213e-05, + 1.1708224519679789e-05, + 1.0570658560027368e-05, + 9.878487617243081e-06, + 9.187580872094259e-06, + 8.566746146243531e-06, + 7.86800683272304e-06, + 7.343615834543016e-06, + 6.898332685523201e-06, + 6.5030772020691074e-06, + 6.1583832575706765e-06, + 5.8692071434052195e-06, + 5.62884088139981e-06, + 5.406636773841456e-06, + 5.2183590923959855e-06, + 5.051136213296559e-06, + 4.907563379674684e-06, + 4.790411367139313e-06, + 4.723475285572931e-06, + 4.633265689335531e-06, + 4.557868578558555e-06, + 4.489865659706993e-06, + 4.38649658462964e-06, + 4.336026449891506e-06, + 4.286298917577369e-06, + 4.247945071256254e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.820503929498955e-05, + "first_to_last_ratio": 3.2848952430521337, + "imbalance_abs_log_ratio": 1.1893347619462469, + "last_quartile_mean": 1.7718994058668613e-05, + "mean": 3.2650613661644456e-05, + "normalized": [ + 2.1450013507239705, + 2.0461366500203595, + 1.9398189683428497, + 1.8522773613526717, + 1.7094590145500226, + 1.6168974254280155, + 1.5255888819635426, + 1.4261239538940118, + 1.2879114356833505, + 1.2060871976977126, + 1.1221510774968064, + 1.0481371490840103, + 0.9613461884209538, + 0.8968067603991513, + 0.8421359861574166, + 0.7933538356930859, + 0.7503585255090798, + 0.7148355279210235, + 0.6855522988186872, + 0.6585248527378349, + 0.6351822812313331, + 0.6146906504361358, + 0.5972743074473386, + 0.5828693520617939, + 0.574802048908203, + 0.5639397943233103, + 0.554845965325609, + 0.5465864044553121, + 0.5340118249846835, + 0.5278506086231465, + 0.5219704528761507, + 0.5174718674324269 + ], + "peak_layer": 1, + "peak_normalized": 2.1450013507239705, + "population_cv": 0.5072019698199706, + "rest_mean": 3.5063350541922434e-05, + "spike_contrast": 0.5596109386261622, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9621834508143365e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.003561040619388e-05, + 6.680761725874618e-05, + 6.333627970889211e-05, + 6.047799251973629e-05, + 5.581488585448824e-05, + 5.279269316815771e-05, + 4.981141319149174e-05, + 4.656382225221023e-05, + 4.205109871691093e-05, + 3.937948713428341e-05, + 3.663892130134627e-05, + 3.422232111915946e-05, + 3.138854299322702e-05, + 2.9281291062943637e-05, + 2.749625673459377e-05, + 2.5903489586198702e-05, + 2.4499666324118152e-05, + 2.3339818653766997e-05, + 2.238370325358119e-05, + 2.1501240553334355e-05, + 2.0739091269206256e-05, + 2.007002694881521e-05, + 1.9501372662489302e-05, + 1.903104202938266e-05, + 1.8767639630823396e-05, + 1.841298035287764e-05, + 1.811606125556864e-05, + 1.7846381524577737e-05, + 1.7435813788324594e-05, + 1.7234646293218248e-05, + 1.704265559965279e-05, + 1.6895774024305865e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05161681119352579, + "first_to_last_ratio": 3.284895287464527, + "imbalance_abs_log_ratio": 1.1893347754664323, + "last_quartile_mean": 0.015713380998931825, + "mean": 0.028954890876775607, + "normalized": [ + 2.145001289725201, + 2.046136691673054, + 1.939818911776489, + 1.8522774119626542, + 1.7094589830823987, + 1.6168973606656405, + 1.5255889998464391, + 1.4261239529824867, + 1.287911366616444, + 1.2060871893436458, + 1.1221510842273104, + 1.0481372445524282, + 0.961346209259541, + 0.8968067750917011, + 0.8421359558072424, + 0.7933538963986676, + 0.7503585064155442, + 0.7148355327095294, + 0.6855522990290889, + 0.6585248332975209, + 0.6351822731624658, + 0.6146906547931704, + 0.5972742941320782, + 0.5828693766065183, + 0.5748020438293477, + 0.5639398000251236, + 0.5548459661026797, + 0.5465864176570202, + 0.534011826874325, + 0.527850568480463, + 0.5219704287182262, + 0.5174718551555549 + ], + "peak_layer": 1, + "peak_normalized": 2.145001289725201, + "population_cv": 0.5072019680935582, + "rest_mean": 0.03109453007992771, + "spike_contrast": 0.5596109391274233, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017400839179754258, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06210827827453613, + 0.05924566462635994, + 0.05616724491119385, + 0.05363249033689499, + 0.049497198313474655, + 0.04681708663702011, + 0.044173263013362885, + 0.04129326343536377, + 0.037291333079338074, + 0.034922122955322266, + 0.032491762191057205, + 0.030348699539899826, + 0.027835674583911896, + 0.025966942310333252, + 0.02438395470380783, + 0.02297147549688816, + 0.021726548671722412, + 0.020697984844446182, + 0.019850092008709908, + 0.019067514687776566, + 0.01839163340628147, + 0.017798300832509995, + 0.017294012010097504, + 0.01687691919505596, + 0.016643330454826355, + 0.01632881537079811, + 0.016065504401922226, + 0.015826350077986717, + 0.015462254174053669, + 0.015283855609595776, + 0.015113596804440022, + 0.014983341097831726 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.355004418561293e-06, + "first_to_last_ratio": 4.2803271552883535, + "imbalance_abs_log_ratio": 1.4540294448239433, + "last_quartile_mean": 1.251073626917787e-06, + "mean": 2.683941971781678e-06, + "normalized": [ + 2.7030485685097614, + 2.4621902558382964, + 2.2386850638453395, + 2.05248673466923, + 1.8575399210018237, + 1.6957758104696088, + 1.5437340831941542, + 1.4081496862748333, + 1.2505669189373825, + 1.154757255828014, + 1.0675003825206393, + 0.9762526930125442, + 0.897424076523062, + 0.8318154411009656, + 0.7746171026378299, + 0.7211559397261628, + 0.6781980327113627, + 0.6399672113971655, + 0.60846523760761, + 0.5811474026512373, + 0.5581318010721549, + 0.5402081583654614, + 0.5217772343602641, + 0.5073418706694675, + 0.49649585255921586, + 0.4853123246471542, + 0.4772675785905054, + 0.4696153655827239, + 0.458545908189773, + 0.453049428909843, + 0.4466290334907211, + 0.4421476251056929 + ], + "peak_layer": 1, + "peak_normalized": 2.7030485685097614, + "population_cv": 0.6457571562979046, + "rest_mean": 2.9201333467830814e-06, + "spike_contrast": 0.48234391361810997, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4085085467741009e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.254825504787732e-06, + 6.608375770156272e-06, + 6.008500804455252e-06, + 5.508755293703871e-06, + 4.985529358236818e-06, + 4.551363872451475e-06, + 4.143292699154699e-06, + 3.7793920455442276e-06, + 3.3564490422577364e-06, + 3.0993014661362395e-06, + 2.8651090815401403e-06, + 2.6202055778412614e-06, + 2.4086341454676585e-06, + 2.232544375146972e-06, + 2.079027353829588e-06, + 1.9355406948307063e-06, + 1.82024416517379e-06, + 1.7176348592329305e-06, + 1.633085389585176e-06, + 1.5597659057675628e-06, + 1.4979933666836587e-06, + 1.4498873497359455e-06, + 1.4004198192196782e-06, + 1.3616761407320155e-06, + 1.332566057499207e-06, + 1.3025501175434329e-06, + 1.280958485949668e-06, + 1.2604203902810696e-06, + 1.2307106089792796e-06, + 1.2159583775428473e-06, + 1.198726408802031e-06, + 1.1866985687447595e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.78605675198196e-05, + "first_to_last_ratio": 2.7793570801926766, + "imbalance_abs_log_ratio": 1.0222196348386712, + "last_quartile_mean": 1.7220013887708774e-05, + "mean": 2.881263026210945e-05, + "normalized": [ + 1.9752570579468534, + 1.888104295535499, + 1.7945647548145947, + 1.7154982752112686, + 1.5967511433618535, + 1.5186186336526764, + 1.441625524284827, + 1.3583546018661343, + 1.2494560102010674, + 1.1811084039550475, + 1.1110423537101946, + 1.0496234480105349, + 0.9817226877095458, + 0.9275068527847158, + 0.8805127756083984, + 0.8381970075463177, + 0.803976865779107, + 0.7714567335637176, + 0.7440882717080923, + 0.7183088339322109, + 0.696644002608348, + 0.6758702747761651, + 0.6578620296540808, + 0.6426087873852322, + 0.6340400523715064, + 0.6220565885755293, + 0.611876165213302, + 0.6024355818665901, + 0.5879093007530156, + 0.5807453084805843, + 0.5738005724905904, + 0.5683768046423998 + ], + "peak_layer": 1, + "peak_normalized": 1.9752570579468534, + "population_cv": 0.4317079414337261, + "rest_mean": 3.0619262057977416e-05, + "spike_contrast": 0.622380073312624, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.905681856442243e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.691235128324479e-05, + 5.440125096356496e-05, + 5.1706130761886016e-05, + 4.9428017518948764e-05, + 4.6006600314285606e-05, + 4.375539720058441e-05, + 4.153702320763841e-05, + 3.9137768908403814e-05, + 3.600011405069381e-05, + 3.4030839742626995e-05, + 3.201205254299566e-05, + 3.0242412321968004e-05, + 2.8286012820899487e-05, + 2.6723912014858797e-05, + 2.5369889044668525e-05, + 2.4150660465238616e-05, + 2.3164688172983006e-05, + 2.2227697627386078e-05, + 2.14391402550973e-05, + 2.069636684609577e-05, + 2.0072146071470343e-05, + 1.9473600332275964e-05, + 1.8954735423903912e-05, + 1.8515249394113198e-05, + 1.8268361600348726e-05, + 1.7923086488735862e-05, + 1.7629761714488268e-05, + 1.7357753677060828e-05, + 1.6939213310251944e-05, + 1.673279984970577e-05, + 1.6532703739358112e-05, + 1.637643072172068e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.7207858895271784e-05, + "first_to_last_ratio": 2.0918563583707814, + "imbalance_abs_log_ratio": 0.7380518814986469, + "last_quartile_mean": 1.778700470822514e-05, + "mean": 2.5014859602379147e-05, + "normalized": [ + 1.766896017685199, + 1.6852067556838466, + 1.5917675487924012, + 1.5166567899677899, + 1.4315811273281913, + 1.3662079609937838, + 1.303579546183775, + 1.2375462607814267, + 1.160212708426433, + 1.1098915366489155, + 1.0567965268051207, + 1.0112629998208633, + 0.9697402160962814, + 0.9378324250878841, + 0.907941204027948, + 0.8814449641361121, + 0.8690589692914835, + 0.8427704073086626, + 0.8201725639084954, + 0.8014260695185076, + 0.784571291095966, + 0.7669834591017225, + 0.7521317253856684, + 0.7398605496651066, + 0.7380371143547211, + 0.7300178093260107, + 0.722929928084616, + 0.7169054508143122, + 0.7047914878526057, + 0.6980364293666598, + 0.6913059490086015, + 0.6864362074508885 + ], + "peak_layer": 1, + "peak_normalized": 1.766896017685199, + "population_cv": 0.3175074310788083, + "rest_mean": 2.614369299812501e-05, + "spike_contrast": 0.7236605504321191, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.8919159265351483e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.419865581439808e-05, + 4.215521039441228e-05, + 3.981784175266512e-05, + 3.79389566660393e-05, + 3.581080090953037e-05, + 3.417550033191219e-05, + 3.260885932832025e-05, + 3.095704596489668e-05, + 2.9022558010183275e-05, + 2.776378096314147e-05, + 2.6435616746312007e-05, + 2.5296601961599663e-05, + 2.4257915356429294e-05, + 2.345974644413218e-05, + 2.2712021745974198e-05, + 2.2049222025088966e-05, + 2.173938810301479e-05, + 2.1081783415866084e-05, + 2.051650153589435e-05, + 2.0047560610692017e-05, + 1.962594069482293e-05, + 1.9185983546776697e-05, + 1.8814469513017684e-05, + 1.8507507775211707e-05, + 1.846189479692839e-05, + 1.8261293007526547e-05, + 1.808399065339472e-05, + 1.793328920030035e-05, + 1.763026011758484e-05, + 1.7461283277953044e-05, + 1.7292921256739646e-05, + 1.7171105355373584e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011250984425714705, + "first_to_last_ratio": 5.136972637567696, + "imbalance_abs_log_ratio": 1.6364639249316602, + "last_quartile_mean": 2.1901974605498253e-05, + "mean": 5.6501408323583746e-05, + "normalized": [ + 2.382164866319327, + 2.279581056980086, + 2.1640865436402454, + 2.0688186882965165, + 1.8923495642236086, + 1.8092596842626791, + 1.7252927191320069, + 1.6086472166206554, + 1.4275667956980447, + 1.3299589734449933, + 1.2272752348601612, + 1.1370747967113455, + 1.007765000781145, + 0.9145523912616853, + 0.8390743254016101, + 0.7615077898186029, + 0.6843792173338624, + 0.6320158419323604, + 0.5900143308948815, + 0.550751795487684, + 0.5037684703042699, + 0.4736161081951874, + 0.4535521893538639, + 0.4358391510862496, + 0.4210578321323425, + 0.40810041507267314, + 0.3968839045479846, + 0.3877734415065673, + 0.37747110329431527, + 0.3721870546903874, + 0.36935648798288095, + 0.3682570087317776 + ], + "peak_layer": 1, + "peak_normalized": 2.382164866319327, + "population_cv": 0.6517494715357345, + "rest_mean": 6.217700879292183e-05, + "spike_contrast": 0.41579944566424976, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5853165789158082e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00013459566980600357, + 0.00012879954010713845, + 0.00012227393744979054, + 0.00011689116945490241, + 0.00010692041541915387, + 0.00010222572018392384, + 9.748146840138361e-05, + 9.089083323488012e-05, + 8.065953443292528e-05, + 7.514455501222983e-05, + 6.934277917025611e-05, + 6.424632738344371e-05, + 5.694014180335216e-05, + 5.16734980919864e-05, + 4.740888107335195e-05, + 4.302626257413067e-05, + 3.866838960675523e-05, + 3.5709785151993856e-05, + 3.3336640626657754e-05, + 3.111825208179653e-05, + 2.8463628041208722e-05, + 2.67599771177629e-05, + 2.5626337446738034e-05, + 2.4625525838928297e-05, + 2.3790360501152463e-05, + 2.3058248189045116e-05, + 2.2424499547923915e-05, + 2.1909745555603877e-05, + 2.1327648937585764e-05, + 2.102909274981357e-05, + 2.086916174448561e-05, + 2.080703961837571e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.002919268561527133, + 0.0065401275642216206, + 0.010618876665830612, + 0.015120964497327805, + 0.0030634324066340923, + 0.007598300464451313, + 0.012669830583035946, + 0.017725100740790367, + 0.0037359180860221386, + 0.009108707308769226, + 0.015153576619923115, + 0.021781234070658684, + 0.005531937349587679, + 0.0119093032553792, + 0.01904802769422531, + 0.02626260370016098, + 0.005659298039972782, + 0.013823030516505241, + 0.02223494090139866, + 0.0319429449737072, + 0.007882392033934593, + 0.01784195750951767, + 0.028768405318260193, + 0.040818072855472565, + 0.012831038795411587, + 0.023826664313673973, + 0.03824474662542343, + 0.053081437945365906, + 0.013431218452751637, + 0.03125866502523422, + 0.04969283565878868, + 0.06860727071762085 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.57084181182654e-05, + "first_to_last_ratio": 3.4725850785839505, + "imbalance_abs_log_ratio": 1.2448992960183491, + "last_quartile_mean": 4.523551695001515e-06, + "mean": 8.68330698722275e-06, + "normalized": [ + 2.2299043393712443, + 2.1042761707719917, + 1.976933189224246, + 1.8613268098100455, + 1.7187475287766918, + 1.6201752163443084, + 1.525403311499459, + 1.435524363342176, + 1.3017427682121105, + 1.2100915045221226, + 1.1332349539879822, + 1.0535572156473645, + 0.9690912612097308, + 0.9035997979344527, + 0.843793758654846, + 0.7926704001101033, + 0.7497625268960744, + 0.7082004131597335, + 0.6752277610326668, + 0.6475699083216085, + 0.6244433954092409, + 0.6007951696153412, + 0.5813694559888941, + 0.5649745160906161, + 0.5550358385641657, + 0.5437760679691019, + 0.5335476779691889, + 0.5248102179054668, + 0.5123899215671036, + 0.50517146191784, + 0.49925770343476866, + 0.49359537473931375 + ], + "peak_layer": 1, + "peak_normalized": 2.2299043393712443, + "population_cv": 0.5276847791721199, + "rest_mean": 9.350114733361235e-06, + "spike_contrast": 0.543581047186555, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.082545158074936e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.9362943930900656e-05, + 1.8272075976710767e-05, + 1.716631777526345e-05, + 1.61624720931286e-05, + 1.4924412425898481e-05, + 1.4068478776607662e-05, + 1.3245545233075973e-05, + 1.2465098734537605e-05, + 1.1303432074782904e-05, + 1.0507596016395837e-05, + 9.840226994128898e-06, + 9.148360732069705e-06, + 8.414916919718962e-06, + 7.846234439057298e-06, + 7.32692024030257e-06, + 6.883000423840713e-06, + 6.510418188554468e-06, + 6.149521595943952e-06, + 5.863209935341729e-06, + 5.6230483096442185e-06, + 5.4222336984821595e-06, + 5.2168888942105696e-06, + 5.0482094593462534e-06, + 4.905847163172439e-06, + 4.819546575163258e-06, + 4.721774530480616e-06, + 4.632958280126331e-06, + 4.557088232104434e-06, + 4.449238986126147e-06, + 4.3865588850167114e-06, + 4.33520790465991e-06, + 4.286040166334715e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.106540149630746e-05, + "first_to_last_ratio": 3.3966554168443395, + "imbalance_abs_log_ratio": 1.2227912465426896, + "last_quartile_mean": 1.7978097275772598e-05, + "mean": 3.421876306219929e-05, + "normalized": [ + 2.125681514851671, + 2.035741091532698, + 1.9424547124055425, + 1.8443747880483625, + 1.7129915891324252, + 1.6251025199478857, + 1.537504861060443, + 1.452620900762345, + 1.3158482592376062, + 1.2267970572702998, + 1.1490758791488886, + 1.0695328816728442, + 0.9835696082276737, + 0.9163259113865182, + 0.85522266858747, + 0.8029399296205194, + 0.7578942872402048, + 0.7156133216460457, + 0.6817566226795772, + 0.6538643195822892, + 0.6299348137903584, + 0.6059197773220268, + 0.5863493060398601, + 0.5697864420541977, + 0.5595343492561913, + 0.5483692204744319, + 0.5380517969111698, + 0.5293376615585488, + 0.5166827361721865, + 0.509484072574387, + 0.5036056858005552, + 0.49803141400477713 + ], + "peak_layer": 1, + "peak_normalized": 2.125681514851671, + "population_cv": 0.512158847941706, + "rest_mean": 3.6814922000134914e-05, + "spike_contrast": 0.5486771042805106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.019950479734689e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.273819210240617e-05, + 6.966054206714034e-05, + 6.646839756285772e-05, + 6.311222387012094e-05, + 5.861645331606269e-05, + 5.560899808187969e-05, + 5.261151454760693e-05, + 4.9706890422385186e-05, + 4.502669980865903e-05, + 4.1979477828135714e-05, + 3.931995524908416e-05, + 3.659809226519428e-05, + 3.365653537912294e-05, + 3.1355539249489084e-05, + 2.926466186181642e-05, + 2.7475611204863526e-05, + 2.5934205041266978e-05, + 2.4487402697559446e-05, + 2.332886833755765e-05, + 2.2374428226612508e-05, + 2.1555590137722902e-05, + 2.073382529488299e-05, + 2.006414797506295e-05, + 1.9497387256706133e-05, + 1.9146573322359473e-05, + 1.8764516426017508e-05, + 1.841146695369389e-05, + 1.8113280020770617e-05, + 1.7680244127404876e-05, + 1.7433914763387293e-05, + 1.7232763639185578e-05, + 1.704201895336155e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05415340978652239, + "first_to_last_ratio": 3.3966554118601855, + "imbalance_abs_log_ratio": 1.2227912450753184, + "last_quartile_mean": 0.015943156788125634, + "mean": 0.030345541599672288, + "normalized": [ + 2.1256815759027154, + 2.0357410833262275, + 1.9424548253937048, + 1.8443746973080621, + 1.712991596235519, + 1.6251025854549026, + 1.5375048897542698, + 1.4526208560186746, + 1.3158482418228976, + 1.2267970397813441, + 1.1490758214573777, + 1.0695328097128658, + 0.9835695720716091, + 0.9163258756382576, + 0.8552226910646754, + 0.8029399206669349, + 0.7578942982897275, + 0.715613345638086, + 0.6817566530269693, + 0.6538643082892013, + 0.6299347939093796, + 0.6059197755459048, + 0.5863493070405259, + 0.5697864549709054, + 0.5595343861542013, + 0.5483692109047464, + 0.5380518325125886, + 0.5293376691592189, + 0.5166827726726805, + 0.5094840492707883, + 0.5036056657399419, + 0.49803139526509604 + ], + "peak_layer": 1, + "peak_normalized": 2.1256815759027154, + "population_cv": 0.51215885272173, + "rest_mean": 0.03264784129957358, + "spike_contrast": 0.5486771102516746, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01791312322020531, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0645049586892128, + 0.06177566573023796, + 0.05894484370946884, + 0.055968549102544785, + 0.05198165774345398, + 0.04931461811065674, + 0.04665641859173775, + 0.04408056661486626, + 0.03993012756109238, + 0.0372278206050396, + 0.03486932814121246, + 0.032455552369356155, + 0.029846951365470886, + 0.02780640497803688, + 0.02595219574868679, + 0.02436564676463604, + 0.022998712956905365, + 0.021715674549341202, + 0.02068827487528324, + 0.019841866567730904, + 0.019115712493658066, + 0.01838696375489235, + 0.017793087288737297, + 0.017290478572249413, + 0.01697937399148941, + 0.01664056070148945, + 0.016327474266290665, + 0.016063038259744644, + 0.015679018571972847, + 0.01546056941151619, + 0.015282186679542065, + 0.015113032422959805 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.484899475050042e-06, + "first_to_last_ratio": 4.317013540640096, + "imbalance_abs_log_ratio": 1.4625638531540355, + "last_quartile_mean": 1.2705309870852943e-06, + "mean": 2.7753340354763623e-06, + "normalized": [ + 2.6423318625200567, + 2.4162339509651107, + 2.211337495963179, + 2.033398824132887, + 1.8430633472212699, + 1.6940067504930827, + 1.5445742745339275, + 1.4254731551929667, + 1.2827132729228703, + 1.1755637695304801, + 1.092360755278899, + 1.0016920978710038, + 0.9227083034090567, + 0.8576286815024052, + 0.7936258208713879, + 0.7371254703657203, + 0.6937452548738572, + 0.6483007348957919, + 0.6136555742307378, + 0.5846345106553689, + 0.5617683465972004, + 0.5391257192612848, + 0.5197876012499301, + 0.5027932393500675, + 0.49022455226643835, + 0.4787926786546725, + 0.46965294739774904, + 0.460864066796148, + 0.45018919494858056, + 0.44356620781452427, + 0.43718650171280415, + 0.43187503652054116 + ], + "peak_layer": 1, + "peak_normalized": 2.6423318625200567, + "population_cv": 0.6359510157609364, + "rest_mean": 3.020622224775488e-06, + "spike_contrast": 0.48029104777209103, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4507778132610838e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.333353551075561e-06, + 6.705856321786996e-06, + 6.137200216471683e-06, + 5.643360964313615e-06, + 5.115116437082179e-06, + 4.701434590970166e-06, + 4.28670955443522e-06, + 3.956164164264919e-06, + 3.5599578041001223e-06, + 3.2625821404508315e-06, + 3.031665983144194e-06, + 2.7800301722891163e-06, + 2.560823759267805e-06, + 2.380206069574342e-06, + 2.2025767520972295e-06, + 2.0457694063225063e-06, + 1.9253748178016394e-06, + 1.7992510947806295e-06, + 1.7030992012223578e-06, + 1.622556055735913e-06, + 1.559094812364492e-06, + 1.496253958066518e-06, + 1.4425842209675466e-06, + 1.3954191899756552e-06, + 1.3605368849312072e-06, + 1.3288096170072095e-06, + 1.3034438097747625e-06, + 1.2790517303074012e-06, + 1.249425395144499e-06, + 1.2310443935348303e-06, + 1.2133385780543904e-06, + 1.1985974879280548e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.00092546644737e-05, + "first_to_last_ratio": 2.8610718587446584, + "imbalance_abs_log_ratio": 1.0511963304057041, + "last_quartile_mean": 1.7479202597314725e-05, + "mean": 3.0017626613698667e-05, + "normalized": [ + 1.9665203436980145, + 1.8844342899401736, + 1.80165076038572, + 1.7146572887379672, + 1.6027118780052416, + 1.5258449550389437, + 1.4518445303422347, + 1.3803062982231078, + 1.2718602647538741, + 1.1969127030718432, + 1.1317702815894917, + 1.0650354822632313, + 0.9973554850676779, + 0.9413624795803887, + 0.8896480124159175, + 0.8445590199428843, + 0.809061501322106, + 0.7713306058151453, + 0.7401513167984001, + 0.7139298020909397, + 0.6922990839843121, + 0.6685043123895319, + 0.6485424460151992, + 0.631323216874115, + 0.6206922556356658, + 0.6084815208074952, + 0.5970366156159529, + 0.5872145081510212, + 0.5725964819678343, + 0.5642525868906152, + 0.5573685431962108, + 0.5507411293887441 + ], + "peak_layer": 1, + "peak_normalized": 1.9665203436980145, + "population_cv": 0.4378481821603461, + "rest_mean": 3.195058242353197e-05, + "spike_contrast": 0.6128109021943274, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9579665240598844e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.903027340536937e-05, + 5.6566244893474504e-05, + 5.4081279813544825e-05, + 5.14699422637932e-05, + 4.810960672330111e-05, + 4.580224413075484e-05, + 4.358092701295391e-05, + 4.1433519072597846e-05, + 3.8178226532181725e-05, + 3.592847861000337e-05, + 3.397305772523396e-05, + 3.196983743691817e-05, + 2.9938244551885873e-05, + 2.825746742018964e-05, + 2.670512185432017e-05, + 2.5351657313876785e-05, + 2.4286106054205447e-05, + 2.3153514121077023e-05, + 2.2217585865291767e-05, + 2.1430478227557614e-05, + 2.0781175408046693e-05, + 2.006691283895634e-05, + 1.9467704987619072e-05, + 1.895082459668629e-05, + 1.8631708371685818e-05, + 1.8265171092934906e-05, + 1.7921622202266008e-05, + 1.7626785847824067e-05, + 1.7187987396027893e-05, + 1.693752346909605e-05, + 1.673088081588503e-05, + 1.6531941582798027e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.890059224431752e-05, + "first_to_last_ratio": 2.166537206678912, + "imbalance_abs_log_ratio": 0.7731301356847547, + "last_quartile_mean": 1.7955192333829473e-05, + "mean": 2.5870607089473197e-05, + "normalized": [ + 1.7831451783784935, + 1.6977459463469209, + 1.6200768511580206, + 1.5334592693492033, + 1.446421368117004, + 1.3744004997462946, + 1.3169700168196983, + 1.2570593821131775, + 1.1875487586493927, + 1.1204242107881341, + 1.071956417312422, + 1.0193758496059768, + 0.9782769348268451, + 0.9375232472756838, + 0.9065756301529028, + 0.8773601959197916, + 0.8648597359592212, + 0.8401947622081907, + 0.814617151100937, + 0.7925804041893227, + 0.77941283303955, + 0.7586878792831788, + 0.7416534116249156, + 0.7273676136251142, + 0.7223613266422326, + 0.7135679461862469, + 0.7058096849258172, + 0.6989156870230683, + 0.6866714405082665, + 0.6814500720405863, + 0.6750012133598903, + 0.6685290817235002 + ], + "peak_layer": 1, + "peak_normalized": 1.7831451783784935, + "population_cv": 0.3297373433822509, + "rest_mean": 2.7087979105999693e-05, + "spike_contrast": 0.7123749662061754, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9296798200230114e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.61310482933186e-05, + 4.392171831568703e-05, + 4.19123716710601e-05, + 3.967152224504389e-05, + 3.741979890037328e-05, + 3.5556575312511995e-05, + 3.407081385375932e-05, + 3.2520889362785965e-05, + 3.0722607334610075e-05, + 2.8986054530832916e-05, + 2.7732163289329037e-05, + 2.6371872081654146e-05, + 2.5308618205599487e-05, + 2.4254295567516237e-05, + 2.3453661924577318e-05, + 2.2697840904584154e-05, + 2.2374446416506544e-05, + 2.1736348571721464e-05, + 2.107464024447836e-05, + 2.0504536223597825e-05, + 2.016388316405937e-05, + 1.962771602848079e-05, + 1.9187024008715525e-05, + 1.8817441741703078e-05, + 1.8687926058191806e-05, + 1.8460435967426747e-05, + 1.825972503866069e-05, + 1.808137312764302e-05, + 1.776460703695193e-05, + 1.762952706485521e-05, + 1.7462691175751388e-05, + 1.7295253201154992e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011785902643168811, + "first_to_last_ratio": 5.2723364637525565, + "imbalance_abs_log_ratio": 1.6624736160576234, + "last_quartile_mean": 2.235423085039656e-05, + "mean": 5.981010565392353e-05, + "normalized": [ + 2.341513132793844, + 2.2440750819699944, + 2.142476081227823, + 2.0324729445905114, + 1.8907305822050442, + 1.7814901971753774, + 1.70505397052119, + 1.6266178018526998, + 1.4513520805328384, + 1.3486545714944895, + 1.25655955310435, + 1.1590271011245943, + 1.0440203382713282, + 0.9503717152908809, + 0.8638113904441559, + 0.7921778834221871, + 0.7124380420896897, + 0.6459278658421956, + 0.5962877600798272, + 0.5569469907381962, + 0.5166814306914113, + 0.4757859010752184, + 0.4474223682075307, + 0.42807794732183224, + 0.41212661540589873, + 0.3977379814905876, + 0.3854116965392957, + 0.3748299777949146, + 0.36298713381472003, + 0.3565353744030076, + 0.3515636207422689, + 0.3488348677420969 + ], + "peak_layer": 1, + "peak_normalized": 2.341513132793844, + "population_cv": 0.6457650964234652, + "rest_mean": 6.583521119157215e-05, + "spike_contrast": 0.4142849283380522, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.7274535750620998e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001400461478624493, + 0.00013421836774796247, + 0.00012814172077924013, + 0.00012156242155469954, + 0.00011308479588478804, + 0.00010655111691448838, + 0.00010197945812251419, + 9.728818258736283e-05, + 8.68055212777108e-05, + 8.066317241173238e-05, + 7.515495963161811e-05, + 6.932153337402269e-05, + 6.244296673685312e-05, + 5.684183270204812e-05, + 5.1664650527527556e-05, + 4.7380242904182523e-05, + 4.2610994569258764e-05, + 3.863301390083507e-05, + 3.566403393051587e-05, + 3.3311058359686285e-05, + 3.090277095907368e-05, + 2.845680501195602e-05, + 2.6760379114421085e-05, + 2.5603387257433496e-05, + 2.4649336410220712e-05, + 2.3788750695530325e-05, + 2.305151429027319e-05, + 2.2418620574171655e-05, + 2.1710298824473284e-05, + 2.1324418412405066e-05, + 2.1027057300671004e-05, + 2.0863850295427255e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.005866481456905603, + 0.010142817161977291, + 0.01482025533914566, + 0.01956234686076641, + 0.0067119961604475975, + 0.011908270418643951, + 0.017094247043132782, + 0.02295343205332756, + 0.007494525518268347, + 0.013766046613454819, + 0.02038806490600109, + 0.02682213857769966, + 0.009193498641252518, + 0.016781754791736603, + 0.024297412484884262, + 0.03140362724661827, + 0.009884056635200977, + 0.018740201368927956, + 0.02799132652580738, + 0.03797563910484314, + 0.012623735703527927, + 0.02351362258195877, + 0.034899674355983734, + 0.047332651913166046, + 0.01744839735329151, + 0.029514454305171967, + 0.044660378247499466, + 0.0601838044822216, + 0.018972838297486305, + 0.0376126803457737, + 0.0565216988325119, + 0.07583927363157272 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5008519199000148e-05, + "first_to_last_ratio": 3.366911210627585, + "imbalance_abs_log_ratio": 1.2139957693612529, + "last_quartile_mean": 4.457652210021479e-06, + "mean": 8.299272167278104e-06, + "normalized": [ + 2.256857417490706, + 2.1152794126450303, + 1.976744865869958, + 1.8670032822761609, + 1.7142943873365317, + 1.61284428729019, + 1.5135327831047092, + 1.4107548139296946, + 1.2736784355647999, + 1.19029358022618, + 1.1070446131205842, + 1.032219511002358, + 0.948055380482258, + 0.88484028406668, + 0.831196979445672, + 0.7835724276747893, + 0.7420372577590195, + 0.7071991859856492, + 0.678238718639715, + 0.6514579423864029, + 0.6287823606114806, + 0.6086317246696183, + 0.5913247143847147, + 0.5772065844972074, + 0.5691335632017953, + 0.558269353765152, + 0.5491862116730458, + 0.5409941785864528, + 0.5285468164414351, + 0.5224629687441996, + 0.516466078557371, + 0.5118498785704394 + ], + "peak_layer": 1, + "peak_normalized": 2.256857417490706, + "population_cv": 0.5242114468317668, + "rest_mean": 8.921693312947389e-06, + "spike_contrast": 0.5535045654951547, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.938197980663972e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8730273950495757e-05, + 1.7555279555381276e-05, + 1.6405543647124432e-05, + 1.5494768376811408e-05, + 1.4227395695343148e-05, + 1.3385433703660965e-05, + 1.256122050108388e-05, + 1.1708238162100315e-05, + 1.0570603990345262e-05, + 9.878570381260943e-06, + 9.187664545606822e-06, + 8.566670658183284e-06, + 7.868169632274657e-06, + 7.343530342041049e-06, + 6.8983299570390955e-06, + 6.5030808400479145e-06, + 6.158369160402799e-06, + 5.869238520972431e-06, + 5.628887720376952e-06, + 5.406626769399736e-06, + 5.218435944698285e-06, + 5.051200332673034e-06, + 4.907564743916737e-06, + 4.7903945414873306e-06, + 4.723394340544473e-06, + 4.63322930954746e-06, + 4.5578458411910105e-06, + 4.489857929002028e-06, + 4.386553882795852e-06, + 4.336062374932226e-06, + 4.286292551114457e-06, + 4.247981451044325e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.8205007462674985e-05, + "first_to_last_ratio": 3.284886533563498, + "imbalance_abs_log_ratio": 1.189332110567881, + "last_quartile_mean": 1.7719031347951386e-05, + "mean": 3.265063452317918e-05, + "normalized": [ + 2.145023601546889, + 2.046116178200582, + 1.9398041355219169, + 1.8522906626530224, + 1.7094551367874427, + 1.616905863157966, + 1.5255793277696308, + 1.4261117891382666, + 1.2879069358899493, + 1.2060838643982497, + 1.1221569343801026, + 1.0481379278736214, + 0.9613520366262555, + 0.8968015077026028, + 0.8421366180143057, + 0.7933562257498653, + 0.7503570432885839, + 0.714837243906681, + 0.6855582118161445, + 0.6585243205638621, + 0.6351846609275962, + 0.6146934889106834, + 0.5972735915655352, + 0.5828673640378894, + 0.5748017373594797, + 0.5639401582431183, + 0.5548451094205582, + 0.5465856652489405, + 0.5340146592968225, + 0.5278533354505021, + 0.5219685037625007, + 0.5174761607904339 + ], + "peak_layer": 1, + "peak_normalized": 2.145023601546889, + "population_cv": 0.5072011380661324, + "rest_mean": 3.5063370281325966e-05, + "spike_contrast": 0.5596113913680666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.962186142918654e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.003638165770099e-05, + 6.680699152639136e-05, + 6.333583587547764e-05, + 6.047846545698121e-05, + 5.581479490501806e-05, + 5.2793002396356314e-05, + 4.981113306712359e-05, + 4.656345481635071e-05, + 4.2050978663610294e-05, + 3.9379403460770845e-05, + 3.663913594209589e-05, + 3.422236841288395e-05, + 3.138875399599783e-05, + 2.928113826783374e-05, + 2.7496294933371246e-05, + 2.5903584173647687e-05, + 2.449963358230889e-05, + 2.3339889594353735e-05, + 2.2383910618373193e-05, + 2.1501236915355548e-05, + 2.0739182218676433e-05, + 2.0070132450200617e-05, + 1.950136174855288e-05, + 1.9030989278689958e-05, + 1.87676414498128e-05, + 1.8413003999739885e-05, + 1.8116044884664007e-05, + 1.7846368791651912e-05, + 1.7435917470720597e-05, + 1.7234746337635443e-05, + 1.7042602848960087e-05, + 1.689592500042636e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05161678325384855, + "first_to_last_ratio": 3.2848865977727097, + "imbalance_abs_log_ratio": 1.189332130114739, + "last_quartile_mean": 0.015713414060883224, + "mean": 0.028954909328604117, + "normalized": [ + 2.1450237245911468, + 2.046116217659094, + 1.9398041664812302, + 1.8522906413119131, + 1.7094550632279355, + 1.6169058509968128, + 1.5255792788818974, + 1.4261118508948372, + 1.2879069434477655, + 1.206083847584953, + 1.1221569306992827, + 1.0481379275281273, + 0.9613519652186596, + 0.8968015075628347, + 0.8421366414011558, + 0.7933562213086081, + 0.7503569989748414, + 0.7148372643644912, + 0.6855582307408538, + 0.6585243493162974, + 0.6351846345398151, + 0.614693479533182, + 0.5972735918667607, + 0.5828673969375834, + 0.5748016775305409, + 0.5639401482692175, + 0.5548451622169414, + 0.5465856833639338, + 0.5340146386986252, + 0.5278533199019263, + 0.5219685200223859, + 0.5174761249263505 + ], + "peak_layer": 1, + "peak_normalized": 2.1450237245911468, + "population_cv": 0.507201147039954, + "rest_mean": 0.031094547602589482, + "spike_contrast": 0.5596113785438716, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017400862649083138, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06210896745324135, + 0.05924510955810547, + 0.05616685375571251, + 0.05363290756940842, + 0.04949711635708809, + 0.0468173623085022, + 0.04417300969362259, + 0.0412929393351078, + 0.03729122877120972, + 0.034922048449516296, + 0.03249195218086243, + 0.03034873865544796, + 0.02783585898578167, + 0.025966806337237358, + 0.024383990094065666, + 0.022971557453274727, + 0.021726518869400024, + 0.020698048174381256, + 0.01985027641057968, + 0.019067512825131416, + 0.018391713500022888, + 0.017798393964767456, + 0.017294002696871758, + 0.01687687262892723, + 0.016643330454826355, + 0.016328835859894753, + 0.01606549136340618, + 0.015826338902115822, + 0.015462345443665981, + 0.015283945016562939, + 0.015113551169633865, + 0.014983474276959896 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.35499910370163e-06, + "first_to_last_ratio": 4.280329422122994, + "imbalance_abs_log_ratio": 1.4540299744175837, + "last_quartile_mean": 1.251071722663255e-06, + "mean": 2.683941740855289e-06, + "normalized": [ + 2.703190107906874, + 2.4622030057009887, + 2.2387169403666407, + 2.052519103467962, + 1.857495181173395, + 1.6956946287041017, + 1.543681691896796, + 1.4080949959710705, + 1.2505201784095583, + 1.154791072279902, + 1.0675304639467325, + 0.9762128756212061, + 0.8974710865803879, + 0.8318006025975303, + 0.7746066644618449, + 0.7211640498251165, + 0.6781926692188598, + 0.6399504079116765, + 0.6084710506699328, + 0.581158338700675, + 0.5581318067356622, + 0.5402181590129327, + 0.5217735093776118, + 0.5073536475319059, + 0.4964690402033343, + 0.48529694803264734, + 0.47725139647875997, + 0.4696126527079396, + 0.4585674232309517, + 0.45305527095834874, + 0.4466379671326055, + 0.4421670631860494 + ], + "peak_layer": 1, + "peak_normalized": 2.703190107906874, + "population_cv": 0.645765617676911, + "rest_mean": 2.9201339657447534e-06, + "spike_contrast": 0.48234216065938457, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4085037264521817e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.255204764078371e-06, + 6.608409421460237e-06, + 6.008585842209868e-06, + 5.508841695700539e-06, + 4.985408850188833e-06, + 4.551145593723049e-06, + 4.143151727475924e-06, + 3.779244934776216e-06, + 3.3563233046152163e-06, + 3.099391960859066e-06, + 2.8651895718212472e-06, + 2.6200984848401276e-06, + 2.408760110483854e-06, + 2.232504357380094e-06, + 2.078999159493833e-06, + 1.9355622953298735e-06, + 1.8202296132585616e-06, + 1.7175896118715173e-06, + 1.6331008509951062e-06, + 1.5597951232848573e-06, + 1.497993252996821e-06, + 1.44991406614281e-06, + 1.400409701091121e-06, + 1.3617076319860644e-06, + 1.3324939800440916e-06, + 1.3025087355345022e-06, + 1.2809149438908207e-06, + 1.2604130006366177e-06, + 1.2307682482060045e-06, + 1.2159739526396152e-06, + 1.1987502830379526e-06, + 1.186750637316436e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.7860596623650054e-05, + "first_to_last_ratio": 2.779352348042467, + "imbalance_abs_log_ratio": 1.022217932231161, + "last_quartile_mean": 1.7220053678101976e-05, + "mean": 2.88126707914671e-05, + "normalized": [ + 1.9752636229242324, + 1.8880994931589683, + 1.7945585688576082, + 1.7155011651574092, + 1.5967516750805444, + 1.5186230631712434, + 1.44161945600051, + 1.358346630504372, + 1.2494545051794677, + 1.1811062374956933, + 1.1110457151250825, + 1.0496228553992981, + 0.981726104768848, + 0.9275032753698961, + 0.8805136203767746, + 0.8381981012329272, + 0.8039749772857018, + 0.7714588681042523, + 0.7440936013254396, + 0.7183075709964174, + 0.6966458004631945, + 0.6758729225629136, + 0.6578613567984469, + 0.6426061157750497, + 0.6340386554461201, + 0.6220565974014273, + 0.6118747994647586, + 0.6024343556608059, + 0.5879125773227442, + 0.5807482794696509, + 0.5737976188800432, + 0.5683818132401579 + ], + "peak_layer": 1, + "peak_normalized": 1.9752636229242324, + "population_cv": 0.4317074018350235, + "rest_mean": 3.0619305444243194e-05, + "spike_contrast": 0.62238001123762, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9056843666476197e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.6912620493676513e-05, + 5.440118911792524e-05, + 5.170602526050061e-05, + 4.942817031405866e-05, + 4.600668034981936e-05, + 4.375558637548238e-05, + 4.1536906792316586e-05, + 3.9137594285421073e-05, + 3.6000121326651424e-05, + 3.4030825190711766e-05, + 3.201219442416914e-05, + 3.0242437787819654e-05, + 2.8286151064094156e-05, + 2.672384653124027e-05, + 2.5369949071318842e-05, + 2.4150725948857144e-05, + 2.3164666345110163e-05, + 2.222779039584566e-05, + 2.1439323973027058e-05, + 2.0696359570138156e-05, + 2.00722261070041e-05, + 1.9473704014671966e-05, + 1.8954742699861526e-05, + 1.85151984624099e-05, + 1.8268347048433498e-05, + 1.7923111954587512e-05, + 1.762974716257304e-05, + 1.7357742763124406e-05, + 1.6939331544563174e-05, + 1.6732908989069983e-05, + 1.653264189371839e-05, + 1.6376598068745807e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.7207921195658855e-05, + "first_to_last_ratio": 2.0918054721335677, + "imbalance_abs_log_ratio": 0.7380275553273271, + "last_quartile_mean": 1.7787467186280992e-05, + "mean": 2.50158799417477e-05, + "normalized": [ + 1.7669834832946238, + 1.6851855745889466, + 1.5913246601136481, + 1.5163874050648884, + 1.4314556947301431, + 1.3660671618942286, + 1.3038144666622677, + 1.2377581340639199, + 1.1604947777085126, + 1.1096820799312197, + 1.0568639468917405, + 1.0113831764859333, + 0.9697099699695068, + 0.9379529791327526, + 0.9077702331627978, + 0.8815756711399201, + 0.8691427723616427, + 0.8426692090660671, + 0.8202682499619266, + 0.8014514792168752, + 0.7847031862420706, + 0.7670164542718427, + 0.7519889963555914, + 0.7399739814225563, + 0.7380541298189486, + 0.7300354427325324, + 0.7228094770104511, + 0.7167792102532495, + 0.7048467977298535, + 0.6982014484170743, + 0.6912475034823692, + 0.6864022468218989 + ], + "peak_layer": 1, + "peak_normalized": 1.7669834832946238, + "population_cv": 0.3174743127828502, + "rest_mean": 2.6144617987921703e-05, + "spike_contrast": 0.7236936680868339, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.8920694492408076e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.420264667714946e-05, + 4.21564000134822e-05, + 3.9808386645745486e-05, + 3.793376527028158e-05, + 3.58091238013003e-05, + 3.417337211431004e-05, + 3.26160661643371e-05, + 3.096360887866467e-05, + 2.9030798032181337e-05, + 2.7759673685068265e-05, + 2.6438381610205397e-05, + 2.5300640118075535e-05, + 2.4258148187072948e-05, + 2.3463719116989523e-05, + 2.2708671167492867e-05, + 2.2053391148801893e-05, + 2.1742371245636605e-05, + 2.1080111764604226e-05, + 2.0519732061075047e-05, + 2.004901398322545e-05, + 1.9630040696938522e-05, + 1.918759153340943e-05, + 1.881166645034682e-05, + 1.851110027928371e-05, + 1.846307350206189e-05, + 1.826247898861766e-05, + 1.8081715097650886e-05, + 1.793086266843602e-05, + 1.763236286933534e-05, + 1.746612360875588e-05, + 1.729216455714777e-05, + 1.7170956198242493e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011250466286583105, + "first_to_last_ratio": 5.136757236447842, + "imbalance_abs_log_ratio": 1.6364219925229777, + "last_quartile_mean": 2.1901884338149102e-05, + "mean": 5.649989418543555e-05, + "normalized": [ + 2.3823389400332498, + 2.2792913554127336, + 2.1636479699592535, + 2.0682892196556057, + 1.89270947379848, + 1.8094109355775225, + 1.7253047001597102, + 1.6089010079061294, + 1.4276863120740728, + 1.3301882974473787, + 1.227270134970172, + 1.1372581289133685, + 1.0077700511385481, + 0.9144159274954295, + 0.8389896038222593, + 0.761607782391717, + 0.6842191357138161, + 0.6319663297187056, + 0.5899873238938539, + 0.550774281755648, + 0.5037238917813971, + 0.4736966989455727, + 0.453500437871209, + 0.43589448694769534, + 0.4210071093140732, + 0.4081032387076438, + 0.39686588746098733, + 0.38782343274226294, + 0.37750089000360326, + 0.37216747430584624, + 0.36939626290542954, + 0.36829327717662536 + ], + "peak_layer": 1, + "peak_normalized": 2.3823389400332498, + "population_cv": 0.6517280428580817, + "rest_mean": 6.217536584127056e-05, + "spike_contrast": 0.4157972678427941, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5852347243926487e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001346018980257213, + 0.00012877972039859742, + 0.00012224588135723025, + 0.00011685812205541879, + 0.00010693788499338552, + 0.00010223152639809996, + 9.747953299665824e-05, + 9.090273670153692e-05, + 8.066412556217983e-05, + 7.515549805248156e-05, + 6.934063276275992e-05, + 6.425496394513175e-05, + 5.6938901252578944e-05, + 5.1664403144968674e-05, + 4.740282383863814e-05, + 4.3030759115936235e-05, + 3.8658308767480776e-05, + 3.570603075786494e-05, + 3.333422137075104e-05, + 3.111868863925338e-05, + 2.8460346584324725e-05, + 2.676381336641498e-05, + 2.5622726752771996e-05, + 2.4627992388559505e-05, + 2.3786857127561234e-05, + 2.3057789803715423e-05, + 2.242288064735476e-05, + 2.1911982912570238e-05, + 2.132876034011133e-05, + 2.1027422917541116e-05, + 2.08708497666521e-05, + 2.080853118968662e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.028571506962180138, + 0.01502692885696888, + 0.015751885250210762, + 0.016715344041585922, + 0.01783851347863674, + 0.012985615991055965, + 0.014028912410140038, + 0.015184907242655754, + 0.016659293323755264, + 0.013609340414404869, + 0.014725575223565102, + 0.016002535820007324, + 0.017315881326794624, + 0.014967667870223522, + 0.016154561191797256, + 0.01737469807267189, + 0.018479224294424057, + 0.01647813245654106, + 0.017671043053269386, + 0.01892261393368244, + 0.020303834229707718, + 0.01861351728439331, + 0.01984807848930359, + 0.021230308338999748, + 0.022779423743486404, + 0.02138628251850605, + 0.02279111184179783, + 0.024287279695272446, + 0.02611272968351841, + 0.024725768715143204, + 0.026407435536384583, + 0.028016721829771996 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.961159703498197e-06, + "first_to_last_ratio": 10.14195644845838, + "imbalance_abs_log_ratio": 2.3166809231867567, + "last_quartile_mean": 4.891718603516892e-07, + "mean": 2.1833543613070105e-06, + "normalized": [ + 2.1149003512951285, + 3.2694277026187555, + 2.6224151681895895, + 2.367649688300867, + 1.872384189095874, + 2.4215723461348544, + 1.997727875442672, + 1.5120426998182623, + 1.407451434597839, + 1.4959940566742542, + 1.146826747734668, + 1.0540980256857684, + 0.9488948361068527, + 0.9447470595114825, + 0.8146834014726949, + 0.6963182667169648, + 0.5482907339349459, + 0.5718154556650352, + 0.5280062120840487, + 0.46712124843842245, + 0.37653152797943845, + 0.3855401511265584, + 0.3587208946748597, + 0.2844717467264502, + 0.29151483884188434, + 0.260094980728135, + 0.263913337696718, + 0.20680766564782654, + 0.1887856679838017, + 0.2117583846844945, + 0.19386962063621954, + 0.1756236837546329 + ], + "peak_layer": 2, + "peak_normalized": 3.2694277026187555, + "population_cv": 0.8462629641613505, + "rest_mean": 2.4504692364475717e-06, + "spike_contrast": 0.3023641450084502, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.409340355479799e-07, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.617576905729948e-06, + 7.13831923349062e-06, + 5.725661594624398e-06, + 5.1694182729988825e-06, + 4.0880781853047665e-06, + 5.2871505431539845e-06, + 4.361747869552346e-06, + 3.30132502313063e-06, + 3.0729652280570008e-06, + 3.2662851481291e-06, + 2.5039291813300224e-06, + 2.3014695216261316e-06, + 2.071773678835598e-06, + 2.0627176127163693e-06, + 1.7787425576898386e-06, + 1.5203095244942233e-06, + 1.197112965201086e-06, + 1.2484757689890102e-06, + 1.1528246659509023e-06, + 1.0198912150372053e-06, + 8.221017537834996e-07, + 8.417707704211352e-07, + 7.832148298803077e-07, + 6.211026288838184e-07, + 6.364801947711385e-07, + 5.678795105268364e-07, + 5.762163368672191e-07, + 4.515344187439041e-07, + 4.121860115446907e-07, + 4.6234359274421877e-07, + 4.2328608174102555e-07, + 3.834487358744809e-07 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.940424550019088e-06, + "first_to_last_ratio": 7.362808193451028, + "imbalance_abs_log_ratio": 1.9964414079930852, + "last_quartile_mean": 1.214268294802423e-06, + "mean": 4.16189458363192e-06, + "normalized": [ + 1.8755411782360838, + 2.990879153140948, + 2.507234682429089, + 2.1623895678054548, + 1.7817542090336904, + 2.3421906478647685, + 1.984613814607898, + 1.5406942028635409, + 1.3930431107090302, + 1.467345705614427, + 1.208878006326314, + 1.0663129583432038, + 0.9225426136071778, + 0.9347168647367088, + 0.8446162613986855, + 0.7123769822844362, + 0.5995323166764556, + 0.6272780709200748, + 0.55800311830824, + 0.5090119191108281, + 0.4207840547535497, + 0.45050976377329044, + 0.4129238940687079, + 0.3527585690707129, + 0.3423379312955225, + 0.33392765114492584, + 0.32776335752361546, + 0.2883552290395252, + 0.26514548144992983, + 0.28609378158280374, + 0.25631521585884554, + 0.23412968642151585 + ], + "peak_layer": 2, + "peak_normalized": 2.990879153140948, + "population_cv": 0.7649777321045976, + "rest_mean": 4.627515910147782e-06, + "spike_contrast": 0.3560310655730732, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6475394204462646e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.805804671079386e-06, + 1.2447723747754935e-05, + 1.0434846444695722e-05, + 8.99963742995169e-06, + 7.415473191940691e-06, + 9.747950571181718e-06, + 8.259753485617694e-06, + 6.412206857930869e-06, + 5.797698577225674e-06, + 6.106938144512242e-06, + 5.0312228268012404e-06, + 4.437882125785109e-06, + 3.839525106741348e-06, + 3.890193056577118e-06, + 3.515203843562631e-06, + 2.9648379040736472e-06, + 2.4951903014880372e-06, + 2.610665205793339e-06, + 2.3223501557367854e-06, + 2.1184539491514442e-06, + 1.7512588783574756e-06, + 1.8749741457213531e-06, + 1.718545718176756e-06, + 1.4681439779451466e-06, + 1.4247743820305914e-06, + 1.3897716826249962e-06, + 1.3641165423905477e-06, + 1.2001040659015416e-06, + 1.1035075431209407e-06, + 1.1906921599802445e-06, + 1.0667569085853756e-06, + 9.744230737851467e-07 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.00792845810065046, + "first_to_last_ratio": 7.362808269910068, + "imbalance_abs_log_ratio": 1.9964414183775794, + "last_quartile_mean": 0.001076825283234939, + "mean": 0.003690809797262773, + "normalized": [ + 1.8755411129192268, + 2.990879196958467, + 2.5072346853780427, + 2.162389622214153, + 1.7817542918869094, + 2.3421908242130387, + 1.9846136654580344, + 1.5406942156700558, + 1.3930430879772915, + 1.4673456922088532, + 1.2088779947214396, + 1.0663129426719855, + 0.9225426097974535, + 0.9347168545387744, + 0.8446162145203677, + 0.7123769861164736, + 0.5995323323039541, + 0.6272780785788562, + 0.5580030712234716, + 0.5090119005316229, + 0.4207840635329223, + 0.45050976081671684, + 0.4129238737745274, + 0.35275859035228563, + 0.3423379200763214, + 0.3339276703286944, + 0.3277633648171629, + 0.2883552364940563, + 0.2651454743312869, + 0.2860937736207281, + 0.25631521203611624, + 0.2341296799307093 + ], + "peak_layer": 2, + "peak_normalized": 2.990879196958467, + "population_cv": 0.7649777429861212, + "rest_mean": 0.004103727453319287, + "spike_contrast": 0.3560310647277095, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0014610544545575976, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.006922265514731407, + 0.011038766242563725, + 0.009253726340830326, + 0.007980968803167343, + 0.006576116196811199, + 0.008644580841064453, + 0.007324831560254097, + 0.005686409305781126, + 0.005141457077115774, + 0.005415693856775761, + 0.004461738746613264, + 0.003935558255761862, + 0.0034049293026328087, + 0.0034498621243983507, + 0.0031173177994787693, + 0.002629247959703207, + 0.002212759805843234, + 0.00231516407802701, + 0.002059483202174306, + 0.001878666109405458, + 0.0015530339442193508, + 0.001662745838984847, + 0.0015240234788507223, + 0.0013019648613408208, + 0.0012635041493922472, + 0.0012324635172262788, + 0.0012097122380509973, + 0.0010642643319442868, + 0.0009786015143617988, + 0.001055917702615261, + 0.0009460106957703829, + 0.000864128116518259 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.5133993492308946e-06, + "first_to_last_ratio": 6.5659297300766655, + "imbalance_abs_log_ratio": 1.881894116932073, + "last_quartile_mean": 5.350954843663658e-07, + "mean": 1.7443735549349526e-06, + "normalized": [ + 1.589774202228725, + 2.5970667163618275, + 2.31809881449688, + 2.0431360816541173, + 1.739210008196666, + 2.257659305838541, + 2.011325443479608, + 1.5567888778424819, + 1.467448725425628, + 1.5902279397678998, + 1.2912623477212033, + 1.1632146422568665, + 1.0012751149296444, + 1.0140542592766522, + 0.8991815616774139, + 0.774122319459461, + 0.6409300827695702, + 0.6678856249973933, + 0.6105498714134762, + 0.5542747505087504, + 0.4535313629747497, + 0.4812253529456957, + 0.4453558428285356, + 0.3783599505586694, + 0.3693486135392654, + 0.34798394288991763, + 0.34559725866794794, + 0.2988109680432934, + 0.27684960189841046, + 0.29841376846953566, + 0.272740676748336, + 0.2442959701328381 + ], + "peak_layer": 2, + "peak_normalized": 2.5970667163618275, + "population_cv": 0.6989714208275406, + "rest_mean": 1.9299347726515677e-06, + "spike_contrast": 0.38464666774479367, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.423429792652314e-07, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.773160076685599e-06, + 4.530254500423325e-06, + 4.043630269734422e-06, + 3.5639925499708625e-06, + 3.033831944776466e-06, + 3.938201189157553e-06, + 3.5085029139736434e-06, + 2.7156213491252856e-06, + 2.559778749855468e-06, + 2.773951564449817e-06, + 2.2524438918480882e-06, + 2.0290808606659994e-06, + 1.746597831697727e-06, + 1.7688894331513438e-06, + 1.5685085372751928e-06, + 1.3503585023499909e-06, + 1.1180214869455085e-06, + 1.1650420219666557e-06, + 1.0650270496626035e-06, + 9.66862216955633e-07, + 7.911281159067585e-07, + 8.394367796427105e-07, + 7.768669547658646e-07, + 6.600010920010391e-07, + 6.442819540097844e-07, + 6.070139875191671e-07, + 6.028507186783827e-07, + 5.212379505792342e-07, + 4.829291242458567e-07, + 5.205450861467398e-07, + 4.7576162387485965e-07, + 4.261434298769018e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.804085307341666e-06, + "first_to_last_ratio": 6.304609762674019, + "imbalance_abs_log_ratio": 1.8412810741903929, + "last_quartile_mean": 7.619956647886283e-07, + "mean": 2.3993721356418973e-06, + "normalized": [ + 1.4988203443996868, + 2.5234715181763763, + 2.2553217408275312, + 1.9958915618525483, + 1.7576503840860649, + 2.319660501024385, + 2.0260662582488056, + 1.6409258036937564, + 1.4769401358690522, + 1.5639642180334559, + 1.318106777079787, + 1.1580760773918135, + 0.96371832012835, + 1.0214962965942758, + 0.8943161393616675, + 0.7727506438550246, + 0.6438659223631747, + 0.6719055424540622, + 0.6043445279969172, + 0.5577548422477306, + 0.4603517050297865, + 0.4900258115330936, + 0.45428577798054076, + 0.38963894059637383, + 0.37546196475112736, + 0.3630494912829247, + 0.35422949063900455, + 0.315438725345089, + 0.2890783675728055, + 0.3094773464831404, + 0.27956031292056627, + 0.2543545101810816 + ], + "peak_layer": 2, + "peak_normalized": 2.5234715181763763, + "population_cv": 0.6891965904370858, + "rest_mean": 2.650882836237374e-06, + "spike_contrast": 0.3927802233252255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.041214352426323e-06, + "top_five_layers": [ + 2, + 6, + 3, + 7, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5962277706858004e-06, + 6.054747245798353e-06, + 5.411356141848955e-06, + 4.788886599271791e-06, + 4.217257355776383e-06, + 5.565728770307032e-06, + 4.861286925006425e-06, + 3.937191650038585e-06, + 3.5437290080153616e-06, + 3.752532165890443e-06, + 3.1626286727259867e-06, + 2.7786554710473865e-06, + 2.312318883923581e-06, + 2.4509497507096967e-06, + 2.145797225239221e-06, + 1.8541163626650814e-06, + 1.5448739532075706e-06, + 1.6121514363476308e-06, + 1.4500474208034575e-06, + 1.3382614270085469e-06, + 1.1045550536437077e-06, + 1.1757542779378127e-06, + 1.090000637304911e-06, + 9.348888170279679e-07, + 9.008729762172152e-07, + 8.710908332432155e-07, + 8.499283694618498e-07, + 7.56854888095404e-07, + 6.936065801710356e-07, + 7.4255132176404e-07, + 6.707692250529362e-07, + 6.102911243033304e-07 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.828203619742453e-06, + "first_to_last_ratio": 5.846579859482356, + "imbalance_abs_log_ratio": 1.765856850870783, + "last_quartile_mean": 4.837364215859452e-07, + "mean": 1.4834222348270032e-06, + "normalized": [ + 1.2646232648739053, + 2.247789759160063, + 2.020880547315863, + 1.842393507943919, + 1.7522315558445254, + 2.3420641180772317, + 2.0534732489830905, + 1.7288628769088255, + 1.548314882558432, + 1.6499586193135367, + 1.4182480230650574, + 1.2414487105462828, + 0.9740236314282787, + 1.1015223442115594, + 0.9385100939682994, + 0.8083610783044318, + 0.6755087261126067, + 0.7006206946100645, + 0.6299107486392663, + 0.5832268076063307, + 0.47664186001684955, + 0.5068198063687983, + 0.4772500226016884, + 0.4085558916985, + 0.3939522492022734, + 0.36559718120224766, + 0.3572742323275962, + 0.3258561263199403, + 0.2984295494995466, + 0.3178990099824609, + 0.2887716969712665, + 0.26097913433726316 + ], + "peak_layer": 6, + "peak_normalized": 2.3420641180772317, + "population_cv": 0.6540082332746273, + "rest_mean": 1.6337852183891368e-06, + "spike_contrast": 0.41098555430286204, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.714621235914819e-07, + "top_five_layers": [ + 6, + 2, + 7, + 3, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.87597026979347e-06, + 3.3344213079544716e-06, + 2.9978191378177144e-06, + 2.7330474949849304e-06, + 2.5992992505052825e-06, + 3.4742699881462613e-06, + 3.0461678761639632e-06, + 2.564633632573532e-06, + 2.2968047233007383e-06, + 2.447585302434163e-06, + 2.1038606519141467e-06, + 1.8415926206216682e-06, + 1.4448883121076506e-06, + 1.6340227375621907e-06, + 1.3922067410021555e-06, + 1.1991407973255264e-06, + 1.002064664135105e-06, + 1.0393163165645092e-06, + 9.34423610488011e-07, + 8.651716143504018e-07, + 7.070611331982946e-07, + 7.518277698181919e-07, + 7.079632950990344e-07, + 6.06060893915128e-07, + 5.843975259267609e-07, + 5.42334987585491e-07, + 5.299885401655047e-07, + 4.833822231375962e-07, + 4.4269702925703314e-07, + 4.7157845983747393e-07, + 4.2837035607590224e-07, + 3.8714225070179964e-07 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 1.4976229522289941e-05, + "first_to_last_ratio": 6.400592033457801, + "imbalance_abs_log_ratio": 1.8563904913150628, + "last_quartile_mean": 2.339819417329636e-06, + "mean": 7.376071508247151e-06, + "normalized": [ + 1.5647793422352725, + 2.5744682287007357, + 2.3807087714141812, + 2.046233253632905, + 1.770279995956188, + 2.2790875337951744, + 2.0129055774210696, + 1.6145788731579245, + 1.4481155829279344, + 1.5649856288248702, + 1.2953241737825878, + 1.1378217116015181, + 0.9671807788478195, + 0.9743671476553158, + 0.8907704628221471, + 0.7647048808528004, + 0.6263127195536261, + 0.6555825424956646, + 0.6004321371903539, + 0.5418622203213719, + 0.44846797056121285, + 0.4882635266638759, + 0.4317412168950189, + 0.383285230653721, + 0.3707675618784486, + 0.36766574152703, + 0.3569460829427414, + 0.31124413981080545, + 0.2846577996070008, + 0.3121781014204342, + 0.2794134285368139, + 0.25486763631343645 + ], + "peak_layer": 2, + "peak_normalized": 2.5744682287007357, + "population_cv": 0.7019462950147708, + "rest_mean": 8.162162531435671e-06, + "spike_contrast": 0.3836213712934221, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.131179983029142e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.154192432295531e-05, + 1.8989461750607006e-05, + 1.756027813826222e-05, + 1.5093162801349536e-05, + 1.3057711839792319e-05, + 1.681071262282785e-05, + 1.484733547840733e-05, + 1.1909249224117957e-05, + 1.068140409188345e-05, + 1.1543445907591376e-05, + 9.554403732181527e-06, + 8.392654308408964e-06, + 7.1339945861836895e-06, + 7.187001756392419e-06, + 6.5703866312105674e-06, + 5.640517883875873e-06, + 4.61972740595229e-06, + 4.835623713006498e-06, + 4.428830379765714e-06, + 3.996814484708011e-06, + 3.307931820017984e-06, + 3.601466687541688e-06, + 3.184554088875302e-06, + 2.8271392693568487e-06, + 2.7348080493538873e-06, + 2.7119288006360875e-06, + 2.632859832374379e-06, + 2.295759031767375e-06, + 2.099656285281526e-06, + 2.302647999385954e-06, + 2.0609734292520443e-06, + 1.8799219105858356e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014360599219799042, + 0.014914286322891712, + 0.015590262599289417, + 0.016720104962587357, + 0.01203431747853756, + 0.01319553330540657, + 0.014193638227880001, + 0.015358141623437405, + 0.012656479142606258, + 0.013699201866984367, + 0.01495310477912426, + 0.01618308201432228, + 0.01409733947366476, + 0.01521329116076231, + 0.016374915838241577, + 0.017517324537038803, + 0.015613679774105549, + 0.016780734062194824, + 0.017834074795246124, + 0.01904122717678547, + 0.017550688236951828, + 0.018688000738620758, + 0.019847052171826363, + 0.021104788407683372, + 0.019968591630458832, + 0.021149026229977608, + 0.02249406836926937, + 0.02368815243244171, + 0.02278727851808071, + 0.024227207526564598, + 0.02562388963997364, + 0.0269329734146595 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.831441280359286e-06, + "first_to_last_ratio": 3.949798443260095, + "imbalance_abs_log_ratio": 1.373664550588347, + "last_quartile_mean": 1.982744535666825e-06, + "mean": 4.823673624088087e-06, + "normalized": [ + 1.646132756330255, + 1.5986846162740493, + 1.5570334831325985, + 1.4112719107036644, + 1.9597644907314193, + 1.739595316367215, + 1.586190473612319, + 1.4896705111709025, + 1.6742871420623147, + 1.4631400905900616, + 1.3644640801393357, + 1.1849574810055368, + 1.2519074382611204, + 1.1640272230786968, + 1.006304156490551, + 0.9127806011294972, + 0.9624471043561035, + 0.8599074596436582, + 0.7423282252089072, + 0.6792100847159988, + 0.7315895600277779, + 0.6384373973703829, + 0.5798636442076341, + 0.5076487157583509, + 0.5180109917100456, + 0.4991175722294294, + 0.4463413444888274, + 0.40238605877214556, + 0.38468977858519854, + 0.36602261687556353, + 0.35225030635593113, + 0.31953736861450793 + ], + "peak_layer": 5, + "peak_normalized": 1.9597644907314193, + "population_cv": 0.4957051427675951, + "rest_mean": 5.185350830658906e-06, + "spike_contrast": 0.5536012513623684, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.870616708605667e-06, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.940407158457674e-06, + 7.711532816756517e-06, + 7.510621344408719e-06, + 6.807515092077665e-06, + 9.45326428336557e-06, + 8.391240044147708e-06, + 7.651265150343534e-06, + 7.185684353316901e-06, + 8.076214726315811e-06, + 7.057710263325134e-06, + 6.5817293943837285e-06, + 5.715848146792268e-06, + 6.038792889739852e-06, + 5.61488741368521e-06, + 4.854082817473682e-06, + 4.402955710247625e-06, + 4.6425307118624914e-06, + 4.147912932239706e-06, + 3.580749080356327e-06, + 3.2762877708591986e-06, + 3.5289492643642006e-06, + 3.079613634326961e-06, + 2.7970729661319638e-06, + 2.448731720505748e-06, + 2.49871595769946e-06, + 2.407580268481979e-06, + 2.1530049707507715e-06, + 1.9409790183999576e-06, + 1.8556179384177085e-06, + 1.7655736428423552e-06, + 1.6991405118460534e-06, + 1.5413439768963144e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.103702101725503e-05, + "first_to_last_ratio": 3.8954992147916703, + "imbalance_abs_log_ratio": 1.3598218392207078, + "last_quartile_mean": 7.967405281306128e-06, + "mean": 1.931944555622067e-05, + "normalized": [ + 1.6032842527864015, + 1.5688395889631517, + 1.533796016047905, + 1.3882386982091592, + 1.9400717090010198, + 1.7314283899901333, + 1.585633146781874, + 1.5008455649726484, + 1.675406616182152, + 1.4714366899853428, + 1.3716626344955323, + 1.1913524891080052, + 1.2645012175425632, + 1.1767240803978434, + 1.0170067664778755, + 0.9256058574210715, + 0.9711818883737663, + 0.8683475684259824, + 0.7494840741626005, + 0.6894104604993242, + 0.7371611971197432, + 0.6427392384511877, + 0.5847748939175224, + 0.5118396580731392, + 0.5189878230973387, + 0.5013130375398106, + 0.4483794446409543, + 0.404418519263501, + 0.386302743223401, + 0.36748312424709273, + 0.35304323396633686, + 0.31929937663562014 + ], + "peak_layer": 5, + "peak_normalized": 1.9400717090010198, + "population_cv": 0.4898474013710311, + "rest_mean": 2.0753733493890008e-05, + "spike_contrast": 0.5576967968781986, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1574290692806243e-05, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.0974562832852826e-05, + 3.0309111025417224e-05, + 2.9632088626385666e-05, + 2.682000194909051e-05, + 3.74811097572092e-05, + 3.345023651490919e-05, + 3.063355325139128e-05, + 2.8995504180784337e-05, + 3.236792690586299e-05, + 2.8427341021597385e-05, + 2.649976158863865e-05, + 2.3016269551590085e-05, + 2.44294624280883e-05, + 2.2733656805939972e-05, + 1.9648006855277345e-05, + 1.7882191968965344e-05, + 1.876269561762456e-05, + 1.677599357208237e-05, + 1.4479616766038816e-05, + 1.3319027857505716e-05, + 1.4241545613913331e-05, + 1.2417365724104457e-05, + 1.1297526725684293e-05, + 9.888458407658618e-06, + 1.0026556992670521e-05, + 9.68508993537398e-06, + 8.662442269269377e-06, + 7.813141564838588e-06, + 7.46315481592319e-06, + 7.099570211721584e-06, + 6.820599537604721e-06, + 6.168686923047062e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.027523940661922097, + "first_to_last_ratio": 3.8954991154967806, + "imbalance_abs_log_ratio": 1.3598218137310631, + "last_quartile_mean": 0.007065574871376157, + "mean": 0.017132677690824494, + "normalized": [ + 1.6032842798207116, + 1.5688395364239456, + 1.5337959695585364, + 1.3882387052917204, + 1.940071709225884, + 1.7314283707937437, + 1.5856331209472507, + 1.5008455668619336, + 1.6754066305691406, + 1.4714366809574604, + 1.3716626672028296, + 1.1913525502963036, + 1.2645012070621602, + 1.176724101825654, + 1.0170067499274977, + 0.9256058277235238, + 0.9711818600927834, + 0.8683475735699788, + 0.7494840504534797, + 0.6894104671183356, + 0.7371612027997355, + 0.6427392553352459, + 0.5847748722004306, + 0.5118396849118217, + 0.5189878410918901, + 0.5013130358668625, + 0.44837945479745345, + 0.40441852259963446, + 0.38630275368745787, + 0.36748313889002426, + 0.3530432360984768, + 0.31929937599809327 + ], + "peak_layer": 5, + "peak_normalized": 1.940071709225884, + "population_cv": 0.4898473968405949, + "rest_mean": 0.018404618560991907, + "spike_contrast": 0.5576968062611827, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01026419699192047, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.027468552812933922, + 0.026878422126173973, + 0.02627803198993206, + 0.023784246295690536, + 0.033238623291254044, + 0.02966400422155857, + 0.02716614119708538, + 0.02571350336074829, + 0.02870420180261135, + 0.02520965039730072, + 0.02350025437772274, + 0.020411059260368347, + 0.021664291620254517, + 0.020160434767603874, + 0.017424048855900764, + 0.015858106315135956, + 0.016638945788145065, + 0.014877119101583958, + 0.01284066867083311, + 0.011811447329819202, + 0.012629545293748379, + 0.011011844500899315, + 0.010018759407103062, + 0.008769184350967407, + 0.008891651406884193, + 0.008588834665715694, + 0.00768194068223238, + 0.006928772199898958, + 0.0066184005700051785, + 0.0062959701754152775, + 0.006048575975000858, + 0.005470453295856714 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0978330894649844e-06, + "first_to_last_ratio": 3.7123595485838843, + "imbalance_abs_log_ratio": 1.3116676713818918, + "last_quartile_mean": 5.650942647150714e-07, + "mean": 1.3341314888393185e-06, + "normalized": [ + 1.5241148267186329, + 1.5504517836489524, + 1.525800361966804, + 1.340018587688921, + 1.9164903896693077, + 1.732090277095866, + 1.5197216980945576, + 1.4707803330175575, + 1.6808225666055663, + 1.5059009907724967, + 1.3766280212942543, + 1.196783029382642, + 1.2615435432700803, + 1.196050358398629, + 1.046816748775451, + 0.9373627935785216, + 0.9843932320660254, + 0.8775573102072941, + 0.7634158116096246, + 0.6842432896373823, + 0.7549093975240115, + 0.6636832533885216, + 0.5908781007193045, + 0.5110061412689536, + 0.5365346292575633, + 0.5196779133499698, + 0.4612664477302504, + 0.4092178350653177, + 0.3933915708760512, + 0.3701864458066216, + 0.3616807560410091, + 0.33658155547385904 + ], + "peak_layer": 5, + "peak_normalized": 1.9164903896693077, + "population_cv": 0.4775581328679472, + "rest_mean": 1.4301389744223006e-06, + "spike_contrast": 0.570357903168614, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.156910666912154e-07, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.0333695829322096e-06, + 2.068506546493154e-06, + 2.035618308582343e-06, + 1.7877609934657812e-06, + 2.5568501769157592e-06, + 2.3108361801860156e-06, + 2.0275085717003094e-06, + 1.9622143554443028e-06, + 2.242438313260209e-06, + 2.009069930863916e-06, + 1.8366027916272287e-06, + 1.596665924807894e-06, + 1.6830649656185415e-06, + 1.5956884453771636e-06, + 1.3965911875857273e-06, + 1.2505652193794958e-06, + 1.3133100082995952e-06, + 1.170776840808685e-06, + 1.018497073346225e-06, + 9.128705187322339e-07, + 1.0071483984575025e-06, + 8.85440726960951e-07, + 7.883090802351944e-07, + 6.817493840571842e-07, + 7.158077437452448e-07, + 6.933186682545056e-07, + 6.153900926619826e-07, + 5.45950399555295e-07, + 5.248360821497045e-07, + 4.938773940921237e-07, + 4.825296855415218e-07, + 4.4904405172019324e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.398965625616256e-05, + "first_to_last_ratio": 3.369027618649736, + "imbalance_abs_log_ratio": 1.2146241622684435, + "last_quartile_mean": 7.120646955627308e-06, + "mean": 1.5399188356468585e-05, + "normalized": [ + 1.5913426512839013, + 1.5418831400755482, + 1.5021141560062705, + 1.3562195006410476, + 1.8805635081013792, + 1.6560198739348433, + 1.512688238851242, + 1.4219848670303201, + 1.6092469390600324, + 1.4165297374033572, + 1.3242582904493114, + 1.1412781561654184, + 1.2305966607275267, + 1.1486603349616307, + 1.006300256787253, + 0.9179301571399356, + 0.9843528747997042, + 0.8966403626677242, + 0.7866772482945813, + 0.7196405302926673, + 0.7789613145990362, + 0.6869940328779693, + 0.6281068759775288, + 0.5617779475196961, + 0.5746692969502841, + 0.5501909649183, + 0.500671447520295, + 0.45822743358408635, + 0.4365896209302579, + 0.4181501204665046, + 0.398078906415543, + 0.36265455356680404 + ], + "peak_layer": 5, + "peak_normalized": 1.8805635081013792, + "population_cv": 0.4466412974933768, + "rest_mean": 1.640840013418778e-05, + "spike_contrast": 0.6063628797090791, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.949444756784941e-06, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.45053852268029e-05, + 2.37437488976866e-05, + 2.3131338821258396e-05, + 2.088467954308726e-05, + 2.8959151677554473e-05, + 2.5501361960778013e-05, + 2.3294171114685014e-05, + 2.1897412807447836e-05, + 2.478109672665596e-05, + 2.181340823881328e-05, + 2.0392502847244032e-05, + 1.7574757293914445e-05, + 1.895018976938445e-05, + 1.7688436855678447e-05, + 1.5496207197429612e-05, + 1.4135379387880675e-05, + 1.5158235328271985e-05, + 1.3807533832732588e-05, + 1.2114191122236662e-05, + 1.108188007492572e-05, + 1.1995372005912941e-05, + 1.0579150512057822e-05, + 9.672336091171019e-06, + 8.650924428366125e-06, + 8.849440746416803e-06, + 8.472494300804101e-06, + 7.7099339250708e-06, + 7.056330559862545e-06, + 6.72312580718426e-06, + 6.439172466343734e-06, + 6.130092060629977e-06, + 5.584585778706241e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7287268860854965e-05, + "first_to_last_ratio": 2.7170527496823964, + "imbalance_abs_log_ratio": 0.9995477449345839, + "last_quartile_mean": 6.362507633639325e-06, + "mean": 1.1818911630712137e-05, + "normalized": [ + 1.5531683121051287, + 1.4809645584609277, + 1.4249957798471882, + 1.2911504637898432, + 1.7654831615088196, + 1.5114625239638928, + 1.4072245252680236, + 1.266979301315116, + 1.5685685065032353, + 1.3590910493816417, + 1.257358469917724, + 1.0915032257961577, + 1.1980939699800082, + 1.1009846962306815, + 1.0217422520554849, + 0.917572741077425, + 1.0166318370670364, + 0.924735094262937, + 0.841915821310778, + 0.7538749370193452, + 0.8515117201007877, + 0.7457322090867645, + 0.703238735238229, + 0.6393538153043691, + 0.6469780758226346, + 0.6196156550317132, + 0.5947215623716378, + 0.5572499311933502, + 0.511086746910104, + 0.5033639872021801, + 0.45209119593165065, + 0.4215551389451849 + ], + "peak_layer": 5, + "peak_normalized": 1.7654831615088196, + "population_cv": 0.3798574342343211, + "rest_mean": 1.2437515848702148e-05, + "spike_contrast": 0.6816834612878747, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.478448853566078e-06, + "top_five_layers": [ + 5, + 9, + 1, + 6, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8356759028392844e-05, + 1.7503389244666323e-05, + 1.6841899196151644e-05, + 1.5259993233485147e-05, + 2.086608947138302e-05, + 1.7863842003862374e-05, + 1.6631862308713607e-05, + 1.4974316400184762e-05, + 1.8538772565079853e-05, + 1.6062977010733448e-05, + 1.4860608644085005e-05, + 1.2900380170322023e-05, + 1.4160166756482795e-05, + 1.301244083151687e-05, + 1.2075881386408582e-05, + 1.0844711141544394e-05, + 1.2015481843263842e-05, + 1.092936236091191e-05, + 9.950528692570515e-06, + 8.909981261240318e-06, + 1.0063941772386897e-05, + 8.813743079372216e-06, + 8.311516467074398e-06, + 7.556466243840987e-06, + 7.646576705155894e-06, + 7.323182671825634e-06, + 7.028961590549443e-06, + 6.586087692994624e-06, + 6.040489097358659e-06, + 5.949214482825482e-06, + 5.343225893739145e-06, + 4.982322934665717e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.156662675493862e-05, + "first_to_last_ratio": 4.432028161582752, + "imbalance_abs_log_ratio": 1.488857303572528, + "last_quartile_mean": 1.3891298635826388e-05, + "mean": 3.828891439638937e-05, + "normalized": [ + 1.5694774864844574, + 1.533915591681154, + 1.4779364465001996, + 1.3758197901662568, + 1.9142257050168354, + 1.758664969175181, + 1.6344745902568742, + 1.5990784946964705, + 1.7383903346505918, + 1.5307661661772007, + 1.4323492558141884, + 1.2429043804879258, + 1.3172537017266024, + 1.2162602046192026, + 1.0552102311500136, + 0.9826538220680813, + 1.0107614019133901, + 0.8826720328909949, + 0.739013117314438, + 0.7006180982562433, + 0.737790003472691, + 0.6192970376209981, + 0.5556544994109884, + 0.47239582046029627, + 0.4740970917174548, + 0.4544292159121539, + 0.39665333426527116, + 0.35034805233110755, + 0.3379110425614577, + 0.30972196454961737, + 0.31015209244222897, + 0.26910402420943297 + ], + "peak_layer": 5, + "peak_normalized": 1.9142257050168354, + "population_cv": 0.5160505486771831, + "rest_mean": 4.132475101801834e-05, + "spike_contrast": 0.5298373517131695, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1895396639592945e-05, + "top_five_layers": [ + 5, + 6, + 9, + 7, + 8 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.009358912706375e-05, + 5.873196278116666e-05, + 5.658858208335005e-05, + 5.2678646170534194e-05, + 7.329362415475771e-05, + 6.733737245667726e-05, + 6.258225766941905e-05, + 6.122697959654033e-05, + 6.656107871094719e-05, + 5.861137469764799e-05, + 5.484309804160148e-05, + 4.758945942739956e-05, + 5.04362142237369e-05, + 4.6569282858399674e-05, + 4.0402854210697114e-05, + 3.76247480744496e-05, + 3.870095679303631e-05, + 3.379655390745029e-05, + 2.8296009986661375e-05, + 2.682590638869442e-05, + 2.8249178285477683e-05, + 2.3712211259407923e-05, + 2.1275407561915927e-05, + 1.8087523130816408e-05, + 1.8152662960346788e-05, + 1.7399601347278804e-05, + 1.5187425560725387e-05, + 1.3414446584647521e-05, + 1.293824698223034e-05, + 1.1858917787321843e-05, + 1.1875386917381547e-05, + 1.0303700946678873e-05 + ] + } + } + } + } + } + }, + "step": 100 + }, + { + "modes": { + "learned": { + "bits_per_byte": 3.0609520355704922, + "forward": { + "activation_sha256": { + "attention_branch_output": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", + "mlp_branch_output": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", + "post_attention_state": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", + "post_mlp_state": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", + "pre_attention_input": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", + "pre_mlp_input": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d" + }, + "logits_sha256": "b2134e9ad4fc330ab7158864584c7289a039b641b1626767dcba8e1a49e62a3d", + "loss_nats": 2.121690273284912, + "mixer_summary_sha256": "24302def7a1bb63aa3c7a0b6a721bf1efbd647790e9d6d8427706d94cca54b03" + }, + "loss_nats": 2.121690273284912, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.658259391784668, + "group": 1, + "latest_source_mass": 0.6112743616104126, + "layer": 1, + "max_source_mass": 0.6112743616104126, + "mixer_index": 1, + "normalized_entropy": 0.9496675601463258, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3887256383895874, + "median": 0.36817848682403564, + "p05": 0.2958398759365082, + "p95": 0.519643247127533 + }, + { + "label": "current_group_partial", + "mean": 0.6112743616104126, + "median": 0.6318215131759644, + "p05": 0.48035678267478943, + "p95": 0.7041600942611694 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6694332361221313, + "group": 1, + "latest_source_mass": 0.5721787214279175, + "layer": 2, + "max_source_mass": 0.5721787214279175, + "mixer_index": 2, + "normalized_entropy": 0.9657880099596494, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42782124876976013, + "median": 0.40344932675361633, + "p05": 0.3339918255805969, + "p95": 0.5918412208557129 + }, + { + "label": "current_group_partial", + "mean": 0.5721787214279175, + "median": 0.596550703048706, + "p05": 0.4081588089466095, + "p95": 0.6660082340240479 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6890689730644226, + "group": 1, + "latest_source_mass": 0.4954703748226166, + "layer": 2, + "max_source_mass": 0.5045296549797058, + "mixer_index": 3, + "normalized_entropy": 0.9941163902704933, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5045296549797058, + "median": 0.50140380859375, + "p05": 0.4251278042793274, + "p95": 0.583143949508667 + }, + { + "label": "current_group_partial", + "mean": 0.4954703748226166, + "median": 0.49859619140625, + "p05": 0.41685599088668823, + "p95": 0.5748721957206726 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6740086078643799, + "group": 1, + "latest_source_mass": 0.5477453470230103, + "layer": 3, + "max_source_mass": 0.5477453470230103, + "mixer_index": 4, + "normalized_entropy": 0.9723888760824149, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.45225459337234497, + "median": 0.42421072721481323, + "p05": 0.3554007411003113, + "p95": 0.6163875460624695 + }, + { + "label": "current_group_partial", + "mean": 0.5477453470230103, + "median": 0.575789213180542, + "p05": 0.3836124539375305, + "p95": 0.644599199295044 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.679557204246521, + "group": 1, + "latest_source_mass": 0.5538821220397949, + "layer": 3, + "max_source_mass": 0.5538821220397949, + "mixer_index": 5, + "normalized_entropy": 0.9803938085668243, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4461178183555603, + "median": 0.43929627537727356, + "p05": 0.3631950318813324, + "p95": 0.562574565410614 + }, + { + "label": "current_group_partial", + "mean": 0.5538821220397949, + "median": 0.5607037544250488, + "p05": 0.4374254047870636, + "p95": 0.63680499792099 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6740646362304688, + "group": 1, + "latest_source_mass": 0.5581084489822388, + "layer": 4, + "max_source_mass": 0.5581084489822388, + "mixer_index": 6, + "normalized_entropy": 0.9724697079283204, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.44189155101776123, + "median": 0.42476987838745117, + "p05": 0.33982762694358826, + "p95": 0.5899009108543396 + }, + { + "label": "current_group_partial", + "mean": 0.5581084489822388, + "median": 0.5752301216125488, + "p05": 0.410099059343338, + "p95": 0.6601723432540894 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6792672276496887, + "group": 1, + "latest_source_mass": 0.5544062852859497, + "layer": 4, + "max_source_mass": 0.5544062852859497, + "mixer_index": 7, + "normalized_entropy": 0.9799754607686005, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4455936849117279, + "median": 0.44227513670921326, + "p05": 0.348201721906662, + "p95": 0.5532940626144409 + }, + { + "label": "current_group_partial", + "mean": 0.5544062852859497, + "median": 0.5577248334884644, + "p05": 0.4467058777809143, + "p95": 0.6517983078956604 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674970269203186, + "group": 2, + "latest_source_mass": 0.5532590746879578, + "layer": 5, + "max_source_mass": 0.5532590746879578, + "mixer_index": 8, + "normalized_entropy": 0.9737762601269252, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.44674086570739746, + "median": 0.43155425786972046, + "p05": 0.3413114547729492, + "p95": 0.5883275270462036 + }, + { + "label": "completed_group_1", + "mean": 0.5532590746879578, + "median": 0.5684457421302795, + "p05": 0.4116724729537964, + "p95": 0.6586886048316956 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.093062162399292, + "group": 2, + "latest_source_mass": 0.35190510749816895, + "layer": 5, + "max_source_mass": 0.35190510749816895, + "mixer_index": 9, + "normalized_entropy": 0.99494805735739, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3082374334335327, + "median": 0.30379241704940796, + "p05": 0.26716136932373047, + "p95": 0.3615886867046356 + }, + { + "label": "completed_group_1", + "mean": 0.33985745906829834, + "median": 0.34154799580574036, + "p05": 0.2781605124473572, + "p95": 0.3903803527355194 + }, + { + "label": "current_group_partial", + "mean": 0.35190510749816895, + "median": 0.35370177030563354, + "p05": 0.3136623501777649, + "p95": 0.3873046040534973 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0853323936462402, + "group": 2, + "latest_source_mass": 0.37136080861091614, + "layer": 6, + "max_source_mass": 0.37136080861091614, + "mixer_index": 10, + "normalized_entropy": 0.9879121186256079, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30180174112319946, + "median": 0.28923720121383667, + "p05": 0.2442891150712967, + "p95": 0.3929156959056854 + }, + { + "label": "completed_group_1", + "mean": 0.3268374800682068, + "median": 0.3262294828891754, + "p05": 0.2930828928947449, + "p95": 0.3670434355735779 + }, + { + "label": "current_group_partial", + "mean": 0.37136080861091614, + "median": 0.38435930013656616, + "p05": 0.2483101636171341, + "p95": 0.4514022171497345 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0921990871429443, + "group": 2, + "latest_source_mass": 0.3605259656906128, + "layer": 6, + "max_source_mass": 0.3605259656906128, + "mixer_index": 11, + "normalized_entropy": 0.9941624524035314, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3034760355949402, + "median": 0.30043965578079224, + "p05": 0.2575032711029053, + "p95": 0.3658854067325592 + }, + { + "label": "completed_group_1", + "mean": 0.3359980285167694, + "median": 0.33590996265411377, + "p05": 0.29249265789985657, + "p95": 0.3839111328125 + }, + { + "label": "current_group_partial", + "mean": 0.3605259656906128, + "median": 0.3592720031738281, + "p05": 0.3142351508140564, + "p95": 0.40569764375686646 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.076724648475647, + "group": 2, + "latest_source_mass": 0.3958718776702881, + "layer": 7, + "max_source_mass": 0.3958718776702881, + "mixer_index": 12, + "normalized_entropy": 0.9800770113185262, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29931002855300903, + "median": 0.2845022678375244, + "p05": 0.24265936017036438, + "p95": 0.3946967124938965 + }, + { + "label": "completed_group_1", + "mean": 0.30481812357902527, + "median": 0.2940869927406311, + "p05": 0.26745280623435974, + "p95": 0.36952510476112366 + }, + { + "label": "current_group_partial", + "mean": 0.3958718776702881, + "median": 0.4253297448158264, + "p05": 0.24267785251140594, + "p95": 0.4795311689376831 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0856893062591553, + "group": 2, + "latest_source_mass": 0.3888435363769531, + "layer": 7, + "max_source_mass": 0.3888435363769531, + "mixer_index": 13, + "normalized_entropy": 0.988236994486361, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2801036834716797, + "median": 0.2685231566429138, + "p05": 0.24713139235973358, + "p95": 0.3503561317920685 + }, + { + "label": "completed_group_1", + "mean": 0.3310527801513672, + "median": 0.330751895904541, + "p05": 0.2913045883178711, + "p95": 0.37040621042251587 + }, + { + "label": "current_group_partial", + "mean": 0.3888435363769531, + "median": 0.3877270221710205, + "p05": 0.3418702185153961, + "p95": 0.44118839502334595 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0752873420715332, + "group": 2, + "latest_source_mass": 0.3948463201522827, + "layer": 8, + "max_source_mass": 0.3948463201522827, + "mixer_index": 14, + "normalized_entropy": 0.9787687186488199, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29463040828704834, + "median": 0.28890830278396606, + "p05": 0.23444202542304993, + "p95": 0.37455716729164124 + }, + { + "label": "completed_group_1", + "mean": 0.3105233311653137, + "median": 0.2965858280658722, + "p05": 0.2587319016456604, + "p95": 0.39421546459198 + }, + { + "label": "current_group_partial", + "mean": 0.3948463201522827, + "median": 0.4135509133338928, + "p05": 0.24363072216510773, + "p95": 0.5001816153526306 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.088913917541504, + "group": 2, + "latest_source_mass": 0.3843393623828888, + "layer": 8, + "max_source_mass": 0.3843393623828888, + "mixer_index": 15, + "normalized_entropy": 0.9911721621661782, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29400017857551575, + "median": 0.2848791778087616, + "p05": 0.2631606459617615, + "p95": 0.35448360443115234 + }, + { + "label": "completed_group_1", + "mean": 0.32166045904159546, + "median": 0.32034605741500854, + "p05": 0.2860022187232971, + "p95": 0.3604046106338501 + }, + { + "label": "current_group_partial", + "mean": 0.3843393623828888, + "median": 0.38500386476516724, + "p05": 0.336969792842865, + "p95": 0.43113112449645996 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.054490327835083, + "group": 3, + "latest_source_mass": 0.42586958408355713, + "layer": 9, + "max_source_mass": 0.42586958408355713, + "mixer_index": 16, + "normalized_entropy": 0.959838460494086, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2850497364997864, + "median": 0.2810911536216736, + "p05": 0.21189084649085999, + "p95": 0.37678587436676025 + }, + { + "label": "completed_group_1", + "mean": 0.2890806794166565, + "median": 0.26693958044052124, + "p05": 0.21384978294372559, + "p95": 0.4104633927345276 + }, + { + "label": "completed_group_2", + "mean": 0.42586958408355713, + "median": 0.45180952548980713, + "p05": 0.2289215475320816, + "p95": 0.5656958818435669 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3700100183486938, + "group": 3, + "latest_source_mass": 0.28284209966659546, + "layer": 9, + "max_source_mass": 0.28284209966659546, + "mixer_index": 17, + "normalized_entropy": 0.9882533297199292, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21610885858535767, + "median": 0.21061542630195618, + "p05": 0.18654318153858185, + "p95": 0.26685822010040283 + }, + { + "label": "completed_group_1", + "mean": 0.22824729979038239, + "median": 0.22523541748523712, + "p05": 0.19356240332126617, + "p95": 0.2682129740715027 + }, + { + "label": "completed_group_2", + "mean": 0.2728016972541809, + "median": 0.27193963527679443, + "p05": 0.21500371396541595, + "p95": 0.3283920884132385 + }, + { + "label": "current_group_partial", + "mean": 0.28284209966659546, + "median": 0.2946581244468689, + "p05": 0.19060052931308746, + "p95": 0.3525015413761139 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3620216846466064, + "group": 3, + "latest_source_mass": 0.2944645285606384, + "layer": 10, + "max_source_mass": 0.2944645285606384, + "mixer_index": 18, + "normalized_entropy": 0.9824909650114454, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22791308164596558, + "median": 0.22349298000335693, + "p05": 0.18910717964172363, + "p95": 0.2777109742164612 + }, + { + "label": "completed_group_1", + "mean": 0.2214205265045166, + "median": 0.19864881038665771, + "p05": 0.15984180569648743, + "p95": 0.34694117307662964 + }, + { + "label": "completed_group_2", + "mean": 0.2562018632888794, + "median": 0.2503240704536438, + "p05": 0.20511329174041748, + "p95": 0.32819920778274536 + }, + { + "label": "current_group_partial", + "mean": 0.2944645285606384, + "median": 0.30793535709381104, + "p05": 0.16727104783058167, + "p95": 0.3644053339958191 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3784711360931396, + "group": 3, + "latest_source_mass": 0.2763957977294922, + "layer": 10, + "max_source_mass": 0.2763957977294922, + "mixer_index": 19, + "normalized_entropy": 0.994356736025074, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23347607254981995, + "median": 0.23056401312351227, + "p05": 0.20095914602279663, + "p95": 0.2848108112812042 + }, + { + "label": "completed_group_1", + "mean": 0.23307976126670837, + "median": 0.2289559543132782, + "p05": 0.205013245344162, + "p95": 0.2760506868362427 + }, + { + "label": "completed_group_2", + "mean": 0.2570483684539795, + "median": 0.2558031678199768, + "p05": 0.21930555999279022, + "p95": 0.29454779624938965 + }, + { + "label": "current_group_partial", + "mean": 0.2763957977294922, + "median": 0.2743210792541504, + "p05": 0.2263108193874359, + "p95": 0.3325941264629364 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3619427680969238, + "group": 3, + "latest_source_mass": 0.302674800157547, + "layer": 11, + "max_source_mass": 0.302674800157547, + "mixer_index": 20, + "normalized_entropy": 0.9824340387540098, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24715131521224976, + "median": 0.24610307812690735, + "p05": 0.21518638730049133, + "p95": 0.2835894525051117 + }, + { + "label": "completed_group_1", + "mean": 0.22003203630447388, + "median": 0.19626209139823914, + "p05": 0.16581544280052185, + "p95": 0.3317301273345947 + }, + { + "label": "completed_group_2", + "mean": 0.23014184832572937, + "median": 0.2278168797492981, + "p05": 0.19933417439460754, + "p95": 0.272026926279068 + }, + { + "label": "current_group_partial", + "mean": 0.302674800157547, + "median": 0.3237387239933014, + "p05": 0.16210556030273438, + "p95": 0.38430726528167725 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3804662227630615, + "group": 3, + "latest_source_mass": 0.28360050916671753, + "layer": 11, + "max_source_mass": 0.28360050916671753, + "mixer_index": 21, + "normalized_entropy": 0.995795886847494, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24302780628204346, + "median": 0.24201302230358124, + "p05": 0.21055130660533905, + "p95": 0.276764452457428 + }, + { + "label": "completed_group_1", + "mean": 0.23108476400375366, + "median": 0.22807812690734863, + "p05": 0.20644722878932953, + "p95": 0.2614594101905823 + }, + { + "label": "completed_group_2", + "mean": 0.24228686094284058, + "median": 0.2414286881685257, + "p05": 0.21957996487617493, + "p95": 0.2676010727882385 + }, + { + "label": "current_group_partial", + "mean": 0.28360050916671753, + "median": 0.2854132354259491, + "p05": 0.246682807803154, + "p95": 0.3173881471157074 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3502743244171143, + "group": 3, + "latest_source_mass": 0.31918564438819885, + "layer": 12, + "max_source_mass": 0.31918564438819885, + "mixer_index": 22, + "normalized_entropy": 0.974017035838133, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2455797642469406, + "median": 0.2419765591621399, + "p05": 0.2023906707763672, + "p95": 0.29628193378448486 + }, + { + "label": "completed_group_1", + "mean": 0.21441738307476044, + "median": 0.19096502661705017, + "p05": 0.1489589512348175, + "p95": 0.3387751877307892 + }, + { + "label": "completed_group_2", + "mean": 0.2208172082901001, + "median": 0.21868902444839478, + "p05": 0.19425739347934723, + "p95": 0.25476256012916565 + }, + { + "label": "current_group_partial", + "mean": 0.31918564438819885, + "median": 0.34166574478149414, + "p05": 0.153193861246109, + "p95": 0.42244112491607666 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3820215463638306, + "group": 3, + "latest_source_mass": 0.27603140473365784, + "layer": 12, + "max_source_mass": 0.27603140473365784, + "mixer_index": 23, + "normalized_entropy": 0.9969178156703975, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24522465467453003, + "median": 0.24368876218795776, + "p05": 0.22978872060775757, + "p95": 0.2647436261177063 + }, + { + "label": "completed_group_1", + "mean": 0.22846508026123047, + "median": 0.22824203968048096, + "p05": 0.21080705523490906, + "p95": 0.24792201817035675 + }, + { + "label": "completed_group_2", + "mean": 0.25027889013290405, + "median": 0.25114861130714417, + "p05": 0.2239076942205429, + "p95": 0.27430635690689087 + }, + { + "label": "current_group_partial", + "mean": 0.27603140473365784, + "median": 0.2767183184623718, + "p05": 0.23810210824012756, + "p95": 0.31143108010292053 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3491160869598389, + "group": 4, + "latest_source_mass": 0.3227139413356781, + "layer": 13, + "max_source_mass": 0.3227139413356781, + "mixer_index": 24, + "normalized_entropy": 0.9731815441202416, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24498310685157776, + "median": 0.24184581637382507, + "p05": 0.20811651647090912, + "p95": 0.2852743864059448 + }, + { + "label": "completed_group_1", + "mean": 0.21178749203681946, + "median": 0.1895410120487213, + "p05": 0.14949846267700195, + "p95": 0.3285062313079834 + }, + { + "label": "completed_group_2", + "mean": 0.2205154001712799, + "median": 0.2162817418575287, + "p05": 0.18882519006729126, + "p95": 0.26559314131736755 + }, + { + "label": "completed_group_3", + "mean": 0.3227139413356781, + "median": 0.34892457723617554, + "p05": 0.15726026892662048, + "p95": 0.42967018485069275 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6032840013504028, + "group": 4, + "latest_source_mass": 0.22103822231292725, + "layer": 13, + "max_source_mass": 0.22103822231292725, + "mixer_index": 25, + "normalized_entropy": 0.9961763600595251, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20074918866157532, + "median": 0.20095838606357574, + "p05": 0.18337497115135193, + "p95": 0.2181507796049118 + }, + { + "label": "completed_group_1", + "mean": 0.17959120869636536, + "median": 0.17876555025577545, + "p05": 0.1576874852180481, + "p95": 0.20805387198925018 + }, + { + "label": "completed_group_2", + "mean": 0.18619459867477417, + "median": 0.18535691499710083, + "p05": 0.1644289195537567, + "p95": 0.21231110394001007 + }, + { + "label": "completed_group_3", + "mean": 0.21242676675319672, + "median": 0.2123909741640091, + "p05": 0.19269250333309174, + "p95": 0.23113366961479187 + }, + { + "label": "current_group_partial", + "mean": 0.22103822231292725, + "median": 0.2211262434720993, + "p05": 0.1813979148864746, + "p95": 0.26087528467178345 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5699318647384644, + "group": 4, + "latest_source_mass": 0.2694075107574463, + "layer": 14, + "max_source_mass": 0.2694075107574463, + "mixer_index": 26, + "normalized_entropy": 0.9754535124403232, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1938496232032776, + "median": 0.19102387130260468, + "p05": 0.16569040715694427, + "p95": 0.23268038034439087 + }, + { + "label": "completed_group_1", + "mean": 0.17019829154014587, + "median": 0.14846302568912506, + "p05": 0.11795832216739655, + "p95": 0.2854715883731842 + }, + { + "label": "completed_group_2", + "mean": 0.1625727415084839, + "median": 0.15400215983390808, + "p05": 0.1330171525478363, + "p95": 0.22402943670749664 + }, + { + "label": "completed_group_3", + "mean": 0.20397180318832397, + "median": 0.2051275074481964, + "p05": 0.14910343289375305, + "p95": 0.25531816482543945 + }, + { + "label": "current_group_partial", + "mean": 0.2694075107574463, + "median": 0.2924001216888428, + "p05": 0.12448020279407501, + "p95": 0.3373839557170868 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6068798303604126, + "group": 4, + "latest_source_mass": 0.21404394507408142, + "layer": 14, + "max_source_mass": 0.21404394507408142, + "mixer_index": 27, + "normalized_entropy": 0.9984105742421472, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19717517495155334, + "median": 0.19712674617767334, + "p05": 0.18285326659679413, + "p95": 0.21078822016716003 + }, + { + "label": "completed_group_1", + "mean": 0.1942940056324005, + "median": 0.19466334581375122, + "p05": 0.18043261766433716, + "p95": 0.20777744054794312 + }, + { + "label": "completed_group_2", + "mean": 0.19104912877082825, + "median": 0.18969203531742096, + "p05": 0.17138653993606567, + "p95": 0.2142438441514969 + }, + { + "label": "completed_group_3", + "mean": 0.20343773066997528, + "median": 0.2029801905155182, + "p05": 0.18992920219898224, + "p95": 0.2184751033782959 + }, + { + "label": "current_group_partial", + "mean": 0.21404394507408142, + "median": 0.21432773768901825, + "p05": 0.1852298229932785, + "p95": 0.24286989867687225 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5761134624481201, + "group": 4, + "latest_source_mass": 0.26560431718826294, + "layer": 15, + "max_source_mass": 0.26560431718826294, + "mixer_index": 28, + "normalized_entropy": 0.979294355048726, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1940101534128189, + "median": 0.19362978637218475, + "p05": 0.1631820648908615, + "p95": 0.2295525223016739 + }, + { + "label": "completed_group_1", + "mean": 0.18250693380832672, + "median": 0.16429191827774048, + "p05": 0.13507941365242004, + "p95": 0.2775976359844208 + }, + { + "label": "completed_group_2", + "mean": 0.16849923133850098, + "median": 0.16119162738323212, + "p05": 0.1375115066766739, + "p95": 0.2266949564218521 + }, + { + "label": "completed_group_3", + "mean": 0.18937936425209045, + "median": 0.18853023648262024, + "p05": 0.1678447425365448, + "p95": 0.21501585841178894 + }, + { + "label": "current_group_partial", + "mean": 0.26560431718826294, + "median": 0.28881219029426575, + "p05": 0.117766834795475, + "p95": 0.34617751836776733 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6027262210845947, + "group": 4, + "latest_source_mass": 0.22770702838897705, + "layer": 15, + "max_source_mass": 0.22770702838897705, + "mixer_index": 29, + "normalized_entropy": 0.9958297916945706, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20452985167503357, + "median": 0.20441509783267975, + "p05": 0.17931760847568512, + "p95": 0.2289508432149887 + }, + { + "label": "completed_group_1", + "mean": 0.18702909350395203, + "median": 0.186691552400589, + "p05": 0.1675943285226822, + "p95": 0.20997506380081177 + }, + { + "label": "completed_group_2", + "mean": 0.17977532744407654, + "median": 0.177423357963562, + "p05": 0.1569271981716156, + "p95": 0.20937731862068176 + }, + { + "label": "completed_group_3", + "mean": 0.20095868408679962, + "median": 0.2005838304758072, + "p05": 0.1845097541809082, + "p95": 0.21870870888233185 + }, + { + "label": "current_group_partial", + "mean": 0.22770702838897705, + "median": 0.22757767140865326, + "p05": 0.18680104613304138, + "p95": 0.2673209607601166 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.577937126159668, + "group": 4, + "latest_source_mass": 0.2687118649482727, + "layer": 16, + "max_source_mass": 0.2687118649482727, + "mixer_index": 30, + "normalized_entropy": 0.9804274610215993, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1962222456932068, + "median": 0.19565686583518982, + "p05": 0.16995516419410706, + "p95": 0.22565382719039917 + }, + { + "label": "completed_group_1", + "mean": 0.1818992793560028, + "median": 0.16701740026474, + "p05": 0.138178750872612, + "p95": 0.2653764486312866 + }, + { + "label": "completed_group_2", + "mean": 0.17167484760284424, + "median": 0.16360890865325928, + "p05": 0.14424335956573486, + "p95": 0.22631293535232544 + }, + { + "label": "completed_group_3", + "mean": 0.18149182200431824, + "median": 0.18099229037761688, + "p05": 0.16264864802360535, + "p95": 0.20300358533859253 + }, + { + "label": "current_group_partial", + "mean": 0.2687118649482727, + "median": 0.29608893394470215, + "p05": 0.1254507452249527, + "p95": 0.339657723903656 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5977532863616943, + "group": 4, + "latest_source_mass": 0.23852142691612244, + "layer": 16, + "max_source_mass": 0.23852142691612244, + "mixer_index": 31, + "normalized_entropy": 0.9927399336239481, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21533392369747162, + "median": 0.21377520263195038, + "p05": 0.1841924786567688, + "p95": 0.24801591038703918 + }, + { + "label": "completed_group_1", + "mean": 0.18402470648288727, + "median": 0.18508511781692505, + "p05": 0.15855011343955994, + "p95": 0.21014195680618286 + }, + { + "label": "completed_group_2", + "mean": 0.17019468545913696, + "median": 0.16691477596759796, + "p05": 0.1446647047996521, + "p95": 0.20511509478092194 + }, + { + "label": "completed_group_3", + "mean": 0.1919252574443817, + "median": 0.191789448261261, + "p05": 0.17424410581588745, + "p95": 0.20975348353385925 + }, + { + "label": "current_group_partial", + "mean": 0.23852142691612244, + "median": 0.24018850922584534, + "p05": 0.18989041447639465, + "p95": 0.27930569648742676 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.578121542930603, + "group": 5, + "latest_source_mass": 0.26854145526885986, + "layer": 17, + "max_source_mass": 0.26854145526885986, + "mixer_index": 32, + "normalized_entropy": 0.9805420456038999, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20422279834747314, + "median": 0.2033933699131012, + "p05": 0.17994040250778198, + "p95": 0.23156529664993286 + }, + { + "label": "completed_group_1", + "mean": 0.1835813671350479, + "median": 0.17072421312332153, + "p05": 0.14088554680347443, + "p95": 0.26194027066230774 + }, + { + "label": "completed_group_2", + "mean": 0.16642534732818604, + "median": 0.1602073311805725, + "p05": 0.13873112201690674, + "p95": 0.22039896249771118 + }, + { + "label": "completed_group_3", + "mean": 0.17722904682159424, + "median": 0.17722415924072266, + "p05": 0.1577582061290741, + "p95": 0.19813896715641022 + }, + { + "label": "completed_group_4", + "mean": 0.26854145526885986, + "median": 0.2872956693172455, + "p05": 0.12619024515151978, + "p95": 0.3464823067188263 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.782798409461975, + "group": 5, + "latest_source_mass": 0.19473549723625183, + "layer": 17, + "max_source_mass": 0.19473549723625183, + "mixer_index": 33, + "normalized_entropy": 0.99499873731939, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15961387753486633, + "median": 0.15747272968292236, + "p05": 0.12699076533317566, + "p95": 0.19599634408950806 + }, + { + "label": "completed_group_1", + "mean": 0.16475997865200043, + "median": 0.16349202394485474, + "p05": 0.1474122852087021, + "p95": 0.1848660409450531 + }, + { + "label": "completed_group_2", + "mean": 0.14714640378952026, + "median": 0.14494061470031738, + "p05": 0.12754470109939575, + "p95": 0.1751968413591385 + }, + { + "label": "completed_group_3", + "mean": 0.15768736600875854, + "median": 0.15663693845272064, + "p05": 0.13646987080574036, + "p95": 0.1821092814207077 + }, + { + "label": "completed_group_4", + "mean": 0.1760568618774414, + "median": 0.17550913989543915, + "p05": 0.15776243805885315, + "p95": 0.19570523500442505 + }, + { + "label": "current_group_partial", + "mean": 0.19473549723625183, + "median": 0.1956680715084076, + "p05": 0.14822278916835785, + "p95": 0.22800904512405396 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7707297801971436, + "group": 5, + "latest_source_mass": 0.20674791932106018, + "layer": 18, + "max_source_mass": 0.20674791932106018, + "mixer_index": 34, + "normalized_entropy": 0.9882631070787802, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16306114196777344, + "median": 0.16283683478832245, + "p05": 0.13633866608142853, + "p95": 0.19143974781036377 + }, + { + "label": "completed_group_1", + "mean": 0.1578725278377533, + "median": 0.14938931167125702, + "p05": 0.13056670129299164, + "p95": 0.21463912725448608 + }, + { + "label": "completed_group_2", + "mean": 0.139655202627182, + "median": 0.1338472068309784, + "p05": 0.11858384311199188, + "p95": 0.18817788362503052 + }, + { + "label": "completed_group_3", + "mean": 0.14299750328063965, + "median": 0.14077699184417725, + "p05": 0.12273441255092621, + "p95": 0.17249858379364014 + }, + { + "label": "completed_group_4", + "mean": 0.18966571986675262, + "median": 0.19467708468437195, + "p05": 0.13032010197639465, + "p95": 0.23041488230228424 + }, + { + "label": "current_group_partial", + "mean": 0.20674791932106018, + "median": 0.21543893218040466, + "p05": 0.11787950992584229, + "p95": 0.24730797111988068 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.781312108039856, + "group": 5, + "latest_source_mass": 0.20407646894454956, + "layer": 18, + "max_source_mass": 0.20407646894454956, + "mixer_index": 35, + "normalized_entropy": 0.9941692167014471, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15919369459152222, + "median": 0.15759795904159546, + "p05": 0.13659372925758362, + "p95": 0.19095730781555176 + }, + { + "label": "completed_group_1", + "mean": 0.1684478521347046, + "median": 0.16807237267494202, + "p05": 0.15075388550758362, + "p95": 0.1875023990869522 + }, + { + "label": "completed_group_2", + "mean": 0.14742717146873474, + "median": 0.14584535360336304, + "p05": 0.12703484296798706, + "p95": 0.1739335060119629 + }, + { + "label": "completed_group_3", + "mean": 0.1539241522550583, + "median": 0.1541770100593567, + "p05": 0.13218382000923157, + "p95": 0.1755600869655609 + }, + { + "label": "completed_group_4", + "mean": 0.1669306606054306, + "median": 0.16647151112556458, + "p05": 0.15041440725326538, + "p95": 0.18558931350708008 + }, + { + "label": "current_group_partial", + "mean": 0.20407646894454956, + "median": 0.20521500706672668, + "p05": 0.1562502235174179, + "p95": 0.24481025338172913 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7736209630966187, + "group": 5, + "latest_source_mass": 0.2125461846590042, + "layer": 19, + "max_source_mass": 0.2125461846590042, + "mixer_index": 36, + "normalized_entropy": 0.9898767069782805, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15695160627365112, + "median": 0.15683278441429138, + "p05": 0.12300770729780197, + "p95": 0.196682408452034 + }, + { + "label": "completed_group_1", + "mean": 0.16567477583885193, + "median": 0.16040010750293732, + "p05": 0.14332951605319977, + "p95": 0.20587366819381714 + }, + { + "label": "completed_group_2", + "mean": 0.1478763222694397, + "median": 0.14480885863304138, + "p05": 0.12841497361660004, + "p95": 0.18284933269023895 + }, + { + "label": "completed_group_3", + "mean": 0.14320014417171478, + "median": 0.14095112681388855, + "p05": 0.12106460332870483, + "p95": 0.17675133049488068 + }, + { + "label": "completed_group_4", + "mean": 0.17375096678733826, + "median": 0.17515000700950623, + "p05": 0.1435358226299286, + "p95": 0.19906878471374512 + }, + { + "label": "current_group_partial", + "mean": 0.2125461846590042, + "median": 0.2186175137758255, + "p05": 0.12126382440328598, + "p95": 0.2633163630962372 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.781569480895996, + "group": 5, + "latest_source_mass": 0.20039017498493195, + "layer": 19, + "max_source_mass": 0.20039017498493195, + "mixer_index": 37, + "normalized_entropy": 0.9943128592274447, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15368077158927917, + "median": 0.1525704562664032, + "p05": 0.12359238415956497, + "p95": 0.20425409078598022 + }, + { + "label": "completed_group_1", + "mean": 0.1725173443555832, + "median": 0.17324355244636536, + "p05": 0.1533699631690979, + "p95": 0.1897408664226532 + }, + { + "label": "completed_group_2", + "mean": 0.14978237450122833, + "median": 0.14767895638942719, + "p05": 0.1299348920583725, + "p95": 0.17630663514137268 + }, + { + "label": "completed_group_3", + "mean": 0.15674246847629547, + "median": 0.15763753652572632, + "p05": 0.13473035395145416, + "p95": 0.17680563032627106 + }, + { + "label": "completed_group_4", + "mean": 0.16688686609268188, + "median": 0.16692788898944855, + "p05": 0.14797358214855194, + "p95": 0.18507987260818481 + }, + { + "label": "current_group_partial", + "mean": 0.20039017498493195, + "median": 0.20044702291488647, + "p05": 0.1619955599308014, + "p95": 0.23957164585590363 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7793617248535156, + "group": 5, + "latest_source_mass": 0.20417512953281403, + "layer": 20, + "max_source_mass": 0.20417512953281403, + "mixer_index": 38, + "normalized_entropy": 0.9930806871193036, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1678246557712555, + "median": 0.17157375812530518, + "p05": 0.13025718927383423, + "p95": 0.19556939601898193 + }, + { + "label": "completed_group_1", + "mean": 0.1683025360107422, + "median": 0.16609728336334229, + "p05": 0.15072931349277496, + "p95": 0.19358061254024506 + }, + { + "label": "completed_group_2", + "mean": 0.14878366887569427, + "median": 0.1468905806541443, + "p05": 0.13322587311267853, + "p95": 0.1712985783815384 + }, + { + "label": "completed_group_3", + "mean": 0.14351718127727509, + "median": 0.1410684883594513, + "p05": 0.12425544857978821, + "p95": 0.1736954152584076 + }, + { + "label": "completed_group_4", + "mean": 0.16739682853221893, + "median": 0.16813713312149048, + "p05": 0.14868372678756714, + "p95": 0.1844603419303894 + }, + { + "label": "current_group_partial", + "mean": 0.20417512953281403, + "median": 0.20674002170562744, + "p05": 0.14016404747962952, + "p95": 0.2501453757286072 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7772380113601685, + "group": 5, + "latest_source_mass": 0.19795814156532288, + "layer": 20, + "max_source_mass": 0.19795814156532288, + "mixer_index": 39, + "normalized_entropy": 0.9918954200509164, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15367703139781952, + "median": 0.15482746064662933, + "p05": 0.11167150735855103, + "p95": 0.2207922786474228 + }, + { + "label": "completed_group_1", + "mean": 0.18459689617156982, + "median": 0.184708833694458, + "p05": 0.160706028342247, + "p95": 0.20910656452178955 + }, + { + "label": "completed_group_2", + "mean": 0.14832255244255066, + "median": 0.1467352658510208, + "p05": 0.12785059213638306, + "p95": 0.17438356578350067 + }, + { + "label": "completed_group_3", + "mean": 0.15016385912895203, + "median": 0.14991271495819092, + "p05": 0.12576226890087128, + "p95": 0.17565840482711792 + }, + { + "label": "completed_group_4", + "mean": 0.1652815043926239, + "median": 0.16595271229743958, + "p05": 0.14006288349628448, + "p95": 0.18747010827064514 + }, + { + "label": "current_group_partial", + "mean": 0.19795814156532288, + "median": 0.19790686666965485, + "p05": 0.1500549614429474, + "p95": 0.24641191959381104 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.780207633972168, + "group": 6, + "latest_source_mass": 0.20078188180923462, + "layer": 21, + "max_source_mass": 0.20078188180923462, + "mixer_index": 40, + "normalized_entropy": 0.9935527979875202, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16951784491539001, + "median": 0.1744978278875351, + "p05": 0.12857238948345184, + "p95": 0.2016112506389618 + }, + { + "label": "completed_group_1", + "mean": 0.17200809717178345, + "median": 0.17267316579818726, + "p05": 0.15072743594646454, + "p95": 0.19161207973957062 + }, + { + "label": "completed_group_2", + "mean": 0.15038041770458221, + "median": 0.14867597818374634, + "p05": 0.13418853282928467, + "p95": 0.17207789421081543 + }, + { + "label": "completed_group_3", + "mean": 0.14669056236743927, + "median": 0.14433687925338745, + "p05": 0.12725189328193665, + "p95": 0.1738971769809723 + }, + { + "label": "completed_group_4", + "mean": 0.16062124073505402, + "median": 0.16014984250068665, + "p05": 0.14366847276687622, + "p95": 0.17978575825691223 + }, + { + "label": "completed_group_5", + "mean": 0.20078188180923462, + "median": 0.20047783851623535, + "p05": 0.14466112852096558, + "p95": 0.25333014130592346 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.937079668045044, + "group": 6, + "latest_source_mass": 0.16695083677768707, + "layer": 21, + "max_source_mass": 0.16695083677768707, + "mixer_index": 41, + "normalized_entropy": 0.9954620304464951, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13008031249046326, + "median": 0.12692952156066895, + "p05": 0.10487981885671616, + "p95": 0.16793549060821533 + }, + { + "label": "completed_group_1", + "mean": 0.15163564682006836, + "median": 0.15097364783287048, + "p05": 0.13223227858543396, + "p95": 0.17222966253757477 + }, + { + "label": "completed_group_2", + "mean": 0.1323973685503006, + "median": 0.131024569272995, + "p05": 0.11479418724775314, + "p95": 0.15471340715885162 + }, + { + "label": "completed_group_3", + "mean": 0.13641127943992615, + "median": 0.13619393110275269, + "p05": 0.11877292394638062, + "p95": 0.15510697662830353 + }, + { + "label": "completed_group_4", + "mean": 0.13806137442588806, + "median": 0.1375710368156433, + "p05": 0.11978909373283386, + "p95": 0.15855638682842255 + }, + { + "label": "completed_group_5", + "mean": 0.1444631814956665, + "median": 0.1438722014427185, + "p05": 0.12040875852108002, + "p95": 0.17076769471168518 + }, + { + "label": "current_group_partial", + "mean": 0.16695083677768707, + "median": 0.17087092995643616, + "p05": 0.13015522062778473, + "p95": 0.195466548204422 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.939245343208313, + "group": 6, + "latest_source_mass": 0.16357660293579102, + "layer": 22, + "max_source_mass": 0.16357660293579102, + "mixer_index": 42, + "normalized_entropy": 0.9965749673230103, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14692580699920654, + "median": 0.1518530696630478, + "p05": 0.11718790233135223, + "p95": 0.16568630933761597 + }, + { + "label": "completed_group_1", + "mean": 0.14671877026557922, + "median": 0.14763988554477692, + "p05": 0.12195808440446854, + "p95": 0.1661134660243988 + }, + { + "label": "completed_group_2", + "mean": 0.13336658477783203, + "median": 0.13267168402671814, + "p05": 0.12229838222265244, + "p95": 0.14625124633312225 + }, + { + "label": "completed_group_3", + "mean": 0.1307726800441742, + "median": 0.12948819994926453, + "p05": 0.11789381504058838, + "p95": 0.1479169726371765 + }, + { + "label": "completed_group_4", + "mean": 0.131200909614563, + "median": 0.1309259533882141, + "p05": 0.11869271099567413, + "p95": 0.14432916045188904 + }, + { + "label": "completed_group_5", + "mean": 0.147438645362854, + "median": 0.14618173241615295, + "p05": 0.13021595776081085, + "p95": 0.16755904257297516 + }, + { + "label": "current_group_partial", + "mean": 0.16357660293579102, + "median": 0.1623307466506958, + "p05": 0.13464656472206116, + "p95": 0.1969548761844635 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9312366247177124, + "group": 6, + "latest_source_mass": 0.1666574329137802, + "layer": 22, + "max_source_mass": 0.16950437426567078, + "mixer_index": 43, + "normalized_entropy": 0.9924593001661848, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12334204465150833, + "median": 0.12106236815452576, + "p05": 0.08947738260030746, + "p95": 0.17906470596790314 + }, + { + "label": "completed_group_1", + "mean": 0.16950437426567078, + "median": 0.17181579768657684, + "p05": 0.1329171359539032, + "p95": 0.19668330252170563 + }, + { + "label": "completed_group_2", + "mean": 0.13870283961296082, + "median": 0.13818645477294922, + "p05": 0.12228117883205414, + "p95": 0.1556926816701889 + }, + { + "label": "completed_group_3", + "mean": 0.1300538182258606, + "median": 0.13022829592227936, + "p05": 0.1119839996099472, + "p95": 0.14633683860301971 + }, + { + "label": "completed_group_4", + "mean": 0.12885478138923645, + "median": 0.1284654438495636, + "p05": 0.11224459111690521, + "p95": 0.14672259986400604 + }, + { + "label": "completed_group_5", + "mean": 0.14288470149040222, + "median": 0.14101536571979523, + "p05": 0.11797811090946198, + "p95": 0.17301785945892334 + }, + { + "label": "current_group_partial", + "mean": 0.1666574329137802, + "median": 0.1637348234653473, + "p05": 0.12865713238716125, + "p95": 0.20903055369853973 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9345850944519043, + "group": 6, + "latest_source_mass": 0.16773909330368042, + "layer": 23, + "max_source_mass": 0.16773909330368042, + "mixer_index": 44, + "normalized_entropy": 0.9941800732120613, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1465124636888504, + "median": 0.14716467261314392, + "p05": 0.12261509895324707, + "p95": 0.17156414687633514 + }, + { + "label": "completed_group_1", + "mean": 0.153631791472435, + "median": 0.159563809633255, + "p05": 0.10715117305517197, + "p95": 0.18623968958854675 + }, + { + "label": "completed_group_2", + "mean": 0.13159149885177612, + "median": 0.13108083605766296, + "p05": 0.11123912036418915, + "p95": 0.1546429693698883 + }, + { + "label": "completed_group_3", + "mean": 0.13120220601558685, + "median": 0.13034585118293762, + "p05": 0.11503913998603821, + "p95": 0.1493607610464096 + }, + { + "label": "completed_group_4", + "mean": 0.12539167702198029, + "median": 0.12449093163013458, + "p05": 0.10983049124479294, + "p95": 0.14342987537384033 + }, + { + "label": "completed_group_5", + "mean": 0.14393125474452972, + "median": 0.1405339539051056, + "p05": 0.1222267672419548, + "p95": 0.17515066266059875 + }, + { + "label": "current_group_partial", + "mean": 0.16773909330368042, + "median": 0.16424956917762756, + "p05": 0.13410812616348267, + "p95": 0.21529987454414368 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9284589290618896, + "group": 6, + "latest_source_mass": 0.16532829403877258, + "layer": 23, + "max_source_mass": 0.169878289103508, + "mixer_index": 45, + "normalized_entropy": 0.9910318469730498, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12412484735250473, + "median": 0.12159939110279083, + "p05": 0.08492134511470795, + "p95": 0.18361014127731323 + }, + { + "label": "completed_group_1", + "mean": 0.169878289103508, + "median": 0.17313054203987122, + "p05": 0.1298009753227234, + "p95": 0.1993248164653778 + }, + { + "label": "completed_group_2", + "mean": 0.13807371258735657, + "median": 0.13714420795440674, + "p05": 0.12058109790086746, + "p95": 0.15811002254486084 + }, + { + "label": "completed_group_3", + "mean": 0.13034150004386902, + "median": 0.13021467626094818, + "p05": 0.11215148121118546, + "p95": 0.14708518981933594 + }, + { + "label": "completed_group_4", + "mean": 0.13028821349143982, + "median": 0.13050158321857452, + "p05": 0.1127624660730362, + "p95": 0.14778561890125275 + }, + { + "label": "completed_group_5", + "mean": 0.1419651210308075, + "median": 0.1401965618133545, + "p05": 0.11490530520677567, + "p95": 0.17368361353874207 + }, + { + "label": "current_group_partial", + "mean": 0.16532829403877258, + "median": 0.16024862229824066, + "p05": 0.1169055625796318, + "p95": 0.22413846850395203 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9329564571380615, + "group": 6, + "latest_source_mass": 0.17254900932312012, + "layer": 24, + "max_source_mass": 0.17254900932312012, + "mixer_index": 46, + "normalized_entropy": 0.9933431191961559, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1454196274280548, + "median": 0.145169198513031, + "p05": 0.12175656855106354, + "p95": 0.17354263365268707 + }, + { + "label": "completed_group_1", + "mean": 0.1566689908504486, + "median": 0.16300347447395325, + "p05": 0.11168468743562698, + "p95": 0.18563294410705566 + }, + { + "label": "completed_group_2", + "mean": 0.1286500096321106, + "median": 0.12701794505119324, + "p05": 0.11111085116863251, + "p95": 0.15331649780273438 + }, + { + "label": "completed_group_3", + "mean": 0.12666445970535278, + "median": 0.12615132331848145, + "p05": 0.11176232993602753, + "p95": 0.14301280677318573 + }, + { + "label": "completed_group_4", + "mean": 0.12782414257526398, + "median": 0.1281316727399826, + "p05": 0.11212349683046341, + "p95": 0.14346082508563995 + }, + { + "label": "completed_group_5", + "mean": 0.14222374558448792, + "median": 0.14010348916053772, + "p05": 0.12181462347507477, + "p95": 0.16952435672283173 + }, + { + "label": "current_group_partial", + "mean": 0.17254900932312012, + "median": 0.1675749123096466, + "p05": 0.1353597342967987, + "p95": 0.2285366952419281 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9259672164916992, + "group": 6, + "latest_source_mass": 0.16209754347801208, + "layer": 24, + "max_source_mass": 0.17791953682899475, + "mixer_index": 47, + "normalized_entropy": 0.989751360013567, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12082186341285706, + "median": 0.11626945436000824, + "p05": 0.08509357273578644, + "p95": 0.18624860048294067 + }, + { + "label": "completed_group_1", + "mean": 0.17791953682899475, + "median": 0.1807878017425537, + "p05": 0.1341632455587387, + "p95": 0.2101013958454132 + }, + { + "label": "completed_group_2", + "mean": 0.1424642950296402, + "median": 0.14065685868263245, + "p05": 0.12273216992616653, + "p95": 0.16812381148338318 + }, + { + "label": "completed_group_3", + "mean": 0.13298648595809937, + "median": 0.13262662291526794, + "p05": 0.11336477845907211, + "p95": 0.15321806073188782 + }, + { + "label": "completed_group_4", + "mean": 0.12929362058639526, + "median": 0.12950879335403442, + "p05": 0.1104152500629425, + "p95": 0.1479472517967224 + }, + { + "label": "completed_group_5", + "mean": 0.1344166398048401, + "median": 0.13280367851257324, + "p05": 0.1057823970913887, + "p95": 0.16791269183158875 + }, + { + "label": "current_group_partial", + "mean": 0.16209754347801208, + "median": 0.15705879032611847, + "p05": 0.1103038564324379, + "p95": 0.22623871266841888 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9201786518096924, + "group": 7, + "latest_source_mass": 0.15931037068367004, + "layer": 25, + "max_source_mass": 0.17355048656463623, + "mixer_index": 48, + "normalized_entropy": 0.9867766262187836, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13756656646728516, + "median": 0.14409756660461426, + "p05": 0.07605770230293274, + "p95": 0.1807018518447876 + }, + { + "label": "completed_group_1", + "mean": 0.17355048656463623, + "median": 0.1850908100605011, + "p05": 0.09468921273946762, + "p95": 0.2254364788532257 + }, + { + "label": "completed_group_2", + "mean": 0.14306488633155823, + "median": 0.1434939205646515, + "p05": 0.12189669907093048, + "p95": 0.16250990331172943 + }, + { + "label": "completed_group_3", + "mean": 0.13006147742271423, + "median": 0.12742848694324493, + "p05": 0.10484378784894943, + "p95": 0.15863747894763947 + }, + { + "label": "completed_group_4", + "mean": 0.11658850312232971, + "median": 0.11437725275754929, + "p05": 0.09685036540031433, + "p95": 0.14346592128276825 + }, + { + "label": "completed_group_5", + "mean": 0.139857679605484, + "median": 0.13671082258224487, + "p05": 0.10990621894598007, + "p95": 0.17913517355918884 + }, + { + "label": "completed_group_6", + "mean": 0.15931037068367004, + "median": 0.15141257643699646, + "p05": 0.10549820214509964, + "p95": 0.2391991913318634 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.043663501739502, + "group": 7, + "latest_source_mass": 0.08876943588256836, + "layer": 25, + "max_source_mass": 0.17510183155536652, + "mixer_index": 49, + "normalized_entropy": 0.9827943997351177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10298450291156769, + "median": 0.09445159137248993, + "p05": 0.06719416379928589, + "p95": 0.17596006393432617 + }, + { + "label": "completed_group_1", + "mean": 0.17510183155536652, + "median": 0.1808546781539917, + "p05": 0.12213055044412613, + "p95": 0.21415141224861145 + }, + { + "label": "completed_group_2", + "mean": 0.1414714753627777, + "median": 0.14097586274147034, + "p05": 0.11696422845125198, + "p95": 0.16771599650382996 + }, + { + "label": "completed_group_3", + "mean": 0.12628035247325897, + "median": 0.12579765915870667, + "p05": 0.10614679008722305, + "p95": 0.14692939817905426 + }, + { + "label": "completed_group_4", + "mean": 0.11767096817493439, + "median": 0.11719106882810593, + "p05": 0.10052644461393356, + "p95": 0.13653282821178436 + }, + { + "label": "completed_group_5", + "mean": 0.12069195508956909, + "median": 0.11935321986675262, + "p05": 0.09329675883054733, + "p95": 0.15252181887626648 + }, + { + "label": "completed_group_6", + "mean": 0.12702946364879608, + "median": 0.12134957313537598, + "p05": 0.08377101272344589, + "p95": 0.18232685327529907 + }, + { + "label": "current_group_partial", + "mean": 0.08876943588256836, + "median": 0.07879889756441116, + "p05": 0.06865477561950684, + "p95": 0.15027400851249695 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0607244968414307, + "group": 7, + "latest_source_mass": 0.11456632614135742, + "layer": 26, + "max_source_mass": 0.1498526930809021, + "mixer_index": 50, + "normalized_entropy": 0.9909990040771789, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1220984011888504, + "median": 0.12646546959877014, + "p05": 0.07202506065368652, + "p95": 0.15947024524211884 + }, + { + "label": "completed_group_1", + "mean": 0.1498526930809021, + "median": 0.15748712420463562, + "p05": 0.08880092203617096, + "p95": 0.19019576907157898 + }, + { + "label": "completed_group_2", + "mean": 0.12913095951080322, + "median": 0.12894359230995178, + "p05": 0.1061355471611023, + "p95": 0.15167440474033356 + }, + { + "label": "completed_group_3", + "mean": 0.1179434061050415, + "median": 0.118401437997818, + "p05": 0.09755907952785492, + "p95": 0.13774089515209198 + }, + { + "label": "completed_group_4", + "mean": 0.10775718837976456, + "median": 0.10657531023025513, + "p05": 0.09242597222328186, + "p95": 0.12714339792728424 + }, + { + "label": "completed_group_5", + "mean": 0.12020602822303772, + "median": 0.11893786489963531, + "p05": 0.10092492401599884, + "p95": 0.14495709538459778 + }, + { + "label": "completed_group_6", + "mean": 0.13844498991966248, + "median": 0.13359181582927704, + "p05": 0.10005055367946625, + "p95": 0.1863655149936676 + }, + { + "label": "current_group_partial", + "mean": 0.11456632614135742, + "median": 0.10762587189674377, + "p05": 0.08914398401975632, + "p95": 0.16366823017597198 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0388383865356445, + "group": 7, + "latest_source_mass": 0.08920245617628098, + "layer": 26, + "max_source_mass": 0.18260905146598816, + "mixer_index": 51, + "normalized_entropy": 0.9804740098096767, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10253133624792099, + "median": 0.09456855058670044, + "p05": 0.07405152916908264, + "p95": 0.16842687129974365 + }, + { + "label": "completed_group_1", + "mean": 0.18260905146598816, + "median": 0.19173294305801392, + "p05": 0.11063728481531143, + "p95": 0.22958147525787354 + }, + { + "label": "completed_group_2", + "mean": 0.14208021759986877, + "median": 0.14284399151802063, + "p05": 0.10792264342308044, + "p95": 0.17174816131591797 + }, + { + "label": "completed_group_3", + "mean": 0.12606358528137207, + "median": 0.12572120130062103, + "p05": 0.10405182838439941, + "p95": 0.14868314564228058 + }, + { + "label": "completed_group_4", + "mean": 0.11357533931732178, + "median": 0.11289471387863159, + "p05": 0.09577979147434235, + "p95": 0.13378244638442993 + }, + { + "label": "completed_group_5", + "mean": 0.11596501618623734, + "median": 0.1148330420255661, + "p05": 0.09200432896614075, + "p95": 0.1449994146823883 + }, + { + "label": "completed_group_6", + "mean": 0.12797299027442932, + "median": 0.12260732054710388, + "p05": 0.08455555140972137, + "p95": 0.1881599724292755 + }, + { + "label": "current_group_partial", + "mean": 0.08920245617628098, + "median": 0.07755134999752045, + "p05": 0.066501684486866, + "p95": 0.1560288667678833 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0634398460388184, + "group": 7, + "latest_source_mass": 0.12262405455112457, + "layer": 27, + "max_source_mass": 0.15090109407901764, + "mixer_index": 52, + "normalized_entropy": 0.9923048110176299, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12216806411743164, + "median": 0.12567293643951416, + "p05": 0.08283672481775284, + "p95": 0.1497604101896286 + }, + { + "label": "completed_group_1", + "mean": 0.15090109407901764, + "median": 0.15809711813926697, + "p05": 0.09385120868682861, + "p95": 0.1825941503047943 + }, + { + "label": "completed_group_2", + "mean": 0.12766355276107788, + "median": 0.12779396772384644, + "p05": 0.10599278658628464, + "p95": 0.14822138845920563 + }, + { + "label": "completed_group_3", + "mean": 0.1134132444858551, + "median": 0.11286742240190506, + "p05": 0.09987462311983109, + "p95": 0.12846460938453674 + }, + { + "label": "completed_group_4", + "mean": 0.10988184064626694, + "median": 0.10929364711046219, + "p05": 0.09590980410575867, + "p95": 0.12525902688503265 + }, + { + "label": "completed_group_5", + "mean": 0.11335283517837524, + "median": 0.11079756915569305, + "p05": 0.09715084731578827, + "p95": 0.14056096971035004 + }, + { + "label": "completed_group_6", + "mean": 0.13999530673027039, + "median": 0.1355634331703186, + "p05": 0.10303047299385071, + "p95": 0.18968813121318817 + }, + { + "label": "current_group_partial", + "mean": 0.12262405455112457, + "median": 0.11672497540712357, + "p05": 0.09879282861948013, + "p95": 0.16369464993476868 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0386834144592285, + "group": 7, + "latest_source_mass": 0.08390261977910995, + "layer": 27, + "max_source_mass": 0.17931805551052094, + "mixer_index": 53, + "normalized_entropy": 0.9803994839943029, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10029368102550507, + "median": 0.09128708392381668, + "p05": 0.07227826118469238, + "p95": 0.17348244786262512 + }, + { + "label": "completed_group_1", + "mean": 0.17931805551052094, + "median": 0.1862885057926178, + "p05": 0.11921355128288269, + "p95": 0.22246700525283813 + }, + { + "label": "completed_group_2", + "mean": 0.14185777306556702, + "median": 0.14203041791915894, + "p05": 0.10853314399719238, + "p95": 0.1728457659482956 + }, + { + "label": "completed_group_3", + "mean": 0.13114282488822937, + "median": 0.1317460834980011, + "p05": 0.10662781447172165, + "p95": 0.15405671298503876 + }, + { + "label": "completed_group_4", + "mean": 0.11872631311416626, + "median": 0.11834143102169037, + "p05": 0.09892343729734421, + "p95": 0.1395292729139328 + }, + { + "label": "completed_group_5", + "mean": 0.11815309524536133, + "median": 0.11647512018680573, + "p05": 0.09220993518829346, + "p95": 0.14916184544563293 + }, + { + "label": "completed_group_6", + "mean": 0.12660562992095947, + "median": 0.12275825440883636, + "p05": 0.08448901027441025, + "p95": 0.1805192083120346 + }, + { + "label": "current_group_partial", + "mean": 0.08390261977910995, + "median": 0.07283803820610046, + "p05": 0.06168072670698166, + "p95": 0.14687296748161316 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0626261234283447, + "group": 7, + "latest_source_mass": 0.1115579903125763, + "layer": 28, + "max_source_mass": 0.15398627519607544, + "mixer_index": 54, + "normalized_entropy": 0.9919134931593667, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11644256114959717, + "median": 0.12025526911020279, + "p05": 0.07829730212688446, + "p95": 0.1473471075296402 + }, + { + "label": "completed_group_1", + "mean": 0.15398627519607544, + "median": 0.1615961492061615, + "p05": 0.09488844126462936, + "p95": 0.18759915232658386 + }, + { + "label": "completed_group_2", + "mean": 0.13596117496490479, + "median": 0.13718228042125702, + "p05": 0.11204125732183456, + "p95": 0.15343619883060455 + }, + { + "label": "completed_group_3", + "mean": 0.12036904692649841, + "median": 0.1201494038105011, + "p05": 0.10638508945703506, + "p95": 0.13539429008960724 + }, + { + "label": "completed_group_4", + "mean": 0.10964121669530869, + "median": 0.10774455964565277, + "p05": 0.09591315686702728, + "p95": 0.1285344362258911 + }, + { + "label": "completed_group_5", + "mean": 0.11886264383792877, + "median": 0.11775418370962143, + "p05": 0.0997074693441391, + "p95": 0.14473959803581238 + }, + { + "label": "completed_group_6", + "mean": 0.13317911326885223, + "median": 0.12975312769412994, + "p05": 0.09947147965431213, + "p95": 0.1780128926038742 + }, + { + "label": "current_group_partial", + "mean": 0.1115579903125763, + "median": 0.1049065813422203, + "p05": 0.0882733166217804, + "p95": 0.15690873563289642 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.020051956176758, + "group": 7, + "latest_source_mass": 0.07486395537853241, + "layer": 28, + "max_source_mass": 0.1913527548313141, + "mixer_index": 55, + "normalized_entropy": 0.9714396465047528, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09590239822864532, + "median": 0.08437805622816086, + "p05": 0.062284864485263824, + "p95": 0.19059056043624878 + }, + { + "label": "completed_group_1", + "mean": 0.1913527548313141, + "median": 0.19941553473472595, + "p05": 0.11809638142585754, + "p95": 0.24551606178283691 + }, + { + "label": "completed_group_2", + "mean": 0.14906750619411469, + "median": 0.14890097081661224, + "p05": 0.1085248589515686, + "p95": 0.18817238509655 + }, + { + "label": "completed_group_3", + "mean": 0.1341754049062729, + "median": 0.13468942046165466, + "p05": 0.10614670813083649, + "p95": 0.16129878163337708 + }, + { + "label": "completed_group_4", + "mean": 0.11847006529569626, + "median": 0.11653131991624832, + "p05": 0.09301862120628357, + "p95": 0.14944709837436676 + }, + { + "label": "completed_group_5", + "mean": 0.11603337526321411, + "median": 0.11465917527675629, + "p05": 0.08672616630792618, + "p95": 0.1505887657403946 + }, + { + "label": "completed_group_6", + "mean": 0.12013453245162964, + "median": 0.11430312693119049, + "p05": 0.07354772090911865, + "p95": 0.18300247192382812 + }, + { + "label": "current_group_partial", + "mean": 0.07486395537853241, + "median": 0.06415845453739166, + "p05": 0.052880410104990005, + "p95": 0.13596120476722717 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0452136993408203, + "group": 8, + "latest_source_mass": 0.09500804543495178, + "layer": 29, + "max_source_mass": 0.15673108398914337, + "mixer_index": 56, + "normalized_entropy": 0.9835398871990577, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11420649290084839, + "median": 0.11760550737380981, + "p05": 0.0455118790268898, + "p95": 0.1930246502161026 + }, + { + "label": "completed_group_1", + "mean": 0.15673108398914337, + "median": 0.1630343794822693, + "p05": 0.0941019207239151, + "p95": 0.2044450342655182 + }, + { + "label": "completed_group_2", + "mean": 0.13583534955978394, + "median": 0.1358843892812729, + "p05": 0.10828851163387299, + "p95": 0.16276802122592926 + }, + { + "label": "completed_group_3", + "mean": 0.12758757174015045, + "median": 0.12776434421539307, + "p05": 0.10106872022151947, + "p95": 0.1536240577697754 + }, + { + "label": "completed_group_4", + "mean": 0.1112617775797844, + "median": 0.10795636475086212, + "p05": 0.0855436623096466, + "p95": 0.1458887755870819 + }, + { + "label": "completed_group_5", + "mean": 0.13417255878448486, + "median": 0.13457609713077545, + "p05": 0.10089332610368729, + "p95": 0.16806170344352722 + }, + { + "label": "completed_group_6", + "mean": 0.12519711256027222, + "median": 0.11858057975769043, + "p05": 0.08283591270446777, + "p95": 0.18291912972927094 + }, + { + "label": "completed_group_7", + "mean": 0.09500804543495178, + "median": 0.086635060608387, + "p05": 0.07067855447530746, + "p95": 0.14996609091758728 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1403071880340576, + "group": 8, + "latest_source_mass": 0.10776971280574799, + "layer": 29, + "max_source_mass": 0.1673685610294342, + "mixer_index": 57, + "normalized_entropy": 0.9740957797899907, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08488726615905762, + "median": 0.07340645045042038, + "p05": 0.057354122400283813, + "p95": 0.1699385792016983 + }, + { + "label": "completed_group_1", + "mean": 0.1673685610294342, + "median": 0.17044895887374878, + "p05": 0.10701967775821686, + "p95": 0.213656947016716 + }, + { + "label": "completed_group_2", + "mean": 0.13726133108139038, + "median": 0.13688170909881592, + "p05": 0.09961593151092529, + "p95": 0.17454354465007782 + }, + { + "label": "completed_group_3", + "mean": 0.12220048904418945, + "median": 0.12267022579908371, + "p05": 0.09608018398284912, + "p95": 0.14777429401874542 + }, + { + "label": "completed_group_4", + "mean": 0.10616929084062576, + "median": 0.10591423511505127, + "p05": 0.08670090138912201, + "p95": 0.12526394426822662 + }, + { + "label": "completed_group_5", + "mean": 0.1037440150976181, + "median": 0.10274936258792877, + "p05": 0.08146549016237259, + "p95": 0.13024258613586426 + }, + { + "label": "completed_group_6", + "mean": 0.10401669144630432, + "median": 0.10373582690954208, + "p05": 0.06632979214191437, + "p95": 0.14398667216300964 + }, + { + "label": "completed_group_7", + "mean": 0.06658264249563217, + "median": 0.05958016216754913, + "p05": 0.04998533055186272, + "p95": 0.10914435982704163 + }, + { + "label": "current_group_partial", + "mean": 0.10776971280574799, + "median": 0.09594395756721497, + "p05": 0.05985837057232857, + "p95": 0.1909799873828888 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.18704891204834, + "group": 8, + "latest_source_mass": 0.1059001013636589, + "layer": 30, + "max_source_mass": 0.12684623897075653, + "mixer_index": 58, + "normalized_entropy": 0.9953688551489734, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10885682702064514, + "median": 0.11496149003505707, + "p05": 0.06692696362733841, + "p95": 0.13488291203975677 + }, + { + "label": "completed_group_1", + "mean": 0.12684623897075653, + "median": 0.12977978587150574, + "p05": 0.0880269706249237, + "p95": 0.15352922677993774 + }, + { + "label": "completed_group_2", + "mean": 0.11236055195331573, + "median": 0.1118544340133667, + "p05": 0.0995602011680603, + "p95": 0.12739220261573792 + }, + { + "label": "completed_group_3", + "mean": 0.10673344880342484, + "median": 0.10557802021503448, + "p05": 0.09419171512126923, + "p95": 0.12314252555370331 + }, + { + "label": "completed_group_4", + "mean": 0.10181289911270142, + "median": 0.10104846954345703, + "p05": 0.0905531719326973, + "p95": 0.1154235228896141 + }, + { + "label": "completed_group_5", + "mean": 0.11441421508789062, + "median": 0.11405807733535767, + "p05": 0.1015545204281807, + "p95": 0.12890149652957916 + }, + { + "label": "completed_group_6", + "mean": 0.11840946972370148, + "median": 0.11774120479822159, + "p05": 0.09769279509782791, + "p95": 0.1386050581932068 + }, + { + "label": "completed_group_7", + "mean": 0.10466624796390533, + "median": 0.10123022645711899, + "p05": 0.08714763075113297, + "p95": 0.1335311383008957 + }, + { + "label": "current_group_partial", + "mean": 0.1059001013636589, + "median": 0.10151755809783936, + "p05": 0.08478552103042603, + "p95": 0.14096680283546448 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1283018589019775, + "group": 8, + "latest_source_mass": 0.07073106616735458, + "layer": 30, + "max_source_mass": 0.1812124252319336, + "mixer_index": 59, + "normalized_entropy": 0.9686319190376982, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08667369186878204, + "median": 0.07598486542701721, + "p05": 0.0599454902112484, + "p95": 0.17076340317726135 + }, + { + "label": "completed_group_1", + "mean": 0.1812124252319336, + "median": 0.18718186020851135, + "p05": 0.11695106327533722, + "p95": 0.22937621176242828 + }, + { + "label": "completed_group_2", + "mean": 0.14178472757339478, + "median": 0.1410643458366394, + "p05": 0.10069074481725693, + "p95": 0.18480442464351654 + }, + { + "label": "completed_group_3", + "mean": 0.12440058588981628, + "median": 0.12452012300491333, + "p05": 0.09520045667886734, + "p95": 0.15427032113075256 + }, + { + "label": "completed_group_4", + "mean": 0.11160320043563843, + "median": 0.11176156997680664, + "p05": 0.08930732309818268, + "p95": 0.1340266317129135 + }, + { + "label": "completed_group_5", + "mean": 0.10330478101968765, + "median": 0.10138857364654541, + "p05": 0.07821988314390182, + "p95": 0.13451263308525085 + }, + { + "label": "completed_group_6", + "mean": 0.11197599023580551, + "median": 0.11038671433925629, + "p05": 0.06996595859527588, + "p95": 0.1621071994304657 + }, + { + "label": "completed_group_7", + "mean": 0.06831350922584534, + "median": 0.06078982725739479, + "p05": 0.049278710037469864, + "p95": 0.11596570909023285 + }, + { + "label": "current_group_partial", + "mean": 0.07073106616735458, + "median": 0.06323730200529099, + "p05": 0.051058635115623474, + "p95": 0.11510380357503891 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.173035144805908, + "group": 8, + "latest_source_mass": 0.097323939204216, + "layer": 31, + "max_source_mass": 0.13639645278453827, + "mixer_index": 60, + "normalized_entropy": 0.9889909148205337, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10752685368061066, + "median": 0.11246659606695175, + "p05": 0.04835075885057449, + "p95": 0.15864257514476776 + }, + { + "label": "completed_group_1", + "mean": 0.13639645278453827, + "median": 0.1424846053123474, + "p05": 0.0804441049695015, + "p95": 0.17550095915794373 + }, + { + "label": "completed_group_2", + "mean": 0.11839058995246887, + "median": 0.11810846626758575, + "p05": 0.0967480018734932, + "p95": 0.13973747193813324 + }, + { + "label": "completed_group_3", + "mean": 0.11235916614532471, + "median": 0.112240731716156, + "p05": 0.09452164173126221, + "p95": 0.12995943427085876 + }, + { + "label": "completed_group_4", + "mean": 0.10138258337974548, + "median": 0.09962283074855804, + "p05": 0.08511637151241302, + "p95": 0.12233416736125946 + }, + { + "label": "completed_group_5", + "mean": 0.11953261494636536, + "median": 0.12015771865844727, + "p05": 0.09630562365055084, + "p95": 0.14168336987495422 + }, + { + "label": "completed_group_6", + "mean": 0.11256720125675201, + "median": 0.11007197946310043, + "p05": 0.0816735252737999, + "p95": 0.14915722608566284 + }, + { + "label": "completed_group_7", + "mean": 0.09452059864997864, + "median": 0.0890384241938591, + "p05": 0.07612448185682297, + "p95": 0.13178984820842743 + }, + { + "label": "current_group_partial", + "mean": 0.097323939204216, + "median": 0.0898042768239975, + "p05": 0.07185541093349457, + "p95": 0.14817608892917633 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1028335094451904, + "group": 8, + "latest_source_mass": 0.06014348566532135, + "layer": 31, + "max_source_mass": 0.19083696603775024, + "mixer_index": 61, + "normalized_entropy": 0.9570407736811942, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08200174570083618, + "median": 0.06842087209224701, + "p05": 0.05465565621852875, + "p95": 0.1860819309949875 + }, + { + "label": "completed_group_1", + "mean": 0.19083696603775024, + "median": 0.19397340714931488, + "p05": 0.1252523958683014, + "p95": 0.24377956986427307 + }, + { + "label": "completed_group_2", + "mean": 0.15512439608573914, + "median": 0.15396007895469666, + "p05": 0.10948964953422546, + "p95": 0.2031564563512802 + }, + { + "label": "completed_group_3", + "mean": 0.1322997808456421, + "median": 0.131678968667984, + "p05": 0.1001092940568924, + "p95": 0.16698354482650757 + }, + { + "label": "completed_group_4", + "mean": 0.11209174990653992, + "median": 0.11124030500650406, + "p05": 0.08621159940958023, + "p95": 0.1389508843421936 + }, + { + "label": "completed_group_5", + "mean": 0.10435720533132553, + "median": 0.10206517577171326, + "p05": 0.07588012516498566, + "p95": 0.13983221352100372 + }, + { + "label": "completed_group_6", + "mean": 0.1024213582277298, + "median": 0.10071991384029388, + "p05": 0.06045639514923096, + "p95": 0.15181156992912292 + }, + { + "label": "completed_group_7", + "mean": 0.06072331964969635, + "median": 0.05212578922510147, + "p05": 0.04197835922241211, + "p95": 0.10935051739215851 + }, + { + "label": "current_group_partial", + "mean": 0.06014348566532135, + "median": 0.054720573127269745, + "p05": 0.043509967625141144, + "p95": 0.09508825838565826 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.147671937942505, + "group": 8, + "latest_source_mass": 0.07905417680740356, + "layer": 32, + "max_source_mass": 0.14984291791915894, + "mixer_index": 62, + "normalized_entropy": 0.9774476219204733, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09926988929510117, + "median": 0.09770013391971588, + "p05": 0.04949922859668732, + "p95": 0.1754719465970993 + }, + { + "label": "completed_group_1", + "mean": 0.14984291791915894, + "median": 0.1559048891067505, + "p05": 0.08648592233657837, + "p95": 0.2032528817653656 + }, + { + "label": "completed_group_2", + "mean": 0.12505242228507996, + "median": 0.12578052282333374, + "p05": 0.09032565355300903, + "p95": 0.15844053030014038 + }, + { + "label": "completed_group_3", + "mean": 0.13004425168037415, + "median": 0.12868037819862366, + "p05": 0.10237257927656174, + "p95": 0.1618403047323227 + }, + { + "label": "completed_group_4", + "mean": 0.1122550517320633, + "median": 0.10850167274475098, + "p05": 0.08462762087583542, + "p95": 0.15187090635299683 + }, + { + "label": "completed_group_5", + "mean": 0.12226949632167816, + "median": 0.12273561954498291, + "p05": 0.08938803523778915, + "p95": 0.15480002760887146 + }, + { + "label": "completed_group_6", + "mean": 0.10689134150743484, + "median": 0.10258689522743225, + "p05": 0.07286856323480606, + "p95": 0.15216295421123505 + }, + { + "label": "completed_group_7", + "mean": 0.07532045990228653, + "median": 0.06964509189128876, + "p05": 0.054946303367614746, + "p95": 0.11525978893041611 + }, + { + "label": "current_group_partial", + "mean": 0.07905417680740356, + "median": 0.072466179728508, + "p05": 0.05414725840091705, + "p95": 0.11476384103298187 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.109609603881836, + "group": 8, + "latest_source_mass": 0.06053228676319122, + "layer": 32, + "max_source_mass": 0.18956288695335388, + "mixer_index": 63, + "normalized_entropy": 0.9601247071609755, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08386656641960144, + "median": 0.07061353325843811, + "p05": 0.055651482194662094, + "p95": 0.18354055285453796 + }, + { + "label": "completed_group_1", + "mean": 0.18956288695335388, + "median": 0.19494932889938354, + "p05": 0.12378176301717758, + "p95": 0.24272672832012177 + }, + { + "label": "completed_group_2", + "mean": 0.1487923562526703, + "median": 0.14935529232025146, + "p05": 0.10487234592437744, + "p95": 0.19193747639656067 + }, + { + "label": "completed_group_3", + "mean": 0.12990039587020874, + "median": 0.1300441324710846, + "p05": 0.09967920184135437, + "p95": 0.160600483417511 + }, + { + "label": "completed_group_4", + "mean": 0.11159682273864746, + "median": 0.11003521829843521, + "p05": 0.08588480204343796, + "p95": 0.14072611927986145 + }, + { + "label": "completed_group_5", + "mean": 0.10738209635019302, + "median": 0.10594583302736282, + "p05": 0.0788765475153923, + "p95": 0.14147546887397766 + }, + { + "label": "completed_group_6", + "mean": 0.10535432398319244, + "median": 0.10358455777168274, + "p05": 0.06328895688056946, + "p95": 0.15448220074176788 + }, + { + "label": "completed_group_7", + "mean": 0.0630122721195221, + "median": 0.05402897298336029, + "p05": 0.044587839394807816, + "p95": 0.11368448287248611 + }, + { + "label": "current_group_partial", + "mean": 0.06053228676319122, + "median": 0.055282287299633026, + "p05": 0.04416987672448158, + "p95": 0.09472700953483582 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1622862815856934, + "group": null, + "latest_source_mass": 0.12579965591430664, + "layer": null, + "max_source_mass": 0.14956718683242798, + "mixer_index": null, + "normalized_entropy": 0.9840988963481907, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12356588989496231, + "median": 0.12339343130588531, + "p05": 0.0884157195687294, + "p95": 0.16069066524505615 + }, + { + "label": "completed_group_1", + "mean": 0.09235234558582306, + "median": 0.09365912526845932, + "p05": 0.06739562749862671, + "p95": 0.11328895390033722 + }, + { + "label": "completed_group_2", + "mean": 0.08736803382635117, + "median": 0.08676370978355408, + "p05": 0.07290066033601761, + "p95": 0.10374386608600616 + }, + { + "label": "completed_group_3", + "mean": 0.08735492825508118, + "median": 0.08669611811637878, + "p05": 0.07205022871494293, + "p95": 0.10462935268878937 + }, + { + "label": "completed_group_4", + "mean": 0.08889827132225037, + "median": 0.08936247229576111, + "p05": 0.07074585556983948, + "p95": 0.10606974363327026 + }, + { + "label": "completed_group_5", + "mean": 0.10504580289125443, + "median": 0.10402922332286835, + "p05": 0.0790465772151947, + "p95": 0.13349474966526031 + }, + { + "label": "completed_group_6", + "mean": 0.14004790782928467, + "median": 0.13968253135681152, + "p05": 0.10069862753152847, + "p95": 0.1819821298122406 + }, + { + "label": "completed_group_7", + "mean": 0.14956718683242798, + "median": 0.14930106699466705, + "p05": 0.12316055595874786, + "p95": 0.17985624074935913 + }, + { + "label": "completed_group_8", + "mean": 0.12579965591430664, + "median": 0.12568724155426025, + "p05": 0.07226289063692093, + "p95": 0.17831499874591827 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", + "optimizer_state_before": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.010549778118729591, + 0.013795096427202225, + 0.012347182258963585, + 0.012820021249353886, + 0.01913420483469963, + 0.018582463264465332, + 0.013614931143820286, + 0.013612493872642517, + 0.019129201769828796, + 0.02525762841105461, + 0.027986859902739525, + 0.014458909630775452, + 0.018231205642223358, + 0.016914349049329758, + 0.022478405386209488, + 0.03932488337159157, + 0.03611915931105614, + 0.0266753900796175, + 0.018954744562506676, + 0.014813725836575031, + 0.013802599161863327, + 0.016533223912119865, + 0.016679473221302032, + 0.01826235093176365, + 0.015529165975749493, + 0.011229476891458035, + 0.008122270926833153, + 0.008894859813153744, + 0.009738311171531677, + 0.007465044502168894, + 0.008605006150901318, + 0.009693782776594162 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.478636267573165e-05, + "first_to_last_ratio": 5.2633491541899735, + "imbalance_abs_log_ratio": 1.6607675454574897, + "last_quartile_mean": 6.60916873584938e-06, + "mean": 1.8399187894146962e-05, + "normalized": [ + 2.32572864627557, + 2.093980749525082, + 2.097253493167976, + 1.9748047942119253, + 1.9001199222002307, + 1.7176970887283356, + 1.5731226694511573, + 1.442465715104865, + 1.388173701809276, + 1.2578149167122274, + 1.163711179168628, + 1.0739337627510765, + 1.0625631931116428, + 0.9721718406735991, + 0.8978727300712377, + 0.830314261296444, + 0.8465768431906301, + 0.7713093251343669, + 0.7043450693146728, + 0.6424476146199775, + 0.6810764492568463, + 0.62147201099682, + 0.5661400138095133, + 0.5212255519238714, + 0.43050775556639687, + 0.41135344603945795, + 0.3923546966297886, + 0.3784948438109706, + 0.3251037643245192, + 0.3141780758593552, + 0.3117257919642976, + 0.30996008329924213 + ], + "peak_layer": 1, + "peak_normalized": 2.32572864627557, + "population_cv": 0.5996620573317034, + "rest_mean": 1.9884464530116895e-05, + "spike_contrast": 0.5219498892811426, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0378694059909322e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.279151835362427e-05, + 3.852754525723867e-05, + 3.858776108245365e-05, + 3.633480446296744e-05, + 3.496066346997395e-05, + 3.160423148074187e-05, + 2.8944179575773887e-05, + 2.6540197723079473e-05, + 2.5541268769302405e-05, + 2.3142772988649085e-05, + 2.141134064004291e-05, + 1.9759509086725302e-05, + 1.955029983946588e-05, + 1.7887172361952253e-05, + 1.65201290656114e-05, + 1.527710810478311e-05, + 1.5576326404698193e-05, + 1.4191465197654907e-05, + 1.295937727263663e-05, + 1.1820514373539481e-05, + 1.2531253560155164e-05, + 1.1434580301283859e-05, + 1.041651648847619e-05, + 9.590126865077764e-06, + 7.92099308455363e-06, + 7.56856934458483e-06, + 7.2190077844425105e-06, + 6.963997748243855e-06, + 5.981645244901301e-06, + 5.7806214499578346e-06, + 5.73550141780288e-06, + 5.7030138123082e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013640912038681563, + "first_to_last_ratio": 5.121983284590993, + "imbalance_abs_log_ratio": 1.6335417243184733, + "last_quartile_mean": 2.663209011188883e-05, + "mean": 7.298096016938871e-05, + "normalized": [ + 2.2789977416441083, + 2.0684974265127174, + 2.076163899895869, + 1.9536947342428674, + 1.8791834481613054, + 1.70558006750568, + 1.5594206835801008, + 1.4313066933158733, + 1.382104599488904, + 1.2645011290875958, + 1.1686177713576213, + 1.080869560986379, + 1.070406696124593, + 0.9820093608320563, + 0.9088928309552108, + 0.8405354206057669, + 0.8564067293095593, + 0.7807789902359659, + 0.7130468773072498, + 0.650906500520924, + 0.6919540061414183, + 0.6322816699001862, + 0.5753535339386485, + 0.5291429877926532, + 0.43715672246183995, + 0.41718055352967753, + 0.3982548633280403, + 0.38497320016438824, + 0.33031538135673594, + 0.31952371692646125, + 0.31683848721825286, + 0.31510371557135003 + ], + "peak_layer": 1, + "peak_normalized": 2.2789977416441083, + "population_cv": 0.5877415594416108, + "rest_mean": 7.874945926973683e-05, + "spike_contrast": 0.5311917747171685, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.183106502750888e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016632344340905547, + 0.00015096092829480767, + 0.00015152043488342315, + 0.00014258251758292317, + 0.00013714461238123477, + 0.00012447487097233534, + 0.00011380801879568025, + 0.00010445813677506521, + 0.00010086732072522864, + 9.228450653608888e-05, + 8.528684702469036e-05, + 7.888289837865159e-05, + 7.811930845491588e-05, + 7.166798604885116e-05, + 6.633187149418518e-05, + 6.134308205218986e-05, + 6.25013854005374e-05, + 5.698200038750656e-05, + 5.2038845751667395e-05, + 4.7503781388513744e-05, + 5.04994677612558e-05, + 4.614452336682007e-05, + 4.198985334369354e-05, + 3.861736331600696e-05, + 3.190411734976806e-05, + 3.044623736059293e-05, + 2.9065022317809053e-05, + 2.8095713787479326e-05, + 2.4106733690132387e-05, + 2.3319147658185102e-05, + 2.3123177015804686e-05, + 2.2996571715339087e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12096897326409817, + "first_to_last_ratio": 5.121983389958884, + "imbalance_abs_log_ratio": 1.6335417448901706, + "last_quartile_mean": 0.02361760358326137, + "mean": 0.06472024595132098, + "normalized": [ + 2.2789977721075467, + 2.0684973368733783, + 2.0761638538672402, + 1.953694825115509, + 1.8791834311760194, + 1.7055800372311585, + 1.5594207388065806, + 1.4313066964307424, + 1.3821046167764135, + 1.264501125605702, + 1.1686178592805807, + 1.080869532341778, + 1.0704067558461303, + 0.9820093835771068, + 0.9088927809148353, + 0.8405354027150204, + 0.8564067368028002, + 0.7807789687785609, + 0.7130468636740094, + 0.6509064717345558, + 0.6919539840797562, + 0.6322817027038486, + 0.5753535315685364, + 0.529143012125947, + 0.4371566943170802, + 0.41718053416692213, + 0.39825484202638917, + 0.3849732137717626, + 0.3303153774147322, + 0.31952373248220817, + 0.3168384772300972, + 0.31510370845705143 + ], + "peak_layer": 1, + "peak_normalized": 2.2789977721075467, + "population_cv": 0.5877415617202747, + "rest_mean": 0.06983580868001338, + "spike_contrast": 0.5311917756455903, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03709620721638203, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.147497296333313, + 0.13387365639209747, + 0.1343698352575302, + 0.12644360959529877, + 0.12162121385335922, + 0.11038555949926376, + 0.10092609375715256, + 0.09263452142477036, + 0.08945015072822571, + 0.08183882385492325, + 0.07563323527574539, + 0.06995414197444916, + 0.06927698850631714, + 0.06355588883161545, + 0.05882376432418823, + 0.05439965799450874, + 0.05542685464024544, + 0.050532206892967224, + 0.04614856839179993, + 0.04212682694196701, + 0.04478343203663826, + 0.04092142730951309, + 0.0372370220720768, + 0.03424626588821411, + 0.028292888775467873, + 0.027000026777386665, + 0.025775151327252388, + 0.024915561079978943, + 0.02137809246778488, + 0.0206796545535326, + 0.020505864173173904, + 0.02039358951151371 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.711734099302703e-06, + "first_to_last_ratio": 4.6783638296609995, + "imbalance_abs_log_ratio": 1.54294843975527, + "last_quartile_mean": 1.648382720986774e-06, + "mean": 4.007230394620365e-06, + "normalized": [ + 3.358984576738256, + 2.372641967350387, + 2.13819011970407, + 1.8393394452465484, + 1.59553806661366, + 1.4487556048903885, + 1.2955565469325476, + 1.3466327346665505, + 1.2739273872501113, + 1.0660089247937763, + 1.0478478795998651, + 1.0914936255170047, + 1.0411021864111147, + 0.9154927446436142, + 0.8065881507886284, + 0.7702055746502616, + 0.8215637644294851, + 0.7364762575936972, + 0.6730206907422001, + 0.6266437238096296, + 0.6693956877918972, + 0.6236755529871887, + 0.5897829924931797, + 0.560318828703654, + 0.45883664783466677, + 0.44720692888245495, + 0.4388849180240322, + 0.4318078866173612, + 0.37877623882612776, + 0.3751519735069292, + 0.3795957753533195, + 0.38055659660739233 + ], + "peak_layer": 1, + "peak_normalized": 3.358984576738256, + "population_cv": 0.6648021914060557, + "rest_mean": 4.31860559678575e-06, + "spike_contrast": 0.5385544594899636, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.3258043029272812e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.346022509096656e-05, + 9.50772300711833e-06, + 8.568220437155105e-06, + 7.3706569310161285e-06, + 6.39368863630807e-06, + 5.805497494293377e-06, + 5.19159357281751e-06, + 5.3962676247465424e-06, + 5.1049205467279535e-06, + 4.271743364370195e-06, + 4.19896787207108e-06, + 4.373866431706119e-06, + 4.171936325292336e-06, + 3.6685903523903107e-06, + 3.232184553780826e-06, + 3.0863911888445728e-06, + 3.292195287940558e-06, + 2.9512300443457207e-06, + 2.6969489681505365e-06, + 2.511105776648037e-06, + 2.6824227461474948e-06, + 2.4992116323119262e-06, + 2.363396333748824e-06, + 2.245326641059364e-06, + 1.838664161368797e-06, + 1.7920611981026013e-06, + 1.7587129832463688e-06, + 1.730353687889874e-06, + 1.5178436569840414e-06, + 1.5033203908387804e-06, + 1.5211277286653058e-06, + 1.5249779607984237e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001145691167039331, + "first_to_last_ratio": 4.836936726468446, + "imbalance_abs_log_ratio": 1.5762816126034256, + "last_quartile_mean": 2.368629634474928e-05, + "mean": 6.211562191538178e-05, + "normalized": [ + 2.348876988858354, + 2.071301897195793, + 2.0294561010843566, + 1.9098237582754058, + 1.8088773728255583, + 1.6555764779765272, + 1.5234729301569887, + 1.4082090466002533, + 1.3570526012658433, + 1.2477509254620376, + 1.1584754738367655, + 1.0770687308168136, + 1.0629673480990054, + 0.9794260409949351, + 0.9103157365029115, + 0.8467425542641047, + 0.8606040430432489, + 0.7903153532861079, + 0.7260518902950809, + 0.6672536282299563, + 0.7127906049054458, + 0.6531196817841458, + 0.5952239503818174, + 0.548639546167493, + 0.4562347554219186, + 0.43716465149452993, + 0.4212539905654889, + 0.40928696694430294, + 0.34099585105581787, + 0.33116538282585856, + 0.32832835587544684, + 0.32617736350768683 + ], + "peak_layer": 1, + "peak_normalized": 2.348876988858354, + "population_cv": 0.5738572576504661, + "rest_mean": 6.679497912248028e-05, + "spike_contrast": 0.5516446517557004, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6847092997049916e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014590195496566594, + 0.00012866020551882684, + 0.00012606092786882073, + 0.0001186298904940486, + 0.00011235954298172146, + 0.00010283716255798936, + 9.463146852795035e-05, + 8.747178071644157e-05, + 8.429416629951447e-05, + 7.750482473056763e-05, + 7.195942453108728e-05, + 6.690279406029731e-05, + 6.602687790291384e-05, + 6.0837657656520605e-05, + 5.6544828112237155e-05, + 5.259594036033377e-05, + 5.3456955356523395e-05, + 4.909092967864126e-05, + 4.509916470851749e-05, + 4.144687409279868e-05, + 4.4275431719142944e-05, + 4.0568935219198465e-05, + 3.697270585689694e-05, + 3.4079086617566645e-05, + 2.8339305572444573e-05, + 2.715475420700386e-05, + 2.6166453608311713e-05, + 2.542311449360568e-05, + 2.1181169358897023e-05, + 2.0570543711073697e-05, + 2.0394320017658174e-05, + 2.026070978899952e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.517959642835194e-05, + "first_to_last_ratio": 4.053573868997912, + "imbalance_abs_log_ratio": 1.39959892878961, + "last_quartile_mean": 2.3480414938603644e-05, + "mean": 5.345439518578132e-05, + "normalized": [ + 2.3487685774234603, + 2.032002289203053, + 1.963516724917696, + 1.8356141378280442, + 1.6872738480579543, + 1.5647260343512426, + 1.4530976571042717, + 1.3596062267041897, + 1.314646683688125, + 1.2118086283799, + 1.12751130576606, + 1.057258217084054, + 1.0421641943948246, + 0.9643812108591268, + 0.9028111172523068, + 0.8481148430618181, + 0.8638417335406758, + 0.7968091313416481, + 0.7398800228203574, + 0.6885877184509301, + 0.7528185246328142, + 0.6965652282580993, + 0.6386689529276455, + 0.5954414082316423, + 0.5183367890031948, + 0.5027841940839927, + 0.4926901944439643, + 0.48501928420581064, + 0.3867176094242744, + 0.3783808574805794, + 0.37618160958425234, + 0.373975045493992 + ], + "peak_layer": 1, + "peak_normalized": 2.3487685774234603, + "population_cv": 0.531886342087635, + "rest_mean": 5.701439672352367e-05, + "spike_contrast": 0.6003814623868411, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.423038688197266e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012555200373753905, + 0.0001086194533854723, + 0.00010495859896764159, + 9.812164353206754e-05, + 9.019220306072384e-05, + 8.364148379769176e-05, + 7.76744564063847e-05, + 7.267692853929475e-05, + 7.027364335954189e-05, + 6.47764973109588e-05, + 6.0270434914855286e-05, + 5.65150985494256e-05, + 5.570825669565238e-05, + 5.155041435500607e-05, + 4.8259222239721566e-05, + 4.533546598395333e-05, + 4.6176137402653694e-05, + 4.259295019437559e-05, + 3.9549839129904285e-05, + 3.6808040022151545e-05, + 4.02414589188993e-05, + 3.7234472983982414e-05, + 3.413966260268353e-05, + 3.182896034559235e-05, + 2.7707379558705725e-05, + 2.687602500373032e-05, + 2.633645635796711e-05, + 2.5926412490662187e-05, + 2.0671755919465795e-05, + 2.0226119886501692e-05, + 2.0108560420339927e-05, + 1.99906098714564e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002512736391508952, + "first_to_last_ratio": 5.815826874804025, + "imbalance_abs_log_ratio": 1.7605829727324667, + "last_quartile_mean": 4.320514426581212e-05, + "mean": 0.00013164091467388062, + "normalized": [ + 2.22465467951502, + 2.0950685860525193, + 2.148742715865945, + 2.011602148255569, + 1.9594409030366386, + 1.7609558829295935, + 1.610634891401338, + 1.4591459078647937, + 1.4125272633741999, + 1.2911637629028558, + 1.1922164013125294, + 1.0885645721953474, + 1.0786559850616189, + 0.9886788153600501, + 0.9103044070794555, + 0.8363940916690796, + 0.853516847878044, + 0.7731887115305843, + 0.7048080081053143, + 0.6396644863918306, + 0.6683488269421659, + 0.6076388883701213, + 0.5570727160911497, + 0.5013742768274315, + 0.394539708177536, + 0.3676631795999013, + 0.3470701351915981, + 0.3326099884380236, + 0.30382715452412723, + 0.2945209687200569, + 0.29298846283259594, + 0.29241662650296524 + ], + "peak_layer": 1, + "peak_normalized": 2.22465467951502, + "population_cv": 0.6145561150438986, + "rest_mean": 0.0001427135030098725, + "spike_contrast": 0.503448770748442, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.184893765952439e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00029285557684488595, + 0.0002757967449724674, + 0.0002828624565154314, + 0.0002648091467563063, + 0.00025794259272515774, + 0.00023181384312920272, + 0.00021202545030973852, + 0.00019208330195397139, + 0.00018594638095237315, + 0.00016996997874230146, + 0.0001569444575579837, + 0.0001432996359653771, + 0.0001419952604919672, + 0.00013015058357268572, + 0.00011983330477960408, + 0.00011010368325514719, + 0.00011235773854423314, + 0.00010178326920140535, + 9.278157085645944e-05, + 8.420601807301864e-05, + 8.798205089988187e-05, + 7.99901390564628e-05, + 7.333356188610196e-05, + 6.600136839551851e-05, + 5.193756805965677e-05, + 4.839951725443825e-05, + 4.568863005260937e-05, + 4.378508310765028e-05, + 3.9996084524318576e-05, + 3.877100971294567e-05, + 3.8569269236177206e-05, + 3.8493992178700864e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.017611408606171608, + 0.014558257535099983, + 0.01080919150263071, + 0.010327119380235672, + 0.008616750128567219, + 0.007770510856062174, + 0.007606723811477423, + 0.008487755432724953, + 0.00865182839334011, + 0.009020541794598103, + 0.008682658895850182, + 0.008706990629434586, + 0.009003226645290852, + 0.009712434373795986, + 0.00981286820024252, + 0.011337134055793285, + 0.010840619914233685, + 0.010956712998449802, + 0.012231985107064247, + 0.014082341454923153, + 0.014436324127018452, + 0.016183899715542793, + 0.017385432496666908, + 0.02254617027938366, + 0.022119678556919098, + 0.022372635081410408, + 0.024847008287906647, + 0.026897024363279343, + 0.026162855327129364, + 0.024674534797668457, + 0.030845578759908676, + 0.029862264171242714 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.2954689231701195e-05, + "first_to_last_ratio": 5.105739778813491, + "imbalance_abs_log_ratio": 1.63036535371595, + "last_quartile_mean": 6.454439642311627e-06, + "mean": 1.73502516958024e-05, + "normalized": [ + 2.118076439435723, + 2.157395185923538, + 2.1393573631676532, + 2.0293387218518246, + 1.9547446735792444, + 1.7470380730869022, + 1.5859544826144767, + 1.463120381926734, + 1.4022787424481573, + 1.2651844959124576, + 1.1596399794189511, + 1.0756457189909705, + 1.0548334272291358, + 0.9617847148778298, + 0.8861896603299768, + 0.8166507566511427, + 0.822632314310604, + 0.7462779082282098, + 0.6809665471807215, + 0.6235405742761265, + 0.6619400891948598, + 0.6046031773324564, + 0.5544108586463486, + 0.5123283891876923, + 0.43840604673891487, + 0.41933885865900805, + 0.40328491124319266, + 0.3940604884962617, + 0.3330148316096749, + 0.3301665032697355, + 0.32877203485044354, + 0.3290236493310315 + ], + "peak_layer": 2, + "peak_normalized": 2.157395185923538, + "population_cv": 0.5996242881005125, + "rest_mean": 1.878216889289669e-05, + "spike_contrast": 0.5120760486362497, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.617898831493221e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.6749159335158765e-05, + 3.7431349483085796e-05, + 3.7118388718226925e-05, + 3.5209537600167096e-05, + 3.391531208762899e-05, + 3.031155029020738e-05, + 2.751670945144724e-05, + 2.538550688768737e-05, + 2.4329889129148796e-05, + 2.195126944570802e-05, + 2.0120045519433916e-05, + 1.8662723960005678e-05, + 1.830162545957137e-05, + 1.6687206880305894e-05, + 1.5375613656942733e-05, + 1.4169096175464801e-05, + 1.4272877706389409e-05, + 1.2948109542776365e-05, + 1.1814940990007017e-05, + 1.0818585906235967e-05, + 1.1484827155072708e-05, + 1.049001730279997e-05, + 9.619167940400075e-06, + 8.88902650331147e-06, + 7.606455255881883e-06, + 7.2756347435642965e-06, + 6.997094715188723e-06, + 6.8370486587809864e-06, + 5.777891146863112e-06, + 5.728471933252877e-06, + 5.704277555196313e-06, + 5.708643129764823e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001289480715058744, + "first_to_last_ratio": 4.956339714424028, + "imbalance_abs_log_ratio": 1.6006675075081214, + "last_quartile_mean": 2.6016794436145574e-05, + "mean": 6.873570305288013e-05, + "normalized": [ + 2.073004605358973, + 2.134365700374101, + 2.1192095069414485, + 2.0040524994244264, + 1.9289936690931089, + 1.7292296834320953, + 1.569549055379467, + 1.4495827774599748, + 1.392480193841659, + 1.2691570699511074, + 1.1657818381111424, + 1.084357575904709, + 1.0620446910222163, + 0.9734770749855297, + 0.8984711717160666, + 0.8270404385478125, + 0.8348284439914535, + 0.7567459964705125, + 0.691239653970941, + 0.6326487613732772, + 0.6743242179293075, + 0.6159548780125667, + 0.5640484616188776, + 0.521373530426787, + 0.4457155128282148, + 0.4257578292711852, + 0.4106916383072167, + 0.4012558017197981, + 0.3390531928847147, + 0.336176289510552, + 0.3343946322908752, + 0.33499360784988247 + ], + "peak_layer": 2, + "peak_normalized": 2.134365700374101, + "population_cv": 0.5868619217766903, + "rest_mean": 7.428186829740837e-05, + "spike_contrast": 0.5221517931823592, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.878641073242761e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001424894289812073, + 0.0001467071269871667, + 0.00014566535537596792, + 0.0001377499575028196, + 0.00013259073602966964, + 0.0001188598180306144, + 0.00010788405779749155, + 9.963809134205803e-05, + 9.571310511091724e-05, + 8.723640348762274e-05, + 8.013083424884826e-05, + 7.4534080340527e-05, + 7.300038851099089e-05, + 6.691263115499169e-05, + 6.175704766064882e-05, + 5.68472059967462e-05, + 5.7382520026294515e-05, + 5.2015468099853024e-05, + 4.751284359372221e-05, + 4.3485557398526e-05, + 4.6350149204954505e-05, + 4.233809158904478e-05, + 3.877026756526902e-05, + 3.5836976167047396e-05, + 3.0636569135822356e-05, + 2.926476372522302e-05, + 2.8229178496985696e-05, + 2.7580599635257386e-05, + 2.330505958525464e-05, + 2.3107313609216362e-05, + 2.2984850147622637e-05, + 2.3026021153782494e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11435244046151638, + "first_to_last_ratio": 4.956339696350975, + "imbalance_abs_log_ratio": 1.6006675038616696, + "last_quartile_mean": 0.023071953793987632, + "mean": 0.0609555093687959, + "normalized": [ + 2.073004695896629, + 2.1343657739438058, + 2.1192095214175937, + 2.004052535226019, + 1.9289935285682593, + 1.7292296759967316, + 1.5695490131514764, + 1.4495827726824428, + 1.3924801813690295, + 1.269157021277261, + 1.165781889125704, + 1.0843576426531487, + 1.0620447122638885, + 0.9734770498631476, + 0.8984712131746528, + 0.8270404152126933, + 0.8348284100671275, + 0.7567460489501358, + 0.691239607862424, + 0.6326487463288526, + 0.6743241627324319, + 0.6159549038523712, + 0.5640484241789988, + 0.5213735345830548, + 0.44571554071981845, + 0.4257578260668156, + 0.4106916263563641, + 0.40125582081652367, + 0.3390531599397356, + 0.3361762979130643, + 0.33439464558088794, + 0.33499360222891106 + ], + "peak_layer": 2, + "peak_normalized": 2.1343657739438058, + "population_cv": 0.5868619269179132, + "rest_mean": 0.06587390429167836, + "spike_contrast": 0.522151786129607, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03439617678523064, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.12636105716228485, + 0.13010135293006897, + 0.1291774958372116, + 0.12215804308652878, + 0.1175827831029892, + 0.10540607571601868, + 0.09567265957593918, + 0.08836005628108978, + 0.08487933874130249, + 0.07736211270093918, + 0.07106082886457443, + 0.06609757244586945, + 0.0647374764084816, + 0.05933878943324089, + 0.05476677045226097, + 0.05041266977787018, + 0.05088739097118378, + 0.046127840876579285, + 0.04213486239314079, + 0.038563426584005356, + 0.041103772819042206, + 0.03754584491252899, + 0.03438185900449753, + 0.03178058937191963, + 0.027168817818164825, + 0.025952285155653954, + 0.025033917278051376, + 0.0244587529450655, + 0.02066715806722641, + 0.020491797477006912, + 0.020383195951581, + 0.020419705659151077 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.461751238224679e-06, + "first_to_last_ratio": 4.607278415178689, + "imbalance_abs_log_ratio": 1.527637317203245, + "last_quartile_mean": 1.6195572669630565e-06, + "mean": 3.872976176211296e-06, + "normalized": [ + 3.025668897657687, + 2.420035163018765, + 2.225735559561749, + 1.8932958237601154, + 1.6740889922677953, + 1.49074225689019, + 1.302073782365782, + 1.3813151404849735, + 1.2950916709377387, + 1.0715944778616184, + 1.0585199120860045, + 1.1104678074006644, + 1.039542517960571, + 0.9298369507481301, + 0.7931357450923319, + 0.7672068262710003, + 0.8115868243354613, + 0.7239540167999846, + 0.6595190440563047, + 0.6140523657384757, + 0.6490685381015874, + 0.6059744156497426, + 0.5735242411534173, + 0.5386197309103437, + 0.45666220052738465, + 0.4487521252190139, + 0.4444189656933061, + 0.44116761367860935, + 0.38204877349263455, + 0.3857109035588071, + 0.3910260087310755, + 0.39556270798873516 + ], + "peak_layer": 1, + "peak_normalized": 3.025668897657687, + "population_cv": 0.6453832806768665, + "rest_mean": 4.185131009156369e-06, + "spike_contrast": 0.5226455452702347, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.187340078307898e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1718343557731714e-05, + 9.372738531965297e-06, + 8.620220796728972e-06, + 7.332689619943267e-06, + 6.483706783910748e-06, + 5.773609245807165e-06, + 5.042900738772005e-06, + 5.349800630938262e-06, + 5.015859187551541e-06, + 4.150259883317631e-06, + 4.09962240155437e-06, + 4.300815362512367e-06, + 4.026123406219995e-06, + 3.601236358008464e-06, + 3.0717958452441962e-06, + 2.971373760374263e-06, + 3.1432564355782233e-06, + 2.8038566597388126e-06, + 2.554301545387716e-06, + 2.3782101834513014e-06, + 2.5138269847957417e-06, + 2.3469244752050145e-06, + 2.221245722466847e-06, + 2.0860613858530996e-06, + 1.7686418232187862e-06, + 1.7380062899974291e-06, + 1.7212240663866396e-06, + 1.7086316574932425e-06, + 1.4796657978877192e-06, + 1.4938491403881926e-06, + 1.5144344160944456e-06, + 1.5320049442379968e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00010837215813808143, + "first_to_last_ratio": 4.663749544487092, + "imbalance_abs_log_ratio": 1.5398197478822726, + "last_quartile_mean": 2.3237130790221272e-05, + "mean": 5.872993017419503e-05, + "normalized": [ + 2.116908005707516, + 2.1257566114370654, + 2.0713368933145166, + 1.958807069139986, + 1.853064595344867, + 1.6774795353818415, + 1.534246942340664, + 1.4245031207093415, + 1.3659761301608329, + 1.2527187093598229, + 1.1583387848911404, + 1.08159561433793, + 1.0590461894491214, + 0.974994469979534, + 0.9039318927032919, + 0.837094157953219, + 0.8447810620709018, + 0.7717232000223526, + 0.7087798354672495, + 0.6523243210626272, + 0.6962576906319006, + 0.6381512392879863, + 0.585489683522781, + 0.5414078956394395, + 0.4665886811070132, + 0.44848929689996414, + 0.43561388426614667, + 0.42649458236277493, + 0.3502420718536938, + 0.3476010806933041, + 0.3452465505067908, + 0.3450102023943839 + ], + "peak_layer": 2, + "peak_normalized": 2.1257566114370654, + "population_cv": 0.5682342302969126, + "rest_mean": 6.323713649989158e-05, + "spike_contrast": 0.5438420826580702, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4391016015433706e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012432585936039686, + 0.0001248455373570323, + 0.00012164947111159563, + 0.000115040602395311, + 0.00010883035429287702, + 9.851825598161668e-05, + 9.010621579363942e-05, + 8.366096881218255e-05, + 8.022368274396285e-05, + 7.357208232861012e-05, + 6.802915595471859e-05, + 6.352203490678221e-05, + 6.219770875759423e-05, + 5.7261357142124325e-05, + 5.308785694069229e-05, + 4.916248144581914e-05, + 4.961393278790638e-05, + 4.532324965111911e-05, + 4.162659024586901e-05, + 3.831096182693727e-05, + 4.089116555405781e-05, + 3.747857772395946e-05, + 3.4385768231004477e-05, + 3.179684790666215e-05, + 2.7402720661484636e-05, + 2.633974509080872e-05, + 2.558357300586067e-05, + 2.5047997041838244e-05, + 2.0569692424032837e-05, + 2.0414587197592482e-05, + 2.0276305804145522e-05, + 2.0262425096007064e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.030398450704524e-05, + "first_to_last_ratio": 3.89627084299362, + "imbalance_abs_log_ratio": 1.360019901586191, + "last_quartile_mean": 2.3177029561338713e-05, + "mean": 5.084823260403937e-05, + "normalized": [ + 2.109723573221063, + 2.0821310668378783, + 1.9942546197542355, + 1.874953814933449, + 1.728017146958865, + 1.5814957309589324, + 1.4629098120324804, + 1.3741245898657877, + 1.3101737824863438, + 1.2101890628861518, + 1.1282931314057325, + 1.0632895987411368, + 1.0440174426577644, + 0.9649796238381606, + 0.9017142254727002, + 0.8410904420982939, + 0.8516872374927745, + 0.7851358108806284, + 0.7291665843144397, + 0.6770579705243002, + 0.7378064582735888, + 0.6786248956615917, + 0.6313013382664657, + 0.5913982993163636, + 0.5312183147012203, + 0.5201489591253866, + 0.5114845807208003, + 0.5059300847390553, + 0.39756423665978435, + 0.3954241221367383, + 0.3931178424325881, + 0.39157560060529917 + ], + "peak_layer": 1, + "peak_normalized": 2.109723573221063, + "population_cv": 0.5213662772063792, + "rest_mean": 5.429395497668776e-05, + "spike_contrast": 0.5938291252788923, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.224133179173805e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010727571498136967, + 0.00010587268479866907, + 0.00010140432277694345, + 9.5338087703567e-05, + 8.786661783233285e-05, + 8.041626279009506e-05, + 7.438637840095907e-05, + 6.987180677242577e-05, + 6.662002124357969e-05, + 6.153597496449947e-05, + 5.7371711591258645e-05, + 5.406639684224501e-05, + 5.308644176693633e-05, + 4.90675083710812e-05, + 4.585057467920706e-05, + 4.276796244084835e-05, + 4.330679075792432e-05, + 3.9922768337419257e-05, + 3.7076832086313516e-05, + 3.4427201171638444e-05, + 3.751615440705791e-05, + 3.450687654549256e-05, + 3.210055729141459e-05, + 3.007155828527175e-05, + 2.7011512429453433e-05, + 2.6448655262356624e-05, + 2.60080869338708e-05, + 2.572565063019283e-05, + 2.021543878072407e-05, + 2.010661773965694e-05, + 1.998934749281034e-05, + 1.991092722164467e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00023694485207670368, + "first_to_last_ratio": 5.643445556811249, + "imbalance_abs_log_ratio": 1.7304947933060113, + "last_quartile_mean": 4.198584883852163e-05, + "mean": 0.00012326974319876172, + "normalized": [ + 2.0534243627746784, + 2.17548929765909, + 2.1923221930833754, + 2.045443753327701, + 2.026125682616779, + 1.7904266625829455, + 1.6133287652163053, + 1.4807635751829418, + 1.4348095534638061, + 1.3014197885977827, + 1.1747669830911103, + 1.0855875327354585, + 1.0626766295217445, + 0.9759316522493466, + 0.8962992674239908, + 0.8231145619826911, + 0.8258276905981559, + 0.7455975627999764, + 0.6776587351559191, + 0.6226196341065754, + 0.6492096639107944, + 0.5955964221479619, + 0.5398643713108281, + 0.4868843463557821, + 0.3972339380481051, + 0.3723774041823533, + 0.3563689360290182, + 0.3460491488507791, + 0.3146328182140966, + 0.31212742697325935, + 0.31138985452451795, + 0.31463178528213015 + ], + "peak_layer": 3, + "peak_normalized": 2.1923221930833754, + "population_cv": 0.61857583014248, + "rest_mean": 0.00013391299553741528, + "spike_contrast": 0.4913352905442922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.579618057003244e-05, + "top_five_layers": [ + 3, + 2, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002531250938773155, + 0.0002681720070540905, + 0.0002702469937503338, + 0.000252141326200217, + 0.00024975999258458614, + 0.0002207054349128157, + 0.00019887462258338928, + 0.00018253334565088153, + 0.00017686860519461334, + 0.00016042568313423544, + 0.00014481322432402521, + 0.0001338200963800773, + 0.0001309958752244711, + 0.00012030284415232018, + 0.00011048658052459359, + 0.00010146512067876756, + 0.00010179956734646112, + 9.19096200959757e-05, + 8.353481825906783e-05, + 7.675016240682453e-05, + 8.002790855243802e-05, + 7.341901800828055e-05, + 6.654894241364673e-05, + 6.001810834277421e-05, + 4.896692553302273e-05, + 4.590286698658019e-05, + 4.3929507228313014e-05, + 4.2657389712985605e-05, + 3.878470670315437e-05, + 3.8475867768283933e-05, + 3.83849474019371e-05, + 3.878457937389612e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.010549778118729591, + 0.02910016104578972, + 0.04255657270550728, + 0.05177471786737442, + 0.01913420483469963, + 0.02246284671127796, + 0.029843268916010857, + 0.036163609474897385, + 0.019129201769828796, + 0.0314205139875412, + 0.03302976116538048, + 0.03871368616819382, + 0.018231205642223358, + 0.02580643631517887, + 0.0349617674946785, + 0.04659247398376465, + 0.03611915931105614, + 0.03664228320121765, + 0.04177439957857132, + 0.04776441678404808, + 0.013802599161863327, + 0.02693362347781658, + 0.03932841867208481, + 0.0514267161488533, + 0.015529165975749493, + 0.03558400645852089, + 0.05707564577460289, + 0.08332383632659912, + 0.009738311171531677, + 0.03187238425016403, + 0.05786185711622238, + 0.09122034162282944 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.478662188172166e-05, + "first_to_last_ratio": 5.263389233448399, + "imbalance_abs_log_ratio": 1.6607751602108825, + "last_quartile_mean": 6.609167655824422e-06, + "mean": 1.839926902391653e-05, + "normalized": [ + 2.3257405363021224, + 2.093986543353821, + 2.0972432569142194, + 1.9748429471145472, + 1.9001738268978305, + 1.7176742899564605, + 1.5731126681977952, + 1.4424450196971237, + 1.3881676796557327, + 1.2578123363623133, + 1.1637254248597446, + 1.0739089583500412, + 1.0625597930502229, + 0.9721654778743629, + 0.897852656475923, + 0.8303134671066331, + 0.8465754829823255, + 0.7713090877071682, + 0.7043469066766334, + 0.642452591912055, + 0.6810739404273755, + 0.621476487606478, + 0.5661443389519891, + 0.5212309648706807, + 0.4305042754919776, + 0.4113531398630616, + 0.3923510140548581, + 0.378496313745133, + 0.3251027262595914, + 0.314172044005768, + 0.31172491175027217, + 0.3099608915257388 + ], + "peak_layer": 1, + "peak_normalized": 2.3257405363021224, + "population_cv": 0.5996654929960216, + "rest_mean": 1.9884546603592485e-05, + "spike_contrast": 0.5219515586938898, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.037877009366639e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.2791925807250664e-05, + 3.852782174362801e-05, + 3.858774289255962e-05, + 3.633566666394472e-05, + 3.4961809433298185e-05, + 3.160395135637373e-05, + 2.8944123187102377e-05, + 2.653993396961596e-05, + 2.554127058829181e-05, + 2.314282755833119e-05, + 2.1411697161966003e-05, + 1.9759139831876382e-05, + 1.9550323486328125e-05, + 1.7887134163174778e-05, + 1.6519832570338622e-05, + 1.5277160855475813e-05, + 1.5576370060443878e-05, + 1.419152340531582e-05, + 1.2959468222106807e-05, + 1.1820658073702361e-05, + 1.2531262655102182e-05, + 1.1434713087510318e-05, + 1.0416641998745035e-05, + 9.59026874625124e-06, + 7.920963980723172e-06, + 7.568597084173234e-06, + 7.2189718594017904e-06, + 6.964055501157418e-06, + 5.9816525208589155e-06, + 5.780535957455868e-06, + 5.735510512749897e-06, + 5.703053830075078e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013640988436236512, + "first_to_last_ratio": 5.122011664762131, + "imbalance_abs_log_ratio": 1.6335472651589908, + "last_quartile_mean": 2.6632091703504557e-05, + "mean": 7.298115508547198e-05, + "normalized": [ + 2.2789966397783896, + 2.0685142340171683, + 2.076140409583827, + 1.9537250082825501, + 1.8792316671609142, + 1.7055529809049683, + 1.5594088420998298, + 1.4313187223467239, + 1.382107687555985, + 1.2644854892365605, + 1.1686221274742359, + 1.0808639824268664, + 1.0704039370079022, + 0.9820006566284564, + 0.9088834247617501, + 0.8405321787627472, + 0.856406336273156, + 0.7807790484250425, + 0.7130462191259114, + 0.650909996157966, + 0.6919497653761542, + 0.6322822243882543, + 0.5753541906206389, + 0.5291452134901508, + 0.4371545081034775, + 0.4171814083388854, + 0.3982526780954131, + 0.3849724710776064, + 0.3303152718064163, + 0.31952201613190695, + 0.31683873767491905, + 0.31510192688522626 + ], + "peak_layer": 1, + "peak_normalized": 2.2789966397783896, + "population_cv": 0.5877439115216546, + "rest_mean": 7.874967775057519e-05, + "spike_contrast": 0.5311911602534674, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1831132693914694e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016632380720693618, + 0.00015096255810931325, + 0.0001515191252110526, + 0.00014258510782383382, + 0.00013714849774260074, + 0.00012447322660591453, + 0.00011380745854694396, + 0.00010445929365232587, + 0.0001008678154903464, + 9.228361159330234e-05, + 8.528739272151142e-05, + 7.8882701927796e-05, + 7.81193157308735e-05, + 7.16675422154367e-05, + 6.633136217715219e-05, + 6.134300929261371e-05, + 6.250152364373207e-05, + 5.6982156820595264e-05, + 5.203893670113757e-05, + 4.750416337628849e-05, + 5.049929313827306e-05, + 4.6144687075866386e-05, + 4.199001341476105e-05, + 3.8617628888459876e-05, + 3.190404095221311e-05, + 3.044638106075581e-05, + 2.9064940463285893e-05, + 2.809573561535217e-05, + 2.4106790078803897e-05, + 2.331908581254538e-05, + 2.312325705133844e-05, + 2.2996502593741752e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12096964661031961, + "first_to_last_ratio": 5.122011546850009, + "imbalance_abs_log_ratio": 1.6335472421383241, + "last_quartile_mean": 0.023617605213075876, + "mean": 0.06472041795495898, + "normalized": [ + 2.278996550367558, + 2.068514172728323, + 2.0761403775307596, + 1.9537250896945753, + 1.879231622178916, + 1.7055529409943957, + 1.5594087663016845, + 1.431318663899547, + 1.382107735687561, + 1.2644854472317082, + 1.1686221211602732, + 1.0808640120306183, + 1.0704040262086971, + 0.9820006724164564, + 0.9088834582355364, + 0.8405322479178425, + 0.8564063602524741, + 0.7807790810183837, + 0.713046234965646, + 0.6509099797949671, + 0.6919497851652032, + 0.6322822095960979, + 0.575354189755429, + 0.529145232117261, + 0.4371544964362311, + 0.41718141126177893, + 0.3982526899729607, + 0.38497247844984545, + 0.33031527661149607, + 0.3195220199067584, + 0.3168387288221828, + 0.3151019212888319 + ], + "peak_layer": 1, + "peak_normalized": 2.278996550367558, + "population_cv": 0.5877438959595169, + "rest_mean": 0.06983600149827975, + "spike_contrast": 0.5311911625115104, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.037096266821026805, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.14749760925769806, + 0.13387510180473328, + 0.1343686729669571, + 0.12644590437412262, + 0.121624656021595, + 0.11038409918546677, + 0.10092558711767197, + 0.09263554215431213, + 0.08945059031248093, + 0.08183802664279938, + 0.0756337121129036, + 0.06995397061109543, + 0.06927699595689774, + 0.06355549395084381, + 0.05882331728935242, + 0.05439959838986397, + 0.05542697757482529, + 0.050532348453998566, + 0.04614865034818649, + 0.04212716594338417, + 0.04478327929973602, + 0.040921568870544434, + 0.03723716363310814, + 0.034246500581502914, + 0.0282928217202425, + 0.027000155299901962, + 0.025775080546736717, + 0.024915579706430435, + 0.02137814275920391, + 0.020679598674178123, + 0.020505934953689575, + 0.020393528044223785 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.71177167280257e-06, + "first_to_last_ratio": 4.678295917137282, + "imbalance_abs_log_ratio": 1.542933923351862, + "last_quartile_mean": 1.6484146811990286e-06, + "mean": 4.007248069370917e-06, + "normalized": [ + 3.3590280906213086, + 2.3726088061078587, + 2.138143013037793, + 1.8393897753025719, + 1.595536022354422, + 1.4487646483149947, + 1.2955498112963995, + 1.3466260007113127, + 1.2739769202114613, + 1.0659628023122865, + 1.0478341793654378, + 1.0914579443825887, + 1.0411457104586188, + 0.9155146938805809, + 0.8066044523853629, + 0.7701707432135287, + 0.8215762550934921, + 0.7364785130598478, + 0.6730090976811546, + 0.6266208737012697, + 0.6693583504854638, + 0.6236588439557476, + 0.5897891291930596, + 0.5603290672032583, + 0.4588325813842428, + 0.4472228580494504, + 0.4388965999812878, + 0.43180771263272905, + 0.37877595830431615, + 0.3751711426274845, + 0.3796103005144005, + 0.3805491021762693 + ], + "peak_layer": 1, + "peak_normalized": 3.3590280906213086, + "population_cv": 0.6648044266247514, + "rest_mean": 4.318630839474348e-06, + "spike_contrast": 0.5385459413556807, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.325781110812386e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.3460458831104916e-05, + 9.507632057648152e-06, + 8.568069461034611e-06, + 7.370891125901835e-06, + 6.393708645191509e-06, + 5.805559339933097e-06, + 5.191589480091352e-06, + 5.396264441515086e-06, + 5.105141553940484e-06, + 4.271577381587122e-06, + 4.198931492283009e-06, + 4.373742740426678e-06, + 4.172129138169112e-06, + 3.6686944895336637e-06, + 3.232264134567231e-06, + 3.086265223828377e-06, + 3.292259862064384e-06, + 2.9512520995922387e-06, + 2.696914407351869e-06, + 2.5110252863669302e-06, + 2.682284957700176e-06, + 2.499155698387767e-06, + 2.3634313492948422e-06, + 2.2453775727626635e-06, + 1.8386559759164811e-06, + 1.7921329344972037e-06, + 1.7587675529284752e-06, + 1.730360622786975e-06, + 1.5178492276390898e-06, + 1.5034038369776681e-06, + 1.5211926438496448e-06, + 1.5249546549966908e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00011456928586994763, + "first_to_last_ratio": 4.836938296613537, + "imbalance_abs_log_ratio": 1.5762819372189676, + "last_quartile_mean": 2.3686323629590333e-05, + "mean": 6.211567796299278e-05, + "normalized": [ + 2.3488758065274955, + 2.0713117417995783, + 2.0294413849686572, + 1.9098281260734524, + 1.808891788235116, + 1.6555687759458058, + 1.5234656987314679, + 1.4082197237907075, + 1.3570554765296423, + 1.2477441770945124, + 1.1584743113969003, + 1.07706541625456, + 1.062967326057557, + 0.9794184805192363, + 0.9103158522005721, + 0.8467437815449563, + 0.8606016851807378, + 0.7903183885158978, + 0.726053695018778, + 0.6672539632450412, + 0.7127881461452974, + 0.6531207323660743, + 0.5952249360680494, + 0.548642506624817, + 0.4562346365956528, + 0.4371660140704956, + 0.4212530247856756, + 0.40928721260242446, + 0.3409968025795472, + 0.33116514257975266, + 0.32832896742236684, + 0.3261762785291743 + ], + "peak_layer": 1, + "peak_normalized": 2.3488758065274955, + "population_cv": 0.5738573169932581, + "rest_mean": 6.679503382690234e-05, + "spike_contrast": 0.5516451476522887, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6847156297881153e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014590201317332685, + 0.00012866093311458826, + 0.00012606012751348317, + 0.00011863026884384453, + 0.0001123605397879146, + 0.00010283677693223581, + 9.463110473006964e-05, + 8.747252286411822e-05, + 8.429442095803097e-05, + 7.750447548460215e-05, + 7.195941725512967e-05, + 6.690264854114503e-05, + 6.602693611057475e-05, + 6.08372429269366e-05, + 5.654488631989807e-05, + 5.259606405161321e-05, + 5.34568571310956e-05, + 4.9091162509284914e-05, + 4.509931750362739e-05, + 4.1446932300459594e-05, + 4.4275318941799924e-05, + 4.0569037082605064e-05, + 3.697280044434592e-05, + 3.4079301258316264e-05, + 2.833932376233861e-05, + 2.7154863346368074e-05, + 2.6166417228523642e-05, + 2.5423152692383155e-05, + 2.1181247575441375e-05, + 2.0570547349052504e-05, + 2.0394376406329684e-05, + 2.0260660676285625e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.518698971078265e-05, + "first_to_last_ratio": 4.0537764308209505, + "imbalance_abs_log_ratio": 1.399648898709643, + "last_quartile_mean": 2.3481065454689087e-05, + "mean": 5.345527347344614e-05, + "normalized": [ + 2.3490757134047047, + 2.03180856162592, + 1.9640798219543283, + 1.8355777168881, + 1.687288592851433, + 1.564711486655573, + 1.4531995506450504, + 1.3597364706007256, + 1.3148292531115187, + 1.211773745536977, + 1.127520479381137, + 1.0571877619151735, + 1.041978223119568, + 0.9643430432278, + 0.9029106186893467, + 0.848047824187126, + 0.863550005931762, + 0.7965670293846996, + 0.7398915500109849, + 0.6885201220036962, + 0.7527468103098911, + 0.696422774701012, + 0.6386525384719246, + 0.5954551044265526, + 0.5183504589787767, + 0.502904662044204, + 0.4927615213178117, + 0.48500410119101467, + 0.3866832502775355, + 0.3782793274311306, + 0.37617794688759315, + 0.3739639328369289 + ], + "peak_layer": 1, + "peak_normalized": 2.3490757134047047, + "population_cv": 0.5319468462674285, + "rest_mean": 5.701573583657456e-05, + "spike_contrast": 0.6003391206010799, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4228776712552644e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012557048466987908, + 0.00010861088230740279, + 0.00010499042400624603, + 9.812130883801728e-05, + 9.019447315949947e-05, + 8.364208042621613e-05, + 7.768117939122021e-05, + 7.268508488778025e-05, + 7.028455729596317e-05, + 6.477569695562124e-05, + 6.027191557222977e-05, + 5.651226092595607e-05, + 5.5699230870231986e-05, + 5.154922109795734e-05, + 4.826533404411748e-05, + 4.5332628360483795e-05, + 4.6161301725078374e-05, + 4.2580708395689726e-05, + 3.955110514652915e-05, + 3.680503141367808e-05, + 4.0238286601379514e-05, + 3.722746987477876e-05, + 3.413934609852731e-05, + 3.1830215448280796e-05, + 2.7708565539796837e-05, + 2.6882906240643933e-05, + 2.6340701879234985e-05, + 2.5926026864908636e-05, + 2.0670258891186677e-05, + 2.022102489718236e-05, + 2.010869502555579e-05, + 1.9990344299003482e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002512551309337141, + "first_to_last_ratio": 5.815353323084221, + "imbalance_abs_log_ratio": 1.7605015447583472, + "last_quartile_mean": 4.320547986935708e-05, + "mean": 0.0001316397593882357, + "normalized": [ + 2.224752247026743, + 2.0950471769673564, + 2.148762457826491, + 2.0117752264198114, + 1.9594249362752976, + 1.7610216345575365, + 1.6099537081187654, + 1.4585175613957049, + 1.4124641586885858, + 1.2915649811078755, + 1.192264449118915, + 1.0889144888939652, + 1.0788791319896314, + 0.988941963176937, + 0.9104012729618977, + 0.8363983367426334, + 0.8532666616213362, + 0.773042726053889, + 0.7048431559693451, + 0.6396986756467042, + 0.668255590231208, + 0.6075232312555597, + 0.5571337058341255, + 0.5014728599784464, + 0.3945136003262428, + 0.3676589998419824, + 0.34703606615034643, + 0.33261343254293874, + 0.3038165004585062, + 0.2945782724854268, + 0.2930231193739135, + 0.29243967096188167 + ], + "peak_layer": 1, + "peak_normalized": 2.224752247026743, + "population_cv": 0.6145445602222892, + "rest_mean": 0.0001427126183073226, + "spike_contrast": 0.503433558134641, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.184632122516632e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00029286585049703717, + 0.0002757915062829852, + 0.00028286257293075323, + 0.0002648296067491174, + 0.00025793822715058923, + 0.00023182046425063163, + 0.0002119339187629521, + 0.00019199890084564686, + 0.00018593644199427217, + 0.00017002130334731191, + 0.00015694940520916134, + 0.00014334444131236523, + 0.00014202338934410363, + 0.00013018408208154142, + 0.00011984500451944768, + 0.0001101032758015208, + 0.00011232381802983582, + 0.00010176315845455974, + 9.278538345824927e-05, + 8.420977974310517e-05, + 8.796900510787964e-05, + 7.997421198524535e-05, + 7.334094698308036e-05, + 6.601376662729308e-05, + 5.193367542233318e-05, + 4.839854227611795e-05, + 4.5683744247071445e-05, + 4.3785152229247615e-05, + 3.999433101853356e-05, + 3.877821291098371e-05, + 3.857349292957224e-05, + 3.849668792099692e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.021530626341700554, + 0.03857670724391937, + 0.04954109713435173, + 0.057694368064403534, + 0.02140580676496029, + 0.025963041931390762, + 0.03339661285281181, + 0.040483180433511734, + 0.02151828072965145, + 0.035047177225351334, + 0.03725159168243408, + 0.043239567428827286, + 0.02124657854437828, + 0.03028978779911995, + 0.03979094699025154, + 0.05222286656498909, + 0.03916332125663757, + 0.04088718071579933, + 0.04756740853190422, + 0.05542457476258278, + 0.020541559904813766, + 0.03680417314171791, + 0.05126549303531647, + 0.06891486048698425, + 0.02905990369617939, + 0.05358703061938286, + 0.07880406081676483, + 0.10752817988395691, + 0.029148343950510025, + 0.054091889411211014, + 0.08628532290458679, + 0.11884573101997375 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.2954461630652077e-05, + "first_to_last_ratio": 5.105695837816317, + "imbalance_abs_log_ratio": 1.6303567474829335, + "last_quartile_mean": 6.454450613091467e-06, + "mean": 1.7350209816413553e-05, + "normalized": [ + 2.118074842246104, + 2.1574284903940977, + 2.139357285099935, + 2.0292905713578766, + 1.9547395369559644, + 1.7470193301554893, + 1.5858965602051887, + 1.4631504379843852, + 1.4022879982430452, + 1.2651865013786623, + 1.1596469721061315, + 1.075647057275218, + 1.0548378604659292, + 0.9617861976864589, + 0.8861851944905509, + 0.81666640942737, + 0.8226291628131417, + 0.7462778224571112, + 0.6809633158339518, + 0.623540506764385, + 0.6619488684789228, + 0.6046004431197245, + 0.5544183299835491, + 0.5123407388295048, + 0.4384090182726579, + 0.4193402639944847, + 0.4032847838635732, + 0.3940637985584808, + 0.33301498018274, + 0.33016832240151506, + 0.32877371956677426, + 0.3290246794070765 + ], + "peak_layer": 2, + "peak_normalized": 2.1574284903940977, + "population_cv": 0.5996208655691896, + "rest_mean": 1.8782105026157632e-05, + "spike_contrast": 0.5120818816847563, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.61797568379552e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.674904291983694e-05, + 3.7431836972245947e-05, + 3.711829776875675e-05, + 3.52086171915289e-05, + 3.391514110262506e-05, + 3.0311151931528002e-05, + 2.7515638066688552e-05, + 2.5385967092006467e-05, + 2.4329990992555395e-05, + 2.1951251255813986e-05, + 2.0120118279010057e-05, + 1.8662702132132836e-05, + 1.8301658201380633e-05, + 1.6687192328390665e-05, + 1.537549906061031e-05, + 1.4169333553581964e-05, + 1.4272788575908635e-05, + 1.2948076800967101e-05, + 1.1814856406999752e-05, + 1.0818558621394914e-05, + 1.148495175584685e-05, + 1.0489944543223828e-05, + 9.619274351280183e-06, + 8.889219316188246e-06, + 7.606488452438498e-06, + 7.27564156477456e-06, + 6.997075615799986e-06, + 6.837089586042566e-06, + 5.77787977817934e-06, + 5.728489668399561e-06, + 5.704293016606243e-06, + 5.708647222490981e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00012894753399450565, + "first_to_last_ratio": 4.956313899688395, + "imbalance_abs_log_ratio": 1.600662299067145, + "last_quartile_mean": 2.601682149361295e-05, + "mean": 6.873563796716553e-05, + "normalized": [ + 2.0729820101052394, + 2.1343785185339303, + 2.1192055857825096, + 2.004033226215171, + 1.9290001532413559, + 1.7292283569198776, + 1.5695265126752398, + 1.449584785193964, + 1.392505223726082, + 1.2691464160417294, + 1.1657826244262586, + 1.0843620958713254, + 1.0620513069452608, + 0.9734787377496757, + 0.8984695878304712, + 0.827044661933948, + 0.8348303459591176, + 0.7567409969045322, + 0.6912396733798123, + 0.6326506836056272, + 0.674324856446311, + 0.615953502956551, + 0.5640541296457265, + 0.5213814868377082, + 0.4457146646251511, + 0.42575966145291916, + 0.4106923182898804, + 0.40125618166856064, + 0.3390529581992065, + 0.3361781427218294, + 0.33439592808042723, + 0.33499466603460004 + ], + "peak_layer": 2, + "peak_normalized": 2.1343785185339303, + "population_cv": 0.5868596491376813, + "rest_mean": 7.428176730981149e-05, + "spike_contrast": 0.5221542367874475, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.8786539516877386e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014248774095904082, + 0.00014670786913484335, + 0.00014566494792234153, + 0.00013774850231129676, + 0.00013259105617180467, + 0.00011885961430380121, + 0.00010788240615511313, + 9.963813499780372e-05, + 9.571473492542282e-05, + 8.723558858036995e-05, + 8.013081242097542e-05, + 7.453432044712827e-05, + 7.300077413674444e-05, + 6.691268208669499e-05, + 6.17568803136237e-05, + 5.684744246536866e-05, + 5.7382596423849463e-05, + 5.201507519814186e-05, + 4.7512799937976524e-05, + 4.3485648347996175e-05, + 4.6350149204954505e-05, + 4.233795698382892e-05, + 3.877062044921331e-05, + 3.5837489122059196e-05, + 3.0636481824330986e-05, + 2.926486195065081e-05, + 2.8229198505869135e-05, + 2.7580599635257386e-05, + 2.3305021386477165e-05, + 2.3107419110601768e-05, + 2.298491745023057e-05, + 2.3026072085485794e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11435196176171303, + "first_to_last_ratio": 4.956313746493952, + "imbalance_abs_log_ratio": 1.6006622681581977, + "last_quartile_mean": 0.023071978008374572, + "mean": 0.06095545022981241, + "normalized": [ + 2.072982016679514, + 2.1343786009397814, + 2.119205465978219, + 2.004033333779568, + 1.9290000448154139, + 1.7292284201771326, + 1.5695265788612092, + 1.4495847902174488, + 1.392505122730498, + 1.2691463963100211, + 1.1657826534776023, + 1.0843621171351583, + 1.0620512430067517, + 0.97347872771027, + 0.8984695791578813, + 0.8270447011604714, + 0.8348303200871255, + 0.7567409772234059, + 0.691239667353491, + 0.6326506435390216, + 0.6743248169617974, + 0.6159534846579819, + 0.5640541661909846, + 0.5213814964456189, + 0.4457146897387812, + 0.4257596753388107, + 0.4106923303842507, + 0.4012562101149584, + 0.33905296941192775, + 0.3361781519455022, + 0.33439594785006815, + 0.3349946606193331 + ], + "peak_layer": 2, + "peak_normalized": 2.1343786009397814, + "population_cv": 0.586859647417989, + "rest_mean": 0.0658738130910529, + "spike_contrast": 0.5221542395241354, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03439629077911377, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.12635955214500427, + 0.1301020085811615, + 0.12917712330818176, + 0.12215675413608551, + 0.11758306622505188, + 0.10540589690208435, + 0.09567119926214218, + 0.08836009353399277, + 0.0848807767033577, + 0.07736138999462128, + 0.07106080651283264, + 0.06609778106212616, + 0.06473781168460846, + 0.05933883413672447, + 0.054766617715358734, + 0.05041288211941719, + 0.05088745802640915, + 0.04612748697400093, + 0.04213482514023781, + 0.038563504815101624, + 0.041103772819042206, + 0.03754572197794914, + 0.0343821756541729, + 0.031781043857336044, + 0.027168739587068558, + 0.025952372699975967, + 0.02503393590450287, + 0.0244587529450655, + 0.020667126402258873, + 0.020491890609264374, + 0.020383255556225777, + 0.02041975036263466 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.461771360794955e-06, + "first_to_last_ratio": 4.607249847489048, + "imbalance_abs_log_ratio": 1.5276311166276264, + "last_quartile_mean": 1.6195716767697377e-06, + "mean": 3.87299716564371e-06, + "normalized": [ + 3.0256593103213434, + 2.4199948075671043, + 2.2257260804728656, + 1.8933484975335824, + 1.6741432062616066, + 1.4907581305464577, + 1.3020525186737137, + 1.38123110006882, + 1.2950838303767327, + 1.0716197853648062, + 1.0585124142900662, + 1.1105323556195759, + 1.0395543790453834, + 0.9297870003858463, + 0.7930926998517134, + 0.7672014355923814, + 0.811551487187536, + 0.7239809147787968, + 0.6594474866461876, + 0.6140918356353618, + 0.649091321445497, + 0.6059854562252807, + 0.5735486080526876, + 0.5386444043869645, + 0.45665673160062437, + 0.44875544656608446, + 0.4444235140257583, + 0.44115113302107534, + 0.3820499025609541, + 0.38571480138297476, + 0.3910437033480733, + 0.3955657011641444 + ], + "peak_layer": 1, + "peak_normalized": 3.0256593103213434, + "population_cv": 0.6453809227293102, + "rest_mean": 4.185142588371322e-06, + "spike_contrast": 0.5226612562717605, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1874118829146026e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1718369933078066e-05, + 9.372633030579891e-06, + 8.620230801170692e-06, + 7.332933364523342e-06, + 6.483951892732875e-06, + 5.773702014266746e-06, + 5.042845714342548e-06, + 5.349504135665484e-06, + 5.015856004320085e-06, + 4.150380391365616e-06, + 4.099615580344107e-06, + 4.30108866567025e-06, + 4.026191163575277e-06, + 3.60106241714675e-06, + 3.0716457786184037e-06, + 2.9713689855270786e-06, + 3.1431366096512647e-06, + 2.8039760309184203e-06, + 2.5540382466715528e-06, + 2.3783759388606995e-06, + 2.5139288482023403e-06, + 2.3469799543818226e-06, + 2.2213521333469544e-06, + 2.086168251480558e-06, + 1.7686302271613386e-06, + 1.7380285726176226e-06, + 1.7212510101671796e-06, + 1.7085770878111362e-06, + 1.4796781897530309e-06, + 1.4938723325030878e-06, + 1.5145111547099077e-06, + 1.5320248394345981e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00010837210174940992, + "first_to_last_ratio": 4.663743740878985, + "imbalance_abs_log_ratio": 1.5398185034733132, + "last_quartile_mean": 2.3237147615873255e-05, + "mean": 5.8729946545099665e-05, + "normalized": [ + 2.116910388942848, + 2.12576419551759, + 2.0713182282290723, + 1.958808009785277, + 1.85306829100999, + 1.6774795633398605, + 1.5342385858151337, + 1.4245037147377415, + 1.3659845454711594, + 1.2527089446494675, + 1.1583413114381589, + 1.0816008878212584, + 1.059050106445956, + 0.9749946937544519, + 0.9039297824077048, + 0.8370924379536968, + 0.8447837999101473, + 0.7717212504681601, + 0.708779947616627, + 0.6523240772837903, + 0.6962576204392495, + 0.6381480261388907, + 0.5854918741973255, + 0.5414119569273494, + 0.4665866617482466, + 0.4484909063213352, + 0.43561453714162146, + 0.42649489708721455, + 0.35024135478224894, + 0.347602098795276, + 0.34524713565577436, + 0.34501016816737656 + ], + "peak_layer": 2, + "peak_normalized": 2.12576419551759, + "population_cv": 0.5682337234037166, + "rest_mean": 6.323715206235647e-05, + "spike_contrast": 0.5438422767363214, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4391036751912904e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001243260339833796, + 0.00012484601757023484, + 0.00012164840882178396, + 0.00011504068970680237, + 0.0001088306016754359, + 9.851828508544713e-05, + 9.010575013235211e-05, + 8.366102701984346e-05, + 8.022419933695346e-05, + 7.357152935583144e-05, + 6.802932330174372e-05, + 6.352236232487485e-05, + 6.219795614015311e-05, + 5.726138624595478e-05, + 5.308774780132808e-05, + 4.916239413432777e-05, + 4.961410741088912e-05, + 4.5323147787712514e-05, + 4.1626608435763046e-05, + 3.8310958188958466e-05, + 4.089117283001542e-05, + 3.747839946299791e-05, + 3.4385906474199146e-05, + 3.1797095289221033e-05, + 2.740260970313102e-05, + 2.6339846954215318e-05, + 2.558361848059576e-05, + 2.5048022507689893e-05, + 2.0569656044244766e-05, + 2.041465268121101e-05, + 2.02763458219124e-05, + 2.026242873398587e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.029894954437623e-05, + "first_to_last_ratio": 3.896020389480804, + "imbalance_abs_log_ratio": 1.3599556192062257, + "last_quartile_mean": 2.3177227149062674e-05, + "mean": 5.08472848537167e-05, + "normalized": [ + 2.109310432443945, + 2.0823500317454187, + 1.994011612294058, + 1.875066462699875, + 1.727617067796283, + 1.5815194849270788, + 1.4631049290688447, + 1.374102981258909, + 1.3102874938711893, + 1.2100951410093623, + 1.1285247251248958, + 1.0633171446519065, + 1.0439918275480478, + 0.9649933174114446, + 0.9016610595399868, + 0.8411183538760414, + 0.8515897815206308, + 0.7853791814165375, + 0.7291252271433669, + 0.6771443554330082, + 0.737694287360078, + 0.6786209457138218, + 0.6314365240390994, + 0.5913748367488056, + 0.5313425485217884, + 0.5201071760800584, + 0.5115574693700321, + 0.5058856398361036, + 0.39753526518974286, + 0.39544594503370456, + 0.39314180453082337, + 0.39154694679511176 + ], + "peak_layer": 1, + "peak_normalized": 2.109310432443945, + "population_cv": 0.5213289902477497, + "rest_mean": 5.429271732282359e-05, + "spike_contrast": 0.5938540399226916, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.224194952053949e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010725270840339363, + 0.00010588184522930533, + 0.00010139007645193487, + 9.53420385485515e-05, + 8.78446371643804e-05, + 8.04159717517905e-05, + 7.439491309924051e-05, + 6.986940570641309e-05, + 6.662456144113094e-05, + 6.153005233500153e-05, + 5.7382418162887916e-05, + 5.406678974395618e-05, + 5.3084149840287864e-05, + 4.906729009235278e-05, + 4.584701673593372e-05, + 4.276858453522436e-05, + 4.3301028199493885e-05, + 3.993439895566553e-05, + 3.707403811858967e-05, + 3.4430951927788556e-05, + 3.750975156435743e-05, + 3.4506032534409314e-05, + 3.210683280485682e-05, + 3.006980477948673e-05, + 2.7017325919587165e-05, + 2.644603773660492e-05, + 2.601130836410448e-05, + 2.572291123215109e-05, + 2.0213588868500665e-05, + 2.0107352611375973e-05, + 1.9990193322882988e-05, + 1.9909099137294106e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002369592275499599, + "first_to_last_ratio": 5.643806467094249, + "imbalance_abs_log_ratio": 1.7305587433768272, + "last_quartile_mean": 4.1985711050074315e-05, + "mean": 0.00012327438730608264, + "normalized": [ + 2.0536938202390447, + 2.174916037645403, + 2.1925488795748866, + 2.045836750341831, + 2.0263612272059848, + 1.7896425612539104, + 1.6137064052293353, + 1.480972211102483, + 1.4347200865083345, + 1.3015150111942828, + 1.1747028946305826, + 1.0857434163548976, + 1.0627794297334374, + 0.9758997849090596, + 0.8960919751718553, + 0.8230876844179131, + 0.8258333502423459, + 0.7456000475898096, + 0.6776422362024666, + 0.6227334054207839, + 0.6492856625079976, + 0.5954634942404196, + 0.5397936868798249, + 0.4867302228264425, + 0.3971938000026409, + 0.37230063475944564, + 0.35633216718416544, + 0.3460424570800059, + 0.31457528168990173, + 0.31217590063925915, + 0.31141034981968424, + 0.31466912740156466 + ], + "peak_layer": 3, + "peak_normalized": 2.1925488795748866, + "population_cv": 0.618597930641205, + "rest_mean": 0.00013391951021427909, + "spike_contrast": 0.49127064082412475, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.57907236018218e-05, + "top_five_layers": [ + 3, + 2, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00025316784740425646, + 0.00026811144198291004, + 0.0002702851197682321, + 0.00025219927192665637, + 0.0002497984387446195, + 0.00022061709023546427, + 0.00019892866839654744, + 0.00018256594194099307, + 0.00017686423962004483, + 0.0001604434655746445, + 0.00014481077960226685, + 0.000133844354422763, + 0.00013101348304189742, + 0.00012030344805680215, + 0.00011046518920920789, + 0.00010146562999580055, + 0.00010180410026805475, + 9.191338904201984e-05, + 8.35359314805828e-05, + 7.67670790082775e-05, + 8.004029223229736e-05, + 7.34053974156268e-05, + 6.654273602180183e-05, + 6.0001370002282783e-05, + 4.896382233710028e-05, + 4.5895132643636316e-05, + 4.3926629587076604e-05, + 4.265817187842913e-05, + 3.877907511196099e-05, + 3.848329288302921e-05, + 3.838892007479444e-05, + 3.8790643884567544e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04393457621335983, + 0.024121688678860664, + 0.030756350606679916, + 0.03544071316719055, + 0.03840360417962074, + 0.026969486847519875, + 0.027978016063570976, + 0.029923241585493088, + 0.03152087330818176, + 0.024161184206604958, + 0.025710709393024445, + 0.02641460672020912, + 0.02742084674537182, + 0.021717891097068787, + 0.0224718376994133, + 0.023445574566721916, + 0.024849263951182365, + 0.01992032676935196, + 0.02058842033147812, + 0.020979218184947968, + 0.02214338816702366, + 0.01903437450528145, + 0.02008410543203354, + 0.021635910496115685, + 0.023218298330903053, + 0.022257626056671143, + 0.024358727037906647, + 0.025900650769472122, + 0.027197737246751785, + 0.02823900803923607, + 0.02913021855056286, + 0.029048049822449684 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.2108061784820165e-05, + "first_to_last_ratio": 4.5038225950610284, + "imbalance_abs_log_ratio": 1.5049265017537072, + "last_quartile_mean": 4.908732819330908e-06, + "mean": 1.424826272966584e-05, + "normalized": [ + 1.5393565072014856, + 1.6436990565699428, + 1.5746209905623882, + 1.4393049982912232, + 1.2296080351406573, + 1.5542168455156227, + 1.5210128581976592, + 1.9112374884411145, + 1.6656665742649008, + 1.4395591772020961, + 1.6324250537476421, + 1.3835491754103626, + 1.3477025444685555, + 1.5824225341301699, + 0.9384151980613074, + 0.8636925984699546, + 0.8034013225116797, + 0.8830691126060858, + 0.8605378947700126, + 0.6634146810472719, + 0.6587076476302224, + 0.7136661453539468, + 0.7077576665676224, + 0.6868400571413836, + 0.5793897205721035, + 0.6202776091342075, + 0.43119680892404477, + 0.3692582531909216, + 0.260043154103081, + 0.12977534522113043, + 0.15751431041740122, + 0.20866063513380237 + ], + "peak_layer": 8, + "peak_normalized": 1.9112374884411145, + "population_cv": 0.5186010202163185, + "rest_mean": 1.5120910120458575e-05, + "spike_contrast": 0.6306476755313104, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.535966819385067e-06, + "top_five_layers": [ + 8, + 9, + 2, + 11, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1933155949227512e-05, + 2.341985600651242e-05, + 2.243561357317958e-05, + 2.050759576377459e-05, + 1.751977833919227e-05, + 2.2144889953779057e-05, + 2.167179081880022e-05, + 2.723181387409568e-05, + 2.3732854970148765e-05, + 2.051121737167705e-05, + 2.3259221052285284e-05, + 1.9713172150659375e-05, + 1.9202419935027137e-05, + 2.2546772015630268e-05, + 1.3370786291488912e-05, + 1.2306119060667697e-05, + 1.1447073120507412e-05, + 1.2582200724864379e-05, + 1.2261170013516676e-05, + 9.452506674278993e-06, + 9.385439625475556e-06, + 1.0168502740270924e-05, + 1.0084317182190716e-05, + 9.786277587409131e-06, + 8.255296961579006e-06, + 8.837878340273164e-06, + 6.143805421743309e-06, + 5.26128860656172e-06, + 3.705163180711679e-06, + 1.8490732145437505e-06, + 2.244305278509273e-06, + 2.9730515507253585e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.226752566362848e-05, + "first_to_last_ratio": 3.2558095109526493, + "imbalance_abs_log_ratio": 1.1804409424216962, + "last_quartile_mean": 1.9125051835544582e-05, + "mean": 4.323053209986938e-05, + "normalized": [ + 1.0342606051842873, + 1.898717126163711, + 1.8089935381392723, + 1.359356421079101, + 1.274089744043777, + 1.682360782557305, + 1.2682690495969076, + 1.1968320759706532, + 1.2588398746694436, + 1.2880627600055157, + 1.2248689931301155, + 1.273330768170305, + 1.3149263319378108, + 1.3464942246682916, + 1.1005868831933079, + 1.0691381511055633, + 1.0591169598492518, + 1.0073228103092708, + 0.9327165506900205, + 0.7338249833201036, + 0.7729257473598179, + 0.8419776567654331, + 0.8893297062436448, + 0.8244833428272487, + 0.7679470877937514, + 0.8196268734117844, + 0.5402279437877997, + 0.47239671566976066, + 0.3242697700833044, + 0.15928933562535502, + 0.20350298326020644, + 0.25191420338787907 + ], + "peak_layer": 2, + "peak_normalized": 1.898717126163711, + "population_cv": 0.42885792050879057, + "rest_mean": 4.467689194387748e-05, + "spike_contrast": 0.7928078118486851, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.542018894222565e-05, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.471163629204966e-05, + 8.208255167119205e-05, + 7.82037532189861e-05, + 5.8765701396623626e-05, + 5.507957757799886e-05, + 7.272935181390494e-05, + 5.482794585986994e-05, + 5.173968747840263e-05, + 5.442031761049293e-05, + 5.568363849306479e-05, + 5.295173832564615e-05, + 5.50467666471377e-05, + 5.684496500180103e-05, + 5.820966180181131e-05, + 4.757895658258349e-05, + 4.621941116056405e-05, + 4.578618973027915e-05, + 4.354710108600557e-05, + 4.0321832784684375e-05, + 3.172364449710585e-05, + 3.341399133205414e-05, + 3.639914211817086e-05, + 3.8446196413133293e-05, + 3.564285361790098e-05, + 3.319876122986898e-05, + 3.543290586094372e-05, + 2.3354341465164907e-05, + 2.0421961380634457e-05, + 1.4018354704603553e-05, + 6.886162736918777e-06, + 8.797542250249535e-06, + 1.0890385055972729e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05521946307271719, + "first_to_last_ratio": 3.255809416547513, + "imbalance_abs_log_ratio": 1.180440913425793, + "last_quartile_mean": 0.016960287292022258, + "mean": 0.03833726760058198, + "normalized": [ + 1.0342606436254083, + 1.89871698336364, + 1.808993472457683, + 1.3593564915846248, + 1.2740897521241208, + 1.682360722725715, + 1.2682690813216775, + 1.1968320831111716, + 1.2588398492110833, + 1.2880628213558492, + 1.224868979767805, + 1.273330745435787, + 1.314926371992558, + 1.3464941899184253, + 1.1005869061077207, + 1.069138220456837, + 1.059116922317307, + 1.0073228539527392, + 0.932716509673011, + 0.7338250308305378, + 0.7729257302043212, + 0.841977653627328, + 0.8893297206164247, + 0.8244832831061564, + 0.7679470991481019, + 0.8196269423990945, + 0.5402279418758984, + 0.4723966869996653, + 0.32426976863400875, + 0.15928933802210024, + 0.20350299592243773, + 0.25191420811076204 + ], + "peak_layer": 2, + "peak_normalized": 1.89871698336364, + "population_cv": 0.42885790694027637, + "rest_mean": 0.039619914031829, + "spike_contrast": 0.7928077998002174, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03141097687184811, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.039650727063417435, + 0.07279162108898163, + 0.06935186684131622, + 0.05211401358246803, + 0.04884511977434158, + 0.06449711322784424, + 0.048621971160173416, + 0.04588327184319496, + 0.04826048016548157, + 0.04938080906867981, + 0.04695812985301018, + 0.0488160215318203, + 0.0504106841981411, + 0.051620908081531525, + 0.042193494737148285, + 0.04098783805966377, + 0.040603648871183395, + 0.03861800581216812, + 0.03575780242681503, + 0.02813284657895565, + 0.029631860554218292, + 0.032279122620821, + 0.034094471484422684, + 0.03160843625664711, + 0.029440993443131447, + 0.03142225742340088, + 0.020710863173007965, + 0.018110398203134537, + 0.012431616894900799, + 0.006106717977672815, + 0.007801748812198639, + 0.009657702408730984 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2073289212821692e-05, + "first_to_last_ratio": 5.118173394564018, + "imbalance_abs_log_ratio": 1.6327976165236144, + "last_quartile_mean": 2.358905859978222e-06, + "mean": 9.608797252980139e-06, + "normalized": [ + 2.2116042312522914, + 1.2252124207577697, + 0.9166421564982928, + 1.186820314517719, + 0.9292558997947784, + 1.1346790160633606, + 1.0583515009014062, + 1.3892974727904446, + 1.5862793081829674, + 1.2655275776533215, + 1.4193695477429429, + 2.042377666614073, + 1.9979815760663124, + 2.116845169548275, + 1.4502152158936208, + 1.0590931016242418, + 1.4220084536973343, + 1.379051361973986, + 1.173597294405293, + 0.9241153337121745, + 0.6121286422299292, + 0.5710108338819657, + 0.461893587104583, + 0.5026871640197357, + 0.30513715456990576, + 0.2951723976971137, + 0.2866930192727529, + 0.2870900856457325, + 0.21497083513437823, + 0.17484004907583897, + 0.18455905309901366, + 0.21549255857844585 + ], + "peak_layer": 1, + "peak_normalized": 2.2116042312522914, + "population_cv": 0.6038429581123452, + "rest_mean": 1.0515277882337546e-05, + "spike_contrast": 0.44828124441370076, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.713801854450139e-06, + "top_five_layers": [ + 1, + 14, + 12, + 13, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1250856661936268e-05, + 1.1772817742894404e-05, + 8.807828635326587e-06, + 1.1403915777918883e-05, + 8.929031537263654e-06, + 1.0902900612563826e-05, + 1.0169484994548839e-05, + 1.3349477740121074e-05, + 1.5242236258927733e-05, + 1.2160197911725845e-05, + 1.3638434211316053e-05, + 1.9624792912509292e-05, + 1.919819987961091e-05, + 2.034033605013974e-05, + 1.3934823982708622e-05, + 1.017661088553723e-05, + 1.3663790923601482e-05, + 1.3251024938654155e-05, + 1.1276858458586503e-05, + 8.879636880010366e-06, + 5.8818200159294065e-06, + 5.4867273320269305e-06, + 4.43824183093966e-06, + 4.830219040741213e-06, + 2.9320010526134865e-06, + 2.836251724147587e-06, + 2.75477509603661e-06, + 2.7585904263105476e-06, + 2.06561117011006e-06, + 1.6800025832708343e-06, + 1.773390522430418e-06, + 2.0706243049062323e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.736539585792343e-05, + "first_to_last_ratio": 5.693189725390274, + "imbalance_abs_log_ratio": 1.7392706754966003, + "last_quartile_mean": 8.319658775235439e-06, + "mean": 3.0428698622131378e-05, + "normalized": [ + 0.9332234373969145, + 2.093244562439342, + 2.000219271237334, + 1.4660610383042352, + 1.4267640458493276, + 1.8877254776078727, + 1.397617962458032, + 1.2479661794130616, + 1.4262249611304942, + 1.4477791439571595, + 1.24181841376056, + 1.2462360629867504, + 1.2738256274260629, + 1.4269714780931013, + 1.1746071387256543, + 1.0297217541028905, + 1.0028420024769986, + 1.0837416535661664, + 0.9989109522318798, + 0.7645609375506638, + 0.8126373951104926, + 0.8905222845518224, + 0.8066853450678224, + 0.7327738890015353, + 0.50846646429871, + 0.4349609059082798, + 0.29076653368375877, + 0.2846228030965179, + 0.21016872042077955, + 0.130190170117833, + 0.14875731368946996, + 0.17938607433847711 + ], + "peak_layer": 2, + "peak_normalized": 2.093244562439342, + "population_cv": 0.5294278422744164, + "rest_mean": 3.183621146052718e-05, + "spike_contrast": 0.7170491791430023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.2828129294794054e-05, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.83967747236602e-05, + 6.369470793288201e-05, + 6.086406938266009e-05, + 4.461032949620858e-05, + 4.3414573156042024e-05, + 5.744102963944897e-05, + 4.2527695768512785e-05, + 3.797398676397279e-05, + 4.339816950960085e-05, + 4.405403524287976e-05, + 3.7786918255733326e-05, + 3.7921341572655365e-05, + 3.876085611409508e-05, + 4.342088504927233e-05, + 3.5741766623687e-05, + 3.133309292024933e-05, + 3.051517705898732e-05, + 3.297684816061519e-05, + 3.0395560315810144e-05, + 2.3264594346983358e-05, + 2.4727498384891078e-05, + 2.7097434212919325e-05, + 2.4546385247958824e-05, + 2.2297355826594867e-05, + 1.547197280160617e-05, + 1.323529431829229e-05, + 8.847647222864907e-06, + 8.660701496410184e-06, + 6.39516065348289e-06, + 3.961517450079555e-06, + 4.52649146609474e-06, + 5.458484793052776e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.615552941482747e-05, + "first_to_last_ratio": 8.764492263828256, + "imbalance_abs_log_ratio": 2.1707085890088735, + "last_quartile_mean": 4.1252280595927004e-06, + "mean": 2.208462114339227e-05, + "normalized": [ + 0.7494933219758998, + 2.2634292925463795, + 2.1410570182439597, + 1.5739498957546751, + 1.5487374547204664, + 2.0376524711677733, + 1.534141636657325, + 1.2486273054515569, + 1.5321016320199576, + 1.6393386005987565, + 1.2261935965091952, + 1.1547399796224558, + 1.219246066181087, + 1.4546728804264264, + 1.2279261343623535, + 1.0312465593605784, + 0.9063241145323925, + 1.1172138768757267, + 1.0304062764426833, + 0.7847973944776799, + 0.8360866099062609, + 0.911401228786939, + 0.6978180640030948, + 0.6390634924568735, + 0.31508360869438684, + 0.26042370730894115, + 0.18982145136915554, + 0.23533143612154772, + 0.14063662904795424, + 0.10306654364310344, + 0.11411045319594963, + 0.13586126753846497 + ], + "peak_layer": 2, + "peak_normalized": 2.2634292925463795, + "population_cv": 0.6059056054769965, + "rest_mean": 2.3393786848289352e-05, + "spike_contrast": 0.6418424872519509, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.5015126336948015e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.6552276065340266e-05, + 4.998697841074318e-05, + 4.7284433094318956e-05, + 3.476008714642376e-05, + 3.420327993808314e-05, + 4.500078284763731e-05, + 3.388093682588078e-05, + 2.757546099019237e-05, + 3.383588409633376e-05, + 3.620417191996239e-05, + 2.708002102735918e-05, + 2.5501994969090447e-05, + 2.6926587452180684e-05, + 3.212589945178479e-05, + 2.7118283469462767e-05, + 2.277468956890516e-05, + 2.0015824702568352e-05, + 2.467324520694092e-05, + 2.2756132239010185e-05, + 1.7331953131360933e-05, + 1.8464656022842973e-05, + 2.0127950847381726e-05, + 1.5411047570523806e-05, + 1.4113475117483176e-05, + 6.958502126508392e-06, + 5.7513589126756415e-06, + 4.19213483837666e-06, + 5.1972056098748e-06, + 3.1059066714078654e-06, + 2.276185568916844e-06, + 2.520086127333343e-06, + 3.0004446216480574e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011569616435735952, + "first_to_last_ratio": 4.483785350026755, + "imbalance_abs_log_ratio": 1.500467633864892, + "last_quartile_mean": 2.5803234393606544e-05, + "mean": 7.895687571135568e-05, + "normalized": [ + 1.113643291734891, + 1.9315033327137208, + 1.8689084538217502, + 1.3013119102609747, + 1.3204401608174607, + 1.7172337521094045, + 1.2105960380108525, + 1.2588293628008012, + 1.3316624978487503, + 1.2436188212370092, + 1.3620564885728002, + 1.3794695319626817, + 1.4052872102652663, + 1.3953280795865155, + 1.1319795963186805, + 0.9796315470123751, + 1.1579414909870167, + 1.0808582581724016, + 0.9982944348550936, + 0.7435043404975695, + 0.8011357344053238, + 0.8526183967497576, + 0.9599446768973036, + 0.8397896829211625, + 0.7011434784757186, + 0.5094988193534132, + 0.32142473628464113, + 0.2742399520171878, + 0.26163364470536055, + 0.152278590465612, + 0.18046309892286552, + 0.21373058921563795 + ], + "peak_layer": 2, + "peak_normalized": 1.9315033327137208, + "population_cv": 0.47743629272692334, + "rest_mean": 8.142900973056115e-05, + "spike_contrast": 0.8056999860950416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.560735200764612e-05, + "top_five_layers": [ + 2, + 3, + 6, + 13, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.79297949722968e-05, + 0.00015250546857714653, + 0.00014756317250430584, + 0.00010274752276018262, + 0.00010425782966194674, + 0.00013558741193264723, + 9.558488090988249e-05, + 9.939323354046792e-05, + 0.00010514391033211723, + 9.819225670071319e-05, + 0.00010754372488008812, + 0.00010891860438277945, + 0.00011095708759967238, + 0.00011017074575647712, + 8.937757229432464e-05, + 7.734864630037919e-05, + 9.142744238488376e-05, + 8.53411911521107e-05, + 7.882220961619169e-05, + 5.870477980352007e-05, + 6.32551746093668e-05, + 6.732008478138596e-05, + 7.579423254355788e-05, + 6.630716961808503e-05, + 5.5360098485834897e-05, + 4.022843495476991e-05, + 2.5378692953381687e-05, + 2.165312980650924e-05, + 2.0657775166910142e-05, + 1.2023441740893759e-05, + 1.4248802472138777e-05, + 1.6875499568413943e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.018067941069602966, + 0.027584951370954514, + 0.03231721371412277, + 0.036101218312978745, + 0.025345133617520332, + 0.027166159823536873, + 0.028526438400149345, + 0.02979987859725952, + 0.02248864620923996, + 0.023894935846328735, + 0.02451697364449501, + 0.025183402001857758, + 0.020499059930443764, + 0.02105838619172573, + 0.0216353852301836, + 0.022715488448739052, + 0.01881815865635872, + 0.0193756353110075, + 0.02001863345503807, + 0.021088071167469025, + 0.018417155370116234, + 0.019219009205698967, + 0.020342160016298294, + 0.02150510996580124, + 0.02099662832915783, + 0.02211105264723301, + 0.02328667603433132, + 0.024160878732800484, + 0.02260657213628292, + 0.024989914149045944, + 0.02536177635192871, + 0.027097374200820923 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2429669027369528e-05, + "first_to_last_ratio": 1.5167074865147177, + "imbalance_abs_log_ratio": 0.4165418581180973, + "last_quartile_mean": 8.195165605684451e-06, + "mean": 1.1438181132916725e-05, + "normalized": [ + 2.3239055235296235, + 1.1190114244144547, + 0.8331917749372405, + 0.7813452753113945, + 1.003803456769089, + 0.9365675136194498, + 0.8842494964053754, + 0.8113839672107822, + 1.0847409217961617, + 1.0682117256400467, + 0.9883998597662562, + 0.9038042770049443, + 1.1938503302540149, + 1.1817190494107024, + 1.0722151723600664, + 0.9843285081524176, + 1.2863323428556752, + 1.156982982287669, + 1.0862279117485159, + 1.0234351273362494, + 1.2841852282798012, + 1.1754321217296801, + 1.070502203826642, + 1.0143774573484126, + 0.8953924191337335, + 0.8270637955966683, + 0.8133202903656793, + 0.7381861493144678, + 0.6730532700250402, + 0.5804717059865397, + 0.6082410306470007, + 0.5960676869362054 + ], + "peak_layer": 1, + "peak_normalized": 2.3239055235296235, + "population_cv": 0.3045451785807339, + "rest_mean": 1.1251827985419098e-05, + "spike_contrast": 1.1059970118215763, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2444488129403908e-05, + "top_five_layers": [ + 1, + 17, + 21, + 13, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.6581252313917503e-05, + 1.2799455362255685e-05, + 9.530198440188542e-06, + 8.937168786360417e-06, + 1.1481685760372784e-05, + 1.0712628863984719e-05, + 1.011420590657508e-05, + 9.280756785301492e-06, + 1.2407463145791553e-05, + 1.22183992061764e-05, + 1.1305496627755929e-05, + 1.0337877029087394e-05, + 1.3655476323037874e-05, + 1.3516716535377782e-05, + 1.2264191354915965e-05, + 1.125892777054105e-05, + 1.4713302334712353e-05, + 1.323378091910854e-05, + 1.2424471606209408e-05, + 1.1706236364261713e-05, + 1.4688743249280378e-05, + 1.3444805517792702e-05, + 1.2244598110555671e-05, + 1.1602633094298653e-05, + 1.0241660675092135e-05, + 9.460105502512306e-06, + 9.302904800279066e-06, + 8.443506885669194e-06, + 7.698505214648321e-06, + 6.639540515607223e-06, + 6.957171081012348e-06, + 6.817930170655018e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.8672892262402456e-05, + "first_to_last_ratio": 1.4790455122350374, + "imbalance_abs_log_ratio": 0.39139695555746173, + "last_quartile_mean": 3.290831273261574e-05, + "mean": 4.5425740495375067e-05, + "normalized": [ + 2.292867781479936, + 1.098844858197334, + 0.8288263395703189, + 0.7584336983627002, + 0.9778795698933019, + 0.9351432056276012, + 0.8751120479081964, + 0.8047536035443427, + 1.088349470440326, + 1.0783459726467945, + 0.9909402014503932, + 0.9019584923345375, + 1.2027204540239667, + 1.175772139842897, + 1.0782465054806212, + 0.9953149147774686, + 1.2799515830354413, + 1.1648392407100392, + 1.0907985090406367, + 1.0337452797487645, + 1.2844092657539465, + 1.1758215530808431, + 1.0768851187191208, + 1.0145044696518641, + 0.9072210422189393, + 0.8359231056438284, + 0.8227296910259355, + 0.7472276249111484, + 0.6761454407882689, + 0.5893866866897062, + 0.6130538257244701, + 0.6038483076763113 + ], + "peak_layer": 1, + "peak_normalized": 2.292867781479936, + "population_cv": 0.29963351102833546, + "rest_mean": 4.465376965796437e-05, + "spike_contrast": 1.1106426937137044, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.9594383017392826e-05, + "top_five_layers": [ + 1, + 21, + 17, + 13, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010415521683171391, + 4.9915841373149306e-05, + 3.765005021705292e-05, + 3.445241236477159e-05, + 4.442090357770212e-05, + 4.2479572584852576e-05, + 3.975261279265396e-05, + 3.655652835732326e-05, + 4.943908061250113e-05, + 4.898466431768611e-05, + 4.501419243752025e-05, + 4.097213241038844e-05, + 5.4634467232972383e-05, + 5.3410320106195286e-05, + 4.89801459480077e-05, + 4.5212917029857635e-05, + 5.814274845761247e-05, + 5.291368506732397e-05, + 4.9550330004421994e-05, + 4.695864481618628e-05, + 5.8345241995994e-05, + 5.341256473911926e-05, + 4.891830394626595e-05, + 4.608461676980369e-05, + 4.1211187635781243e-05, + 3.797242607106455e-05, + 3.737310544238426e-05, + 3.394336818018928e-05, + 3.071440733037889e-05, + 2.6773326680995524e-05, + 2.784842399705667e-05, + 2.7430256523075514e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.0431636068969965, + "first_to_last_ratio": 1.4790455355871048, + "imbalance_abs_log_ratio": 0.39139697134606793, + "last_quartile_mean": 0.02918341988697648, + "mean": 0.04028400045353919, + "normalized": [ + 2.2928676956325664, + 1.0988448461233453, + 0.8288263129280055, + 0.7584337335279033, + 0.9778795372444193, + 0.9351432892874852, + 0.8751120548755073, + 0.8047535990009899, + 1.0883495038762179, + 1.0783460397661777, + 0.9909401548295442, + 0.9019584843211922, + 1.2027204645548106, + 1.1757721269107668, + 1.0782465359354751, + 0.9953149017803241, + 1.2799516204237265, + 1.1648392822258575, + 1.090798537285597, + 1.0337452971674015, + 1.2844092255829724, + 1.1758215089234016, + 1.0768851089878908, + 1.014504529948594, + 0.9072209977912502, + 0.8359230814928427, + 0.8227296688325424, + 0.7472276232112787, + 0.676145406434352, + 0.5893866663000006, + 0.6130538743742001, + 0.6038482904233612 + ], + "peak_layer": 1, + "peak_normalized": 2.2928676956325664, + "population_cv": 0.29963350187956755, + "rest_mean": 0.0395994091199504, + "spike_contrast": 1.110642674533265, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.04398079365491867, + "top_five_layers": [ + 1, + 21, + 17, + 13, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.09236588329076767, + 0.04426586627960205, + 0.03338843956589699, + 0.03055274486541748, + 0.039392899721860886, + 0.03767131268978119, + 0.03525301441550255, + 0.03241869434714317, + 0.04384307190775871, + 0.043440092355012894, + 0.03991903364658356, + 0.03633449599146843, + 0.04845039173960686, + 0.04736480489373207, + 0.04343608394265175, + 0.040095265954732895, + 0.05156157165765762, + 0.04692438617348671, + 0.04394172877073288, + 0.04164339601993561, + 0.05174114182591438, + 0.04736679419875145, + 0.04338124021887779, + 0.04086830094456673, + 0.036546491086483, + 0.03367432579398155, + 0.03314284235239029, + 0.030101317912340164, + 0.027237841859459877, + 0.023742852732539177, + 0.024696262553334236, + 0.024325424805283546 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.470277960104795e-06, + "first_to_last_ratio": 1.5769587704873496, + "imbalance_abs_log_ratio": 0.4554981633698566, + "last_quartile_mean": 2.2006142614827695e-06, + "mean": 2.887214563429552e-06, + "normalized": [ + 2.7519535335351715, + 1.313168413337379, + 0.9399430071407856, + 0.8753790372729023, + 1.0813834937551676, + 0.9567319703631147, + 0.8229162441441861, + 0.8740977436183472, + 1.0566599601043785, + 0.9454608388064315, + 0.9224389765316489, + 0.8092920053718847, + 1.0583487949182648, + 1.0534209725986503, + 0.9874597862575576, + 0.8539122103086818, + 1.169801497382018, + 1.081662118029895, + 1.0127159978603695, + 0.961060648114576, + 1.1999729309980693, + 1.1228483457198037, + 1.0226342502719021, + 1.0291945210343731, + 0.8967756962809945, + 0.8510820239929227, + 0.8261498765197046, + 0.7721004685625728, + 0.7451436093586472, + 0.6285081646269207, + 0.6879375096468019, + 0.6898453535358771 + ], + "peak_layer": 1, + "peak_normalized": 2.7519535335351715, + "population_cv": 0.35016697743385605, + "rest_mean": 2.8581899574352e-06, + "spike_contrast": 1.0649912990844543, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.043947435799055e-06, + "top_five_layers": [ + 1, + 2, + 21, + 17, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.945480319904163e-06, + 3.7913989672233583e-06, + 2.7138171390106436e-06, + 2.5274071049352642e-06, + 3.12218617182225e-06, + 2.7622904781310353e-06, + 2.3759357645758428e-06, + 2.523707735235803e-06, + 3.0508040254062507e-06, + 2.729748302954249e-06, + 2.6632792469172273e-06, + 2.3365996639768127e-06, + 3.055680053876131e-06, + 3.041452373508946e-06, + 2.851008275683853e-06, + 2.4654277694935445e-06, + 3.3774679195630597e-06, + 3.122990619885968e-06, + 2.92392837764055e-06, + 2.774788299575448e-06, + 3.4645793220988708e-06, + 3.2419040962849976e-06, + 2.952564500446897e-06, + 2.9715054097323446e-06, + 2.589183850432164e-06, + 2.4572564143454656e-06, + 2.3852719550632173e-06, + 2.229219717264641e-06, + 2.1513894807867473e-06, + 1.814637926145224e-06, + 1.9862231965817045e-06, + 1.9917315512429923e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.1570810026314575e-05, + "first_to_last_ratio": 1.5044497617163743, + "imbalance_abs_log_ratio": 0.40842722451739255, + "last_quartile_mean": 2.763190309451602e-05, + "mean": 3.7813772848949156e-05, + "normalized": [ + 2.388009071075982, + 1.1470596649494917, + 0.8534302761319221, + 0.7776431816025992, + 0.9983567968228081, + 0.9421045940312636, + 0.8787540856715802, + 0.8094928604716568, + 1.0777400416324399, + 1.0654722038935143, + 0.9718539610468895, + 0.8865810688989143, + 1.168850821337844, + 1.1332939684204937, + 1.0417735360252487, + 0.9621498679953465, + 1.2282401671332226, + 1.126146404473492, + 1.059853477638917, + 1.0196054375892694, + 1.2877065753533457, + 1.1949291884271547, + 1.0946140188339069, + 1.0404469270679417, + 0.9334039480335107, + 0.8520038997158634, + 0.8392353967355263, + 0.7615998622978171, + 0.6682831346061853, + 0.5860512257947129, + 0.6091233405529298, + 0.5961909957382107 + ], + "peak_layer": 1, + "peak_normalized": 2.388009071075982, + "population_cv": 0.309055923286973, + "rest_mean": 3.704195080866123e-05, + "spike_contrast": 1.13335315635395, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.198161186650395e-05, + "top_five_layers": [ + 1, + 21, + 17, + 22, + 13 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 9.029963257489726e-05, + 4.337465361459181e-05, + 3.227141860406846e-05, + 2.9405622626654804e-05, + 3.775163713726215e-05, + 3.562452911864966e-05, + 3.322900738567114e-05, + 3.060997914872132e-05, + 4.0753417124506086e-05, + 4.0289523894898593e-05, + 3.674946492537856e-05, + 3.3524975151522085e-05, + 4.419865945237689e-05, + 4.2854120692936704e-05, + 3.9393387851305306e-05, + 3.638251655502245e-05, + 4.644439468393102e-05, + 4.258384433342144e-05, + 4.007705865660682e-05, + 3.855512841255404e-05, + 4.869304393650964e-05, + 4.518478090176359e-05, + 4.139148586546071e-05, + 3.934322376153432e-05, + 3.5295524867251515e-05, + 3.2217481930274516e-05, + 3.173465665895492e-05, + 2.879896419472061e-05, + 2.5270306650782004e-05, + 2.2160807930049486e-05, + 2.303325163666159e-05, + 2.2544230887433514e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.563517248039716e-05, + "first_to_last_ratio": 1.4352355648633195, + "imbalance_abs_log_ratio": 0.3613289924385252, + "last_quartile_mean": 2.4828796995279845e-05, + "mean": 3.227010705586508e-05, + "normalized": [ + 2.4014447290909864, + 1.1673853090110469, + 0.8593097269193264, + 0.7832036205871891, + 1.0067937101766118, + 0.9335226700834045, + 0.8775072266073977, + 0.8050578010660164, + 1.053006259394381, + 1.0264307263592312, + 0.9338631869266799, + 0.8595966381479759, + 1.1212921464302679, + 1.0758485643993823, + 0.9942738962286088, + 0.9189156755903263, + 1.1746749961886236, + 1.0800838025282244, + 1.019430881526982, + 0.9950392558872734, + 1.2898668670070823, + 1.2311059818213306, + 1.13592608464973, + 1.1011764563471198, + 0.9942343261534591, + 0.8955937948027416, + 0.9014365249588188, + 0.831025803802073, + 0.6870790019632969, + 0.6046383015495036, + 0.6312332814164576, + 0.610002752378451 + ], + "peak_layer": 1, + "peak_normalized": 2.4014447290909864, + "population_cv": 0.30444570497962015, + "rest_mean": 3.1370954645724315e-05, + "spike_contrast": 1.1834364140297289, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.7125530070625246e-05, + "top_five_layers": [ + 1, + 21, + 22, + 17, + 2 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.749487849650905e-05, + 3.7671648897230625e-05, + 2.7730016881832853e-05, + 2.527406468288973e-05, + 3.2489340810570866e-05, + 3.0124876502668485e-05, + 2.8317252144915983e-05, + 2.5979301426559687e-05, + 3.398062472115271e-05, + 3.312302942504175e-05, + 3.0135865017655306e-05, + 2.77392755378969e-05, + 3.618421760620549e-05, + 3.4717748349066824e-05, + 3.2085325074149296e-05, + 2.965350722661242e-05, + 3.790688788285479e-05, + 3.4854419936891645e-05, + 3.289714368293062e-05, + 3.211002331227064e-05, + 4.162414188613184e-05, + 3.972792183049023e-05, + 3.6656456359196454e-05, + 3.55350821337197e-05, + 3.2084048143588006e-05, + 2.8900907636852935e-05, + 2.9089453164488077e-05, + 2.6817291654879227e-05, + 2.2172112949192524e-05, + 1.9511742721078917e-05, + 2.03699655685341e-05, + 1.9684854123624973e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.83701450220542e-05, + "first_to_last_ratio": 1.5137756203780361, + "imbalance_abs_log_ratio": 0.41460694084716315, + "last_quartile_mean": 5.837730759594706e-05, + "mean": 8.360470872048609e-05, + "normalized": [ + 2.260874919572962, + 1.046876661671232, + 0.8108813008392315, + 0.7410270778432692, + 0.9650502462422242, + 0.9363082641888973, + 0.8839048255705971, + 0.8110736329048943, + 1.115418327104151, + 1.1154856868411749, + 1.0274711394986145, + 0.9306550068796721, + 1.2562444747790198, + 1.2247146350825069, + 1.1283567053258787, + 1.0398138975133757, + 1.305825331548614, + 1.176867029634062, + 1.1082958616806637, + 1.034638772497703, + 1.2882363465735436, + 1.1524683581694375, + 1.0651417678309785, + 0.9883391355795494, + 0.8670300800809992, + 0.8013878422949675, + 0.7767996236633684, + 0.7006926068874338, + 0.6674164616571702, + 0.5747719332171776, + 0.5954528519609661, + 0.6024791948656639 + ], + "peak_layer": 1, + "peak_normalized": 2.260874919572962, + "population_cv": 0.30484696934594563, + "rest_mean": 8.24862149278892e-05, + "spike_contrast": 1.0867825039478176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.964457520050928e-05, + "top_five_layers": [ + 1, + 17, + 21, + 13, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.0001890197891043499, + 8.752381836529821e-05, + 6.77934949635528e-05, + 6.195335299707949e-05, + 8.068274473771453e-05, + 7.82797797000967e-05, + 7.389860547846183e-05, + 6.780957482988015e-05, + 9.325422433903441e-05, + 9.325985593022779e-05, + 8.590142533648759e-05, + 7.780714076943696e-05, + 0.00010502795339561999, + 0.0001023919103317894, + 9.433593368157744e-05, + 8.693333802511916e-05, + 0.00010917314648395404, + 9.839162521529943e-05, + 9.265875269193202e-05, + 8.650067320559174e-05, + 0.00010770262451842427, + 9.635178139433265e-05, + 8.905086724553257e-05, + 8.262980554718524e-05, + 7.248779729707167e-05, + 6.69997971272096e-05, + 6.494410627055913e-05, + 5.858120130142197e-05, + 5.579915887210518e-05, + 4.8053640057332814e-05, + 4.978266224497929e-05, + 5.037009759689681e-05 + ] + } + } + } + } + } + }, + "step": 500 + }, + { + "modes": { + "learned": { + "bits_per_byte": 2.073383110809672, + "forward": { + "activation_sha256": { + "attention_branch_output": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", + "mlp_branch_output": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", + "post_attention_state": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", + "post_mlp_state": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", + "pre_attention_input": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", + "pre_mlp_input": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93" + }, + "logits_sha256": "9adeddfaa352b7946853a2fe6e55c7269e6a01f6272498cdfb911b26410503c6", + "loss_nats": 1.4371596574783325, + "mixer_summary_sha256": "739af9a4ced9bd8d3e1cd2555b418bfc5db56ce4da6de7d1c45506ae68b1e3e1" + }, + "loss_nats": 1.4371596574783325, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.4738388657569885, + "group": 1, + "latest_source_mass": 0.19323334097862244, + "layer": 1, + "max_source_mass": 0.8067666888237, + "mixer_index": 1, + "normalized_entropy": 0.6836049818080586, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8067666888237, + "median": 0.8265105485916138, + "p05": 0.6604017019271851, + "p95": 0.8949732184410095 + }, + { + "label": "current_group_partial", + "mean": 0.19323334097862244, + "median": 0.17348942160606384, + "p05": 0.10502681136131287, + "p95": 0.33959829807281494 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6344536542892456, + "group": 1, + "latest_source_mass": 0.5104025602340698, + "layer": 2, + "max_source_mass": 0.5104025602340698, + "mixer_index": 2, + "normalized_entropy": 0.9153231407169755, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4895974397659302, + "median": 0.45956116914749146, + "p05": 0.2515213191509247, + "p95": 0.7871550917625427 + }, + { + "label": "current_group_partial", + "mean": 0.5104025602340698, + "median": 0.5404388308525085, + "p05": 0.21284496784210205, + "p95": 0.7484787106513977 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.41773349046707153, + "group": 1, + "latest_source_mass": 0.15382957458496094, + "layer": 2, + "max_source_mass": 0.8461704254150391, + "mixer_index": 3, + "normalized_entropy": 0.6026620351100812, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8461704254150391, + "median": 0.8643104434013367, + "p05": 0.7285550832748413, + "p95": 0.9108630418777466 + }, + { + "label": "current_group_partial", + "mean": 0.15382957458496094, + "median": 0.13568954169750214, + "p05": 0.08913692831993103, + "p95": 0.2714449167251587 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6868985891342163, + "group": 1, + "latest_source_mass": 0.49918031692504883, + "layer": 3, + "max_source_mass": 0.500819742679596, + "mixer_index": 4, + "normalized_entropy": 0.9909851881375595, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.500819742679596, + "median": 0.50103759765625, + "p05": 0.41182032227516174, + "p95": 0.5924672484397888 + }, + { + "label": "current_group_partial", + "mean": 0.49918031692504883, + "median": 0.49896240234375, + "p05": 0.40753278136253357, + "p95": 0.5881797075271606 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5841774344444275, + "group": 1, + "latest_source_mass": 0.315071702003479, + "layer": 3, + "max_source_mass": 0.684928297996521, + "mixer_index": 5, + "normalized_entropy": 0.8427898876722131, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.684928297996521, + "median": 0.6976089477539062, + "p05": 0.4487886428833008, + "p95": 0.8860265612602234 + }, + { + "label": "current_group_partial", + "mean": 0.315071702003479, + "median": 0.3023911118507385, + "p05": 0.1139734610915184, + "p95": 0.5512113571166992 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6896515488624573, + "group": 1, + "latest_source_mass": 0.49074798822402954, + "layer": 4, + "max_source_mass": 0.5092520117759705, + "mixer_index": 6, + "normalized_entropy": 0.9949568694852597, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5092520117759705, + "median": 0.509276270866394, + "p05": 0.4409506916999817, + "p95": 0.5760948657989502 + }, + { + "label": "current_group_partial", + "mean": 0.49074798822402954, + "median": 0.49072372913360596, + "p05": 0.4239051342010498, + "p95": 0.5590493679046631 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5611414909362793, + "group": 1, + "latest_source_mass": 0.2719496786594391, + "layer": 4, + "max_source_mass": 0.7280503511428833, + "mixer_index": 7, + "normalized_entropy": 0.8095560462108093, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7280503511428833, + "median": 0.7264741063117981, + "p05": 0.5577248930931091, + "p95": 0.8873021602630615 + }, + { + "label": "current_group_partial", + "mean": 0.2719496786594391, + "median": 0.2735258936882019, + "p05": 0.11269781738519669, + "p95": 0.44227516651153564 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6817618608474731, + "group": 2, + "latest_source_mass": 0.4686949551105499, + "layer": 5, + "max_source_mass": 0.5313050746917725, + "mixer_index": 8, + "normalized_entropy": 0.9835744557118811, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5313050746917725, + "median": 0.532880961894989, + "p05": 0.4224134683609009, + "p95": 0.6363531351089478 + }, + { + "label": "completed_group_1", + "mean": 0.4686949551105499, + "median": 0.4671190679073334, + "p05": 0.363646924495697, + "p95": 0.5775865316390991 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0218309164047241, + "group": 2, + "latest_source_mass": 0.2602040767669678, + "layer": 5, + "max_source_mass": 0.4974328279495239, + "mixer_index": 9, + "normalized_entropy": 0.9301105830916285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4974328279495239, + "median": 0.4930803179740906, + "p05": 0.35206541419029236, + "p95": 0.6587448716163635 + }, + { + "label": "completed_group_1", + "mean": 0.2423631250858307, + "median": 0.23208677768707275, + "p05": 0.15309149026870728, + "p95": 0.3585616946220398 + }, + { + "label": "current_group_partial", + "mean": 0.2602040767669678, + "median": 0.261092871427536, + "p05": 0.1870650351047516, + "p95": 0.33141374588012695 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0932960510253906, + "group": 2, + "latest_source_mass": 0.34662574529647827, + "layer": 6, + "max_source_mass": 0.3533782958984375, + "mixer_index": 10, + "normalized_entropy": 0.9951609519595268, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3533782958984375, + "median": 0.3537653684616089, + "p05": 0.300750195980072, + "p95": 0.40760302543640137 + }, + { + "label": "completed_group_1", + "mean": 0.29999595880508423, + "median": 0.2998189330101013, + "p05": 0.277343213558197, + "p95": 0.3256012797355652 + }, + { + "label": "current_group_partial", + "mean": 0.34662574529647827, + "median": 0.34948140382766724, + "p05": 0.3000357747077942, + "p95": 0.3795154392719269 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0456881523132324, + "group": 2, + "latest_source_mass": 0.3350909352302551, + "layer": 6, + "max_source_mass": 0.4338536858558655, + "mixer_index": 11, + "normalized_entropy": 0.9518263750544431, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4338536858558655, + "median": 0.43320339918136597, + "p05": 0.2745198607444763, + "p95": 0.6123260259628296 + }, + { + "label": "completed_group_1", + "mean": 0.23105540871620178, + "median": 0.22906750440597534, + "p05": 0.15816110372543335, + "p95": 0.3054558038711548 + }, + { + "label": "current_group_partial", + "mean": 0.3350909352302551, + "median": 0.33941850066185, + "p05": 0.2199106216430664, + "p95": 0.4273326098918915 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0891339778900146, + "group": 2, + "latest_source_mass": 0.37785181403160095, + "layer": 7, + "max_source_mass": 0.37785181403160095, + "mixer_index": 12, + "normalized_entropy": 0.9913724697276178, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33998239040374756, + "median": 0.3426361680030823, + "p05": 0.2884981632232666, + "p95": 0.38276687264442444 + }, + { + "label": "completed_group_1", + "mean": 0.2821658253669739, + "median": 0.2775929868221283, + "p05": 0.2536381781101227, + "p95": 0.3205426335334778 + }, + { + "label": "current_group_partial", + "mean": 0.37785181403160095, + "median": 0.3805788457393646, + "p05": 0.34096643328666687, + "p95": 0.40283167362213135 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.038641333580017, + "group": 2, + "latest_source_mass": 0.4193001687526703, + "layer": 7, + "max_source_mass": 0.4193001687526703, + "mixer_index": 13, + "normalized_entropy": 0.9454120842205417, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34165239334106445, + "median": 0.308063805103302, + "p05": 0.2054935097694397, + "p95": 0.6325749754905701 + }, + { + "label": "completed_group_1", + "mean": 0.23904743790626526, + "median": 0.244267076253891, + "p05": 0.1361692249774933, + "p95": 0.30873122811317444 + }, + { + "label": "current_group_partial", + "mean": 0.4193001687526703, + "median": 0.44425007700920105, + "p05": 0.22311054170131683, + "p95": 0.5117462277412415 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0754427909851074, + "group": 2, + "latest_source_mass": 0.3816372752189636, + "layer": 8, + "max_source_mass": 0.3816372752189636, + "mixer_index": 14, + "normalized_entropy": 0.9789102143476917, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3781457245349884, + "median": 0.37959107756614685, + "p05": 0.3339657485485077, + "p95": 0.42053842544555664 + }, + { + "label": "completed_group_1", + "mean": 0.24021697044372559, + "median": 0.23580573499202728, + "p05": 0.21105526387691498, + "p95": 0.2833511531352997 + }, + { + "label": "current_group_partial", + "mean": 0.3816372752189636, + "median": 0.38216716051101685, + "p05": 0.3468884229660034, + "p95": 0.4153265357017517 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0433809757232666, + "group": 2, + "latest_source_mass": 0.4086577892303467, + "layer": 8, + "max_source_mass": 0.4086577892303467, + "mixer_index": 15, + "normalized_entropy": 0.9497262924195011, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.37802231311798096, + "median": 0.3628900945186615, + "p05": 0.27504366636276245, + "p95": 0.5507957339286804 + }, + { + "label": "completed_group_1", + "mean": 0.21331989765167236, + "median": 0.2002047300338745, + "p05": 0.13782089948654175, + "p95": 0.3198853135108948 + }, + { + "label": "current_group_partial", + "mean": 0.4086577892303467, + "median": 0.41623425483703613, + "p05": 0.29855865240097046, + "p95": 0.47824543714523315 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0922105312347412, + "group": 3, + "latest_source_mass": 0.3464643955230713, + "layer": 9, + "max_source_mass": 0.3464643955230713, + "mixer_index": 16, + "normalized_entropy": 0.994172869264798, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3460429310798645, + "median": 0.3446507453918457, + "p05": 0.28670766949653625, + "p95": 0.40976202487945557 + }, + { + "label": "completed_group_1", + "mean": 0.3074926435947418, + "median": 0.306111603975296, + "p05": 0.2859768867492676, + "p95": 0.3337686061859131 + }, + { + "label": "completed_group_2", + "mean": 0.3464643955230713, + "median": 0.3486866354942322, + "p05": 0.27475792169570923, + "p95": 0.40797942876815796 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3404325246810913, + "group": 3, + "latest_source_mass": 0.3108460605144501, + "layer": 9, + "max_source_mass": 0.3108460605144501, + "mixer_index": 17, + "normalized_entropy": 0.9669176780018418, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2880150079727173, + "median": 0.27969539165496826, + "p05": 0.18933802843093872, + "p95": 0.45879441499710083 + }, + { + "label": "completed_group_1", + "mean": 0.16260020434856415, + "median": 0.16053099930286407, + "p05": 0.11509333550930023, + "p95": 0.22219550609588623 + }, + { + "label": "completed_group_2", + "mean": 0.2385387420654297, + "median": 0.24041923880577087, + "p05": 0.1582493633031845, + "p95": 0.2989943325519562 + }, + { + "label": "current_group_partial", + "mean": 0.3108460605144501, + "median": 0.3127351701259613, + "p05": 0.2580314874649048, + "p95": 0.3545264005661011 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3765960931777954, + "group": 3, + "latest_source_mass": 0.24758204817771912, + "layer": 10, + "max_source_mass": 0.28703436255455017, + "mixer_index": 18, + "normalized_entropy": 0.9930041784673634, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28703436255455017, + "median": 0.2873149514198303, + "p05": 0.2421720325946808, + "p95": 0.32964712381362915 + }, + { + "label": "completed_group_1", + "mean": 0.2552259564399719, + "median": 0.2551601231098175, + "p05": 0.2324453592300415, + "p95": 0.27810582518577576 + }, + { + "label": "completed_group_2", + "mean": 0.2101576328277588, + "median": 0.20999819040298462, + "p05": 0.17491281032562256, + "p95": 0.24711720645427704 + }, + { + "label": "current_group_partial", + "mean": 0.24758204817771912, + "median": 0.2449823021888733, + "p05": 0.21694661676883698, + "p95": 0.2901700437068939 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3274900913238525, + "group": 3, + "latest_source_mass": 0.26038026809692383, + "layer": 10, + "max_source_mass": 0.33539247512817383, + "mixer_index": 19, + "normalized_entropy": 0.9575816857910796, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33539247512817383, + "median": 0.3373180031776428, + "p05": 0.16021503508090973, + "p95": 0.5567492246627808 + }, + { + "label": "completed_group_1", + "mean": 0.1987970769405365, + "median": 0.19616791605949402, + "p05": 0.13216234743595123, + "p95": 0.2684130072593689 + }, + { + "label": "completed_group_2", + "mean": 0.20543015003204346, + "median": 0.20896214246749878, + "p05": 0.11321981251239777, + "p95": 0.2748231887817383 + }, + { + "label": "current_group_partial", + "mean": 0.26038026809692383, + "median": 0.2554967999458313, + "p05": 0.1821153312921524, + "p95": 0.3462901711463928 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3747133016586304, + "group": 3, + "latest_source_mass": 0.2833673655986786, + "layer": 11, + "max_source_mass": 0.2833673655986786, + "mixer_index": 20, + "normalized_entropy": 0.9916460314734998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27988842129707336, + "median": 0.28038889169692993, + "p05": 0.24265244603157043, + "p95": 0.31478387117385864 + }, + { + "label": "completed_group_1", + "mean": 0.2360810488462448, + "median": 0.2335505485534668, + "p05": 0.21358957886695862, + "p95": 0.2655458450317383 + }, + { + "label": "completed_group_2", + "mean": 0.20066317915916443, + "median": 0.2006566822528839, + "p05": 0.18401089310646057, + "p95": 0.21715889871120453 + }, + { + "label": "current_group_partial", + "mean": 0.2833673655986786, + "median": 0.28357356786727905, + "p05": 0.2585292458534241, + "p95": 0.30662211775779724 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3239198923110962, + "group": 3, + "latest_source_mass": 0.2723969519138336, + "layer": 11, + "max_source_mass": 0.3361120820045471, + "mixer_index": 21, + "normalized_entropy": 0.9550063315857346, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3361120820045471, + "median": 0.3591133654117584, + "p05": 0.14645838737487793, + "p95": 0.49824467301368713 + }, + { + "label": "completed_group_1", + "mean": 0.20296622812747955, + "median": 0.19982439279556274, + "p05": 0.13988786935806274, + "p95": 0.27185869216918945 + }, + { + "label": "completed_group_2", + "mean": 0.1885247528553009, + "median": 0.18602871894836426, + "p05": 0.12637801468372345, + "p95": 0.2539179027080536 + }, + { + "label": "current_group_partial", + "mean": 0.2723969519138336, + "median": 0.25681596994400024, + "p05": 0.19296783208847046, + "p95": 0.3959115445613861 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3692121505737305, + "group": 3, + "latest_source_mass": 0.2974509000778198, + "layer": 12, + "max_source_mass": 0.2974509000778198, + "mixer_index": 22, + "normalized_entropy": 0.9876777897788168, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28167375922203064, + "median": 0.27825769782066345, + "p05": 0.23325958847999573, + "p95": 0.3442809581756592 + }, + { + "label": "completed_group_1", + "mean": 0.21946553885936737, + "median": 0.21849599480628967, + "p05": 0.18336281180381775, + "p95": 0.2548777461051941 + }, + { + "label": "completed_group_2", + "mean": 0.20140981674194336, + "median": 0.2013496309518814, + "p05": 0.1822294294834137, + "p95": 0.2219773232936859 + }, + { + "label": "current_group_partial", + "mean": 0.2974509000778198, + "median": 0.297912061214447, + "p05": 0.26495662331581116, + "p95": 0.32783159613609314 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3342361450195312, + "group": 3, + "latest_source_mass": 0.23993611335754395, + "layer": 12, + "max_source_mass": 0.36468517780303955, + "mixer_index": 23, + "normalized_entropy": 0.9624479348972428, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.36468517780303955, + "median": 0.36720338463783264, + "p05": 0.24960748851299286, + "p95": 0.4882795810699463 + }, + { + "label": "completed_group_1", + "mean": 0.20593920350074768, + "median": 0.2046196460723877, + "p05": 0.16298335790634155, + "p95": 0.25410282611846924 + }, + { + "label": "completed_group_2", + "mean": 0.18943950533866882, + "median": 0.1881066858768463, + "p05": 0.13994796574115753, + "p95": 0.24192847311496735 + }, + { + "label": "current_group_partial", + "mean": 0.23993611335754395, + "median": 0.232496976852417, + "p05": 0.17571662366390228, + "p95": 0.320015549659729 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3645737171173096, + "group": 4, + "latest_source_mass": 0.30504196882247925, + "layer": 13, + "max_source_mass": 0.30504196882247925, + "mixer_index": 24, + "normalized_entropy": 0.9843318673062809, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2597482204437256, + "median": 0.2599886655807495, + "p05": 0.23272822797298431, + "p95": 0.2871353030204773 + }, + { + "label": "completed_group_1", + "mean": 0.22607669234275818, + "median": 0.21320533752441406, + "p05": 0.1705966293811798, + "p95": 0.32829806208610535 + }, + { + "label": "completed_group_2", + "mean": 0.209133118391037, + "median": 0.20912399888038635, + "p05": 0.19090500473976135, + "p95": 0.2265017032623291 + }, + { + "label": "completed_group_3", + "mean": 0.30504196882247925, + "median": 0.3161492347717285, + "p05": 0.20431193709373474, + "p95": 0.37756454944610596 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5910131931304932, + "group": 4, + "latest_source_mass": 0.19691801071166992, + "layer": 13, + "max_source_mass": 0.2617476284503937, + "mixer_index": 25, + "normalized_entropy": 0.988552078237214, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2617476284503937, + "median": 0.2677137851715088, + "p05": 0.20086795091629028, + "p95": 0.31516557931900024 + }, + { + "label": "completed_group_1", + "mean": 0.18299485743045807, + "median": 0.1818893998861313, + "p05": 0.1605435013771057, + "p95": 0.20783139765262604 + }, + { + "label": "completed_group_2", + "mean": 0.17935845255851746, + "median": 0.18049052357673645, + "p05": 0.14385172724723816, + "p95": 0.2110004425048828 + }, + { + "label": "completed_group_3", + "mean": 0.17898105084896088, + "median": 0.17628446221351624, + "p05": 0.14925207197666168, + "p95": 0.21800483763217926 + }, + { + "label": "current_group_partial", + "mean": 0.19691801071166992, + "median": 0.1964835375547409, + "p05": 0.16946497559547424, + "p95": 0.22626768052577972 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5884685516357422, + "group": 4, + "latest_source_mass": 0.26269519329071045, + "layer": 14, + "max_source_mass": 0.26269519329071045, + "mixer_index": 26, + "normalized_entropy": 0.9869710035805953, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2046908289194107, + "median": 0.2052469551563263, + "p05": 0.18031299114227295, + "p95": 0.2295369803905487 + }, + { + "label": "completed_group_1", + "mean": 0.17294223606586456, + "median": 0.16847725212574005, + "p05": 0.15012723207473755, + "p95": 0.21394622325897217 + }, + { + "label": "completed_group_2", + "mean": 0.15044090151786804, + "median": 0.1493840515613556, + "p05": 0.1370292603969574, + "p95": 0.1660556048154831 + }, + { + "label": "completed_group_3", + "mean": 0.20923087000846863, + "median": 0.20989257097244263, + "p05": 0.17921577394008636, + "p95": 0.23829039931297302 + }, + { + "label": "current_group_partial", + "mean": 0.26269519329071045, + "median": 0.2656410038471222, + "p05": 0.23202235996723175, + "p95": 0.28119558095932007 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.599259853363037, + "group": 4, + "latest_source_mass": 0.21076934039592743, + "layer": 14, + "max_source_mass": 0.21657392382621765, + "mixer_index": 27, + "normalized_entropy": 0.9936760163331371, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21657392382621765, + "median": 0.21635186672210693, + "p05": 0.15601962804794312, + "p95": 0.2927207350730896 + }, + { + "label": "completed_group_1", + "mean": 0.20580385625362396, + "median": 0.20571252703666687, + "p05": 0.18009206652641296, + "p95": 0.2311524748802185 + }, + { + "label": "completed_group_2", + "mean": 0.19366896152496338, + "median": 0.194598987698555, + "p05": 0.16221287846565247, + "p95": 0.22272710502147675 + }, + { + "label": "completed_group_3", + "mean": 0.1731838881969452, + "median": 0.17174121737480164, + "p05": 0.14892742037773132, + "p95": 0.19922053813934326 + }, + { + "label": "current_group_partial", + "mean": 0.21076934039592743, + "median": 0.20835596323013306, + "p05": 0.1771993488073349, + "p95": 0.2505151629447937 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5786192417144775, + "group": 4, + "latest_source_mass": 0.2778070569038391, + "layer": 15, + "max_source_mass": 0.2778070569038391, + "mixer_index": 28, + "normalized_entropy": 0.980851283245209, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20599736273288727, + "median": 0.20488622784614563, + "p05": 0.17115657031536102, + "p95": 0.25322195887565613 + }, + { + "label": "completed_group_1", + "mean": 0.20593789219856262, + "median": 0.20346379280090332, + "p05": 0.18622605502605438, + "p95": 0.2330799102783203 + }, + { + "label": "completed_group_2", + "mean": 0.13473063707351685, + "median": 0.13439436256885529, + "p05": 0.11930938065052032, + "p95": 0.15121281147003174 + }, + { + "label": "completed_group_3", + "mean": 0.17552703619003296, + "median": 0.1757315993309021, + "p05": 0.1576612889766693, + "p95": 0.1928577572107315 + }, + { + "label": "current_group_partial", + "mean": 0.2778070569038391, + "median": 0.2790508270263672, + "p05": 0.24338077008724213, + "p95": 0.3088063597679138 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.580681324005127, + "group": 4, + "latest_source_mass": 0.2450503408908844, + "layer": 15, + "max_source_mass": 0.2450503408908844, + "mixer_index": 29, + "normalized_entropy": 0.9821325270103262, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21277281641960144, + "median": 0.20640528202056885, + "p05": 0.12015408277511597, + "p95": 0.3659750819206238 + }, + { + "label": "completed_group_1", + "mean": 0.18522509932518005, + "median": 0.1863747239112854, + "p05": 0.1475280076265335, + "p95": 0.21663622558116913 + }, + { + "label": "completed_group_2", + "mean": 0.18489867448806763, + "median": 0.1888006329536438, + "p05": 0.12606219947338104, + "p95": 0.22459235787391663 + }, + { + "label": "completed_group_3", + "mean": 0.17205308377742767, + "median": 0.17103317379951477, + "p05": 0.13283535838127136, + "p95": 0.20961183309555054 + }, + { + "label": "current_group_partial", + "mean": 0.2450503408908844, + "median": 0.24511924386024475, + "p05": 0.19535338878631592, + "p95": 0.2939102053642273 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5705549716949463, + "group": 4, + "latest_source_mass": 0.28460633754730225, + "layer": 16, + "max_source_mass": 0.28460633754730225, + "mixer_index": 30, + "normalized_entropy": 0.9758406705603525, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2125052809715271, + "median": 0.21049857139587402, + "p05": 0.16735027730464935, + "p95": 0.2640833854675293 + }, + { + "label": "completed_group_1", + "mean": 0.20224158465862274, + "median": 0.20027190446853638, + "p05": 0.16782984137535095, + "p95": 0.24093782901763916 + }, + { + "label": "completed_group_2", + "mean": 0.12915468215942383, + "median": 0.12795840203762054, + "p05": 0.1122070699930191, + "p95": 0.14902588725090027 + }, + { + "label": "completed_group_3", + "mean": 0.1714920997619629, + "median": 0.1704360991716385, + "p05": 0.15070456266403198, + "p95": 0.19576270878314972 + }, + { + "label": "current_group_partial", + "mean": 0.28460633754730225, + "median": 0.28339242935180664, + "p05": 0.2418026626110077, + "p95": 0.33516958355903625 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5599372386932373, + "group": 4, + "latest_source_mass": 0.2475198209285736, + "layer": 16, + "max_source_mass": 0.2665032148361206, + "mixer_index": 31, + "normalized_entropy": 0.9692435021205642, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2665032148361206, + "median": 0.2523844242095947, + "p05": 0.16442818939685822, + "p95": 0.41760408878326416 + }, + { + "label": "completed_group_1", + "mean": 0.181569442152977, + "median": 0.18072238564491272, + "p05": 0.1432785987854004, + "p95": 0.2251572459936142 + }, + { + "label": "completed_group_2", + "mean": 0.15844660997390747, + "median": 0.15800738334655762, + "p05": 0.11516115069389343, + "p95": 0.20221972465515137 + }, + { + "label": "completed_group_3", + "mean": 0.14596089720726013, + "median": 0.14446985721588135, + "p05": 0.10661658644676208, + "p95": 0.18743650615215302 + }, + { + "label": "current_group_partial", + "mean": 0.2475198209285736, + "median": 0.24823713302612305, + "p05": 0.1857534945011139, + "p95": 0.30610787868499756 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.562828540802002, + "group": 5, + "latest_source_mass": 0.26428917050361633, + "layer": 17, + "max_source_mass": 0.26428917050361633, + "mixer_index": 32, + "normalized_entropy": 0.9710399691271056, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21806953847408295, + "median": 0.21765722334384918, + "p05": 0.1832873821258545, + "p95": 0.256405234336853 + }, + { + "label": "completed_group_1", + "mean": 0.22324788570404053, + "median": 0.2111721932888031, + "p05": 0.1587892323732376, + "p95": 0.33203551173210144 + }, + { + "label": "completed_group_2", + "mean": 0.11591021716594696, + "median": 0.11326268315315247, + "p05": 0.09832467138767242, + "p95": 0.13705819845199585 + }, + { + "label": "completed_group_3", + "mean": 0.17848318815231323, + "median": 0.1790149211883545, + "p05": 0.1343652456998825, + "p95": 0.21781212091445923 + }, + { + "label": "completed_group_4", + "mean": 0.26428917050361633, + "median": 0.27161818742752075, + "p05": 0.18240834772586823, + "p95": 0.3144339919090271 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.761409044265747, + "group": 5, + "latest_source_mass": 0.14106416702270508, + "layer": 17, + "max_source_mass": 0.20728391408920288, + "mixer_index": 33, + "normalized_entropy": 0.9830611053081897, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1612042486667633, + "median": 0.14052782952785492, + "p05": 0.09785228967666626, + "p95": 0.30037230253219604 + }, + { + "label": "completed_group_1", + "mean": 0.16519047319889069, + "median": 0.16112321615219116, + "p05": 0.13238948583602905, + "p95": 0.2119981199502945 + }, + { + "label": "completed_group_2", + "mean": 0.17529632151126862, + "median": 0.18006199598312378, + "p05": 0.12263078987598419, + "p95": 0.20985063910484314 + }, + { + "label": "completed_group_3", + "mean": 0.14996090531349182, + "median": 0.1468663513660431, + "p05": 0.11909312009811401, + "p95": 0.18974041938781738 + }, + { + "label": "completed_group_4", + "mean": 0.20728391408920288, + "median": 0.2042727917432785, + "p05": 0.1449805498123169, + "p95": 0.27828457951545715 + }, + { + "label": "current_group_partial", + "mean": 0.14106416702270508, + "median": 0.1405012309551239, + "p05": 0.10205969214439392, + "p95": 0.1835198849439621 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7366840839385986, + "group": 5, + "latest_source_mass": 0.2530929446220398, + "layer": 18, + "max_source_mass": 0.2530929446220398, + "mixer_index": 34, + "normalized_entropy": 0.9692618422085502, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18514017760753632, + "median": 0.18704235553741455, + "p05": 0.13617590069770813, + "p95": 0.22916007041931152 + }, + { + "label": "completed_group_1", + "mean": 0.1663871705532074, + "median": 0.16571956872940063, + "p05": 0.1462152749300003, + "p95": 0.18810859322547913 + }, + { + "label": "completed_group_2", + "mean": 0.09063515067100525, + "median": 0.08811916410923004, + "p05": 0.07649004459381104, + "p95": 0.11148498952388763 + }, + { + "label": "completed_group_3", + "mean": 0.11971023678779602, + "median": 0.11747527122497559, + "p05": 0.10390099883079529, + "p95": 0.1423199474811554 + }, + { + "label": "completed_group_4", + "mean": 0.18503433465957642, + "median": 0.18477493524551392, + "p05": 0.15245835483074188, + "p95": 0.22021599113941193 + }, + { + "label": "current_group_partial", + "mean": 0.2530929446220398, + "median": 0.25188589096069336, + "p05": 0.22554033994674683, + "p95": 0.2833186984062195 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.777672290802002, + "group": 5, + "latest_source_mass": 0.19777557253837585, + "layer": 18, + "max_source_mass": 0.19777557253837585, + "mixer_index": 35, + "normalized_entropy": 0.9921377960222963, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15483248233795166, + "median": 0.14712893962860107, + "p05": 0.12276758253574371, + "p95": 0.22418904304504395 + }, + { + "label": "completed_group_1", + "mean": 0.16576308012008667, + "median": 0.16587528586387634, + "p05": 0.13462601602077484, + "p95": 0.19553470611572266 + }, + { + "label": "completed_group_2", + "mean": 0.16957050561904907, + "median": 0.17099428176879883, + "p05": 0.14377880096435547, + "p95": 0.19032810628414154 + }, + { + "label": "completed_group_3", + "mean": 0.14225168526172638, + "median": 0.14081379771232605, + "p05": 0.12333962321281433, + "p95": 0.1642245650291443 + }, + { + "label": "completed_group_4", + "mean": 0.16980668902397156, + "median": 0.16821962594985962, + "p05": 0.13770604133605957, + "p95": 0.20901428163051605 + }, + { + "label": "current_group_partial", + "mean": 0.19777557253837585, + "median": 0.19705063104629517, + "p05": 0.1461065709590912, + "p95": 0.2508929371833801 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7414222955703735, + "group": 5, + "latest_source_mass": 0.25494134426116943, + "layer": 19, + "max_source_mass": 0.25494134426116943, + "mixer_index": 36, + "normalized_entropy": 0.9719062884710925, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18020270764827728, + "median": 0.180809885263443, + "p05": 0.1366642266511917, + "p95": 0.22788876295089722 + }, + { + "label": "completed_group_1", + "mean": 0.1900298148393631, + "median": 0.1896573156118393, + "p05": 0.17211011052131653, + "p95": 0.20767614245414734 + }, + { + "label": "completed_group_2", + "mean": 0.09944289177656174, + "median": 0.09800659120082855, + "p05": 0.08568179607391357, + "p95": 0.11607980728149414 + }, + { + "label": "completed_group_3", + "mean": 0.11960913985967636, + "median": 0.11915834993124008, + "p05": 0.10471149533987045, + "p95": 0.1360406130552292 + }, + { + "label": "completed_group_4", + "mean": 0.1557741016149521, + "median": 0.15596863627433777, + "p05": 0.13432671129703522, + "p95": 0.1770717203617096 + }, + { + "label": "current_group_partial", + "mean": 0.25494134426116943, + "median": 0.25551509857177734, + "p05": 0.22669723629951477, + "p95": 0.2819991707801819 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.733978509902954, + "group": 5, + "latest_source_mass": 0.2618475556373596, + "layer": 19, + "max_source_mass": 0.2618475556373596, + "mixer_index": 37, + "normalized_entropy": 0.9677518325883359, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15204599499702454, + "median": 0.1308840811252594, + "p05": 0.09369519352912903, + "p95": 0.32571133971214294 + }, + { + "label": "completed_group_1", + "mean": 0.14166373014450073, + "median": 0.14088967442512512, + "p05": 0.11298352479934692, + "p95": 0.17192427814006805 + }, + { + "label": "completed_group_2", + "mean": 0.1616208553314209, + "median": 0.1641538441181183, + "p05": 0.11296901851892471, + "p95": 0.1948811560869217 + }, + { + "label": "completed_group_3", + "mean": 0.1256796270608902, + "median": 0.122653529047966, + "p05": 0.09500046819448471, + "p95": 0.16056732833385468 + }, + { + "label": "completed_group_4", + "mean": 0.1571422517299652, + "median": 0.1528569459915161, + "p05": 0.11623523384332657, + "p95": 0.21299627423286438 + }, + { + "label": "current_group_partial", + "mean": 0.2618475556373596, + "median": 0.26793089509010315, + "p05": 0.15509119629859924, + "p95": 0.339982271194458 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7344707250595093, + "group": 5, + "latest_source_mass": 0.2615981698036194, + "layer": 20, + "max_source_mass": 0.2615981698036194, + "mixer_index": 38, + "normalized_entropy": 0.9680265430977588, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1829337775707245, + "median": 0.183739572763443, + "p05": 0.13839715719223022, + "p95": 0.2284226417541504 + }, + { + "label": "completed_group_1", + "mean": 0.18416839838027954, + "median": 0.18364182114601135, + "p05": 0.16665373742580414, + "p95": 0.20437687635421753 + }, + { + "label": "completed_group_2", + "mean": 0.09014187008142471, + "median": 0.08851997554302216, + "p05": 0.07715612649917603, + "p95": 0.10642829537391663 + }, + { + "label": "completed_group_3", + "mean": 0.12335794419050217, + "median": 0.12144016474485397, + "p05": 0.10556001961231232, + "p95": 0.14631223678588867 + }, + { + "label": "completed_group_4", + "mean": 0.15779981017112732, + "median": 0.15796968340873718, + "p05": 0.1340107023715973, + "p95": 0.18188069760799408 + }, + { + "label": "current_group_partial", + "mean": 0.2615981698036194, + "median": 0.262384295463562, + "p05": 0.230574831366539, + "p95": 0.28982529044151306 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.735896348953247, + "group": 5, + "latest_source_mass": 0.27010154724121094, + "layer": 20, + "max_source_mass": 0.27010154724121094, + "mixer_index": 39, + "normalized_entropy": 0.9688221989423192, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17276549339294434, + "median": 0.16886113584041595, + "p05": 0.10281073302030563, + "p95": 0.2923668622970581 + }, + { + "label": "completed_group_1", + "mean": 0.15101772546768188, + "median": 0.14904022216796875, + "p05": 0.13034172356128693, + "p95": 0.17705516517162323 + }, + { + "label": "completed_group_2", + "mean": 0.13220664858818054, + "median": 0.1315174400806427, + "p05": 0.10168404132127762, + "p95": 0.16267849504947662 + }, + { + "label": "completed_group_3", + "mean": 0.11932596564292908, + "median": 0.11731794476509094, + "p05": 0.09613680094480515, + "p95": 0.1465223878622055 + }, + { + "label": "completed_group_4", + "mean": 0.15458261966705322, + "median": 0.15350666642189026, + "p05": 0.12296099960803986, + "p95": 0.19313591718673706 + }, + { + "label": "current_group_partial", + "mean": 0.27010154724121094, + "median": 0.27393096685409546, + "p05": 0.19484001398086548, + "p95": 0.3264923095703125 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7644665241241455, + "group": 6, + "latest_source_mass": 0.2094498574733734, + "layer": 21, + "max_source_mass": 0.2094498574733734, + "mixer_index": 40, + "normalized_entropy": 0.9847675173076283, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19160009920597076, + "median": 0.196849524974823, + "p05": 0.12982682883739471, + "p95": 0.24405300617218018 + }, + { + "label": "completed_group_1", + "mean": 0.1915181279182434, + "median": 0.1907939314842224, + "p05": 0.17317205667495728, + "p95": 0.2111247181892395 + }, + { + "label": "completed_group_2", + "mean": 0.11520625650882721, + "median": 0.11408726871013641, + "p05": 0.10055835545063019, + "p95": 0.13332504034042358 + }, + { + "label": "completed_group_3", + "mean": 0.1341419219970703, + "median": 0.13221079111099243, + "p05": 0.11641807854175568, + "p95": 0.15773889422416687 + }, + { + "label": "completed_group_4", + "mean": 0.1580837219953537, + "median": 0.15754032135009766, + "p05": 0.13662610948085785, + "p95": 0.18392926454544067 + }, + { + "label": "completed_group_5", + "mean": 0.2094498574733734, + "median": 0.20958366990089417, + "p05": 0.1805649697780609, + "p95": 0.23726987838745117 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6835086345672607, + "group": 6, + "latest_source_mass": 0.43279585242271423, + "layer": 21, + "max_source_mass": 0.43279585242271423, + "mixer_index": 41, + "normalized_entropy": 0.8651522966692777, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10065847635269165, + "median": 0.09005473554134369, + "p05": 0.05753578990697861, + "p95": 0.19214311242103577 + }, + { + "label": "completed_group_1", + "mean": 0.09339895844459534, + "median": 0.09201203286647797, + "p05": 0.07428346574306488, + "p95": 0.11678388714790344 + }, + { + "label": "completed_group_2", + "mean": 0.09682720899581909, + "median": 0.09625889360904694, + "p05": 0.07657647132873535, + "p95": 0.11835116147994995 + }, + { + "label": "completed_group_3", + "mean": 0.08155890554189682, + "median": 0.08032877743244171, + "p05": 0.06701995432376862, + "p95": 0.09893998503684998 + }, + { + "label": "completed_group_4", + "mean": 0.09126688539981842, + "median": 0.09069330245256424, + "p05": 0.07231517881155014, + "p95": 0.1128317192196846 + }, + { + "label": "completed_group_5", + "mean": 0.10349369794130325, + "median": 0.10402766615152359, + "p05": 0.07336798310279846, + "p95": 0.1318768411874771 + }, + { + "label": "current_group_partial", + "mean": 0.43279585242271423, + "median": 0.4367910623550415, + "p05": 0.3485929071903229, + "p95": 0.5040479302406311 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7937740087509155, + "group": 6, + "latest_source_mass": 0.3452628254890442, + "layer": 22, + "max_source_mass": 0.3452628254890442, + "mixer_index": 42, + "normalized_entropy": 0.9218174896830382, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13723835349082947, + "median": 0.1412133425474167, + "p05": 0.08910952508449554, + "p95": 0.17562061548233032 + }, + { + "label": "completed_group_1", + "mean": 0.12937891483306885, + "median": 0.13051339983940125, + "p05": 0.10541269928216934, + "p95": 0.1487061083316803 + }, + { + "label": "completed_group_2", + "mean": 0.07450413703918457, + "median": 0.07415075600147247, + "p05": 0.06578464806079865, + "p95": 0.08426731824874878 + }, + { + "label": "completed_group_3", + "mean": 0.08537043631076813, + "median": 0.08443015813827515, + "p05": 0.07344960421323776, + "p95": 0.09956531971693039 + }, + { + "label": "completed_group_4", + "mean": 0.09880410879850388, + "median": 0.09825065732002258, + "p05": 0.08438277244567871, + "p95": 0.11530908197164536 + }, + { + "label": "completed_group_5", + "mean": 0.12944120168685913, + "median": 0.12968069314956665, + "p05": 0.10858328640460968, + "p95": 0.15059794485569 + }, + { + "label": "current_group_partial", + "mean": 0.3452628254890442, + "median": 0.33866745233535767, + "p05": 0.288330078125, + "p95": 0.42612025141716003 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5790722370147705, + "group": 6, + "latest_source_mass": 0.49195876717567444, + "layer": 22, + "max_source_mass": 0.49195876717567444, + "mixer_index": 43, + "normalized_entropy": 0.8114826050839847, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09500400722026825, + "median": 0.07923799753189087, + "p05": 0.045484717935323715, + "p95": 0.2259966880083084 + }, + { + "label": "completed_group_1", + "mean": 0.08892714232206345, + "median": 0.0876917839050293, + "p05": 0.06900187581777573, + "p95": 0.11303731799125671 + }, + { + "label": "completed_group_2", + "mean": 0.09081116318702698, + "median": 0.0898551493883133, + "p05": 0.06699725985527039, + "p95": 0.11749734729528427 + }, + { + "label": "completed_group_3", + "mean": 0.07538282871246338, + "median": 0.07381957024335861, + "p05": 0.05869410187005997, + "p95": 0.09544046223163605 + }, + { + "label": "completed_group_4", + "mean": 0.07685283571481705, + "median": 0.07580418884754181, + "p05": 0.056799523532390594, + "p95": 0.10076010227203369 + }, + { + "label": "completed_group_5", + "mean": 0.08106326311826706, + "median": 0.08067160844802856, + "p05": 0.05696551501750946, + "p95": 0.10726240277290344 + }, + { + "label": "current_group_partial", + "mean": 0.49195876717567444, + "median": 0.49396610260009766, + "p05": 0.3897285759449005, + "p95": 0.583945631980896 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8034920692443848, + "group": 6, + "latest_source_mass": 0.33595213294029236, + "layer": 23, + "max_source_mass": 0.33595213294029236, + "mixer_index": 44, + "normalized_entropy": 0.926811584861681, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.142653226852417, + "median": 0.14694267511367798, + "p05": 0.08785572648048401, + "p95": 0.18721188604831696 + }, + { + "label": "completed_group_1", + "mean": 0.13419847190380096, + "median": 0.13438312709331512, + "p05": 0.11451410502195358, + "p95": 0.15392597019672394 + }, + { + "label": "completed_group_2", + "mean": 0.07437049597501755, + "median": 0.07391499727964401, + "p05": 0.06574448198080063, + "p95": 0.08430380374193192 + }, + { + "label": "completed_group_3", + "mean": 0.09070361405611038, + "median": 0.08887749910354614, + "p05": 0.07508289068937302, + "p95": 0.11117327958345413 + }, + { + "label": "completed_group_4", + "mean": 0.1013825386762619, + "median": 0.10140319168567657, + "p05": 0.08447366952896118, + "p95": 0.11889617145061493 + }, + { + "label": "completed_group_5", + "mean": 0.12073953449726105, + "median": 0.12033741921186447, + "p05": 0.10022275149822235, + "p95": 0.1415012627840042 + }, + { + "label": "current_group_partial", + "mean": 0.33595213294029236, + "median": 0.330989271402359, + "p05": 0.27680477499961853, + "p95": 0.41842275857925415 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5278494358062744, + "group": 6, + "latest_source_mass": 0.51820969581604, + "layer": 23, + "max_source_mass": 0.51820969581604, + "mixer_index": 45, + "normalized_entropy": 0.7851592924514033, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10045768320560455, + "median": 0.08486579358577728, + "p05": 0.04896760731935501, + "p95": 0.2206544280052185 + }, + { + "label": "completed_group_1", + "mean": 0.08844950795173645, + "median": 0.08638796210289001, + "p05": 0.06730383634567261, + "p95": 0.11565113067626953 + }, + { + "label": "completed_group_2", + "mean": 0.08561251312494278, + "median": 0.08505219221115112, + "p05": 0.0625130906701088, + "p95": 0.11068116128444672 + }, + { + "label": "completed_group_3", + "mean": 0.06598107516765594, + "median": 0.06433478742837906, + "p05": 0.04859655350446701, + "p95": 0.08642670512199402 + }, + { + "label": "completed_group_4", + "mean": 0.06767018139362335, + "median": 0.06678327918052673, + "p05": 0.047009289264678955, + "p95": 0.09129223972558975 + }, + { + "label": "completed_group_5", + "mean": 0.07361933588981628, + "median": 0.07290077209472656, + "p05": 0.049975596368312836, + "p95": 0.09868258237838745 + }, + { + "label": "current_group_partial", + "mean": 0.51820969581604, + "median": 0.5205609798431396, + "p05": 0.40315571427345276, + "p95": 0.6213710308074951 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8435695171356201, + "group": 6, + "latest_source_mass": 0.29524505138397217, + "layer": 24, + "max_source_mass": 0.29524505138397217, + "mixer_index": 46, + "normalized_entropy": 0.9474073188993969, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14839638769626617, + "median": 0.15232303738594055, + "p05": 0.08978340029716492, + "p95": 0.19894656538963318 + }, + { + "label": "completed_group_1", + "mean": 0.1482572853565216, + "median": 0.14857065677642822, + "p05": 0.13154153525829315, + "p95": 0.16430144011974335 + }, + { + "label": "completed_group_2", + "mean": 0.08113027364015579, + "median": 0.08067925274372101, + "p05": 0.07102686911821365, + "p95": 0.09267604351043701 + }, + { + "label": "completed_group_3", + "mean": 0.09534022957086563, + "median": 0.09320184588432312, + "p05": 0.0778813511133194, + "p95": 0.11805401742458344 + }, + { + "label": "completed_group_4", + "mean": 0.10612697899341583, + "median": 0.10644470155239105, + "p05": 0.08815888315439224, + "p95": 0.12415381520986557 + }, + { + "label": "completed_group_5", + "mean": 0.12550383806228638, + "median": 0.1254296898841858, + "p05": 0.10509618371725082, + "p95": 0.14616522192955017 + }, + { + "label": "current_group_partial", + "mean": 0.29524505138397217, + "median": 0.29100558161735535, + "p05": 0.24107839167118073, + "p95": 0.37045177817344666 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.6421003341674805, + "group": 6, + "latest_source_mass": 0.4530346691608429, + "layer": 24, + "max_source_mass": 0.4530346691608429, + "mixer_index": 47, + "normalized_entropy": 0.8438726397334819, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11422763019800186, + "median": 0.09945468604564667, + "p05": 0.06323201954364777, + "p95": 0.23352834582328796 + }, + { + "label": "completed_group_1", + "mean": 0.10543527454137802, + "median": 0.1044958084821701, + "p05": 0.08478721231222153, + "p95": 0.1284603774547577 + }, + { + "label": "completed_group_2", + "mean": 0.09285669028759003, + "median": 0.09177995473146439, + "p05": 0.06902404874563217, + "p95": 0.12035554647445679 + }, + { + "label": "completed_group_3", + "mean": 0.07746599614620209, + "median": 0.07653152942657471, + "p05": 0.058040909469127655, + "p95": 0.09812526404857635 + }, + { + "label": "completed_group_4", + "mean": 0.07571054995059967, + "median": 0.07457360625267029, + "p05": 0.05648855119943619, + "p95": 0.0989091694355011 + }, + { + "label": "completed_group_5", + "mean": 0.08126918971538544, + "median": 0.08049014210700989, + "p05": 0.05931335315108299, + "p95": 0.10596252232789993 + }, + { + "label": "current_group_partial", + "mean": 0.4530346691608429, + "median": 0.455755352973938, + "p05": 0.3413957357406616, + "p95": 0.5519019365310669 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8733552694320679, + "group": 7, + "latest_source_mass": 0.26451510190963745, + "layer": 25, + "max_source_mass": 0.26451510190963745, + "mixer_index": 48, + "normalized_entropy": 0.9627141676307774, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14798463881015778, + "median": 0.14657661318778992, + "p05": 0.09472498297691345, + "p95": 0.2258487045764923 + }, + { + "label": "completed_group_1", + "mean": 0.160526841878891, + "median": 0.16084568202495575, + "p05": 0.1358838975429535, + "p95": 0.18423213064670563 + }, + { + "label": "completed_group_2", + "mean": 0.09668850898742676, + "median": 0.09659601747989655, + "p05": 0.08555105328559875, + "p95": 0.1084243580698967 + }, + { + "label": "completed_group_3", + "mean": 0.10477185994386673, + "median": 0.10316876322031021, + "p05": 0.08876212686300278, + "p95": 0.12562818825244904 + }, + { + "label": "completed_group_4", + "mean": 0.10857057571411133, + "median": 0.1074000895023346, + "p05": 0.09068692475557327, + "p95": 0.12911471724510193 + }, + { + "label": "completed_group_5", + "mean": 0.11694245785474777, + "median": 0.11679476499557495, + "p05": 0.0976894274353981, + "p95": 0.13528725504875183 + }, + { + "label": "completed_group_6", + "mean": 0.26451510190963745, + "median": 0.26442092657089233, + "p05": 0.21035388112068176, + "p95": 0.31906524300575256 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.784506916999817, + "group": 7, + "latest_source_mass": 0.27427971363067627, + "layer": 25, + "max_source_mass": 0.29875272512435913, + "mixer_index": 49, + "normalized_entropy": 0.8581664265292297, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08589775860309601, + "median": 0.07514797151088715, + "p05": 0.03144541755318642, + "p95": 0.20497030019760132 + }, + { + "label": "completed_group_1", + "mean": 0.08786046504974365, + "median": 0.08646859228610992, + "p05": 0.06123892217874527, + "p95": 0.11937689781188965 + }, + { + "label": "completed_group_2", + "mean": 0.07714234292507172, + "median": 0.07644426077604294, + "p05": 0.05296650156378746, + "p95": 0.10448302328586578 + }, + { + "label": "completed_group_3", + "mean": 0.06314892321825027, + "median": 0.06132982298731804, + "p05": 0.044174306094646454, + "p95": 0.08466912060976028 + }, + { + "label": "completed_group_4", + "mean": 0.05684571713209152, + "median": 0.055630408227443695, + "p05": 0.036537639796733856, + "p95": 0.08176163583993912 + }, + { + "label": "completed_group_5", + "mean": 0.056072384119033813, + "median": 0.0549161471426487, + "p05": 0.03825453296303749, + "p95": 0.07769857347011566 + }, + { + "label": "completed_group_6", + "mean": 0.29875272512435913, + "median": 0.29761478304862976, + "p05": 0.17369259893894196, + "p95": 0.4214131534099579 + }, + { + "label": "current_group_partial", + "mean": 0.27427971363067627, + "median": 0.26517564058303833, + "p05": 0.11691330373287201, + "p95": 0.45670589804649353 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9675533771514893, + "group": 7, + "latest_source_mass": 0.24859297275543213, + "layer": 26, + "max_source_mass": 0.24859297275543213, + "mixer_index": 50, + "normalized_entropy": 0.9461931666335953, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11549945175647736, + "median": 0.1186310350894928, + "p05": 0.0548042394220829, + "p95": 0.17459861934185028 + }, + { + "label": "completed_group_1", + "mean": 0.12592050433158875, + "median": 0.12912830710411072, + "p05": 0.08620737493038177, + "p95": 0.15710318088531494 + }, + { + "label": "completed_group_2", + "mean": 0.07530814409255981, + "median": 0.07495854794979095, + "p05": 0.06174474209547043, + "p95": 0.0894969180226326 + }, + { + "label": "completed_group_3", + "mean": 0.08277833461761475, + "median": 0.08158879727125168, + "p05": 0.06734651327133179, + "p95": 0.10064717382192612 + }, + { + "label": "completed_group_4", + "mean": 0.08296641707420349, + "median": 0.08213280141353607, + "p05": 0.06705781072378159, + "p95": 0.10095006227493286 + }, + { + "label": "completed_group_5", + "mean": 0.09016789495944977, + "median": 0.08980530500411987, + "p05": 0.07541801035404205, + "p95": 0.10646115988492966 + }, + { + "label": "completed_group_6", + "mean": 0.17876628041267395, + "median": 0.17772917449474335, + "p05": 0.13528534770011902, + "p95": 0.2264900654554367 + }, + { + "label": "current_group_partial", + "mean": 0.24859297275543213, + "median": 0.245039701461792, + "p05": 0.14620795845985413, + "p95": 0.3620130121707916 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7119876146316528, + "group": 7, + "latest_source_mass": 0.3658455014228821, + "layer": 26, + "max_source_mass": 0.3658455014228821, + "mixer_index": 51, + "normalized_entropy": 0.8232920138974705, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08804735541343689, + "median": 0.08122549951076508, + "p05": 0.054039862006902695, + "p95": 0.1443495899438858 + }, + { + "label": "completed_group_1", + "mean": 0.07527871429920197, + "median": 0.0738755539059639, + "p05": 0.05371527001261711, + "p95": 0.10149700939655304 + }, + { + "label": "completed_group_2", + "mean": 0.05666808784008026, + "median": 0.05508599430322647, + "p05": 0.03937675803899765, + "p95": 0.07947445660829544 + }, + { + "label": "completed_group_3", + "mean": 0.050419021397829056, + "median": 0.04877372086048126, + "p05": 0.03257015347480774, + "p95": 0.0718536451458931 + }, + { + "label": "completed_group_4", + "mean": 0.05142981931567192, + "median": 0.0498797781765461, + "p05": 0.034862153232097626, + "p95": 0.07353642582893372 + }, + { + "label": "completed_group_5", + "mean": 0.05313318967819214, + "median": 0.051505643874406815, + "p05": 0.037266045808792114, + "p95": 0.07490143924951553 + }, + { + "label": "completed_group_6", + "mean": 0.2591783106327057, + "median": 0.2531449794769287, + "p05": 0.15121260285377502, + "p95": 0.386227548122406 + }, + { + "label": "current_group_partial", + "mean": 0.3658455014228821, + "median": 0.3656759262084961, + "p05": 0.19218485057353973, + "p95": 0.5332037210464478 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9094492197036743, + "group": 7, + "latest_source_mass": 0.2766643464565277, + "layer": 27, + "max_source_mass": 0.2766643464565277, + "mixer_index": 52, + "normalized_entropy": 0.918250973365264, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10337945818901062, + "median": 0.10516610741615295, + "p05": 0.05207033082842827, + "p95": 0.1573200225830078 + }, + { + "label": "completed_group_1", + "mean": 0.10583682358264923, + "median": 0.10595200955867767, + "p05": 0.0643514022231102, + "p95": 0.14548130333423615 + }, + { + "label": "completed_group_2", + "mean": 0.08642244338989258, + "median": 0.08549235016107559, + "p05": 0.06607823818922043, + "p95": 0.11075147986412048 + }, + { + "label": "completed_group_3", + "mean": 0.08043928444385529, + "median": 0.07886679470539093, + "p05": 0.05936145409941673, + "p95": 0.10334464907646179 + }, + { + "label": "completed_group_4", + "mean": 0.0717030018568039, + "median": 0.07051880657672882, + "p05": 0.05472247302532196, + "p95": 0.09299077093601227 + }, + { + "label": "completed_group_5", + "mean": 0.0645938515663147, + "median": 0.06373226642608643, + "p05": 0.04421529173851013, + "p95": 0.08712958544492722 + }, + { + "label": "completed_group_6", + "mean": 0.21096080541610718, + "median": 0.20807480812072754, + "p05": 0.14586198329925537, + "p95": 0.2868158519268036 + }, + { + "label": "current_group_partial", + "mean": 0.2766643464565277, + "median": 0.27722808718681335, + "p05": 0.16094130277633667, + "p95": 0.3963583707809448 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7860023975372314, + "group": 7, + "latest_source_mass": 0.3244447112083435, + "layer": 27, + "max_source_mass": 0.3244447112083435, + "mixer_index": 53, + "normalized_entropy": 0.8588856006475877, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09247160702943802, + "median": 0.08733023703098297, + "p05": 0.04943867400288582, + "p95": 0.16303153336048126 + }, + { + "label": "completed_group_1", + "mean": 0.0784597396850586, + "median": 0.07711629569530487, + "p05": 0.05584436655044556, + "p95": 0.10506026446819305 + }, + { + "label": "completed_group_2", + "mean": 0.07236532866954803, + "median": 0.07097338140010834, + "p05": 0.050456635653972626, + "p95": 0.09917968511581421 + }, + { + "label": "completed_group_3", + "mean": 0.05986309051513672, + "median": 0.057845089584589005, + "p05": 0.04075033217668533, + "p95": 0.08416998386383057 + }, + { + "label": "completed_group_4", + "mean": 0.05389959365129471, + "median": 0.05301224812865257, + "p05": 0.03848717734217644, + "p95": 0.07268592715263367 + }, + { + "label": "completed_group_5", + "mean": 0.06059400737285614, + "median": 0.05963316559791565, + "p05": 0.04356392100453377, + "p95": 0.08111730217933655 + }, + { + "label": "completed_group_6", + "mean": 0.2579019069671631, + "median": 0.2530352473258972, + "p05": 0.16421708464622498, + "p95": 0.3691956102848053 + }, + { + "label": "current_group_partial", + "mean": 0.3244447112083435, + "median": 0.3252047896385193, + "p05": 0.17263495922088623, + "p95": 0.47815465927124023 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9232951402664185, + "group": 7, + "latest_source_mass": 0.2509572505950928, + "layer": 28, + "max_source_mass": 0.2509572505950928, + "mixer_index": 54, + "normalized_entropy": 0.9249094536760685, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1160542219877243, + "median": 0.1222282201051712, + "p05": 0.061863381415605545, + "p95": 0.16598135232925415 + }, + { + "label": "completed_group_1", + "mean": 0.11346358060836792, + "median": 0.11177559196949005, + "p05": 0.05926967412233353, + "p95": 0.16970883309841156 + }, + { + "label": "completed_group_2", + "mean": 0.08816123753786087, + "median": 0.08707751333713531, + "p05": 0.06823235750198364, + "p95": 0.1114775538444519 + }, + { + "label": "completed_group_3", + "mean": 0.08138670027256012, + "median": 0.0797908753156662, + "p05": 0.06149819865822792, + "p95": 0.10523255914449692 + }, + { + "label": "completed_group_4", + "mean": 0.07161372900009155, + "median": 0.07095701992511749, + "p05": 0.055086471140384674, + "p95": 0.09069444239139557 + }, + { + "label": "completed_group_5", + "mean": 0.06013647839426994, + "median": 0.05960702896118164, + "p05": 0.04596705362200737, + "p95": 0.076435886323452 + }, + { + "label": "completed_group_6", + "mean": 0.21822679042816162, + "median": 0.21632584929466248, + "p05": 0.14715570211410522, + "p95": 0.2940847873687744 + }, + { + "label": "current_group_partial", + "mean": 0.2509572505950928, + "median": 0.24700899422168732, + "p05": 0.14031530916690826, + "p95": 0.37019768357276917 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8294003009796143, + "group": 7, + "latest_source_mass": 0.20826560258865356, + "layer": 28, + "max_source_mass": 0.31431835889816284, + "mixer_index": 55, + "normalized_entropy": 0.879755580674689, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08831655979156494, + "median": 0.06431648135185242, + "p05": 0.03252303972840309, + "p95": 0.2706584930419922 + }, + { + "label": "completed_group_1", + "mean": 0.10072161257266998, + "median": 0.1009141132235527, + "p05": 0.06054181233048439, + "p95": 0.14067479968070984 + }, + { + "label": "completed_group_2", + "mean": 0.0818098932504654, + "median": 0.08032949268817902, + "p05": 0.054586175829172134, + "p95": 0.1145491972565651 + }, + { + "label": "completed_group_3", + "mean": 0.07514279335737228, + "median": 0.07337591052055359, + "p05": 0.04998338222503662, + "p95": 0.10431315004825592 + }, + { + "label": "completed_group_4", + "mean": 0.06651485711336136, + "median": 0.06524930149316788, + "p05": 0.04414192587137222, + "p95": 0.09315875917673111 + }, + { + "label": "completed_group_5", + "mean": 0.06491032987833023, + "median": 0.06368150562047958, + "p05": 0.04332352057099342, + "p95": 0.09065432846546173 + }, + { + "label": "completed_group_6", + "mean": 0.31431835889816284, + "median": 0.3147735297679901, + "p05": 0.18699732422828674, + "p95": 0.44058555364608765 + }, + { + "label": "current_group_partial", + "mean": 0.20826560258865356, + "median": 0.1914007067680359, + "p05": 0.08225662261247635, + "p95": 0.38793477416038513 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9393219947814941, + "group": 8, + "latest_source_mass": 0.15344023704528809, + "layer": 29, + "max_source_mass": 0.2604207992553711, + "mixer_index": 56, + "normalized_entropy": 0.9326167415193847, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11041727662086487, + "median": 0.10045937448740005, + "p05": 0.024797730147838593, + "p95": 0.279465913772583 + }, + { + "label": "completed_group_1", + "mean": 0.12080886960029602, + "median": 0.12561748921871185, + "p05": 0.0680447518825531, + "p95": 0.16192926466464996 + }, + { + "label": "completed_group_2", + "mean": 0.09995828568935394, + "median": 0.09907965362071991, + "p05": 0.07490158826112747, + "p95": 0.1276247799396515 + }, + { + "label": "completed_group_3", + "mean": 0.09934850037097931, + "median": 0.09762611985206604, + "p05": 0.06941570341587067, + "p95": 0.134530171751976 + }, + { + "label": "completed_group_4", + "mean": 0.08031868189573288, + "median": 0.07758662104606628, + "p05": 0.05694210156798363, + "p95": 0.11248748749494553 + }, + { + "label": "completed_group_5", + "mean": 0.075287364423275, + "median": 0.07397153228521347, + "p05": 0.05265223979949951, + "p95": 0.10342685133218765 + }, + { + "label": "completed_group_6", + "mean": 0.2604207992553711, + "median": 0.2616186738014221, + "p05": 0.16666486859321594, + "p95": 0.34887272119522095 + }, + { + "label": "completed_group_7", + "mean": 0.15344023704528809, + "median": 0.13995957374572754, + "p05": 0.05727577582001686, + "p95": 0.29449376463890076 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.9925593137741089, + "group": 8, + "latest_source_mass": 0.18462148308753967, + "layer": 29, + "max_source_mass": 0.2503892779350281, + "mixer_index": 57, + "normalized_entropy": 0.9068528243889232, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08698172122240067, + "median": 0.07842238247394562, + "p05": 0.02795225754380226, + "p95": 0.1876923143863678 + }, + { + "label": "completed_group_1", + "mean": 0.08857265114784241, + "median": 0.08714614808559418, + "p05": 0.06365615874528885, + "p95": 0.11900493502616882 + }, + { + "label": "completed_group_2", + "mean": 0.07299254834651947, + "median": 0.07189398258924484, + "p05": 0.05586059391498566, + "p95": 0.0933830514550209 + }, + { + "label": "completed_group_3", + "mean": 0.060478877276182175, + "median": 0.05862775444984436, + "p05": 0.04474102333188057, + "p95": 0.07903359085321426 + }, + { + "label": "completed_group_4", + "mean": 0.054115522652864456, + "median": 0.052696481347084045, + "p05": 0.03536852449178696, + "p95": 0.07748772948980331 + }, + { + "label": "completed_group_5", + "mean": 0.056821923702955246, + "median": 0.05573669448494911, + "p05": 0.0398341529071331, + "p95": 0.07687154412269592 + }, + { + "label": "completed_group_6", + "mean": 0.2503892779350281, + "median": 0.2475910186767578, + "p05": 0.15396791696548462, + "p95": 0.357724130153656 + }, + { + "label": "completed_group_7", + "mean": 0.14502598345279694, + "median": 0.13307344913482666, + "p05": 0.06183449178934097, + "p95": 0.2673647403717041 + }, + { + "label": "current_group_partial", + "mean": 0.18462148308753967, + "median": 0.17473986744880676, + "p05": 0.08267536759376526, + "p95": 0.3240666389465332 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.068819999694824, + "group": 8, + "latest_source_mass": 0.15494346618652344, + "layer": 30, + "max_source_mass": 0.21415996551513672, + "mixer_index": 58, + "normalized_entropy": 0.9415605582761754, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09429103136062622, + "median": 0.0943363606929779, + "p05": 0.03544093295931816, + "p95": 0.16596338152885437 + }, + { + "label": "completed_group_1", + "mean": 0.09028280526399612, + "median": 0.090491384267807, + "p05": 0.05409309267997742, + "p95": 0.12526515126228333 + }, + { + "label": "completed_group_2", + "mean": 0.09188266098499298, + "median": 0.09010649472475052, + "p05": 0.06878700107336044, + "p95": 0.12181076407432556 + }, + { + "label": "completed_group_3", + "mean": 0.0735517293214798, + "median": 0.07178769260644913, + "p05": 0.0548262856900692, + "p95": 0.09550250321626663 + }, + { + "label": "completed_group_4", + "mean": 0.06746294349431992, + "median": 0.06608930230140686, + "p05": 0.050633110105991364, + "p95": 0.08874846994876862 + }, + { + "label": "completed_group_5", + "mean": 0.060166649520397186, + "median": 0.060052454471588135, + "p05": 0.039401471614837646, + "p95": 0.08220525085926056 + }, + { + "label": "completed_group_6", + "mean": 0.21415996551513672, + "median": 0.2118481993675232, + "p05": 0.15205052495002747, + "p95": 0.2836942672729492 + }, + { + "label": "completed_group_7", + "mean": 0.15325874090194702, + "median": 0.14970678091049194, + "p05": 0.08526452630758286, + "p95": 0.23864519596099854 + }, + { + "label": "current_group_partial", + "mean": 0.15494346618652344, + "median": 0.1473558098077774, + "p05": 0.0697651207447052, + "p95": 0.2689872086048126 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0139355659484863, + "group": 8, + "latest_source_mass": 0.1372738927602768, + "layer": 30, + "max_source_mass": 0.24272148311138153, + "mixer_index": 59, + "normalized_entropy": 0.916581576012616, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0956040769815445, + "median": 0.09956102818250656, + "p05": 0.03933291882276535, + "p95": 0.16697832942008972 + }, + { + "label": "completed_group_1", + "mean": 0.09831400960683823, + "median": 0.09554082155227661, + "p05": 0.05786377936601639, + "p95": 0.14979025721549988 + }, + { + "label": "completed_group_2", + "mean": 0.0714857280254364, + "median": 0.06985784322023392, + "p05": 0.04893886297941208, + "p95": 0.10043824464082718 + }, + { + "label": "completed_group_3", + "mean": 0.0677608773112297, + "median": 0.06590595096349716, + "p05": 0.044468365609645844, + "p95": 0.09555256366729736 + }, + { + "label": "completed_group_4", + "mean": 0.05654233694076538, + "median": 0.054802894592285156, + "p05": 0.037319183349609375, + "p95": 0.08175043016672134 + }, + { + "label": "completed_group_5", + "mean": 0.05388312786817551, + "median": 0.05288306996226311, + "p05": 0.036485109478235245, + "p95": 0.07530424743890762 + }, + { + "label": "completed_group_6", + "mean": 0.24272148311138153, + "median": 0.23883119225502014, + "p05": 0.1493627429008484, + "p95": 0.35197100043296814 + }, + { + "label": "completed_group_7", + "mean": 0.17641445994377136, + "median": 0.17445214092731476, + "p05": 0.0891575887799263, + "p95": 0.276222825050354 + }, + { + "label": "current_group_partial", + "mean": 0.1372738927602768, + "median": 0.12184794992208481, + "p05": 0.05821039155125618, + "p95": 0.2647005021572113 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0349321365356445, + "group": 8, + "latest_source_mass": 0.13534162938594818, + "layer": 31, + "max_source_mass": 0.2261163890361786, + "mixer_index": 60, + "normalized_entropy": 0.9261375270991514, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10256819427013397, + "median": 0.10616284608840942, + "p05": 0.019877789542078972, + "p95": 0.19627508521080017 + }, + { + "label": "completed_group_1", + "mean": 0.0938093364238739, + "median": 0.09388002753257751, + "p05": 0.034650709480047226, + "p95": 0.15219232439994812 + }, + { + "label": "completed_group_2", + "mean": 0.0912778377532959, + "median": 0.08955617249011993, + "p05": 0.06106609106063843, + "p95": 0.1285511702299118 + }, + { + "label": "completed_group_3", + "mean": 0.08394120633602142, + "median": 0.0820196121931076, + "p05": 0.05809478089213371, + "p95": 0.11390723288059235 + }, + { + "label": "completed_group_4", + "mean": 0.06161237880587578, + "median": 0.059880051761865616, + "p05": 0.044582173228263855, + "p95": 0.08485961705446243 + }, + { + "label": "completed_group_5", + "mean": 0.053358763456344604, + "median": 0.052702393382787704, + "p05": 0.03554790839552879, + "p95": 0.07320070266723633 + }, + { + "label": "completed_group_6", + "mean": 0.2261163890361786, + "median": 0.22380158305168152, + "p05": 0.1451866328716278, + "p95": 0.3134670853614807 + }, + { + "label": "completed_group_7", + "mean": 0.15197426080703735, + "median": 0.1527099758386612, + "p05": 0.0639471560716629, + "p95": 0.2459363490343094 + }, + { + "label": "current_group_partial", + "mean": 0.13534162938594818, + "median": 0.11519233882427216, + "p05": 0.04325530678033829, + "p95": 0.2904844284057617 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.046657085418701, + "group": 8, + "latest_source_mass": 0.09365127235651016, + "layer": 31, + "max_source_mass": 0.2572976052761078, + "mixer_index": 61, + "normalized_entropy": 0.9314737813009277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08380690217018127, + "median": 0.06641843914985657, + "p05": 0.039923232048749924, + "p95": 0.23694062232971191 + }, + { + "label": "completed_group_1", + "mean": 0.1217331737279892, + "median": 0.12164830416440964, + "p05": 0.08828366547822952, + "p95": 0.1561325639486313 + }, + { + "label": "completed_group_2", + "mean": 0.10242430865764618, + "median": 0.10224691033363342, + "p05": 0.06860878318548203, + "p95": 0.13651791214942932 + }, + { + "label": "completed_group_3", + "mean": 0.09225838631391525, + "median": 0.09045033901929855, + "p05": 0.0655013769865036, + "p95": 0.1235189214348793 + }, + { + "label": "completed_group_4", + "mean": 0.06788390874862671, + "median": 0.0664045438170433, + "p05": 0.047665324062108994, + "p95": 0.09357043355703354 + }, + { + "label": "completed_group_5", + "mean": 0.06336672604084015, + "median": 0.06275692582130432, + "p05": 0.044327184557914734, + "p95": 0.08375845849514008 + }, + { + "label": "completed_group_6", + "mean": 0.2572976052761078, + "median": 0.25880685448646545, + "p05": 0.14942821860313416, + "p95": 0.36036571860313416 + }, + { + "label": "completed_group_7", + "mean": 0.11757771670818329, + "median": 0.10582176595926285, + "p05": 0.05813705176115036, + "p95": 0.21660257875919342 + }, + { + "label": "current_group_partial", + "mean": 0.09365127235651016, + "median": 0.08298523724079132, + "p05": 0.04494771361351013, + "p95": 0.18065699934959412 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.094015121459961, + "group": 8, + "latest_source_mass": 0.09723767638206482, + "layer": 32, + "max_source_mass": 0.2018478661775589, + "mixer_index": 62, + "normalized_entropy": 0.9530273523513089, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10284869372844696, + "median": 0.09173205494880676, + "p05": 0.03822793811559677, + "p95": 0.22231025993824005 + }, + { + "label": "completed_group_1", + "mean": 0.13057854771614075, + "median": 0.13079363107681274, + "p05": 0.0711527168750763, + "p95": 0.19180890917778015 + }, + { + "label": "completed_group_2", + "mean": 0.10152992606163025, + "median": 0.09965524077415466, + "p05": 0.07341106235980988, + "p95": 0.13697651028633118 + }, + { + "label": "completed_group_3", + "mean": 0.10079469531774521, + "median": 0.10007210820913315, + "p05": 0.07479676604270935, + "p95": 0.12990058958530426 + }, + { + "label": "completed_group_4", + "mean": 0.0796234980225563, + "median": 0.07812212407588959, + "p05": 0.05766180902719498, + "p95": 0.10730832070112228 + }, + { + "label": "completed_group_5", + "mean": 0.07438632845878601, + "median": 0.07310014963150024, + "p05": 0.05309801548719406, + "p95": 0.10082833468914032 + }, + { + "label": "completed_group_6", + "mean": 0.2018478661775589, + "median": 0.20017066597938538, + "p05": 0.13963642716407776, + "p95": 0.2733708918094635 + }, + { + "label": "completed_group_7", + "mean": 0.1111527606844902, + "median": 0.10317055881023407, + "p05": 0.05416924133896828, + "p95": 0.1888933926820755 + }, + { + "label": "current_group_partial", + "mean": 0.09723767638206482, + "median": 0.08349506556987762, + "p05": 0.03931424394249916, + "p95": 0.1855989694595337 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.057201385498047, + "group": 8, + "latest_source_mass": 0.09480181336402893, + "layer": 32, + "max_source_mass": 0.24551761150360107, + "mixer_index": 63, + "normalized_entropy": 0.9362726990757002, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09265108406543732, + "median": 0.07600092887878418, + "p05": 0.04942645877599716, + "p95": 0.20456968247890472 + }, + { + "label": "completed_group_1", + "mean": 0.11481083929538727, + "median": 0.1135297492146492, + "p05": 0.08239293098449707, + "p95": 0.15063267946243286 + }, + { + "label": "completed_group_2", + "mean": 0.09007447212934494, + "median": 0.0894046500325203, + "p05": 0.06493785232305527, + "p95": 0.11702875047922134 + }, + { + "label": "completed_group_3", + "mean": 0.08496076613664627, + "median": 0.08339293301105499, + "p05": 0.06288711726665497, + "p95": 0.11049411445856094 + }, + { + "label": "completed_group_4", + "mean": 0.06770835816860199, + "median": 0.06692098081111908, + "p05": 0.0488700196146965, + "p95": 0.09013402462005615 + }, + { + "label": "completed_group_5", + "mean": 0.05925413966178894, + "median": 0.058933183550834656, + "p05": 0.04426975175738335, + "p95": 0.07536415755748749 + }, + { + "label": "completed_group_6", + "mean": 0.24551761150360107, + "median": 0.24382448196411133, + "p05": 0.15310630202293396, + "p95": 0.3403708338737488 + }, + { + "label": "completed_group_7", + "mean": 0.15022093057632446, + "median": 0.14221730828285217, + "p05": 0.0825236365199089, + "p95": 0.24877692759037018 + }, + { + "label": "current_group_partial", + "mean": 0.09480181336402893, + "median": 0.08117073774337769, + "p05": 0.04724843427538872, + "p95": 0.17635159194469452 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.8435449600219727, + "group": null, + "latest_source_mass": 0.20717504620552063, + "layer": null, + "max_source_mass": 0.31774085760116577, + "mixer_index": null, + "normalized_entropy": 0.839033469331102, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1112452894449234, + "median": 0.11452806740999222, + "p05": 0.04704529047012329, + "p95": 0.1625424325466156 + }, + { + "label": "completed_group_1", + "mean": 0.06898629665374756, + "median": 0.06820575892925262, + "p05": 0.04699244350194931, + "p95": 0.09449291229248047 + }, + { + "label": "completed_group_2", + "mean": 0.05835210159420967, + "median": 0.05662037432193756, + "p05": 0.039408810436725616, + "p95": 0.081703320145607 + }, + { + "label": "completed_group_3", + "mean": 0.042265649884939194, + "median": 0.040285609662532806, + "p05": 0.026811392977833748, + "p95": 0.06341734528541565 + }, + { + "label": "completed_group_4", + "mean": 0.041034016758203506, + "median": 0.039777740836143494, + "p05": 0.027301941066980362, + "p95": 0.05862141400575638 + }, + { + "label": "completed_group_5", + "mean": 0.04826357960700989, + "median": 0.047516219317913055, + "p05": 0.03253086656332016, + "p95": 0.06693053245544434 + }, + { + "label": "completed_group_6", + "mean": 0.10493715852499008, + "median": 0.09417732059955597, + "p05": 0.04745320975780487, + "p95": 0.19990575313568115 + }, + { + "label": "completed_group_7", + "mean": 0.31774085760116577, + "median": 0.3171391487121582, + "p05": 0.10361962765455246, + "p95": 0.5363480448722839 + }, + { + "label": "completed_group_8", + "mean": 0.20717504620552063, + "median": 0.19020375609397888, + "p05": 0.03896218538284302, + "p95": 0.4246461093425751 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", + "optimizer_state_before": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.04804428666830063, + 0.12356939166784286, + 0.05923242121934891, + 0.04508936405181885, + 0.0799286738038063, + 0.03290615975856781, + 0.03170505166053772, + 0.03961387276649475, + 0.04508088901638985, + 0.0312613770365715, + 0.029994890093803406, + 0.03058313950896263, + 0.0497574545443058, + 0.03759258985519409, + 0.030690157786011696, + 0.03000541403889656, + 0.079553984105587, + 0.0414140485227108, + 0.04122982919216156, + 0.05830265209078789, + 0.03440643846988678, + 0.033897366374731064, + 0.02957344427704811, + 0.03406442329287529, + 0.025225287303328514, + 0.01915411651134491, + 0.014526215381920338, + 0.011566666886210442, + 0.0242468249052763, + 0.013597902841866016, + 0.020146220922470093, + 0.02710210345685482 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.406048522767378e-05, + "first_to_last_ratio": 1.2121174427344374, + "imbalance_abs_log_ratio": 0.19236878289853168, + "last_quartile_mean": 1.1599936385664478e-05, + "mean": 1.385052358671146e-05, + "normalized": [ + 1.1035462446531923, + 1.2453829183001823, + 1.1319013131596403, + 1.0360400584890102, + 0.9887153549881315, + 0.9324565992202007, + 0.881809246461942, + 0.8014211622598989, + 0.7197327796271318, + 0.6642367228932815, + 0.63144486698815, + 0.6054650987106983, + 0.6612979512556375, + 0.6318736594713652, + 0.6080689788224007, + 0.5826508433334544, + 0.71610531341614, + 0.685873407734308, + 0.733241450138544, + 0.7670951091759252, + 2.8592552563843903, + 2.5014923465112644, + 2.101621489466362, + 1.7092008525685705, + 1.57037557605485, + 1.4158002854110978, + 1.11083466622777, + 0.8417375710978994, + 0.5124454971351892, + 0.45441040448215947, + 0.4041964735899746, + 0.3902705019712373 + ], + "peak_layer": 21, + "peak_normalized": 2.8592552563843903, + "population_cv": 0.5792971940134687, + "rest_mean": 1.0905006850710683e-05, + "spike_contrast": 2.7286836559095273, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.9756313961115665e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.5284693290595897e-05, + 1.724920548440423e-05, + 1.5677425835747272e-05, + 1.4349697266879957e-05, + 1.369422534480691e-05, + 1.2915012121084146e-05, + 1.2213519767101388e-05, + 1.1100102710770443e-05, + 9.96867584035499e-06, + 9.20002639759332e-06, + 8.745842023927253e-06, + 8.38600863062311e-06, + 9.159322871710174e-06, + 8.75178102432983e-06, + 8.422073733527213e-06, + 8.070019248407334e-06, + 9.91843353403965e-06, + 9.499705811322201e-06, + 1.0155777999898419e-05, + 1.0624668902892154e-05, + 3.960218236898072e-05, + 3.464697874733247e-05, + 2.910855801019352e-05, + 2.3673326722928323e-05, + 2.1750523956143297e-05, + 1.9609575247159228e-05, + 1.538564174552448e-05, + 1.165850608231267e-05, + 7.0976384449750185e-06, + 6.293822025327245e-06, + 5.598332791123539e-06, + 5.405450792750344e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.784132736152969e-05, + "first_to_last_ratio": 1.247704596204738, + "imbalance_abs_log_ratio": 0.22130554016992907, + "last_quartile_mean": 4.635819050236023e-05, + "mean": 5.578696777774894e-05, + "normalized": [ + 1.1172279996665548, + 1.2820128122064964, + 1.1617993468115064, + 1.0568066843619655, + 1.0159130322231686, + 0.9528514141416237, + 0.8952059773271221, + 0.8127833779089165, + 0.726905325923399, + 0.6687158255328219, + 0.6353938715547037, + 0.6079598423303039, + 0.663249561462659, + 0.632156813919224, + 0.6072625305936415, + 0.5816757612181556, + 0.7067342161190907, + 0.6745011066249388, + 0.7095591340205663, + 0.7592836625042008, + 2.8322187742699385, + 2.4767130392204146, + 2.0818487421172924, + 1.6933329620054323, + 1.5551938962366243, + 1.4028337924372432, + 1.095017319874792, + 0.8328031307001514, + 0.5108569494896386, + 0.4540995368567512, + 0.4053097243347225, + 0.3917738360059398 + ], + "peak_layer": 21, + "peak_normalized": 2.8322187742699385, + "population_cv": 0.5731363623317645, + "rest_mean": 4.413512107459138e-05, + "spike_contrast": 2.6896253388356377, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011870693997479975, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.2326762417797e-05, + 7.151960744522512e-05, + 6.481326272478327e-05, + 5.8956040447810665e-05, + 5.667470759362914e-05, + 5.315669113770127e-05, + 4.994082701159641e-05, + 4.534272011369467e-05, + 4.055184399476275e-05, + 3.730562821147032e-05, + 3.5446697438601404e-05, + 3.391623613424599e-05, + 3.700068191392347e-05, + 3.526611180859618e-05, + 3.387733522686176e-05, + 3.2449926948174834e-05, + 3.942655894206837e-05, + 3.762837150134146e-05, + 3.9584152546012774e-05, + 4.235813321429305e-05, + 0.00015800089749973267, + 0.00013816831051371992, + 0.00011614002869464457, + 9.446591138839722e-05, + 8.675955177750438e-05, + 7.825984357623383e-05, + 6.108769593993202e-05, + 4.645956141757779e-05, + 2.8499160180217586e-05, + 2.5332836230518296e-05, + 2.2611000531469472e-05, + 2.1855874365428463e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.051294267643243074, + "first_to_last_ratio": 1.247704590273494, + "imbalance_abs_log_ratio": 0.22130553541620454, + "last_quartile_mean": 0.04111090721562505, + "mean": 0.04947244143113494, + "normalized": [ + 1.1172280253192406, + 1.2820128195205216, + 1.1617993344751307, + 1.056806678091042, + 1.0159130361962296, + 0.9528513588115517, + 0.895205957602592, + 0.8127833650766507, + 0.726905316511733, + 0.6687157952448587, + 0.6353938256699311, + 0.6079598134280393, + 0.6632495949619723, + 0.6321568158374328, + 0.6072625701847152, + 0.5816757499080466, + 0.7067342453085952, + 0.6745011182519967, + 0.7095591365309404, + 0.759283697871088, + 2.8322188463523386, + 2.476713119523555, + 2.0818486723693663, + 1.6933329451602226, + 1.555193919529888, + 1.4028338274869276, + 1.095017294552204, + 0.832803082651722, + 0.5108569355338682, + 0.4540995509098777, + 0.40530972999858744, + 0.391773821129134 + ], + "peak_layer": 21, + "peak_normalized": 2.8322188463523386, + "population_cv": 0.5731363736273818, + "rest_mean": 0.039139466981093086, + "spike_contrast": 2.6896253725737616, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10527050346136094, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.055271998047828674, + 0.06342430412769318, + 0.05747704952955246, + 0.05228280648589134, + 0.05025969818234444, + 0.047139883041381836, + 0.0442880243062973, + 0.04021037742495537, + 0.035961780697107315, + 0.033083003014326096, + 0.03143448382616043, + 0.0300772562623024, + 0.03281257674098015, + 0.03127434104681015, + 0.03004276193678379, + 0.028776919469237328, + 0.03496386855840683, + 0.033369217067956924, + 0.03510362282395363, + 0.037563618272542953, + 0.14011678099632263, + 0.1225290447473526, + 0.10299413651227951, + 0.08377331495285034, + 0.07693924009799957, + 0.06940161436796188, + 0.05417317897081375, + 0.041200801730155945, + 0.025273339822888374, + 0.022465413436293602, + 0.020051661878824234, + 0.01938200742006302 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.2204174854323355e-06, + "first_to_last_ratio": 0.735976396978067, + "imbalance_abs_log_ratio": 0.306557230090771, + "last_quartile_mean": 3.0169683356007226e-06, + "mean": 2.8222587751258743e-06, + "normalized": [ + 1.1437692142315412, + 0.8934875077918236, + 0.843634317775851, + 0.8332025934148901, + 0.6691897239669076, + 0.649206320818976, + 0.6345003713598615, + 0.6270252408101402, + 0.548073507234572, + 0.5298155928110359, + 0.5235851424549667, + 0.5213985029212593, + 0.5319259781097652, + 0.5365763585413705, + 0.538492744687615, + 0.5418387466813859, + 0.5975925562684307, + 0.6133961155504078, + 0.6328406634942718, + 0.6938494895757952, + 2.792643580637443, + 2.792207888188673, + 2.5604162828657895, + 2.199406095874773, + 1.9619910931916569, + 1.8148930082375383, + 1.4357221480925244, + 1.0849940773120685, + 0.6411781557251384, + 0.5785168725769138, + 0.5245362458580646, + 0.5100938629385489 + ], + "peak_layer": 21, + "peak_normalized": 2.792643580637443, + "population_cv": 0.7010168394314351, + "rest_mean": 2.0585069512014378e-06, + "spike_contrast": 3.3745422235584535, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.946518624317832e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.2280127015837934e-06, + 2.5216529593308223e-06, + 2.3809543563402258e-06, + 2.3515133307228098e-06, + 1.8886265706896666e-06, + 1.8322282357985387e-06, + 1.7907242408909951e-06, + 1.7696274881018326e-06, + 1.5468052652067854e-06, + 1.495276706009463e-06, + 1.4776927628190606e-06, + 1.4715215002070181e-06, + 1.5012327594376984e-06, + 1.5143573364184704e-06, + 1.5197658740362385e-06, + 1.5292091575247468e-06, + 1.6865608358784812e-06, + 1.7311625697402633e-06, + 1.7860401158031891e-06, + 1.958222810571897e-06, + 7.881562851252966e-06, + 7.880333214416169e-06, + 7.2261573222931474e-06, + 6.207293154147919e-06, + 5.5372465794789605e-06, + 5.122097718412988e-06, + 4.051979431096697e-06, + 3.062134055653587e-06, + 1.809570676414296e-06, + 1.6327243201885722e-06, + 1.4803770227445057e-06, + 1.4396168808161747e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.569237398754922e-05, + "first_to_last_ratio": 1.4673112290466532, + "imbalance_abs_log_ratio": 0.38343163006552816, + "last_quartile_mean": 3.114020603334211e-05, + "mean": 4.183441066629712e-05, + "normalized": [ + 1.1974296567066158, + 1.344289107904746, + 1.230408462214659, + 1.1234591616583915, + 1.0623659972849115, + 0.9983070144562474, + 0.9335099902486312, + 0.8479894563327729, + 0.7756304429380606, + 0.7156796904410139, + 0.6795815361308393, + 0.6491943107046658, + 0.7109629912124332, + 0.6750948063174392, + 0.6474411689349395, + 0.6200717213988419, + 0.7662657439847727, + 0.7307653188389935, + 0.7545343036423542, + 0.7927280102241547, + 2.8287575533506253, + 2.4215604626110183, + 1.9807235727824308, + 1.5583037155212895, + 1.4117479335225582, + 1.2389201568217543, + 0.9856291732269274, + 0.776844076198889, + 0.43766749474230543, + 0.39060885635754505, + 0.36143095960303334, + 0.352097153686139 + ], + "peak_layer": 21, + "peak_normalized": 2.8287575533506253, + "population_cv": 0.551495636352156, + "rest_mean": 3.3775719169137085e-05, + "spike_contrast": 2.5270030320761303, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.535134475096129e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.009376400266774e-05, + 5.623754259431735e-05, + 5.1473412895575166e-05, + 4.699925193563104e-05, + 4.444345540832728e-05, + 4.176358561380766e-05, + 3.905284029315226e-05, + 3.547513915691525e-05, + 3.244804247515276e-05, + 2.994003807543777e-05, + 2.842989306373056e-05, + 2.7158661396242678e-05, + 2.9742717742919922e-05, + 2.8242193366168067e-05, + 2.708531974349171e-05, + 2.5940335035556927e-05, + 3.205627581337467e-05, + 3.0571136448998004e-05, + 3.156549792038277e-05, + 3.316330912639387e-05, + 0.00011833940516225994, + 0.00010130455484613776, + 8.286240336019546e-05, + 6.519071757793427e-05, + 5.905964280827902e-05, + 5.1829494623234496e-05, + 4.1233215597458184e-05, + 3.249881410738453e-05, + 1.830956171033904e-05, + 1.63408913067542e-05, + 1.512025119154714e-05, + 1.4729776921740267e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.5050525639235275e-05, + "first_to_last_ratio": 1.5890428234254472, + "imbalance_abs_log_ratio": 0.46313183711305195, + "last_quartile_mean": 2.2057634396333015e-05, + "mean": 3.131621542706853e-05, + "normalized": [ + 1.247642034455602, + 1.3675088708957686, + 1.2617968995203173, + 1.1676693179472686, + 1.0797799047255963, + 1.0154085918080462, + 0.9450439951530737, + 0.8691124556208352, + 0.8151155724977966, + 0.7531778294737046, + 0.7147915273060139, + 0.6827252927557514, + 0.7417922051240886, + 0.7100839458764344, + 0.679513505581043, + 0.6487030016923234, + 0.8192115232383504, + 0.7896113859496233, + 0.7975027577835305, + 0.8129328118460493, + 2.8268246118214386, + 2.3622917424367986, + 1.850934565013104, + 1.4060108194738914, + 1.2902617154939733, + 1.123282471468924, + 0.9231284485726192, + 0.7885511099194735, + 0.4159101173561302, + 0.376982553975174, + 0.35968054987980413, + 0.3570178653374513 + ], + "peak_layer": 21, + "peak_normalized": 2.8268246118214386, + "population_cv": 0.5277183722759345, + "rest_mean": 2.5822744107615478e-05, + "spike_contrast": 2.3615213122965844, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.0980960552114996e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 2 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.907142672687769e-05, + 4.282520239939913e-05, + 3.951470353058539e-05, + 3.656698390841484e-05, + 3.3814620110206306e-05, + 3.179875420755707e-05, + 2.959520134027116e-05, + 2.721731289057061e-05, + 2.5526334866299294e-05, + 2.3586679162690416e-05, + 2.238456545455847e-05, + 2.138037234544754e-05, + 2.3230124497786164e-05, + 2.223714182036929e-05, + 2.1279791326378472e-05, + 2.03149229491828e-05, + 2.5654604542069137e-05, + 2.4727640266064554e-05, + 2.4974768166430295e-05, + 2.5457979063503444e-05, + 8.852544851833954e-05, + 7.397803710773587e-05, + 5.796426557935774e-05, + 4.403093771543354e-05, + 4.040611383970827e-05, + 3.517695586197078e-05, + 2.8908889362355694e-05, + 2.469443643349223e-05, + 1.3024730833421927e-05, + 1.1805666872533038e-05, + 1.1263833584962413e-05, + 1.1180448382219765e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011513851586641977, + "first_to_last_ratio": 1.1690722131114868, + "imbalance_abs_log_ratio": 0.15621045398792427, + "last_quartile_mean": 9.848708623394486e-05, + "mean": 0.00011408905686494109, + "normalized": [ + 1.0878105091943224, + 1.2333698943486997, + 1.1325669787296346, + 1.0350609428916697, + 0.9847532971789725, + 0.9327565235282874, + 0.875161061795294, + 0.7921095504711437, + 0.7054552974920509, + 0.6480774339061697, + 0.6173745394266138, + 0.5959457136906282, + 0.6486626275163402, + 0.6167276122272274, + 0.5953045899588231, + 0.5685580660104519, + 0.6801746911900812, + 0.6508152675683995, + 0.6710893308044551, + 0.7221277670118063, + 2.8258795357154587, + 2.493070695082837, + 2.1580463328702972, + 1.8231222239658644, + 1.644278740520026, + 1.5034127662796515, + 1.171932855948766, + 0.8973146194608761, + 0.48937000596181734, + 0.4306716621780954, + 0.38968515069330856, + 0.3793137163819305 + ], + "peak_layer": 21, + "peak_normalized": 2.8258795357154587, + "population_cv": 0.5949686549165712, + "rest_mean": 8.89708825087399e-05, + "spike_contrast": 2.806841871708938, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002497271983884275, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012410727504175156, + 0.00014071400801185519, + 0.0001292134984396398, + 0.00011808912677224725, + 0.00011234957491979003, + 0.00010641731205396354, + 9.98463001451455e-05, + 9.037103154696524e-05, + 8.048472955124453e-05, + 7.393854320980608e-05, + 7.043567893560976e-05, + 6.799088441766798e-05, + 7.400530739687383e-05, + 7.036187162157148e-05, + 6.79177392157726e-05, + 6.486625352408737e-05, + 7.760048902127892e-05, + 7.425090007018298e-05, + 7.656394882360473e-05, + 8.238687587436289e-05, + 0.0003224019310437143, + 0.00028443208429962397, + 0.0002462094707880169, + 0.00020799829508177936, + 0.00018759421072900295, + 0.00017152294458355755, + 0.00013370471424423158, + 0.00010237377864541486, + 5.583176243817434e-05, + 4.913492375635542e-05, + 4.4458811316872016e-05, + 4.327554415795021e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.07808538526296616, + 0.03881331905722618, + 0.01480295043438673, + 0.02500341460108757, + 0.015180142596364021, + 0.013721407391130924, + 0.016902483999729156, + 0.03325798362493515, + 0.026172807440161705, + 0.027887089177966118, + 0.03240044787526131, + 0.03682815283536911, + 0.029301028698682785, + 0.0338764488697052, + 0.0393521711230278, + 0.046694882214069366, + 0.029503758996725082, + 0.03615153953433037, + 0.030687781050801277, + 0.05279085040092468, + 0.024021649733185768, + 0.03336174041032791, + 0.04255227372050285, + 0.0520954355597496, + 0.03984410688281059, + 0.04249191656708717, + 0.05309459939599037, + 0.06231515854597092, + 0.06388300657272339, + 0.0749494656920433, + 0.10156872868537903, + 0.11044161766767502 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.387307077038713e-05, + "first_to_last_ratio": 1.3885610977109948, + "imbalance_abs_log_ratio": 0.32826802946929257, + "last_quartile_mean": 9.990968919737497e-06, + "mean": 1.2854331032485788e-05, + "normalized": [ + 1.1519258940103256, + 1.3312041641813053, + 1.2167309231197612, + 1.1117514471382886, + 1.0587713135165344, + 0.9928224052891267, + 0.9274023828418168, + 0.8434126323121538, + 0.7336839389102785, + 0.6915061646833663, + 0.6596107083150127, + 0.6376340360074726, + 0.6952642607649582, + 0.6629197998266377, + 0.6345516401532968, + 0.6122226131721931, + 0.7648990252904823, + 0.729764382313612, + 0.7866676809835089, + 0.8716429632898921, + 2.8889885153646824, + 2.3385372141657035, + 1.899664892928511, + 1.5404582652892223, + 1.5315814746576397, + 1.1978299957950485, + 0.9082536044284508, + 0.816087598131252, + 0.48962660632970234, + 0.43639989519040423, + 0.42103372910248354, + 0.4171498324968764 + ], + "peak_layer": 21, + "peak_normalized": 2.8889885153646824, + "population_cv": 0.5419279223583527, + "rest_mean": 1.0379048506837097e-05, + "spike_contrast": 2.5263256698064365, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6220856670988724e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.4807236766500864e-05, + 1.7111738998210058e-05, + 1.5640262063243426e-05, + 1.4290821127360687e-05, + 1.3609796951641329e-05, + 1.2762067854055203e-05, + 1.1921137229364831e-05, + 1.0841505172720645e-05, + 9.4310162239708e-06, + 8.888849151844624e-06, + 8.478854397253599e-06, + 8.196358976420015e-06, + 8.937156962929294e-06, + 8.521390554960817e-06, + 8.156736839737277e-06, + 7.869712135288864e-06, + 9.832265277509578e-06, + 9.380632945976686e-06, + 1.0112086783919949e-05, + 1.120438719226513e-05, + 3.7136014725547284e-05, + 3.0060331482673064e-05, + 2.441892138449475e-05, + 1.9801560483756475e-05, + 1.9687455278472044e-05, + 1.5397303286590613e-05, + 1.1674992492771707e-05, + 1.0490260137885343e-05, + 6.293822480074596e-06, + 5.609628715319559e-06, + 5.412106929725269e-06, + 5.3621820370608475e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.7066908539127326e-05, + "first_to_last_ratio": 1.4278402189707298, + "imbalance_abs_log_ratio": 0.3561629661799943, + "last_quartile_mean": 3.99672931052919e-05, + "mean": 5.182235634038079e-05, + "normalized": [ + 1.1683197832075227, + 1.369358043457492, + 1.2477885257362629, + 1.1320828936276526, + 1.0861971638477104, + 1.0136921452638399, + 0.9398491667147963, + 0.8523323061186183, + 0.7376381293423032, + 0.6943242114792568, + 0.6618878610984286, + 0.637951161315391, + 0.6961169332186252, + 0.6619150990682322, + 0.6320240246440271, + 0.6099217459665006, + 0.753292310075495, + 0.715605419809301, + 0.7584607148457314, + 0.8619138236046655, + 2.8636215539276253, + 2.323250279031787, + 1.8821006100447468, + 1.5304640359562964, + 1.5177704400182797, + 1.1794085853576253, + 0.8974069340864187, + 0.8090474514501561, + 0.4887596394765995, + 0.4369840848318672, + 0.4222231761264232, + 0.4182917472503225 + ], + "peak_layer": 21, + "peak_normalized": 2.8636215539276253, + "population_cv": 0.5384897216371255, + "rest_mean": 4.2000662991389014e-05, + "spike_contrast": 2.4966153616773794, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010485950042493641, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.0545084124896675e-05, + 7.09633604856208e-05, + 6.466334161814302e-05, + 5.866720312042162e-05, + 5.6289296480827034e-05, + 5.2531915571307763e-05, + 4.870519842370413e-05, + 4.416986848809756e-05, + 3.822614598902874e-05, + 3.598151670303196e-05, + 3.430058859521523e-05, + 3.306013240944594e-05, + 3.607441976782866e-05, + 3.430200013099238e-05, + 3.275297422078438e-05, + 3.1607582059223205e-05, + 3.9037382521200925e-05, + 3.708435906446539e-05, + 3.930522143491544e-05, + 4.466640530154109e-05, + 0.00014839961659163237, + 0.00012039630382787436, + 9.753488848218694e-05, + 7.931225263746455e-05, + 7.865444058552384e-05, + 6.111973198130727e-05, + 4.650574192055501e-05, + 4.192674532532692e-05, + 2.5328676201752387e-05, + 2.264554495923221e-05, + 2.188059988839086e-05, + 2.1676863980246708e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05060750572010875, + "first_to_last_ratio": 1.4278402181986947, + "imbalance_abs_log_ratio": 0.356162965639293, + "last_quartile_mean": 0.03544339560903609, + "mean": 0.045956584566738456, + "normalized": [ + 1.1683197838664416, + 1.3693581481609842, + 1.2477884983122534, + 1.1320828879264566, + 1.086197129718159, + 1.0136921293230559, + 0.9398491347700728, + 0.852332271168493, + 0.7376380598562373, + 0.6943242075392044, + 0.6618878602537814, + 0.6379511317995364, + 0.6961168732850856, + 0.66191505624554, + 0.6320240673482492, + 0.6099217105646539, + 0.7532923321079824, + 0.7156054155999519, + 0.7584607053972269, + 0.8619137716807654, + 2.8636216906024767, + 2.3232503851911614, + 1.8821005956556751, + 1.530464123018414, + 1.5177704445282252, + 1.1794086151290946, + 0.8974069322276481, + 0.8090474387174796, + 0.48875963581745097, + 0.43698406072332996, + 0.42222316261516873, + 0.41829174084974413 + ], + "peak_layer": 21, + "peak_normalized": 2.8636216906024767, + "population_cv": 0.5384897545714166, + "rest_mean": 0.03724660800287017, + "spike_contrast": 2.496615477158658, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0929904580116272, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.053691986948251724, + 0.06293102353811264, + 0.05734409764409065, + 0.0520266629755497, + 0.04991791024804115, + 0.04658582806587219, + 0.04319225624203682, + 0.0391702800989151, + 0.033899325877428055, + 0.03190876916050911, + 0.03041810542345047, + 0.029318055137991905, + 0.031991153955459595, + 0.030419355258345604, + 0.029045667499303818, + 0.028029918670654297, + 0.034618742763996124, + 0.03288678079843521, + 0.03485626354813576, + 0.0396106131374836, + 0.13160227239131927, + 0.1067686527967453, + 0.08649491518735886, + 0.07033490389585495, + 0.0697515457868576, + 0.05420159175992012, + 0.041241757571697235, + 0.037181057035923004, + 0.022461723536252975, + 0.020082294940948486, + 0.019403934478759766, + 0.019223259761929512 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.187219166671639e-06, + "first_to_last_ratio": 0.8437183001333121, + "imbalance_abs_log_ratio": 0.16993660771354646, + "last_quartile_mean": 2.5923571485009234e-06, + "mean": 2.5683367930184886e-06, + "normalized": [ + 1.1919543817128102, + 0.9792836739870964, + 0.924042037649071, + 0.9164810837361373, + 0.7326981846944463, + 0.7055637920842305, + 0.6863211882787902, + 0.6765290243634249, + 0.5799994679651287, + 0.5635357621627565, + 0.5597470962238182, + 0.5639314892075872, + 0.5739189935534712, + 0.5756380159008986, + 0.5748207554502599, + 0.5781987328520684, + 0.6522355446992087, + 0.6638052489880939, + 0.6844497180630856, + 0.7655738065181559, + 2.843883820310657, + 2.5944552048335017, + 2.2989551083840256, + 2.0391579131550635, + 1.9644428765002167, + 1.5269231604355886, + 1.179167971687766, + 1.0934476508277742, + 0.6272800411331483, + 0.5710254497958045, + 0.5563528736513774, + 0.556179931194537 + ], + "peak_layer": 21, + "peak_normalized": 2.843883820310657, + "population_cv": 0.6360421157910927, + "rest_mean": 1.9271187023116874e-06, + "spike_contrast": 3.129498185867219, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.030914482835214e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0613402941526147e-06, + 2.5151302907033823e-06, + 2.3732511635898845e-06, + 2.35383208746498e-06, + 1.8818157059286023e-06, + 1.8121254470315762e-06, + 1.7627039596845862e-06, + 1.7375543848174857e-06, + 1.4896339735059883e-06, + 1.4473496321443236e-06, + 1.4376190620168927e-06, + 1.4483659924735548e-06, + 1.4740172673555207e-06, + 1.4784322956984397e-06, + 1.4763332956135855e-06, + 1.4850090792606352e-06, + 1.675160547165433e-06, + 1.7048754443749203e-06, + 1.757897393872554e-06, + 1.9662513750517974e-06, + 7.304051450773841e-06, + 6.663434760412201e-06, + 5.9044909903605e-06, + 5.2372442951309495e-06, + 5.045350917498581e-06, + 3.921652933058795e-06, + 3.028500486834673e-06, + 2.808341832860606e-06, + 1.6110664091684157e-06, + 1.4665856724604964e-06, + 1.4289015553003992e-06, + 1.4284573808254208e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.514474267125479e-05, + "first_to_last_ratio": 1.655519660200222, + "imbalance_abs_log_ratio": 0.5041149536218862, + "last_quartile_mean": 2.726922775764251e-05, + "mean": 3.928292341015549e-05, + "normalized": [ + 1.2428531532199, + 1.4224719349524337, + 1.3075943550391356, + 1.1908940314839356, + 1.1240062299860272, + 1.0512639226040563, + 0.9724687394121546, + 0.8822119975062471, + 0.7835780621436017, + 0.7378068411250925, + 0.7017901972736155, + 0.675554292822624, + 0.7400896696370771, + 0.7012250467261758, + 0.6687293301443596, + 0.6449026664130145, + 0.8086596332888067, + 0.7678950765711053, + 0.7972794784799546, + 0.884192918474659, + 2.849023696568842, + 2.282552164711599, + 1.7952274549858118, + 1.4143282104887471, + 1.3505434814026465, + 1.05691913246549, + 0.8328089954359738, + 0.7560703953177289, + 0.41728160395883906, + 0.38644818475535425, + 0.376537560284359, + 0.3767915423206315 + ], + "peak_layer": 21, + "peak_normalized": 2.849023696568842, + "population_cv": 0.5250932744673414, + "rest_mean": 3.245689711345929e-05, + "spike_contrast": 2.34598720716096, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.614346541231498e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.8822905228007585e-05, + 5.587885607383214e-05, + 5.136612890055403e-05, + 4.6781799028394744e-05, + 4.415425064507872e-05, + 4.129672015551478e-05, + 3.820141500909813e-05, + 3.4655866329558194e-05, + 3.0781237001065165e-05, + 2.898320963140577e-05, + 2.7568370569497347e-05, + 2.6537747544352897e-05, + 2.907288580900058e-05, + 2.754616980382707e-05, + 2.6269643058185466e-05, + 2.5333662051707506e-05, + 3.176651443936862e-05, + 3.016516347997822e-05, + 3.131946868961677e-05, + 3.4733682696241885e-05, + 0.0001119179796660319, + 8.966532186605036e-05, + 7.0521782618016e-05, + 5.555894676945172e-05, + 5.305329614202492e-05, + 4.151887333136983e-05, + 3.271517198299989e-05, + 2.9700655431952327e-05, + 1.639204128878191e-05, + 1.51808144437382e-05, + 1.479149614169728e-05, + 1.4801473298575729e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.469924399723823e-05, + "first_to_last_ratio": 1.7510149859622042, + "imbalance_abs_log_ratio": 0.5601956117834839, + "last_quartile_mean": 1.981664593131427e-05, + "mean": 2.9772959720730796e-05, + "normalized": [ + 1.2826990789388135, + 1.4313521308000512, + 1.325603166494873, + 1.2222410753351367, + 1.1310087328328617, + 1.056511628684211, + 0.9758416336275024, + 0.898436085327916, + 0.8214143991142518, + 0.7703245121438669, + 0.7298179894195511, + 0.7040392340287683, + 0.7681958890441348, + 0.7313866125384519, + 0.6969281018261846, + 0.6697043803055042, + 0.8568340618774071, + 0.8208415080145076, + 0.8328878615188123, + 0.8883662275011015, + 2.8312616617599446, + 2.239848895619399, + 1.6996416088520014, + 1.290076953440798, + 1.215112714243769, + 0.9719493099553681, + 0.8309033623504332, + 0.7782938507320865, + 0.3989932047584423, + 0.3786265198784562, + 0.3759890641537673, + 0.37486854488162713 + ], + "peak_layer": 21, + "peak_normalized": 2.8312616617599446, + "population_cv": 0.5072717351311633, + "rest_mean": 2.505786920260612e-05, + "spike_contrast": 2.2042755539987984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.523444851860404e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 3 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.8189748011063784e-05, + 4.261558933649212e-05, + 3.9467129681725055e-05, + 3.638973430497572e-05, + 3.367347744642757e-05, + 3.14554781652987e-05, + 2.905369365180377e-05, + 2.67491013801191e-05, + 2.445593781885691e-05, + 2.293484067195095e-05, + 2.172884160245303e-05, + 2.096133175655268e-05, + 2.287146526214201e-05, + 2.177554415538907e-05, + 2.0749612303916365e-05, + 1.9939081539632753e-05, + 2.55104860116262e-05, + 2.443888115521986e-05, + 2.4797536752885208e-05, + 2.6449291908647865e-05, + 8.429503941442817e-05, + 6.668693094979972e-05, + 5.0603361160028726e-05, + 3.840940917143598e-05, + 3.61775018973276e-05, + 2.8937807655893266e-05, + 2.473845233907923e-05, + 2.3172111468738876e-05, + 1.1879208614118397e-05, + 1.1272832125541754e-05, + 1.1194307262485381e-05, + 1.1160946087329648e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011354441994626541, + "first_to_last_ratio": 1.3378984746048481, + "imbalance_abs_log_ratio": 0.2911000803554069, + "last_quartile_mean": 8.486774004268227e-05, + "mean": 0.00010590371198304638, + "normalized": [ + 1.1416862116188895, + 1.318287506150763, + 1.2188213004978723, + 1.1129317281051134, + 1.0503469451456144, + 0.9936779473327889, + 0.9152461482494265, + 0.8261837163725879, + 0.7163781808438098, + 0.6746444347634984, + 0.649936107552919, + 0.6258996320477875, + 0.6816391399615429, + 0.6448073899118816, + 0.6202888218188445, + 0.5944257945762098, + 0.72764205396545, + 0.6903920368963828, + 0.7174952313478369, + 0.8067894001918988, + 2.8841338501609886, + 2.3452358192551364, + 1.972412467214386, + 1.6597621298267584, + 1.6194268438614279, + 1.2623302767875173, + 0.9611591980513982, + 0.8707097795471348, + 0.4633879176416832, + 0.4198531088291576, + 0.4079853661334264, + 0.40608351533986714 + ], + "peak_layer": 21, + "peak_normalized": 2.8841338501609886, + "population_cv": 0.5599416174982532, + "rest_mean": 8.440537176620974e-05, + "spike_contrast": 2.630102142892712, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022199474915396422, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012090880773030221, + 0.00013961154036223888, + 0.0001290776999667287, + 0.00011786360119003803, + 0.00011123564036097378, + 0.0001052341831382364, + 9.692796447779983e-05, + 8.749592234380543e-05, + 7.586710853502154e-05, + 7.144734991015866e-05, + 6.883064634166658e-05, + 6.62850943626836e-05, + 7.218811515485868e-05, + 6.828749610576779e-05, + 6.569088873220608e-05, + 6.295189814409241e-05, + 7.70599945099093e-05, + 7.311507943086326e-05, + 7.598540832987055e-05, + 8.544199226889759e-05, + 0.000305440480588004, + 0.00024836917873471975, + 0.00020888580183964223, + 0.00017577497055754066, + 0.00017150331404991448, + 0.00013368546206038445, + 0.0001017903268802911, + 9.221139771398157e-05, + 4.9074500566348433e-05, + 4.4464002712629735e-05, + 4.320716470829211e-05, + 4.3005751649616286e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04804428666830063, + 0.18500657379627228, + 0.22299833595752716, + 0.2331935465335846, + 0.0799286738038063, + 0.09402555972337723, + 0.10843002796173096, + 0.12457633018493652, + 0.04508088901638985, + 0.0760042667388916, + 0.10498681664466858, + 0.13159531354904175, + 0.0497574545443058, + 0.07735268771648407, + 0.10224150121212006, + 0.13362281024456024, + 0.079553984105587, + 0.10536550730466843, + 0.14260481297969818, + 0.19080305099487305, + 0.03440643846988678, + 0.060867179185152054, + 0.08364810794591904, + 0.11119203269481659, + 0.025225287303328514, + 0.05530283972620964, + 0.08037404716014862, + 0.11502843350172043, + 0.0242468249052763, + 0.07164724916219711, + 0.1371813863515854, + 0.23144805431365967 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.406055810093676e-05, + "first_to_last_ratio": 1.2121225310488972, + "imbalance_abs_log_ratio": 0.19237298076220427, + "last_quartile_mean": 1.1599947811191669e-05, + "mean": 1.3850583115981863e-05, + "normalized": [ + 1.1035219335667126, + 1.245398841062786, + 1.1319237648162956, + 1.0360361966114253, + 0.9887036854141024, + 0.9324573194178568, + 0.8818082800629979, + 0.8014300627506455, + 0.7197404552580381, + 0.6642341963531577, + 0.63144215306254, + 0.605460329509273, + 0.6612975386173722, + 0.6318610283296698, + 0.6080617688348535, + 0.5826465005104303, + 0.7161183891450303, + 0.6858808349031477, + 0.733242501237421, + 0.767094241825568, + 2.859288144745241, + 2.5015026079052487, + 2.101601950422847, + 1.7091984970083665, + 1.5703665940407991, + 1.4157861892600545, + 1.1108502479674358, + 0.8417170775086074, + 0.5124528160490821, + 0.4544107825414821, + 0.40419536018403524, + 0.39026971107747666 + ], + "peak_layer": 21, + "peak_normalized": 2.859288144745241, + "population_cv": 0.5793000123245652, + "rest_mean": 1.0905047424279877e-05, + "spike_contrast": 2.7286883489310063, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.975647585117258e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.5284422261174768e-05, + 1.7249500160687603e-05, + 1.567780418554321e-05, + 1.4349705452332273e-05, + 1.369412257190561e-05, + 1.2915077604702674e-05, + 1.2213558875373565e-05, + 1.1100273695774376e-05, + 9.968824997486081e-06, + 9.200030945066828e-06, + 8.745842023927253e-06, + 8.385978617297951e-06, + 9.15935652301414e-06, + 8.751643690629862e-06, + 8.42201006889809e-06, + 8.069993782555684e-06, + 9.918657269736286e-06, + 9.499849511485081e-06, + 1.0155836207559332e-05, + 1.062470255419612e-05, + 3.960280810133554e-05, + 3.4647269785637036e-05, + 2.9108412491041236e-05, + 2.3673395844525658e-05, + 2.1750493033323437e-05, + 1.960946428880561e-05, + 1.538592368888203e-05, + 1.1658272342174314e-05, + 7.0977703217067756e-06, + 6.293854312389158e-06, + 5.598341431323206e-06, + 5.405463070928818e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.784146014775615e-05, + "first_to_last_ratio": 1.2477003862842722, + "imbalance_abs_log_ratio": 0.22130216603186706, + "last_quartile_mean": 4.635845334632904e-05, + "mean": 5.5787147289265704e-05, + "normalized": [ + 1.1172174922096436, + 1.2820167732139756, + 1.1618030425233523, + 1.0568058922472814, + 1.015898285952758, + 0.952837653331755, + 0.8952147696466347, + 0.8127990870482149, + 0.7269196158944218, + 0.6687128912021425, + 0.6353920226237901, + 0.6079547558759868, + 0.6632448840034066, + 0.632152432145331, + 0.6072547727004206, + 0.5816761067060003, + 0.7067464190080565, + 0.6745056530323595, + 0.7095637632538165, + 0.7592902185109378, + 2.8322130517871535, + 2.4767089823758397, + 2.0818425648522476, + 1.6933243830446072, + 1.5552023255692202, + 1.402829278407864, + 1.095030881819868, + 0.8327931471930783, + 0.5108623485290222, + 0.45409986898200716, + 0.4053123002322658, + 0.3917743360765406 + ], + "peak_layer": 21, + "peak_normalized": 2.8322130517871535, + "population_cv": 0.5731345570971216, + "rest_mean": 4.41352963712739e-05, + "spike_contrast": 2.6896192391648626, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011870714224642143, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.232637679204345e-05, + 7.15200585545972e-05, + 6.481367745436728e-05, + 5.895618596696295e-05, + 5.667406730935909e-05, + 5.315609450917691e-05, + 4.994147820980288e-05, + 4.534374238573946e-05, + 4.055277167935856e-05, + 3.7305584555724636e-05, + 3.5446708352537826e-05, + 3.391606151126325e-05, + 3.7000540032749996e-05, + 3.5265980841359124e-05, + 3.387701144674793e-05, + 3.2450050639454275e-05, + 3.942736657336354e-05, + 3.762874621315859e-05, + 3.9584538171766326e-05, + 4.2358635255368426e-05, + 0.00015800108667463064, + 0.00013816852879244834, + 0.00011614005779847503, + 9.446573676541448e-05, + 8.676030120113865e-05, + 7.825984357623383e-05, + 6.108864909037948e-05, + 4.6459153963951394e-05, + 2.8499553081928752e-05, + 2.533293627493549e-05, + 2.2611216991208494e-05, + 2.1855972590856254e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.051294384989887476, + "first_to_last_ratio": 1.2477003995429623, + "imbalance_abs_log_ratio": 0.2213021766583686, + "last_quartile_mean": 0.04111113934777677, + "mean": 0.04947259998880327, + "normalized": [ + 1.117217517042371, + 1.282016692529182, + 1.1618030656626175, + 1.0568059265711638, + 1.0158982593254349, + 0.9528376876517187, + 0.8952147600149789, + 0.8127990580563501, + 0.7269196281255159, + 0.6687128990372552, + 0.6353920151588079, + 0.6079547776386781, + 0.6632449090716109, + 0.6321524554983756, + 0.6072547505273581, + 0.5816761070130537, + 0.706746437867777, + 0.674505658209711, + 0.7095637900293472, + 0.7592902251062212, + 2.8322130824094, + 2.4767089467624213, + 2.081842602529718, + 1.693324355494102, + 1.5552023386093694, + 1.4028291808614948, + 1.0950308028824434, + 0.8327931847449266, + 0.5108623388148443, + 0.45409990274101625, + 0.40531230895107145, + 0.3917743350616629 + ], + "peak_layer": 21, + "peak_normalized": 2.8322130824094, + "population_cv": 0.5731345524386317, + "rest_mean": 0.03913962206354848, + "spike_contrast": 2.689619246048363, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10527068078517914, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.055271655321121216, + 0.06342469900846481, + 0.05747741833329201, + 0.05228293687105179, + 0.05025912821292877, + 0.04713935777544975, + 0.044288601726293564, + 0.040211282670497894, + 0.035962603986263275, + 0.03308296576142311, + 0.031434495002031326, + 0.030077103525400162, + 0.03281245008111, + 0.0312742255628109, + 0.03004247136414051, + 0.028777029365301132, + 0.034964583814144135, + 0.03336954861879349, + 0.03510396555066109, + 0.03756406158208847, + 0.14011694490909576, + 0.12252923101186752, + 0.1029941663146019, + 0.0837731584906578, + 0.0769399031996727, + 0.06940160691738129, + 0.054174020886421204, + 0.04120044410228729, + 0.02527368813753128, + 0.022465502843260765, + 0.020051853731274605, + 0.019382094964385033 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.2204297493999547e-06, + "first_to_last_ratio": 0.7359606295430359, + "imbalance_abs_log_ratio": 0.3065786541527352, + "last_quartile_mean": 3.0170496359005483e-06, + "mean": 2.8223206918198684e-06, + "normalized": [ + 1.1437255119946472, + 0.8934281083094181, + 0.843667370045445, + 0.833234424409991, + 0.6691350034907739, + 0.6491575975670105, + 0.6344872974587108, + 0.6270766601877794, + 0.5480904457549984, + 0.5298089242035497, + 0.5236022556745512, + 0.5213819889179987, + 0.5319070176791533, + 0.5365159271613865, + 0.5384760973551012, + 0.5417839198031603, + 0.5976312076631317, + 0.6133914803428778, + 0.632878138794897, + 0.6938486884695898, + 2.7927450515890637, + 2.792273921126163, + 2.5604223062656137, + 2.1993623563278324, + 1.9620758229843447, + 1.8148751057591102, + 1.435693067783232, + 1.0849680187180202, + 0.6411889832806804, + 0.5785351975720235, + 0.5245373868045311, + 0.5100947165052133 + ], + "peak_layer": 21, + "peak_normalized": 2.7927450515890637, + "population_cv": 0.7010322041721825, + "rest_mean": 2.058529688568982e-06, + "spike_contrast": 3.374637804812921, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.946792109374656e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.227960178264766e-06, + 2.5215406367351534e-06, + 2.3810998754925095e-06, + 2.3516547571489355e-06, + 1.8885135659729713e-06, + 1.832130919865449e-06, + 1.7907266283145873e-06, + 1.7698114334052661e-06, + 1.5468870060431072e-06, + 1.4952906894905027e-06, + 1.477773480473843e-06, + 1.4715071756654652e-06, + 1.5012121821200708e-06, + 1.5142200027185027e-06, + 1.519752231615712e-06, + 1.5290879673557356e-06, + 1.6867069234649534e-06, + 1.7311874671577243e-06, + 1.7861850665212842e-06, + 1.958263510459801e-06, + 7.882022146077361e-06, + 7.880692464823369e-06, + 7.22633285477059e-06, + 6.207305887073744e-06, + 5.5376071941282135e-06, + 5.1221595640527084e-06, + 4.0519862523069605e-06, + 3.0621276891906746e-06, + 1.809640934880008e-06, + 1.6328118590536178e-06, + 1.4804127204115503e-06, + 1.4396508731806534e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.56924158243055e-05, + "first_to_last_ratio": 1.4673115279536288, + "imbalance_abs_log_ratio": 0.38343183377619244, + "last_quartile_mean": 3.1140228202275466e-05, + "mean": 4.183447202876778e-05, + "normalized": [ + 1.1974206825409346, + 1.3442935712461421, + 1.2304058748098903, + 1.1234622096860003, + 1.0623591343774659, + 0.9983032021931795, + 0.9335117515893174, + 0.8479976043257199, + 0.7756355664609215, + 0.7156793363778972, + 0.6795817133031917, + 0.6491928801846335, + 0.7109607744004338, + 0.6750928595197824, + 0.6474397844670364, + 0.6200735511625727, + 0.7662731422367131, + 0.7307687689442663, + 0.7545377188834692, + 0.7927345870100259, + 2.8287570565248865, + 2.421556562840323, + 1.9807276243744367, + 1.558292733686825, + 1.4117490803489046, + 1.238912861023533, + 0.9856350322605589, + 0.7768378929765343, + 0.43766980945827866, + 0.39060663115145766, + 0.36143264697135674, + 0.3520973546633113 + ], + "peak_layer": 21, + "peak_normalized": 2.8287570565248865, + "population_cv": 0.551494708887264, + "rest_mean": 3.377578448433067e-05, + "spike_contrast": 2.5269993301954115, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.535138476872817e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.009346205042675e-05, + 5.623781180474907e-05, + 5.14733801537659e-05, + 4.699944838648662e-05, + 4.444323349162005e-05, + 4.176348738837987e-05, + 3.905297126038931e-05, + 3.547553205862641e-05, + 3.244830440962687e-05, + 2.9940067179268226e-05, + 2.8429942176444456e-05, + 2.715864138735924e-05, + 2.9742668630206026e-05, + 2.824215334840119e-05, + 2.7085301553597674e-05, + 2.594044963188935e-05, + 3.205663233529776e-05, + 3.057132562389597e-05, + 3.156568709528074e-05, + 3.31636329065077e-05, + 0.00011833955795736983, + 0.00010130454029422253, + 8.286269439850003e-05, + 6.519035378005356e-05, + 5.9059777413494885e-05, + 5.182926543056965e-05, + 4.123352118767798e-05, + 3.249860310461372e-05, + 1.830968540161848e-05, + 1.6340822185156867e-05, + 1.5120343960006721e-05, + 1.4729806935065426e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.505123777358676e-05, + "first_to_last_ratio": 1.5889493494930538, + "imbalance_abs_log_ratio": 0.4630730113341737, + "last_quartile_mean": 2.2059380171413068e-05, + "mean": 3.131689976498819e-05, + "normalized": [ + 1.2479593211749789, + 1.367487352044708, + 1.261680923885286, + 1.1674855830446973, + 1.0794756507539984, + 1.0156514953231304, + 0.945160188314607, + 0.8690478102601445, + 0.8149681766862981, + 0.7530914967772463, + 0.7148635553828698, + 0.6827279730652124, + 0.7417389963838543, + 0.7102014979770842, + 0.6794125192711595, + 0.6486601330546842, + 0.8192809210261347, + 0.7895373258521271, + 0.7975483511256357, + 0.812938977938045, + 2.826829287271584, + 2.3623544295272914, + 1.8510256189591003, + 1.4057347513501408, + 1.2902085448265628, + 1.123507683724877, + 0.9230685473509727, + 0.7887132397564032, + 0.4159196155359273, + 0.3769758263062851, + 0.35976736593467973, + 0.35697684011427405 + ], + "peak_layer": 21, + "peak_normalized": 2.826829287271584, + "population_cv": 0.5277213122295441, + "rest_mean": 2.5823506533655477e-05, + "spike_contrast": 2.3614617610009216, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.0981223214184865e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 2 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.908221697201952e-05, + 4.282546433387324e-05, + 3.9511935028713197e-05, + 3.656202898127958e-05, + 3.380583075340837e-05, + 3.180705607519485e-05, + 2.9599486879305914e-05, + 2.7215883164899424e-05, + 2.5522276700939983e-05, + 2.358449091843795e-05, + 2.238731030956842e-05, + 2.1380923499236815e-05, + 2.3228965801536106e-05, + 2.2241309125092812e-05, + 2.1277093765093014e-05, + 2.031402436841745e-05, + 2.5657338483142667e-05, + 2.4725861294427887e-05, + 2.4976741769933142e-05, + 2.5458728487137705e-05, + 8.85275294422172e-05, + 7.398161687888205e-05, + 5.796838377136737e-05, + 4.402325430419296e-05, + 4.0405331674264744e-05, + 3.518477751640603e-05, + 2.890764517360367e-05, + 2.4700053472770378e-05, + 1.3025312910031062e-05, + 1.180571416625753e-05, + 1.1266798537690192e-05, + 1.1179407920280937e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011513973367982544, + "first_to_last_ratio": 1.1689761361783517, + "imbalance_abs_log_ratio": 0.15612826840504843, + "last_quartile_mean": 9.849622256297152e-05, + "mean": 0.00011409382295823889, + "normalized": [ + 1.087599643787199, + 1.2332404432021893, + 1.1325955558216252, + 1.0348100003758545, + 0.9846631201892967, + 0.9329597002283465, + 0.8753802278299959, + 0.7920881953113688, + 0.705656362899583, + 0.6479209049109275, + 0.6172280934926675, + 0.5959629720755263, + 0.6485619380890737, + 0.616624813163851, + 0.5952030046676504, + 0.5686106501058988, + 0.6801552059812119, + 0.6508138445342289, + 0.6710903132095474, + 0.7219939084470286, + 2.8256801162653837, + 2.4931864357424143, + 2.15835309890828, + 1.8232898013061003, + 1.6443057108717645, + 1.5033517491402877, + 1.1720956223235564, + 0.8974197290603315, + 0.48941489739710603, + 0.4307568860388074, + 0.3896112863482277, + 0.3793757682746689 + ], + "peak_layer": 21, + "peak_normalized": 2.8256801162653837, + "population_cv": 0.5949861940504896, + "rest_mean": 8.897283448647984e-05, + "spike_contrast": 2.80700465650208, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00024974716070573776, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012408840120770037, + 0.00014070511679165065, + 0.00012922215682920069, + 0.00011806542897829786, + 0.00011234397970838472, + 0.0001064449388650246, + 9.987547673517838e-05, + 9.037237032316625e-05, + 8.05110321380198e-05, + 7.392377301584929e-05, + 7.042191282380372e-05, + 6.799569382565096e-05, + 7.399691094178706e-05, + 7.035308226477355e-05, + 6.790898623876274e-05, + 6.487496284535155e-05, + 7.76015076553449e-05, + 7.425383955705911e-05, + 7.656725938431919e-05, + 8.237504516728222e-05, + 0.0003223926469217986, + 0.00028445717180147767, + 0.0002462547563482076, + 0.00020802610379178077, + 0.00018760512466542423, + 0.00017152314831037074, + 0.00013372887042351067, + 0.00010239004768664017, + 5.583921665675007e-05, + 4.9146699893753976e-05, + 4.445224112714641e-05, + 4.328443174017593e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.09451264888048172, + 0.2031717449426651, + 0.22856928408145905, + 0.2430499792098999, + 0.08243431895971298, + 0.09709372371435165, + 0.11282631754875183, + 0.13645486533641815, + 0.05442482605576515, + 0.0885990634560585, + 0.12215005606412888, + 0.15440259873867035, + 0.060873307287693024, + 0.09471380710601807, + 0.1260664463043213, + 0.163865864276886, + 0.08853588253259659, + 0.12131853401660919, + 0.15660560131072998, + 0.2097783088684082, + 0.04455159604549408, + 0.07411240041255951, + 0.10008715838193893, + 0.13061410188674927, + 0.04960336163640022, + 0.07874798029661179, + 0.11358420550823212, + 0.1580832600593567, + 0.06955241411924362, + 0.13302887976169586, + 0.22457914054393768, + 0.32859006524086 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3873141369913355e-05, + "first_to_last_ratio": 1.3885721694739201, + "imbalance_abs_log_ratio": 0.32827600298864523, + "last_quartile_mean": 9.990940100124135e-06, + "mean": 1.2854364840109156e-05, + "normalized": [ + 1.1519312840854947, + 1.3311758992301272, + 1.2167315437597699, + 1.1117686172453958, + 1.058772632613286, + 0.9928324590403058, + 0.927417632168206, + 0.8434123244463263, + 0.7336779055664342, + 0.6915156665898262, + 0.6596055773251193, + 0.6376390806056413, + 0.6952626444464219, + 0.6629130327970861, + 0.6345504665270513, + 0.6122283613875565, + 0.7648834288449639, + 0.7297709534484661, + 0.7866551404509706, + 0.8716621799696268, + 2.8889843133608784, + 2.338532478770988, + 1.899676736110628, + 1.5404711947102903, + 1.5315734843108408, + 1.1978159493624418, + 0.9082426544704836, + 0.8160818433380076, + 0.4896280426071342, + 0.4364018959793389, + 0.4210420673902388, + 0.4171425090406535 + ], + "peak_layer": 21, + "peak_normalized": 2.8889843133608784, + "population_cv": 0.5419273520370343, + "rest_mean": 1.037907206948687e-05, + "spike_contrast": 2.5263285220415503, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6220945801469498e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.4807344996370375e-05, + 1.7111420675064437e-05, + 1.5640311175957322e-05, + 1.429107942385599e-05, + 1.3609849702334031e-05, + 1.2762230653606821e-05, + 1.1921364603040274e-05, + 1.0841529729077592e-05, + 9.430963473278098e-06, + 8.888994670996908e-06, + 8.478810741507914e-06, + 8.196445378416684e-06, + 8.937159691413399e-06, + 8.521325980836991e-06, + 8.15674320620019e-06, + 7.869806722737849e-06, + 9.832090654526837e-06, + 9.380742085340898e-06, + 1.0111952178704087e-05, + 1.120466367865447e-05, + 3.713605838129297e-05, + 3.00603496725671e-05, + 2.4419137844233774e-05, + 1.98017787624849e-05, + 1.9687404346768744e-05, + 1.539716322440654e-05, + 1.1674882443912793e-05, + 1.0490213753655553e-05, + 6.293857495620614e-06, + 5.609669187833788e-06, + 5.4122283472679555e-06, + 5.3621020015270915e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.706737192667788e-05, + "first_to_last_ratio": 1.4278557447018398, + "imbalance_abs_log_ratio": 0.3561738396982594, + "last_quartile_mean": 3.996718305643299e-05, + "mean": 5.182250379220932e-05, + "normalized": [ + 1.1683241108409874, + 1.3693593420455452, + 1.2478029467661282, + 1.1320859203517522, + 1.0862154844943164, + 1.0136862423485675, + 0.9398593392770029, + 0.8523331101914637, + 0.7376332926932397, + 0.6943335382230071, + 0.6618887156422048, + 0.6379567874173152, + 0.6961101086882286, + 0.6619092142621851, + 0.6320190672968202, + 0.6099299790483219, + 0.7532799174046017, + 0.7156061917099562, + 0.758449009476584, + 0.8619189528616946, + 2.863607509132394, + 2.3232710469251585, + 1.8820910428129736, + 1.530471615417879, + 1.517753064129064, + 1.1793980690808934, + 0.8973983434078874, + 0.8090471852696426, + 0.4887601793176453, + 0.43698385938136236, + 0.4222309955613731, + 0.41828581852380436 + ], + "peak_layer": 21, + "peak_normalized": 2.863607509132394, + "population_cv": 0.5384894543542103, + "rest_mean": 4.200080675892187e-05, + "spike_contrast": 2.4966108002124767, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010485966777196154, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.054548066458665e-05, + 7.096362969605252e-05, + 6.466427294071764e-05, + 5.866752690053545e-05, + 5.6290406064363196e-05, + 5.253175913821906e-05, + 4.870586417382583e-05, + 4.417003583512269e-05, + 3.822600410785526e-05, + 3.59821024176199e-05, + 3.430073047638871e-05, + 3.306051803519949e-05, + 3.607416874729097e-05, + 3.430179276620038e-05, + 3.275281051173806e-05, + 3.160809865221381e-05, + 3.903685137629509e-05, + 3.708450458361767e-05, + 3.9304726669797674e-05, + 4.4666798203252256e-05, + 0.00014839931100141257, + 0.00012039772263960913, + 9.753467020345852e-05, + 7.931287109386176e-05, + 7.865376392146572e-05, + 6.111936090746894e-05, + 4.65054290543776e-05, + 4.1926850826712325e-05, + 2.5328776246169582e-05, + 2.2645597709924914e-05, + 2.1881067368667573e-05, + 2.167661841667723e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05060791550204158, + "first_to_last_ratio": 1.4278557661609772, + "imbalance_abs_log_ratio": 0.3561738547271846, + "last_quartile_mean": 0.035443296656012535, + "mean": 0.045956714369822294, + "normalized": [ + 1.1683241037091443, + 1.3693593062308924, + 1.2478029694871473, + 1.1320859320845278, + 1.0862154618577222, + 1.013686266934737, + 0.9398592878142475, + 0.8523331062206766, + 0.7376333014160883, + 0.6943335139035525, + 0.6618887468442457, + 0.6379567875276879, + 0.6961101245379498, + 0.6619092147086973, + 0.632019080315496, + 0.6099299583472245, + 0.75327990973307, + 0.7156062315279661, + 0.7584489979720741, + 0.861918956956209, + 2.863607441781645, + 2.3232710596995285, + 1.8820912266878924, + 1.5304716351616434, + 1.5177530257827039, + 1.179397988455519, + 0.8973983179704046, + 0.8090471801133733, + 0.48876020079385396, + 0.43698383973821214, + 0.4222310083457578, + 0.41828581734010906 + ], + "peak_layer": 21, + "peak_normalized": 2.863607441781645, + "population_cv": 0.538489454771283, + "rest_mean": 0.037246735076661465, + "spike_contrast": 2.496610840158122, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09299060255289078, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.05369233712553978, + 0.06293125450611115, + 0.05734492465853691, + 0.05202694982290268, + 0.04991889372467995, + 0.04658569023013115, + 0.043192844837903976, + 0.03917042911052704, + 0.033899202942848206, + 0.031909286975860596, + 0.030418232083320618, + 0.029318397864699364, + 0.031990934163331985, + 0.03041917271912098, + 0.02904552035033703, + 0.028030376881361008, + 0.03461826965212822, + 0.03288691118359566, + 0.03485582396388054, + 0.03961096331477165, + 0.1316019892692566, + 0.10676990449428558, + 0.08649472892284393, + 0.07033544778823853, + 0.06975094228982925, + 0.05420125648379326, + 0.04124147817492485, + 0.037181150168180466, + 0.02246181294322014, + 0.020082341507077217, + 0.019404349848628044, + 0.019223041832447052 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.187229483752162e-06, + "first_to_last_ratio": 0.843721752674449, + "imbalance_abs_log_ratio": 0.16993251566757275, + "last_quartile_mean": 2.592358768538361e-06, + "mean": 2.5683487088201673e-06, + "normalized": [ + 1.191975321874382, + 0.9792375219295126, + 0.9240659028379828, + 0.9165026822465744, + 0.7327411746199551, + 0.7055071798348216, + 0.68634443005073, + 0.6764996809984228, + 0.5799709708281261, + 0.5635565193402196, + 0.5596940376847175, + 0.5639570693866405, + 0.5739412518179093, + 0.5756289711426894, + 0.5747974170292999, + 0.5782423953100819, + 0.652186394989175, + 0.6637611802901628, + 0.6844201166273622, + 0.7655503355998897, + 2.843845838013164, + 2.5945154076809365, + 2.2989812705463573, + 2.039289390897498, + 1.9644018920207635, + 1.5269105875005642, + 1.1791879088220734, + 1.0934639133198125, + 0.6272666844428548, + 0.5710249694984988, + 0.5563916798315591, + 0.5561399029872623 + ], + "peak_layer": 21, + "peak_normalized": 2.843845838013164, + "population_cv": 0.6360528130844023, + "rest_mean": 1.9271144327393374e-06, + "spike_contrast": 3.129556655898081, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.031013799656648e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.061408278881572e-06, + 2.515023425075924e-06, + 2.3733234684186755e-06, + 2.3538984805782093e-06, + 1.8819348497345345e-06, + 1.8119884543921216e-06, + 1.762771830726706e-06, + 1.7374870822095545e-06, + 1.4895676940795965e-06, + 1.4474096587946406e-06, + 1.43748945902189e-06, + 1.4484384109891835e-06, + 1.474081273045158e-06, + 1.4784159247938078e-06, + 1.4762802038603695e-06, + 1.4851281093797297e-06, + 1.675042085480527e-06, + 1.7047701703631901e-06, + 1.7578295228304341e-06, + 1.9662002159748226e-06, + 7.303987786144717e-06, + 6.663620297331363e-06, + 5.904585577809485e-06, + 5.237606274022255e-06, + 5.0452690629754215e-06, + 3.921638835890917e-06, + 3.0285657430795254e-06, + 2.8083966299163876e-06, + 1.6110395790747134e-06, + 1.4665912431155448e-06, + 1.429007852493669e-06, + 1.4283612017607084e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.514497459240374e-05, + "first_to_last_ratio": 1.6555269986325893, + "imbalance_abs_log_ratio": 0.5041193863183254, + "last_quartile_mean": 2.7269246970718086e-05, + "mean": 3.9283020413449776e-05, + "normalized": [ + 1.2428521215956514, + 1.4224732380710257, + 1.3076049249463846, + 1.1908965547141857, + 1.1240104927458778, + 1.0512648458306981, + 0.9724745802930264, + 0.8822121342574466, + 0.7835766828764287, + 0.7378119649346744, + 0.7017902701963611, + 0.6755581812134592, + 0.7400830264097682, + 0.701223315160251, + 0.6687269842506862, + 0.6449060748365787, + 0.8086534690047941, + 0.7678965143140377, + 0.7972719531557511, + 0.8841907350981015, + 2.849012586533336, + 2.2825637536620915, + 1.7952254297971346, + 1.4143290706697607, + 1.3505415355877917, + 1.056910410344958, + 0.832807309381792, + 0.7560735292299187, + 0.4172799715855694, + 0.3864473693959841, + 0.37654214074567177, + 0.3767888291608037 + ], + "peak_layer": 21, + "peak_normalized": 2.849012586533336, + "population_cv": 0.5250931691969987, + "rest_mean": 3.245698109013675e-05, + "spike_contrast": 2.345986293299445, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.61436327593401e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.882298526354134e-05, + 5.587904524873011e-05, + 5.136667095939629e-05, + 4.678201366914436e-05, + 4.415452713146806e-05, + 4.1296858398709446e-05, + 3.820173878921196e-05, + 3.465595727902837e-05, + 3.078125882893801e-05, + 2.8983482479816303e-05, + 2.7568441510084085e-05, + 2.6537965823081322e-05, + 2.9072696634102613e-05, + 2.754616980382707e-05, + 2.6269615773344412e-05, + 2.533385850256309e-05, + 3.17663507303223e-05, + 3.0165294447215274e-05, + 3.1319250410888344e-05, + 3.4733682696241885e-05, + 0.00011191781959496439, + 8.966599853010848e-05, + 7.052187720546499e-05, + 5.5559117754455656e-05, + 5.3053350711707026e-05, + 4.1518633224768564e-05, + 3.271518653491512e-05, + 2.970085188280791e-05, + 1.6392017641919665e-05, + 1.518081990070641e-05, + 1.4791712601436302e-05, + 1.4801403267483693e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.469784724074998e-05, + "first_to_last_ratio": 1.7508907324210148, + "imbalance_abs_log_ratio": 0.5601246483988392, + "last_quartile_mean": 1.9817254496956593e-05, + "mean": 2.9772389552817913e-05, + "normalized": [ + 1.2825657703659485, + 1.4312331552120288, + 1.325820640442136, + 1.2221823686225868, + 1.130955243942597, + 1.0564145565086833, + 0.9757410003641811, + 0.8985840377367545, + 0.8214467481888739, + 0.7702253806339946, + 0.7297568784639149, + 0.7040532057825352, + 0.7682184821342395, + 0.7314383768963997, + 0.6969099228187515, + 0.6697241707472397, + 0.8568233441617242, + 0.8208320560978474, + 0.8328912261837936, + 0.8882556709751984, + 2.8316971252422305, + 2.2395765326860197, + 1.6996117178665944, + 1.2900403186629539, + 1.2152436368088633, + 0.9720044593739222, + 0.8310137912106149, + 0.7782207156702814, + 0.3990111711474371, + 0.3786791961781316, + 0.37596186733999515, + 0.3748672315335271 + ], + "peak_layer": 21, + "peak_normalized": 2.8316971252422305, + "population_cv": 0.507296052444767, + "rest_mean": 2.505713847414073e-05, + "spike_contrast": 2.204351683440534, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5234745377674696e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 3 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.818504774244502e-05, + 4.2611231037881225e-05, + 3.94728485844098e-05, + 3.638728958321735e-05, + 3.367124008946121e-05, + 3.145198570564389e-05, + 2.9050141165498644e-05, + 2.675299401744269e-05, + 2.4456432583974674e-05, + 2.2931450075702742e-05, + 2.1726606064476073e-05, + 2.096134630846791e-05, + 2.2871699911775067e-05, + 2.1776668290840462e-05, + 2.0748673705384135e-05, + 1.9939288904424757e-05, + 2.5509678380331025e-05, + 2.44381317315856e-05, + 2.4797162041068077e-05, + 2.644549385877326e-05, + 8.43063899083063e-05, + 6.667754496447742e-05, + 5.06015021528583e-05, + 3.840758290607482e-05, + 3.618070695665665e-05, + 2.8938895411556587e-05, + 2.4741266315686516e-05, + 2.3169490305008367e-05, + 1.1879516023327596e-05, + 1.127418454416329e-05, + 1.1193283171451185e-05, + 1.1160693247802556e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011355107744748238, + "first_to_last_ratio": 1.3380154347456796, + "imbalance_abs_log_ratio": 0.2911874973238657, + "last_quartile_mean": 8.486529713991331e-05, + "mean": 0.00010590487272565952, + "normalized": [ + 1.14177249304275, + 1.318239805267508, + 1.2190343862407116, + 1.1130801571737208, + 1.0501716457006294, + 0.9937227743286174, + 0.9154089043884999, + 0.8261602336216923, + 0.7161833889440711, + 0.6746260480574641, + 0.6498212581625004, + 0.6259761084931398, + 0.6817894792861978, + 0.6448396206511546, + 0.6202148320012568, + 0.5943358056681826, + 0.7276123000587434, + 0.6905927768196939, + 0.7174064355718048, + 0.8068892453627287, + 2.8834514867746446, + 2.3459559521323445, + 1.9722659474769593, + 1.659767709595541, + 1.6192051847503666, + 1.2622818151535096, + 0.9611298389601171, + 0.8706981752777411, + 0.4632426508072287, + 0.41992689699434765, + 0.4080288833982593, + 0.4061677598378732 + ], + "peak_layer": 21, + "peak_normalized": 2.8834514867746446, + "population_cv": 0.5599097168924846, + "rest_mean": 8.44075709917686e-05, + "spike_contrast": 2.6299809303873802, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022199030208867044, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012091927055735141, + 0.00013960801879875362, + 0.00012910168152302504, + 0.00011788061237893999, + 0.00011121829447802156, + 0.0001052400839398615, + 9.69462635111995e-05, + 8.749439439270645e-05, + 7.584731065435335e-05, + 7.14461857569404e-05, + 6.881923764012754e-05, + 6.62939200992696e-05, + 7.220482802949846e-05, + 6.82916579535231e-05, + 6.56837728456594e-05, + 6.294305785559118e-05, + 7.70576880313456e-05, + 7.313714013434947e-05, + 7.597683725180104e-05, + 8.545350283384323e-05, + 0.00030537156271748245, + 0.00024844816653057933, + 0.00020887257414869964, + 0.00017577748803887516, + 0.00017148171900771558, + 0.0001336817949777469, + 0.00010178833326790482, + 9.221117943525314e-05, + 4.905965397483669e-05, + 4.447230458026752e-05, + 4.321224696468562e-05, + 4.30151449108962e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05595625191926956, + 0.0632273480296135, + 0.11160287261009216, + 0.12218338996171951, + 0.12365243583917618, + 0.0926642045378685, + 0.093480683863163, + 0.08749273419380188, + 0.09927791357040405, + 0.07820390164852142, + 0.07669360190629959, + 0.07840896397829056, + 0.08920575678348541, + 0.06850729882717133, + 0.07440610229969025, + 0.07867594808340073, + 0.09058429300785065, + 0.07234266400337219, + 0.07771008461713791, + 0.08416424691677094, + 0.08947562426328659, + 0.056279782205820084, + 0.057985786348581314, + 0.06238175556063652, + 0.06515856087207794, + 0.05041429027915001, + 0.05083387345075607, + 0.05600840970873833, + 0.06128782033920288, + 0.056301049888134, + 0.06424970924854279, + 0.06844019144773483 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 9.178191703540506e-06, + "first_to_last_ratio": 2.0927096232778837, + "imbalance_abs_log_ratio": 0.7384596967524485, + "last_quartile_mean": 4.385793232586366e-06, + "mean": 9.837752486419049e-06, + "normalized": [ + 1.3396520059806467, + 1.2909822759863991, + 0.8248929730174986, + 0.8734689592802515, + 0.4594300774035242, + 0.8015983028871521, + 0.902358022333612, + 0.9712665986901348, + 0.6517279620702138, + 0.5032471363266136, + 0.3886238838550298, + 0.3309759496712648, + 0.3194854781922982, + 0.5459377912807375, + 0.43221827830345944, + 0.4611789434475917, + 1.0400100603479894, + 1.0312967006898324, + 1.4985429159426815, + 1.6347346226131838, + 2.6918571666331577, + 4.297443405515621, + 2.6798531616309957, + 2.4627171634441143, + 1.010352002858167, + 0.8524077751550129, + 0.3921416774715023, + 0.37035603642720544, + 0.29698661283582345, + 0.19544726726737693, + 0.19990393116510585, + 0.24890486127580272 + ], + "peak_layer": 22, + "peak_normalized": 4.297443405515621, + "population_cv": 0.8934604862364398, + "rest_mean": 6.871042353816813e-06, + "spike_contrast": 3.763328163463758, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5857987202471124e-05, + "top_five_layers": [ + 22, + 21, + 23, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.3179164852772374e-05, + 1.2700364095508121e-05, + 8.115092896332499e-06, + 8.592971425969154e-06, + 4.5197593863122165e-06, + 7.88592569733737e-06, + 8.877174877852667e-06, + 9.555080396239646e-06, + 6.411538379325066e-06, + 4.950820766680408e-06, + 3.823185579676647e-06, + 3.2560594718233915e-06, + 3.1430190574610606e-06, + 5.370800863602199e-06, + 4.252056442055618e-06, + 4.536964297585655e-06, + 1.0231361557089258e-05, + 1.014564168144716e-05, + 1.4742294297320768e-05, + 1.6082114598248154e-05, + 2.6481824534130283e-05, + 4.227718454785645e-05, + 2.636373210407328e-05, + 2.4227601898019202e-05, + 9.939592928276397e-06, + 8.385776709474158e-06, + 3.8577927625738084e-06, + 3.6434710182220442e-06, + 2.9216807888587937e-06, + 1.922761839523446e-06, + 1.9666053958644625e-06, + 2.4486644178978167e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.4749664109767764e-05, + "first_to_last_ratio": 2.002663558914822, + "imbalance_abs_log_ratio": 0.6944780739856734, + "last_quartile_mean": 1.735172338612756e-05, + "mean": 3.858046457594355e-05, + "normalized": [ + 0.8375866313273315, + 1.2899140602905792, + 0.8694344042805302, + 0.9158393863926719, + 0.4754536346534661, + 0.8638099380048725, + 0.9821647500595908, + 0.9714468918970042, + 0.6838803886511606, + 0.5360010655021746, + 0.3851151654093992, + 0.3481231986646324, + 0.3212694297422333, + 0.5688836414280135, + 0.4682021401391152, + 0.46212538368136635, + 1.0036599670842112, + 1.0844213697059686, + 1.4666068093127032, + 1.6452318694807484, + 2.683098703810988, + 4.385873690297744, + 2.767792621757238, + 2.3860317964922024, + 1.0350167413971205, + 0.871092691603006, + 0.39147360723073643, + 0.341414306210179, + 0.30371057059916196, + 0.20069739154416044, + 0.20302718339269, + 0.25160056995699964 + ], + "peak_layer": 22, + "peak_normalized": 4.385873690297744, + "population_cv": 0.8995046127848129, + "rest_mean": 2.6780824454261972e-05, + "spike_contrast": 3.8198421190406635, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010229852123302407, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.2314481359208e-05, + 4.9765483709052205e-05, + 3.354318323545158e-05, + 3.533350900397636e-05, + 1.8343222109251656e-05, + 3.332618871354498e-05, + 3.7892372347414494e-05, + 3.7478872400242835e-05, + 2.6384423108538613e-05, + 2.0679170120274648e-05, + 1.4857921996735968e-05, + 1.3430754734145012e-05, + 1.239472385350382e-05, + 2.194779517594725e-05, + 1.8063456082018092e-05, + 1.7829011994763277e-05, + 3.8721667806385085e-05, + 4.183748023933731e-05, + 5.658237205352634e-05, + 6.34738098597154e-05, + 0.00010351519449613988, + 0.00016920904454309493, + 0.00010678272519726306, + 9.205421520164236e-05, + 3.9931426726980135e-05, + 3.3607160730753094e-05, + 1.5103233636182267e-05, + 1.3171922546462156e-05, + 1.171729491034057e-05, + 7.742998604953755e-06, + 7.832883056835271e-06, + 9.706866876513232e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.030816350365057588, + "first_to_last_ratio": 2.002663649499559, + "imbalance_abs_log_ratio": 0.6944781192178017, + "last_quartile_mean": 0.015387681487482041, + "mean": 0.03421354205056559, + "normalized": [ + 0.8375866167785538, + 1.2899141113192616, + 0.8694344424609419, + 0.915839393595925, + 0.475453642772802, + 0.8638099562451347, + 0.9821647993180688, + 0.9714468517291538, + 0.6838804035566444, + 0.5360010365403223, + 0.3851151782472681, + 0.3481232059212625, + 0.321269430974011, + 0.5688836365906816, + 0.46820216481780275, + 0.4621253765317527, + 1.0036599271213709, + 1.0844213253993487, + 1.4666067752392928, + 1.645231903565359, + 2.6830987807805085, + 4.385873655623207, + 2.767792506758732, + 2.3860319203463485, + 1.0350167426483041, + 0.8710926293236593, + 0.3914735937130686, + 0.34141432168112545, + 0.3037105506770517, + 0.20069737952502648, + 0.20302717344287544, + 0.25160056675513426 + ], + "peak_layer": 22, + "peak_normalized": 4.385873655623207, + "population_cv": 0.8995046133337352, + "rest_mean": 0.023749503048343792, + "spike_contrast": 3.8198421448186797, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09071935266256333, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.02865680493414402, + 0.0441325306892395, + 0.02974643185734749, + 0.03133410960435867, + 0.016266953200101852, + 0.02955399826169014, + 0.03360333666205406, + 0.03323663771152496, + 0.02339797094464302, + 0.01833849400281906, + 0.013176154345273972, + 0.01191052794456482, + 0.010991765186190605, + 0.019463524222373962, + 0.016018854454159737, + 0.015810946002602577, + 0.03433876112103462, + 0.03710189461708069, + 0.050177812576293945, + 0.05628921091556549, + 0.0917983129620552, + 0.15005627274513245, + 0.09469598531723022, + 0.08163460344076157, + 0.035411588847637177, + 0.029803164303302765, + 0.013393698260188103, + 0.011680993251502514, + 0.010391013696789742, + 0.006866568233817816, + 0.006946278735995293, + 0.008608146570622921 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.8313342426145027e-06, + "first_to_last_ratio": 0.9841100436889928, + "imbalance_abs_log_ratio": 0.01601755516546022, + "last_quartile_mean": 2.87705044854647e-06, + "mean": 2.8823285838086576e-06, + "normalized": [ + 3.860850182514526, + 1.3779145113400497, + 0.37073650468044217, + 0.4054583128312923, + 0.3411145989186331, + 0.46814184382330143, + 0.46288850930954656, + 0.5713590261909561, + 0.3765635727895922, + 0.3810005221832118, + 0.35155981621084625, + 0.393783350864445, + 0.33878649322165827, + 0.47478509983905887, + 0.4584607500663275, + 0.5226177027680562, + 0.4037194811557118, + 0.6360221076907124, + 0.6562906885789336, + 0.9724019253458765, + 1.6772106521914107, + 3.0942458588650434, + 2.9339099985078207, + 2.4848281321882983, + 1.7824259625811416, + 1.6267190981510171, + 1.2975773719544916, + 1.1633314414023308, + 0.5790326834473365, + 0.5326514143129327, + 0.49977516979896786, + 0.503837216276031 + ], + "peak_layer": 1, + "peak_normalized": 3.860850182514526, + "population_cv": 0.9094699254837627, + "rest_mean": 2.1379810404040554e-06, + "spike_contrast": 3.2281882709722924, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.90180531819351e-06, + "top_five_layers": [ + 1, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.112823883886449e-05, + 3.971602382080164e-06, + 1.0685844245017506e-06, + 1.1686640846164664e-06, + 9.83204358817602e-07, + 1.3493386177287903e-06, + 1.334196781499486e-06, + 1.6468444528072723e-06, + 1.0853799494725536e-06, + 1.0981686955346959e-06, + 1.0133109071830404e-06, + 1.1350130080245435e-06, + 9.764939932210837e-07, + 1.3684866644325666e-06, + 1.3214345244705328e-06, + 1.5063559430927853e-06, + 1.163652200375509e-06, + 1.8332247009311686e-06, + 1.8916454109785263e-06, + 2.8027818643749924e-06, + 4.834272203879664e-06, + 8.918633284338284e-06, + 8.456492651021108e-06, + 7.16209115125821e-06, + 5.137537300470285e-06, + 4.688738954428118e-06, + 3.74004434888775e-06, + 3.3531034659972647e-06, + 1.6689624544596882e-06, + 1.535276396680274e-06, + 1.4405162573893904e-06, + 1.4522244100589887e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.3673139480706595e-05, + "first_to_last_ratio": 2.257300425332895, + "imbalance_abs_log_ratio": 0.8141695972328834, + "last_quartile_mean": 1.0487367660516611e-05, + "mean": 2.6041475749138954e-05, + "normalized": [ + 0.6771788519354571, + 1.2979325532283608, + 0.9303271057055151, + 0.9639489120456851, + 0.5271438854731791, + 0.9092278846604233, + 1.028243959121235, + 0.9384386122687313, + 0.6892881365249813, + 0.5698927083153338, + 0.44101071224628413, + 0.3889922048793135, + 0.37294517808508393, + 0.6424365166247796, + 0.5244231346818359, + 0.4778644993700143, + 1.0150499086102556, + 1.1538261154181413, + 1.4466435534537925, + 1.6137729430394836, + 2.6468294071482537, + 4.220741079232897, + 2.9670008977788065, + 2.335098275492907, + 0.997666272782542, + 0.8091786594449493, + 0.3691954363618722, + 0.3097208222153953, + 0.23815117281925108, + 0.1573714206968098, + 0.14298175278225358, + 0.19747742755617637 + ], + "peak_layer": 22, + "peak_normalized": 4.220741079232897, + "population_cv": 0.8898631795325547, + "rest_mean": 1.816408757804311e-05, + "spike_contrast": 3.7755473033479423, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.857937187305651e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.7634736650506966e-05, + 3.380007910891436e-05, + 2.4227090761996806e-05, + 2.5102652216446586e-05, + 1.3727604709856678e-05, + 2.3677635908825323e-05, + 2.6776990125654265e-05, + 2.443832636345178e-05, + 1.7950080291484483e-05, + 1.4840847143204883e-05, + 1.1484569768072106e-05, + 1.0129931069968734e-05, + 9.712042810861021e-06, + 1.6729994968045503e-05, + 1.3656752344104461e-05, + 1.2444296771718655e-05, + 2.643339757923968e-05, + 3.004733480338473e-05, + 3.767273301491514e-05, + 4.202502896077931e-05, + 6.892734381835908e-05, + 0.00010991432645823807, + 7.726508192718029e-05, + 6.080940511310473e-05, + 2.5980702048400417e-05, + 2.1072206436656415e-05, + 9.614394002710469e-06, + 8.065587280725595e-06, + 6.201807991601527e-06, + 4.098184035683516e-06, + 3.723455847648438e-06, + 5.142603640706511e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.5893040995251795e-05, + "first_to_last_ratio": 2.4728739688838175, + "imbalance_abs_log_ratio": 0.9053810243954281, + "last_quartile_mean": 6.426951472349174e-06, + "mean": 1.7339248245207273e-05, + "normalized": [ + 0.5512149265120582, + 1.2985625489006958, + 0.9783452688543264, + 0.9971647574760376, + 0.5962146785243507, + 0.9290000522923118, + 1.0861697535899135, + 0.8960754411880637, + 0.726825196328683, + 0.6059411861934751, + 0.5099587939140718, + 0.44057324634523176, + 0.4391978251197769, + 0.7191858964012756, + 0.5811495621714656, + 0.4881322826391351, + 1.0011626163335512, + 1.1990369599246455, + 1.4118886136322046, + 1.554688514867236, + 2.6196553178384634, + 3.9391535171303675, + 3.1321858798359785, + 2.3332437535315793, + 0.9448004540825558, + 0.7676606562766751, + 0.3665181536820326, + 0.2919617681864151, + 0.18930565608900826, + 0.13559488549785165, + 0.11350226730002422, + 0.15592956934053803 + ], + "peak_layer": 22, + "peak_normalized": 3.9391535171303675, + "population_cv": 0.8670378924383861, + "rest_mean": 1.2221576239594637e-05, + "spike_contrast": 3.679940802546367, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.49746770755155e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.55765244725626e-06, + 2.2516098397318274e-05, + 1.6963771486189216e-05, + 1.7290087271248922e-05, + 1.0337914318370167e-05, + 1.610816252650693e-05, + 1.8833366993931122e-05, + 1.5537274521193467e-05, + 1.2602602510014549e-05, + 1.0506564649404027e-05, + 8.842302122502588e-06, + 7.639208888576832e-06, + 7.615360118506942e-06, + 1.2470142792153638e-05, + 1.0076696526084561e-05, + 8.463846825179644e-06, + 1.735940713842865e-05, + 2.0790399503312074e-05, + 2.4481087166350335e-05, + 2.6957130103255622e-05, + 4.542285387287848e-05, + 6.830196070950478e-05, + 5.430974852060899e-05, + 4.0456692659063265e-05, + 1.638212961552199e-05, + 1.3310658687260002e-05, + 6.355149253067793e-06, + 5.062397576693911e-06, + 3.282417765149148e-06, + 2.351113380427705e-06, + 1.968043989108992e-06, + 2.7037015115638496e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.000305004112306e-05, + "first_to_last_ratio": 2.193525307703265, + "imbalance_abs_log_ratio": 0.7855099791473917, + "last_quartile_mean": 3.191349094322504e-05, + "mean": 7.690838336316119e-05, + "normalized": [ + 0.8923720562068232, + 1.3111973164248405, + 0.8460717362092315, + 0.9445716926262052, + 0.4741848629196519, + 0.8656844065975011, + 0.9924506003335006, + 0.9551754577499273, + 0.6234358560310297, + 0.5152936103919111, + 0.3846693562402596, + 0.34526537659024253, + 0.30979982474526946, + 0.5791064055325631, + 0.47493901082375733, + 0.4350256040989574, + 1.011874248892453, + 1.1362510856108166, + 1.416617477431854, + 1.6892273518644711, + 2.692715672842754, + 4.577499824936365, + 2.8907687343273323, + 2.31616526773872, + 1.0250323630610032, + 0.8340050859269844, + 0.36589411026523266, + 0.29901912364743294, + 0.2532838824218693, + 0.16506308776319403, + 0.15276161475417316, + 0.22457789499367237 + ], + "peak_layer": 22, + "peak_normalized": 4.577499824936365, + "population_cv": 0.9353823752601559, + "rest_mean": 5.269026995072771e-05, + "spike_contrast": 3.9416422801499356, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000207686195790302, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 6.863089220132679e-05, + 0.00010084206587634981, + 6.507000944111496e-05, + 7.264548185048625e-05, + 3.646879122243263e-05, + 6.657838821411133e-05, + 7.632777123944834e-05, + 7.346100028371438e-05, + 4.7947443817975e-05, + 3.9630398532608524e-05, + 2.9584298317786306e-05, + 2.6553801944828592e-05, + 2.3826203687349334e-05, + 4.453813744476065e-05, + 3.652679151855409e-05, + 3.34571159328334e-05, + 7.782161264913157e-05, + 8.738723408896476e-05, + 0.00010894976003328338, + 0.00012991574476473033, + 0.00020709240925498307, + 0.0003520481113810092, + 0.00022232435003388673, + 0.00017813252634368837, + 7.883358193794265e-05, + 6.414198287529871e-05, + 2.8140324502601288e-05, + 2.2997077394393273e-05, + 1.9479653929010965e-05, + 1.2694735232798848e-05, + 1.174864883068949e-05, + 1.7271922843065113e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.047033343464136124, + 0.060984931886196136, + 0.09079429507255554, + 0.08531255275011063, + 0.0786692351102829, + 0.08076046407222748, + 0.08904207497835159, + 0.08403915911912918, + 0.06012389436364174, + 0.06841155141592026, + 0.07131828367710114, + 0.07358616590499878, + 0.06597626209259033, + 0.07235056161880493, + 0.07261744141578674, + 0.07352335751056671, + 0.07520031183958054, + 0.07378771901130676, + 0.07715436071157455, + 0.08649503439664841, + 0.04556623473763466, + 0.04546353965997696, + 0.05167657509446144, + 0.058827146887779236, + 0.04798740893602371, + 0.046526361256837845, + 0.051699329167604446, + 0.059578344225883484, + 0.05090610682964325, + 0.05821637064218521, + 0.06188955530524254, + 0.06796324998140335 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.364372159670893e-06, + "first_to_last_ratio": 0.2681115118992595, + "imbalance_abs_log_ratio": 1.3163522958360352, + "last_quartile_mean": 1.627819756322424e-05, + "mean": 1.0259845446114468e-05, + "normalized": [ + 1.0134993414800566, + 0.522056581911947, + 0.2644061262673726, + 0.3833634649429024, + 0.2440106441209935, + 0.2461172286895863, + 0.27872130983193855, + 0.4508957935090263, + 0.4833194134691065, + 0.39025737906993885, + 0.3708598519154414, + 0.36981229946243677, + 0.42398498232424564, + 0.41928891386718925, + 0.4397458055599365, + 0.45615006609958, + 0.4114730365687318, + 0.5108641768312265, + 0.48188059954330154, + 0.7614115358485232, + 2.335493650100753, + 2.8449380275824225, + 2.7201337953566025, + 2.484573069357449, + 2.4148997615580035, + 2.4501695400243877, + 2.1889991494613, + 1.6914475251239836, + 1.1580207584918483, + 1.0537166329047938, + 0.8749014303613178, + 0.8605881083636573 + ], + "peak_layer": 22, + "peak_normalized": 2.8449380275824225, + "population_cv": 0.8504187150143712, + "rest_mean": 7.295875539731006e-06, + "spike_contrast": 3.6000179550148346, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6265282940585166e-05, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.0398346603324171e-05, + 5.356219844543375e-06, + 2.7127659905090695e-06, + 3.933249900001101e-06, + 2.503511495888233e-06, + 2.525124727981165e-06, + 2.8596375614142744e-06, + 4.626121153705753e-06, + 4.9587824832997285e-06, + 4.00398039346328e-06, + 3.804964762821328e-06, + 3.7942170365568018e-06, + 4.350020390120335e-06, + 4.301839453546563e-06, + 4.511724000622053e-06, + 4.680029178416589e-06, + 4.221649760438595e-06, + 5.241387498244876e-06, + 4.944020474795252e-06, + 7.811964678694494e-06, + 2.3961803890415467e-05, + 2.9188624466769397e-05, + 2.7908152333111502e-05, + 2.5491335691185668e-05, + 2.4776498321443796e-05, + 2.5138360797427595e-05, + 2.2458792955148965e-05, + 1.735399018798489e-05, + 1.1881114005518612e-05, + 1.081096979760332e-05, + 8.976353456091601e-06, + 8.829500984575134e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.76496694166417e-05, + "first_to_last_ratio": 0.27059749062388033, + "imbalance_abs_log_ratio": 1.3071228366665688, + "last_quartile_mean": 6.522480816784082e-05, + "mean": 4.117927500146834e-05, + "normalized": [ + 1.007463472142432, + 0.5209651751221482, + 0.2643970395144652, + 0.3829104157055737, + 0.2536844256534051, + 0.25437837481906395, + 0.28207653195017285, + 0.46296966564632774, + 0.49565891190391403, + 0.39385639261196637, + 0.3771016062127897, + 0.37530908825337306, + 0.4278590182802805, + 0.42183720924091433, + 0.44371630887119135, + 0.4598267075919064, + 0.41216967133849985, + 0.5079852336529339, + 0.47800548121281095, + 0.7481511452590028, + 2.3310574965471607, + 2.8352891763903005, + 2.718777452615949, + 2.4731685677354274, + 2.4074108105775847, + 2.4550622811024825, + 2.180382977806297, + 1.6846495469817935, + 1.161664542480338, + 1.0519342613419045, + 0.8725636415138176, + 0.8577173699237725 + ], + "peak_layer": 22, + "peak_normalized": 2.8352891763903005, + "population_cv": 0.8467657487851442, + "rest_mean": 2.9335347587928278e-05, + "spike_contrast": 3.5839521832647088, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010513648303458467, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.148661537328735e-05, + 2.1452968212543055e-05, + 1.0887678399740253e-05, + 1.576797330926638e-05, + 1.0446540727571119e-05, + 1.0475117051100824e-05, + 1.1615707080636639e-05, + 1.906475517898798e-05, + 2.0410874640219845e-05, + 1.6218720702454448e-05, + 1.552877074573189e-05, + 1.5454956155736e-05, + 1.761892417562194e-05, + 1.7370950445183553e-05, + 1.8271915905643255e-05, + 1.8935330444946885e-05, + 1.697284824331291e-05, + 2.0918463633279316e-05, + 1.968391916307155e-05, + 3.080832175328396e-05, + 9.599125769454986e-05, + 0.00011675515270326287, + 0.00011195728438906372, + 0.00010184328857576475, + 9.913543181028217e-05, + 0.0001010976848192513, + 8.978659025160596e-05, + 6.937264697626233e-05, + 4.783650365425274e-05, + 4.3317890231264755e-05, + 3.593153815018013e-05, + 3.5320179449627176e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.015651903115212917, + "first_to_last_ratio": 0.2705974872956941, + "imbalance_abs_log_ratio": 1.307122848965967, + "last_quartile_mean": 0.05784201202914119, + "mean": 0.03651819290826097, + "normalized": [ + 1.0074634186476772, + 0.5209651919181703, + 0.2643970221469926, + 0.382910419856966, + 0.2536844208302265, + 0.25437835679028203, + 0.2820765262777231, + 0.4629696917045884, + 0.4956588954855505, + 0.393856398901423, + 0.3771015816519749, + 0.37530907952450476, + 0.42785903141736564, + 0.4218372178649866, + 0.4437163224543097, + 0.4598267050189124, + 0.41216967747329053, + 0.5079852485753572, + 0.4780054808986325, + 0.7481511542287224, + 2.331057435590359, + 2.835289207255424, + 2.7187775380829278, + 2.4731685834004287, + 2.4074106915648685, + 2.455062286976735, + 2.180383000719517, + 1.6846496365458605, + 1.1616645069657927, + 1.0519342829703067, + 0.8725635941676015, + 0.857717394092523 + ], + "peak_layer": 22, + "peak_normalized": 2.835289207255424, + "population_cv": 0.8467657515631118, + "rest_mean": 0.026014879690828146, + "spike_contrast": 3.5839521608577614, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09323608428239823, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.036790743470191956, + 0.01902470737695694, + 0.009655301459133625, + 0.013983196578919888, + 0.00926409661769867, + 0.009289437904953957, + 0.010300925001502037, + 0.016906816512346268, + 0.018100567162036896, + 0.01438292395323515, + 0.013771068304777145, + 0.013705609366297722, + 0.01562463864684105, + 0.015404732897877693, + 0.01620371825993061, + 0.016792040318250656, + 0.01505169179290533, + 0.018550703302025795, + 0.017455896362662315, + 0.02732112817466259, + 0.08512600511312485, + 0.10353963822126389, + 0.09928484261035919, + 0.09031564742326736, + 0.08791428804397583, + 0.08965443819761276, + 0.07962364703416824, + 0.06152036041021347, + 0.042421888560056686, + 0.038414739072322845, + 0.03186444565653801, + 0.03132228925824165 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.036145782506992e-06, + "first_to_last_ratio": 0.23813875303874632, + "imbalance_abs_log_ratio": 1.434901779225667, + "last_quartile_mean": 4.35101708262664e-06, + "mean": 2.601542721691885e-06, + "normalized": [ + 1.0445105925849996, + 0.4868956651441851, + 0.2653781288407282, + 0.38704215458190727, + 0.2311435484604591, + 0.22762453648926426, + 0.22559249684218238, + 0.3180631703688383, + 0.41371995096278646, + 0.35456417415204133, + 0.35816018513093856, + 0.3384166475711158, + 0.38409460459920464, + 0.37901778296480176, + 0.4237575276229331, + 0.4302249635723777, + 0.38747010650156044, + 0.47861136725136255, + 0.44057460034262785, + 0.49825323727916515, + 2.2524328318137035, + 2.8621193651579846, + 2.8149870576895797, + 2.617539402453915, + 2.4626516595822796, + 2.6269385253668815, + 2.362864616381361, + 1.8136241394740558, + 1.1783341687752071, + 1.0893413201795874, + 0.9454878584900481, + 0.9005636133719171 + ], + "peak_layer": 22, + "peak_normalized": 2.8621193651579846, + "population_cv": 0.9010662730896349, + "rest_mean": 1.8297776991689548e-06, + "spike_contrast": 3.6993968427913813, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.76907384331571e-06, + "top_five_layers": [ + 22, + 23, + 26, + 24, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.717338929869584e-06, + 1.266679873879184e-06, + 6.903925395818078e-07, + 1.0069067002405063e-06, + 6.013298161633429e-07, + 5.921749561821343e-07, + 5.868885182280792e-07, + 8.274549259112973e-07, + 1.0763101272459608e-06, + 9.224138466379372e-07, + 9.317690228272113e-07, + 8.80405366388004e-07, + 9.992385230361833e-07, + 9.860309546638746e-07, + 1.1024233117495896e-06, + 1.1192486226718756e-06, + 1.0080200354423141e-06, + 1.245127918991784e-06, + 1.1461736448836746e-06, + 1.296227083003032e-06, + 5.859800239704782e-06, + 7.445925803040154e-06, + 7.323309091589181e-06, + 6.809640581195708e-06, + 6.406693501048721e-06, + 6.8340928010002244e-06, + 6.147093245090218e-06, + 4.718220679933438e-06, + 3.0654866804979974e-06, + 2.8339679829514353e-06, + 2.4597270567028318e-06, + 2.342854713788256e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.2745702917982271e-05, + "first_to_last_ratio": 0.3040242464136832, + "imbalance_abs_log_ratio": 1.1906478228166633, + "last_quartile_mean": 4.1923310618585674e-05, + "mean": 2.8222844818515114e-05, + "normalized": [ + 1.092738547299169, + 0.5627549631144219, + 0.24809914749976855, + 0.3821239330736069, + 0.2717732575074452, + 0.2708922129031054, + 0.29822547774823177, + 0.4862679168736732, + 0.544040724965591, + 0.43215299071277247, + 0.4170763968939518, + 0.42015115877685144, + 0.47871065526968887, + 0.46826286484802104, + 0.4900402959326792, + 0.5070735564042542, + 0.4633460623422733, + 0.5693312811282049, + 0.5194930950029935, + 0.7472397737749802, + 2.399371006579266, + 2.8986292719666995, + 2.739777767666698, + 2.408916709158605, + 2.333408286273547, + 2.300446003604439, + 2.062640270367703, + 1.6162771645238014, + 1.0435838635901262, + 0.9412033869198291, + 0.8018738201820883, + 0.7840781370955142 + ], + "peak_layer": 22, + "peak_normalized": 2.8986292719666995, + "population_cv": 0.8223755795225939, + "rest_mean": 2.0090376640129965e-05, + "spike_contrast": 3.5906829560228823, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.213817298179492e-05, + "top_five_layers": [ + 22, + 23, + 24, + 21, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0840190447634086e-05, + 1.5882545994827524e-05, + 7.0020637394918595e-06, + 1.0784624464577064e-05, + 7.670214472454973e-06, + 7.6453488873085e-06, + 8.416771379415877e-06, + 1.3723863958148286e-05, + 1.5354376955656335e-05, + 1.219658679474378e-05, + 1.1771082427003421e-05, + 1.1857860954478383e-05, + 1.3510576536646113e-05, + 1.3215710168879014e-05, + 1.3830331226927228e-05, + 1.4311058293969836e-05, + 1.3076944014756009e-05, + 1.606814839760773e-05, + 1.4661573004559614e-05, + 2.1089232177473605e-05, + 6.771707558073103e-05, + 8.18075641291216e-05, + 7.732432277407497e-05, + 6.79864824633114e-05, + 6.58554199617356e-05, + 6.492513057310134e-05, + 5.821357626700774e-05, + 4.561593959806487e-05, + 2.9452905437210575e-05, + 2.6563437131699175e-05, + 2.263116039102897e-05, + 2.2128915588837117e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.982105043742195e-06, + "first_to_last_ratio": 0.3255748540372994, + "imbalance_abs_log_ratio": 1.122162877451286, + "last_quartile_mean": 2.758844834716001e-05, + "mean": 1.9313689406885715e-05, + "normalized": [ + 1.153292392223287, + 0.5797609618245384, + 0.23076786066825913, + 0.37581241874572735, + 0.2819306093436648, + 0.2851688939012761, + 0.30549556090944335, + 0.5082847079850745, + 0.583859969769007, + 0.45907810601554777, + 0.45116352890905354, + 0.46235698273638653, + 0.522619204122844, + 0.5090842134923546, + 0.5336639455934807, + 0.5440001607440246, + 0.5095787126873097, + 0.6267236930541729, + 0.5514908746704607, + 0.7356132069066491, + 2.430614280460028, + 2.906404914690865, + 2.7136484598097654, + 2.3120655741144565, + 2.2482332201724016, + 2.1318721582988647, + 1.9232163840966459, + 1.5686764250532175, + 1.0367548344938609, + 0.9157520446280638, + 0.8238220024989565, + 0.7791936973803123 + ], + "peak_layer": 22, + "peak_normalized": 2.906404914690865, + "population_cv": 0.7900897792080465, + "rest_mean": 1.3869398959219257e-05, + "spike_contrast": 3.5122544219484153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.8712857824284585e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2274331058724783e-05, + 1.119732314691646e-05, + 4.456978786038235e-06, + 7.258324330905452e-06, + 5.4451202231575735e-06, + 5.5076634453143924e-06, + 5.900246378587326e-06, + 9.816852980293334e-06, + 1.1276490113232285e-05, + 8.866491953085642e-06, + 8.713632269063964e-06, + 8.92981915967539e-06, + 1.0093704986502416e-05, + 9.832294381340034e-06, + 1.0307019692845643e-05, + 1.0506650141905993e-05, + 9.841844985203352e-06, + 1.2104346751584671e-05, + 1.0651323464117013e-05, + 1.4207405001798179e-05, + 4.694412928074598e-05, + 5.613340181298554e-05, + 5.2410563512239605e-05, + 4.4654516386799514e-05, + 4.3421678128652275e-05, + 4.117431672057137e-05, + 3.714440390467644e-05, + 3.0296929253381677e-05, + 2.0023560864501633e-05, + 1.768655056366697e-05, + 1.5911042282823473e-05, + 1.5049105059006251e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.531509560161794e-05, + "first_to_last_ratio": 0.2551422661109217, + "imbalance_abs_log_ratio": 1.36593398307824, + "last_quartile_mean": 0.00013841334930475568, + "mean": 8.573219730578785e-05, + "normalized": [ + 0.9822788357613834, + 0.5147377658277125, + 0.24077984042300446, + 0.3633612864660074, + 0.2505573845679934, + 0.23871348492801034, + 0.2720502738875204, + 0.4329080853308244, + 0.4899097166580031, + 0.3804980486342757, + 0.36593190899850525, + 0.3643305688887499, + 0.412819132135824, + 0.40296081445560644, + 0.4305724210670739, + 0.4502160271371867, + 0.3962072388182212, + 0.4765863055500514, + 0.454434200643566, + 0.7197132038245179, + 2.2718289409811727, + 2.8618271883484865, + 2.7794936015049534, + 2.53140346971279, + 2.4848609397336863, + 2.575834812444881, + 2.3585524326140215, + 1.8181964500308416, + 1.0732112926751614, + 0.9812549829334379, + 0.8189418899610229, + 0.8050274550555072 + ], + "peak_layer": 22, + "peak_normalized": 2.8618271883484865, + "population_cv": 0.8817284503784036, + "rest_mean": 6.055419369119961e-05, + "spike_contrast": 3.6610745401896616, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022169341682456433, + "top_five_layers": [ + 22, + 23, + 26, + 24, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.42129229567945e-05, + 4.4129599700681865e-05, + 2.064258478640113e-05, + 3.115176150458865e-05, + 2.1480835130205378e-05, + 2.0465431589400396e-05, + 2.3323467758018523e-05, + 3.7114161386853084e-05, + 4.200103649054654e-05, + 3.262093377998099e-05, + 3.137214662274346e-05, + 3.123486021650024e-05, + 3.539189128787257e-05, + 3.454671605140902e-05, + 3.691391975735314e-05, + 3.859800926875323e-05, + 3.3967717172345147e-05, + 4.08587911806535e-05, + 3.895964255207218e-05, + 6.170259439386427e-05, + 0.00019476888701319695, + 0.00024535073316656053, + 0.00023829209385439754, + 0.0002170227817259729, + 0.0002130325883626938, + 0.00022083197836764157, + 0.0002022038825089112, + 0.00015587797679472715, + 9.200876229442656e-05, + 8.412514580413699e-05, + 7.020968769211322e-05, + 6.9016772613395e-05 + ] + } + } + } + } + } + }, + "step": 2000 + }, + { + "modes": { + "learned": { + "bits_per_byte": 1.9433676664104056, + "forward": { + "activation_sha256": { + "attention_branch_output": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", + "mlp_branch_output": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", + "post_attention_state": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", + "post_mlp_state": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", + "pre_attention_input": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", + "pre_mlp_input": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c" + }, + "logits_sha256": "000b42d2e473ee11592a1ced2fe607cfcfbbe21b9cb8637453b9d2bc3414fcfe", + "loss_nats": 1.347039818763733, + "mixer_summary_sha256": "7935548f1014624ed4c9fa535e6d7a55432ce9be32776c3985f675572d6d61e8" + }, + "loss_nats": 1.347039818763733, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.24809812009334564, + "group": 1, + "latest_source_mass": 0.07119978964328766, + "layer": 1, + "max_source_mass": 0.9288002252578735, + "mixer_index": 1, + "normalized_entropy": 0.35792992751254427, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9288002252578735, + "median": 0.9386287331581116, + "p05": 0.858244001865387, + "p95": 0.9685006141662598 + }, + { + "label": "current_group_partial", + "mean": 0.07119978964328766, + "median": 0.06137128174304962, + "p05": 0.031499382108449936, + "p95": 0.14175599813461304 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6067080497741699, + "group": 1, + "latest_source_mass": 0.43957531452178955, + "layer": 2, + "max_source_mass": 0.5604246854782104, + "mixer_index": 2, + "normalized_entropy": 0.8752946946766094, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5604246854782104, + "median": 0.5413482189178467, + "p05": 0.26798251271247864, + "p95": 0.8787311315536499 + }, + { + "label": "current_group_partial", + "mean": 0.43957531452178955, + "median": 0.4586517810821533, + "p05": 0.1212688684463501, + "p95": 0.7320175170898438 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3847596049308777, + "group": 1, + "latest_source_mass": 0.13518528640270233, + "layer": 2, + "max_source_mass": 0.8648147583007812, + "mixer_index": 3, + "normalized_entropy": 0.555090773968174, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8648147583007812, + "median": 0.8810019493103027, + "p05": 0.7563575506210327, + "p95": 0.9309777617454529 + }, + { + "label": "current_group_partial", + "mean": 0.13518528640270233, + "median": 0.11899800598621368, + "p05": 0.06902224570512772, + "p95": 0.24364244937896729 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6843481659889221, + "group": 1, + "latest_source_mass": 0.5056107044219971, + "layer": 3, + "max_source_mass": 0.5056107044219971, + "mixer_index": 4, + "normalized_entropy": 0.9873057053136751, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.49438929557800293, + "median": 0.4973145127296448, + "p05": 0.3897787928581238, + "p95": 0.6022077798843384 + }, + { + "label": "current_group_partial", + "mean": 0.5056107044219971, + "median": 0.502685546875, + "p05": 0.3977922201156616, + "p95": 0.6102211475372314 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5728601813316345, + "group": 1, + "latest_source_mass": 0.30453476309776306, + "layer": 3, + "max_source_mass": 0.6954652070999146, + "mixer_index": 5, + "normalized_entropy": 0.8264625427299015, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6954652070999146, + "median": 0.7084143161773682, + "p05": 0.453019917011261, + "p95": 0.9048184752464294 + }, + { + "label": "current_group_partial", + "mean": 0.30453476309776306, + "median": 0.29158568382263184, + "p05": 0.09518152475357056, + "p95": 0.5469801425933838 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.686518132686615, + "group": 1, + "latest_source_mass": 0.4736926555633545, + "layer": 4, + "max_source_mass": 0.5263072848320007, + "mixer_index": 6, + "normalized_entropy": 0.9904363055073309, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5263072848320007, + "median": 0.5291417241096497, + "p05": 0.4370424747467041, + "p95": 0.6044135093688965 + }, + { + "label": "current_group_partial", + "mean": 0.4736926555633545, + "median": 0.47085827589035034, + "p05": 0.3955864906311035, + "p95": 0.5629575252532959 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5044870376586914, + "group": 1, + "latest_source_mass": 0.2221563160419464, + "layer": 4, + "max_source_mass": 0.777843713760376, + "mixer_index": 7, + "normalized_entropy": 0.7278209474229579, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.777843713760376, + "median": 0.7819972634315491, + "p05": 0.6222297549247742, + "p95": 0.9289263486862183 + }, + { + "label": "current_group_partial", + "mean": 0.2221563160419464, + "median": 0.21800270676612854, + "p05": 0.07107366621494293, + "p95": 0.3777702748775482 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6724032163619995, + "group": 2, + "latest_source_mass": 0.4305936098098755, + "layer": 5, + "max_source_mass": 0.5694063901901245, + "mixer_index": 8, + "normalized_entropy": 0.9700727857232454, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5694063901901245, + "median": 0.5728721022605896, + "p05": 0.45058131217956543, + "p95": 0.6837627291679382 + }, + { + "label": "completed_group_1", + "mean": 0.4305936098098755, + "median": 0.4271278977394104, + "p05": 0.31623730063438416, + "p95": 0.5494186878204346 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9718273282051086, + "group": 2, + "latest_source_mass": 0.23954525589942932, + "layer": 5, + "max_source_mass": 0.5544795989990234, + "mixer_index": 9, + "normalized_entropy": 0.8845953556402437, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5544795989990234, + "median": 0.5475102663040161, + "p05": 0.40941980481147766, + "p95": 0.7249865531921387 + }, + { + "label": "completed_group_1", + "mean": 0.20597517490386963, + "median": 0.20141024887561798, + "p05": 0.11906205117702484, + "p95": 0.3046763241291046 + }, + { + "label": "current_group_partial", + "mean": 0.23954525589942932, + "median": 0.2397485226392746, + "p05": 0.14606745541095734, + "p95": 0.33606359362602234 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0856361389160156, + "group": 2, + "latest_source_mass": 0.36206769943237305, + "layer": 6, + "max_source_mass": 0.3703838884830475, + "mixer_index": 10, + "normalized_entropy": 0.9881885994850598, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3703838884830475, + "median": 0.3719092607498169, + "p05": 0.31337594985961914, + "p95": 0.42367395758628845 + }, + { + "label": "completed_group_1", + "mean": 0.26754841208457947, + "median": 0.2691181004047394, + "p05": 0.24584907293319702, + "p95": 0.28693482279777527 + }, + { + "label": "current_group_partial", + "mean": 0.36206769943237305, + "median": 0.3628925681114197, + "p05": 0.31465041637420654, + "p95": 0.4040696620941162 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.022249460220337, + "group": 2, + "latest_source_mass": 0.32249319553375244, + "layer": 6, + "max_source_mass": 0.46441033482551575, + "mixer_index": 11, + "normalized_entropy": 0.9304915580906614, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46441033482551575, + "median": 0.47390004992485046, + "p05": 0.25647592544555664, + "p95": 0.6490365266799927 + }, + { + "label": "completed_group_1", + "mean": 0.21309645473957062, + "median": 0.2148953229188919, + "p05": 0.13959459960460663, + "p95": 0.27812671661376953 + }, + { + "label": "current_group_partial", + "mean": 0.32249319553375244, + "median": 0.31052517890930176, + "p05": 0.20389017462730408, + "p95": 0.47258833050727844 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0742855072021484, + "group": 2, + "latest_source_mass": 0.36498188972473145, + "layer": 7, + "max_source_mass": 0.38770636916160583, + "mixer_index": 12, + "normalized_entropy": 0.9778568092521033, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38770636916160583, + "median": 0.4003474712371826, + "p05": 0.2879825532436371, + "p95": 0.45421719551086426 + }, + { + "label": "completed_group_1", + "mean": 0.24731171131134033, + "median": 0.24118143320083618, + "p05": 0.21154659986495972, + "p95": 0.2994430363178253 + }, + { + "label": "current_group_partial", + "mean": 0.36498188972473145, + "median": 0.35881954431533813, + "p05": 0.31933724880218506, + "p95": 0.42228373885154724 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0207322835922241, + "group": 2, + "latest_source_mass": 0.4325985312461853, + "layer": 7, + "max_source_mass": 0.4325985312461853, + "mixer_index": 13, + "normalized_entropy": 0.9291105644100317, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3404029607772827, + "median": 0.2972515821456909, + "p05": 0.1890696883201599, + "p95": 0.675995409488678 + }, + { + "label": "completed_group_1", + "mean": 0.22699852287769318, + "median": 0.2352103292942047, + "p05": 0.11332616209983826, + "p95": 0.2928347587585449 + }, + { + "label": "current_group_partial", + "mean": 0.4325985312461853, + "median": 0.4621005654335022, + "p05": 0.2054932415485382, + "p95": 0.5461148023605347 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0628657341003418, + "group": 2, + "latest_source_mass": 0.361979603767395, + "layer": 8, + "max_source_mass": 0.41766518354415894, + "mixer_index": 14, + "normalized_entropy": 0.9674620838156608, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.41766518354415894, + "median": 0.4145643711090088, + "p05": 0.36852169036865234, + "p95": 0.47784724831581116 + }, + { + "label": "completed_group_1", + "mean": 0.22035519778728485, + "median": 0.2179521918296814, + "p05": 0.191657155752182, + "p95": 0.25349321961402893 + }, + { + "label": "current_group_partial", + "mean": 0.361979603767395, + "median": 0.363964319229126, + "p05": 0.3155593276023865, + "p95": 0.39730241894721985 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.029329538345337, + "group": 2, + "latest_source_mass": 0.3858758807182312, + "layer": 8, + "max_source_mass": 0.42153143882751465, + "mixer_index": 15, + "normalized_entropy": 0.936936122927619, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42153143882751465, + "median": 0.41038113832473755, + "p05": 0.29627615213394165, + "p95": 0.6068807244300842 + }, + { + "label": "completed_group_1", + "mean": 0.19259268045425415, + "median": 0.18828678131103516, + "p05": 0.12331291288137436, + "p95": 0.2707396447658539 + }, + { + "label": "current_group_partial", + "mean": 0.3858758807182312, + "median": 0.39191579818725586, + "p05": 0.2700805068016052, + "p95": 0.4676054120063782 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0849454402923584, + "group": 3, + "latest_source_mass": 0.3364688754081726, + "layer": 9, + "max_source_mass": 0.37970075011253357, + "mixer_index": 16, + "normalized_entropy": 0.9875598985040298, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.37970075011253357, + "median": 0.382883220911026, + "p05": 0.29113224148750305, + "p95": 0.46186232566833496 + }, + { + "label": "completed_group_1", + "mean": 0.2838304042816162, + "median": 0.282575786113739, + "p05": 0.25098085403442383, + "p95": 0.31989675760269165 + }, + { + "label": "completed_group_2", + "mean": 0.3364688754081726, + "median": 0.33596307039260864, + "p05": 0.28113189339637756, + "p95": 0.3942520022392273 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3144235610961914, + "group": 3, + "latest_source_mass": 0.30927807092666626, + "layer": 9, + "max_source_mass": 0.3205832242965698, + "mixer_index": 17, + "normalized_entropy": 0.9481561766105434, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3205832242965698, + "median": 0.3228584825992584, + "p05": 0.20318874716758728, + "p95": 0.49334216117858887 + }, + { + "label": "completed_group_1", + "mean": 0.13048580288887024, + "median": 0.1293262243270874, + "p05": 0.08803658187389374, + "p95": 0.17583128809928894 + }, + { + "label": "completed_group_2", + "mean": 0.2396528720855713, + "median": 0.23850613832473755, + "p05": 0.15605276823043823, + "p95": 0.3097888231277466 + }, + { + "label": "current_group_partial", + "mean": 0.30927807092666626, + "median": 0.3110916018486023, + "p05": 0.24646314978599548, + "p95": 0.36450129747390747 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3647149801254272, + "group": 3, + "latest_source_mass": 0.24869562685489655, + "layer": 10, + "max_source_mass": 0.3188267946243286, + "mixer_index": 18, + "normalized_entropy": 0.9844337670269171, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3188267946243286, + "median": 0.32193225622177124, + "p05": 0.2530626356601715, + "p95": 0.38645845651626587 + }, + { + "label": "completed_group_1", + "mean": 0.2379944920539856, + "median": 0.23598769307136536, + "p05": 0.21630848944187164, + "p95": 0.2653774321079254 + }, + { + "label": "completed_group_2", + "mean": 0.19448311626911163, + "median": 0.19229988753795624, + "p05": 0.16299879550933838, + "p95": 0.23252975940704346 + }, + { + "label": "current_group_partial", + "mean": 0.24869562685489655, + "median": 0.24343957006931305, + "p05": 0.21172824501991272, + "p95": 0.29676955938339233 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2778496742248535, + "group": 3, + "latest_source_mass": 0.2590479850769043, + "layer": 10, + "max_source_mass": 0.37896889448165894, + "mixer_index": 19, + "normalized_entropy": 0.9217736940028869, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.37896889448165894, + "median": 0.3914150297641754, + "p05": 0.14146459102630615, + "p95": 0.6288160681724548 + }, + { + "label": "completed_group_1", + "mean": 0.16353653371334076, + "median": 0.162285715341568, + "p05": 0.08837783336639404, + "p95": 0.23340767621994019 + }, + { + "label": "completed_group_2", + "mean": 0.1984466314315796, + "median": 0.19906753301620483, + "p05": 0.09471487253904343, + "p95": 0.29035234451293945 + }, + { + "label": "current_group_partial", + "mean": 0.2590479850769043, + "median": 0.2469818890094757, + "p05": 0.1562758982181549, + "p95": 0.3916419744491577 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3567583560943604, + "group": 3, + "latest_source_mass": 0.3088001608848572, + "layer": 11, + "max_source_mass": 0.3088001608848572, + "mixer_index": 20, + "normalized_entropy": 0.978694276010998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29770222306251526, + "median": 0.29406338930130005, + "p05": 0.24821004271507263, + "p95": 0.3566911518573761 + }, + { + "label": "completed_group_1", + "mean": 0.21492168307304382, + "median": 0.21014970541000366, + "p05": 0.18107125163078308, + "p95": 0.26316314935684204 + }, + { + "label": "completed_group_2", + "mean": 0.17857596278190613, + "median": 0.17864474654197693, + "p05": 0.15527817606925964, + "p95": 0.2019544541835785 + }, + { + "label": "current_group_partial", + "mean": 0.3088001608848572, + "median": 0.309349000453949, + "p05": 0.2659364938735962, + "p95": 0.3488244414329529 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2773678302764893, + "group": 3, + "latest_source_mass": 0.24117937684059143, + "layer": 11, + "max_source_mass": 0.3856483995914459, + "mixer_index": 21, + "normalized_entropy": 0.9214261170654932, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3856483995914459, + "median": 0.4236947298049927, + "p05": 0.12469026446342468, + "p95": 0.588103175163269 + }, + { + "label": "completed_group_1", + "mean": 0.18032220005989075, + "median": 0.17527258396148682, + "p05": 0.10318915545940399, + "p95": 0.2606950104236603 + }, + { + "label": "completed_group_2", + "mean": 0.1928499937057495, + "median": 0.18568618595600128, + "p05": 0.12013933062553406, + "p95": 0.28692901134490967 + }, + { + "label": "current_group_partial", + "mean": 0.24117937684059143, + "median": 0.2192210555076599, + "p05": 0.14662782847881317, + "p95": 0.4063720405101776 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3655540943145752, + "group": 3, + "latest_source_mass": 0.29187142848968506, + "layer": 12, + "max_source_mass": 0.29819542169570923, + "mixer_index": 22, + "normalized_entropy": 0.9850390599666288, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29819542169570923, + "median": 0.29867103695869446, + "p05": 0.2453947365283966, + "p95": 0.34859126806259155 + }, + { + "label": "completed_group_1", + "mean": 0.20063453912734985, + "median": 0.19907723367214203, + "p05": 0.16811034083366394, + "p95": 0.23844538629055023 + }, + { + "label": "completed_group_2", + "mean": 0.20929867029190063, + "median": 0.21011404693126678, + "p05": 0.18459835648536682, + "p95": 0.23381994664669037 + }, + { + "label": "current_group_partial", + "mean": 0.29187142848968506, + "median": 0.29417213797569275, + "p05": 0.24766536056995392, + "p95": 0.32690057158470154 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2754871845245361, + "group": 3, + "latest_source_mass": 0.22957558929920197, + "layer": 12, + "max_source_mass": 0.42651012539863586, + "mixer_index": 23, + "normalized_entropy": 0.9200695179154873, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42651012539863586, + "median": 0.44201791286468506, + "p05": 0.24115881323814392, + "p95": 0.5910159945487976 + }, + { + "label": "completed_group_1", + "mean": 0.1713070571422577, + "median": 0.170497864484787, + "p05": 0.12993231415748596, + "p95": 0.21631143987178802 + }, + { + "label": "completed_group_2", + "mean": 0.1726072132587433, + "median": 0.16530022025108337, + "p05": 0.11561842262744904, + "p95": 0.2528795301914215 + }, + { + "label": "current_group_partial", + "mean": 0.22957558929920197, + "median": 0.2182570993900299, + "p05": 0.14037305116653442, + "p95": 0.3425685167312622 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3596348762512207, + "group": 4, + "latest_source_mass": 0.3104133605957031, + "layer": 13, + "max_source_mass": 0.3104133605957031, + "mixer_index": 24, + "normalized_entropy": 0.9807692466936578, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2750997543334961, + "median": 0.2747533321380615, + "p05": 0.2329672873020172, + "p95": 0.31678152084350586 + }, + { + "label": "completed_group_1", + "mean": 0.22746169567108154, + "median": 0.215274840593338, + "p05": 0.18571145832538605, + "p95": 0.3119947016239166 + }, + { + "label": "completed_group_2", + "mean": 0.18702521920204163, + "median": 0.1861300766468048, + "p05": 0.16768434643745422, + "p95": 0.20858913660049438 + }, + { + "label": "completed_group_3", + "mean": 0.3104133605957031, + "median": 0.32007908821105957, + "p05": 0.2161477506160736, + "p95": 0.3817093074321747 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5752925872802734, + "group": 4, + "latest_source_mass": 0.20874352753162384, + "layer": 13, + "max_source_mass": 0.28211504220962524, + "mixer_index": 25, + "normalized_entropy": 0.9787843166300303, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28211504220962524, + "median": 0.2998196482658386, + "p05": 0.19197332859039307, + "p95": 0.3606031835079193 + }, + { + "label": "completed_group_1", + "mean": 0.1675817370414734, + "median": 0.16741523146629333, + "p05": 0.14403808116912842, + "p95": 0.19205224514007568 + }, + { + "label": "completed_group_2", + "mean": 0.1759152114391327, + "median": 0.17603915929794312, + "p05": 0.138168603181839, + "p95": 0.21206450462341309 + }, + { + "label": "completed_group_3", + "mean": 0.16564446687698364, + "median": 0.16148409247398376, + "p05": 0.12781348824501038, + "p95": 0.21234022080898285 + }, + { + "label": "current_group_partial", + "mean": 0.20874352753162384, + "median": 0.2069983333349228, + "p05": 0.17109251022338867, + "p95": 0.24858909845352173 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5704567432403564, + "group": 4, + "latest_source_mass": 0.29884350299835205, + "layer": 14, + "max_source_mass": 0.29884350299835205, + "mixer_index": 26, + "normalized_entropy": 0.975779637789948, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21000900864601135, + "median": 0.21329987049102783, + "p05": 0.17762595415115356, + "p95": 0.2425374686717987 + }, + { + "label": "completed_group_1", + "mean": 0.18079037964344025, + "median": 0.17888139188289642, + "p05": 0.15963314473628998, + "p95": 0.20616433024406433 + }, + { + "label": "completed_group_2", + "mean": 0.13544103503227234, + "median": 0.1345825046300888, + "p05": 0.12005693465471268, + "p95": 0.1533820480108261 + }, + { + "label": "completed_group_3", + "mean": 0.1749160885810852, + "median": 0.17272356152534485, + "p05": 0.15442612767219543, + "p95": 0.20184364914894104 + }, + { + "label": "current_group_partial", + "mean": 0.29884350299835205, + "median": 0.3006042242050171, + "p05": 0.2674207091331482, + "p95": 0.3227018713951111 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.579707384109497, + "group": 4, + "latest_source_mass": 0.24509160220623016, + "layer": 14, + "max_source_mass": 0.24509160220623016, + "mixer_index": 27, + "normalized_entropy": 0.98152738412901, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21335941553115845, + "median": 0.21055345237255096, + "p05": 0.12470410019159317, + "p95": 0.33470356464385986 + }, + { + "label": "completed_group_1", + "mean": 0.18788450956344604, + "median": 0.18773096799850464, + "p05": 0.14719606935977936, + "p95": 0.22808396816253662 + }, + { + "label": "completed_group_2", + "mean": 0.20250459015369415, + "median": 0.20350772142410278, + "p05": 0.15922458469867706, + "p95": 0.24294495582580566 + }, + { + "label": "completed_group_3", + "mean": 0.1511598825454712, + "median": 0.14913730323314667, + "p05": 0.12029191851615906, + "p95": 0.1872422844171524 + }, + { + "label": "current_group_partial", + "mean": 0.24509160220623016, + "median": 0.23680704832077026, + "p05": 0.18374761939048767, + "p95": 0.3342677652835846 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5663042068481445, + "group": 4, + "latest_source_mass": 0.29828590154647827, + "layer": 15, + "max_source_mass": 0.29828590154647827, + "mixer_index": 28, + "normalized_entropy": 0.9731995218624366, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21676908433437347, + "median": 0.2142370641231537, + "p05": 0.1767536848783493, + "p95": 0.2722740173339844 + }, + { + "label": "completed_group_1", + "mean": 0.19467836618423462, + "median": 0.19520726799964905, + "p05": 0.16570945084095, + "p95": 0.2210082858800888 + }, + { + "label": "completed_group_2", + "mean": 0.13713446259498596, + "median": 0.13748224079608917, + "p05": 0.12025479972362518, + "p95": 0.15284715592861176 + }, + { + "label": "completed_group_3", + "mean": 0.15313217043876648, + "median": 0.153159961104393, + "p05": 0.13474825024604797, + "p95": 0.1712232530117035 + }, + { + "label": "current_group_partial", + "mean": 0.29828590154647827, + "median": 0.29950791597366333, + "p05": 0.2589420676231384, + "p95": 0.33298033475875854 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5384478569030762, + "group": 4, + "latest_source_mass": 0.29070425033569336, + "layer": 15, + "max_source_mass": 0.29070425033569336, + "mixer_index": 29, + "normalized_entropy": 0.9558913984922479, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21904908120632172, + "median": 0.20143723487854004, + "p05": 0.09117002040147781, + "p95": 0.4469224512577057 + }, + { + "label": "completed_group_1", + "mean": 0.15864285826683044, + "median": 0.15922614932060242, + "p05": 0.10943489521741867, + "p95": 0.2047940045595169 + }, + { + "label": "completed_group_2", + "mean": 0.18178996443748474, + "median": 0.18768036365509033, + "p05": 0.10846641659736633, + "p95": 0.23086166381835938 + }, + { + "label": "completed_group_3", + "mean": 0.14981383085250854, + "median": 0.14754682779312134, + "p05": 0.10115746408700943, + "p95": 0.19903366267681122 + }, + { + "label": "current_group_partial", + "mean": 0.29070425033569336, + "median": 0.2898399233818054, + "p05": 0.20691633224487305, + "p95": 0.37558960914611816 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.552668571472168, + "group": 4, + "latest_source_mass": 0.30334532260894775, + "layer": 16, + "max_source_mass": 0.30334532260894775, + "mixer_index": 30, + "normalized_entropy": 0.9647272252484255, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22710487246513367, + "median": 0.2245989590883255, + "p05": 0.16175971925258636, + "p95": 0.2988150715827942 + }, + { + "label": "completed_group_1", + "mean": 0.1922147125005722, + "median": 0.1916719675064087, + "p05": 0.15179888904094696, + "p95": 0.23553937673568726 + }, + { + "label": "completed_group_2", + "mean": 0.13047492504119873, + "median": 0.12986470758914948, + "p05": 0.10791919380426407, + "p95": 0.15449906885623932 + }, + { + "label": "completed_group_3", + "mean": 0.14686015248298645, + "median": 0.14501844346523285, + "p05": 0.12235601246356964, + "p95": 0.17693395912647247 + }, + { + "label": "current_group_partial", + "mean": 0.30334532260894775, + "median": 0.29679566621780396, + "p05": 0.24902349710464478, + "p95": 0.3774981200695038 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5210195779800415, + "group": 4, + "latest_source_mass": 0.282525897026062, + "layer": 16, + "max_source_mass": 0.282525897026062, + "mixer_index": 31, + "normalized_entropy": 0.9450625999481175, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2812899947166443, + "median": 0.2745441794395447, + "p05": 0.15230709314346313, + "p95": 0.4724813997745514 + }, + { + "label": "completed_group_1", + "mean": 0.14511607587337494, + "median": 0.14084157347679138, + "p05": 0.10224629938602448, + "p95": 0.2002197802066803 + }, + { + "label": "completed_group_2", + "mean": 0.15909329056739807, + "median": 0.15865719318389893, + "p05": 0.10898765176534653, + "p95": 0.21094731986522675 + }, + { + "label": "completed_group_3", + "mean": 0.1319747418165207, + "median": 0.1301102191209793, + "p05": 0.08913854509592056, + "p95": 0.17862913012504578 + }, + { + "label": "current_group_partial", + "mean": 0.282525897026062, + "median": 0.28236913681030273, + "p05": 0.19806379079818726, + "p95": 0.36864838004112244 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5376527309417725, + "group": 5, + "latest_source_mass": 0.287123441696167, + "layer": 17, + "max_source_mass": 0.287123441696167, + "mixer_index": 32, + "normalized_entropy": 0.9553973589551147, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23905543982982635, + "median": 0.23701313138008118, + "p05": 0.18620719015598297, + "p95": 0.2925828695297241 + }, + { + "label": "completed_group_1", + "mean": 0.21673071384429932, + "median": 0.2044925093650818, + "p05": 0.15778198838233948, + "p95": 0.321443647146225 + }, + { + "label": "completed_group_2", + "mean": 0.0950920432806015, + "median": 0.09378060698509216, + "p05": 0.07889896631240845, + "p95": 0.11442621052265167 + }, + { + "label": "completed_group_3", + "mean": 0.16199833154678345, + "median": 0.16117054224014282, + "p05": 0.12811943888664246, + "p95": 0.19840528070926666 + }, + { + "label": "completed_group_4", + "mean": 0.287123441696167, + "median": 0.29111331701278687, + "p05": 0.2107674777507782, + "p95": 0.34516024589538574 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7209875583648682, + "group": 5, + "latest_source_mass": 0.1365262269973755, + "layer": 17, + "max_source_mass": 0.25328749418258667, + "mixer_index": 33, + "normalized_entropy": 0.9605014444859178, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1703755259513855, + "median": 0.1390259861946106, + "p05": 0.08010496944189072, + "p95": 0.3672885000705719 + }, + { + "label": "completed_group_1", + "mean": 0.14151468873023987, + "median": 0.1361694037914276, + "p05": 0.10555511713027954, + "p95": 0.19257718324661255 + }, + { + "label": "completed_group_2", + "mean": 0.17087554931640625, + "median": 0.17344608902931213, + "p05": 0.10519347339868546, + "p95": 0.22031483054161072 + }, + { + "label": "completed_group_3", + "mean": 0.12742051482200623, + "median": 0.12381769716739655, + "p05": 0.0888485461473465, + "p95": 0.17263755202293396 + }, + { + "label": "completed_group_4", + "mean": 0.25328749418258667, + "median": 0.25078505277633667, + "p05": 0.163196861743927, + "p95": 0.35207411646842957 + }, + { + "label": "current_group_partial", + "mean": 0.1365262269973755, + "median": 0.13269345462322235, + "p05": 0.0852486714720726, + "p95": 0.20064347982406616 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7200093269348145, + "group": 5, + "latest_source_mass": 0.27521812915802, + "layer": 18, + "max_source_mass": 0.27521812915802, + "mixer_index": 34, + "normalized_entropy": 0.9599554831295783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18992580473423004, + "median": 0.19383516907691956, + "p05": 0.1371862292289734, + "p95": 0.23474527895450592 + }, + { + "label": "completed_group_1", + "mean": 0.16036251187324524, + "median": 0.1590055525302887, + "p05": 0.14324960112571716, + "p95": 0.1809857338666916 + }, + { + "label": "completed_group_2", + "mean": 0.08439569175243378, + "median": 0.08315694332122803, + "p05": 0.07186377048492432, + "p95": 0.10051116347312927 + }, + { + "label": "completed_group_3", + "mean": 0.11250905692577362, + "median": 0.11015664041042328, + "p05": 0.09692119807004929, + "p95": 0.13486211001873016 + }, + { + "label": "completed_group_4", + "mean": 0.1775888353586197, + "median": 0.1758042573928833, + "p05": 0.1488552987575531, + "p95": 0.21211811900138855 + }, + { + "label": "current_group_partial", + "mean": 0.27521812915802, + "median": 0.2752543091773987, + "p05": 0.25022760033607483, + "p95": 0.3000151515007019 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.753357172012329, + "group": 5, + "latest_source_mass": 0.24494445323944092, + "layer": 18, + "max_source_mass": 0.24494445323944092, + "mixer_index": 35, + "normalized_entropy": 0.978567269839924, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15539543330669403, + "median": 0.14567650854587555, + "p05": 0.1150466799736023, + "p95": 0.23550888895988464 + }, + { + "label": "completed_group_1", + "mean": 0.1388990879058838, + "median": 0.1368958055973053, + "p05": 0.11385980248451233, + "p95": 0.17140795290470123 + }, + { + "label": "completed_group_2", + "mean": 0.15745997428894043, + "median": 0.15778976678848267, + "p05": 0.13043276965618134, + "p95": 0.1834889054298401 + }, + { + "label": "completed_group_3", + "mean": 0.12427005916833878, + "median": 0.12222100794315338, + "p05": 0.10104347765445709, + "p95": 0.1512138545513153 + }, + { + "label": "completed_group_4", + "mean": 0.17903099954128265, + "median": 0.1781059056520462, + "p05": 0.14208883047103882, + "p95": 0.21939483284950256 + }, + { + "label": "current_group_partial", + "mean": 0.24494445323944092, + "median": 0.24745827913284302, + "p05": 0.16898170113563538, + "p95": 0.3117744028568268 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7199265956878662, + "group": 5, + "latest_source_mass": 0.2730581760406494, + "layer": 19, + "max_source_mass": 0.2730581760406494, + "mixer_index": 36, + "normalized_entropy": 0.9599093099415087, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19207912683486938, + "median": 0.19417056441307068, + "p05": 0.14205873012542725, + "p95": 0.2466108500957489 + }, + { + "label": "completed_group_1", + "mean": 0.17904852330684662, + "median": 0.17943383753299713, + "p05": 0.15607136487960815, + "p95": 0.2006712406873703 + }, + { + "label": "completed_group_2", + "mean": 0.08410340547561646, + "median": 0.08364531397819519, + "p05": 0.07357755303382874, + "p95": 0.09571512043476105 + }, + { + "label": "completed_group_3", + "mean": 0.1121990978717804, + "median": 0.11130240559577942, + "p05": 0.0962604433298111, + "p95": 0.13100717961788177 + }, + { + "label": "completed_group_4", + "mean": 0.15951167047023773, + "median": 0.1591540426015854, + "p05": 0.13084900379180908, + "p95": 0.1905163675546646 + }, + { + "label": "current_group_partial", + "mean": 0.2730581760406494, + "median": 0.27290621399879456, + "p05": 0.241956889629364, + "p95": 0.3052297830581665 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6756346225738525, + "group": 5, + "latest_source_mass": 0.3124367296695709, + "layer": 19, + "max_source_mass": 0.3124367296695709, + "mixer_index": 37, + "normalized_entropy": 0.9351894890756556, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15863081812858582, + "median": 0.1342058777809143, + "p05": 0.07548131048679352, + "p95": 0.3739146590232849 + }, + { + "label": "completed_group_1", + "mean": 0.11242728680372238, + "median": 0.11213450133800507, + "p05": 0.08389876782894135, + "p95": 0.14075449109077454 + }, + { + "label": "completed_group_2", + "mean": 0.15970337390899658, + "median": 0.16262288391590118, + "p05": 0.10122501850128174, + "p95": 0.2050495743751526 + }, + { + "label": "completed_group_3", + "mean": 0.09774985909461975, + "median": 0.09510388225317001, + "p05": 0.06857988238334656, + "p95": 0.12727102637290955 + }, + { + "label": "completed_group_4", + "mean": 0.15905189514160156, + "median": 0.1553158164024353, + "p05": 0.11221202462911606, + "p95": 0.2187761515378952 + }, + { + "label": "current_group_partial", + "mean": 0.3124367296695709, + "median": 0.3196638822555542, + "p05": 0.18551334738731384, + "p95": 0.4139381945133209 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7271867990493774, + "group": 5, + "latest_source_mass": 0.2614278793334961, + "layer": 20, + "max_source_mass": 0.2614278793334961, + "mixer_index": 38, + "normalized_entropy": 0.9639613065884912, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19560003280639648, + "median": 0.1968384087085724, + "p05": 0.1501731276512146, + "p95": 0.2440420687198639 + }, + { + "label": "completed_group_1", + "mean": 0.18333327770233154, + "median": 0.18228237330913544, + "p05": 0.1655699908733368, + "p95": 0.2043052464723587 + }, + { + "label": "completed_group_2", + "mean": 0.08475088328123093, + "median": 0.08410285413265228, + "p05": 0.07304102182388306, + "p95": 0.09892263263463974 + }, + { + "label": "completed_group_3", + "mean": 0.11598999798297882, + "median": 0.1141754686832428, + "p05": 0.10139824450016022, + "p95": 0.1362377256155014 + }, + { + "label": "completed_group_4", + "mean": 0.15889792144298553, + "median": 0.15817126631736755, + "p05": 0.13111311197280884, + "p95": 0.18958830833435059 + }, + { + "label": "current_group_partial", + "mean": 0.2614278793334961, + "median": 0.2622074484825134, + "p05": 0.2309473156929016, + "p95": 0.2893397808074951 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6916961669921875, + "group": 5, + "latest_source_mass": 0.3091394901275635, + "layer": 20, + "max_source_mass": 0.3091394901275635, + "mixer_index": 39, + "normalized_entropy": 0.9441536076943532, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19065655767917633, + "median": 0.1910099983215332, + "p05": 0.09996911883354187, + "p95": 0.313973069190979 + }, + { + "label": "completed_group_1", + "mean": 0.12983909249305725, + "median": 0.12771058082580566, + "p05": 0.10737630724906921, + "p95": 0.15882958471775055 + }, + { + "label": "completed_group_2", + "mean": 0.11857748031616211, + "median": 0.11902420967817307, + "p05": 0.08811743557453156, + "p95": 0.14735448360443115 + }, + { + "label": "completed_group_3", + "mean": 0.09883742034435272, + "median": 0.0951075479388237, + "p05": 0.07493072748184204, + "p95": 0.1301351934671402 + }, + { + "label": "completed_group_4", + "mean": 0.1529499888420105, + "median": 0.15091747045516968, + "p05": 0.11475001275539398, + "p95": 0.19942505657672882 + }, + { + "label": "current_group_partial", + "mean": 0.3091394901275635, + "median": 0.3146688640117645, + "p05": 0.2252732366323471, + "p95": 0.3756124973297119 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7504222393035889, + "group": 6, + "latest_source_mass": 0.2152354121208191, + "layer": 21, + "max_source_mass": 0.2152354121208191, + "mixer_index": 40, + "normalized_entropy": 0.9769292527069633, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20809361338615417, + "median": 0.21471115946769714, + "p05": 0.13097207248210907, + "p95": 0.2735797166824341 + }, + { + "label": "completed_group_1", + "mean": 0.19103102385997772, + "median": 0.19057458639144897, + "p05": 0.16917173564434052, + "p95": 0.2132101207971573 + }, + { + "label": "completed_group_2", + "mean": 0.10743936896324158, + "median": 0.10638970136642456, + "p05": 0.09155704826116562, + "p95": 0.12698283791542053 + }, + { + "label": "completed_group_3", + "mean": 0.1194620281457901, + "median": 0.11578401178121567, + "p05": 0.10080886632204056, + "p95": 0.14768129587173462 + }, + { + "label": "completed_group_4", + "mean": 0.15873853862285614, + "median": 0.15696901082992554, + "p05": 0.13252565264701843, + "p95": 0.19312383234500885 + }, + { + "label": "completed_group_5", + "mean": 0.2152354121208191, + "median": 0.21616290509700775, + "p05": 0.18420100212097168, + "p95": 0.24399688839912415 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5379457473754883, + "group": 6, + "latest_source_mass": 0.515034556388855, + "layer": 21, + "max_source_mass": 0.515034556388855, + "mixer_index": 41, + "normalized_entropy": 0.7903477702308708, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09571269154548645, + "median": 0.08428268879652023, + "p05": 0.04900683835148811, + "p95": 0.1901935338973999 + }, + { + "label": "completed_group_1", + "mean": 0.07078731805086136, + "median": 0.06967149674892426, + "p05": 0.05498572438955307, + "p95": 0.08918046206235886 + }, + { + "label": "completed_group_2", + "mean": 0.0824701264500618, + "median": 0.08152244985103607, + "p05": 0.06332080066204071, + "p95": 0.10414593666791916 + }, + { + "label": "completed_group_3", + "mean": 0.060012347996234894, + "median": 0.058421529829502106, + "p05": 0.04666804522275925, + "p95": 0.07636430114507675 + }, + { + "label": "completed_group_4", + "mean": 0.08145605027675629, + "median": 0.0806138813495636, + "p05": 0.061518602073192596, + "p95": 0.10454171150922775 + }, + { + "label": "completed_group_5", + "mean": 0.09452696144580841, + "median": 0.09397821128368378, + "p05": 0.06939592957496643, + "p95": 0.12165899574756622 + }, + { + "label": "current_group_partial", + "mean": 0.515034556388855, + "median": 0.5195302963256836, + "p05": 0.41491252183914185, + "p95": 0.5988897681236267 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7319592237472534, + "group": 6, + "latest_source_mass": 0.3903183937072754, + "layer": 22, + "max_source_mass": 0.3903183937072754, + "mixer_index": 42, + "normalized_entropy": 0.8900509741357138, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13678354024887085, + "median": 0.14219121634960175, + "p05": 0.0800691545009613, + "p95": 0.18056681752204895 + }, + { + "label": "completed_group_1", + "mean": 0.11740624904632568, + "median": 0.11779546737670898, + "p05": 0.09323577582836151, + "p95": 0.13856223225593567 + }, + { + "label": "completed_group_2", + "mean": 0.06668868660926819, + "median": 0.06673324108123779, + "p05": 0.05682401731610298, + "p95": 0.07633707672357559 + }, + { + "label": "completed_group_3", + "mean": 0.07360264658927917, + "median": 0.07287117838859558, + "p05": 0.06238633394241333, + "p95": 0.0858106017112732 + }, + { + "label": "completed_group_4", + "mean": 0.09287553280591965, + "median": 0.0919380784034729, + "p05": 0.07644987106323242, + "p95": 0.11259186267852783 + }, + { + "label": "completed_group_5", + "mean": 0.12232495099306107, + "median": 0.12153243273496628, + "p05": 0.10421743988990784, + "p95": 0.14276890456676483 + }, + { + "label": "current_group_partial", + "mean": 0.3903183937072754, + "median": 0.382642537355423, + "p05": 0.32760486006736755, + "p95": 0.4824257791042328 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4663677215576172, + "group": 6, + "latest_source_mass": 0.5484944581985474, + "layer": 22, + "max_source_mass": 0.5484944581985474, + "mixer_index": 43, + "normalized_entropy": 0.7535639414129677, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09485072642564774, + "median": 0.07708277553319931, + "p05": 0.041287682950496674, + "p95": 0.2252470850944519 + }, + { + "label": "completed_group_1", + "mean": 0.07167200744152069, + "median": 0.07091978192329407, + "p05": 0.05350632965564728, + "p95": 0.09384822845458984 + }, + { + "label": "completed_group_2", + "mean": 0.0841417908668518, + "median": 0.08349095284938812, + "p05": 0.060824211686849594, + "p95": 0.11007672548294067 + }, + { + "label": "completed_group_3", + "mean": 0.05784624069929123, + "median": 0.056044802069664, + "p05": 0.04343996196985245, + "p95": 0.07539206743240356 + }, + { + "label": "completed_group_4", + "mean": 0.07052354514598846, + "median": 0.06955709308385849, + "p05": 0.048022691160440445, + "p95": 0.09658979624509811 + }, + { + "label": "completed_group_5", + "mean": 0.0724712610244751, + "median": 0.07230593264102936, + "p05": 0.05036160722374916, + "p95": 0.09567554295063019 + }, + { + "label": "current_group_partial", + "mean": 0.5484944581985474, + "median": 0.5530485510826111, + "p05": 0.4294937252998352, + "p95": 0.6537213325500488 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.749481201171875, + "group": 6, + "latest_source_mass": 0.37420469522476196, + "layer": 23, + "max_source_mass": 0.37420469522476196, + "mixer_index": 44, + "normalized_entropy": 0.8990554892892669, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14530612528324127, + "median": 0.1497679054737091, + "p05": 0.08296400308609009, + "p95": 0.20158618688583374 + }, + { + "label": "completed_group_1", + "mean": 0.12230369448661804, + "median": 0.12149038910865784, + "p05": 0.10498286038637161, + "p95": 0.14227813482284546 + }, + { + "label": "completed_group_2", + "mean": 0.0660901814699173, + "median": 0.06578399986028671, + "p05": 0.05611872673034668, + "p95": 0.07736622542142868 + }, + { + "label": "completed_group_3", + "mean": 0.07630561292171478, + "median": 0.07523077726364136, + "p05": 0.06339914351701736, + "p95": 0.09089048206806183 + }, + { + "label": "completed_group_4", + "mean": 0.09453228116035461, + "median": 0.09349468350410461, + "p05": 0.07686570286750793, + "p95": 0.11586424708366394 + }, + { + "label": "completed_group_5", + "mean": 0.12125740945339203, + "median": 0.12067118287086487, + "p05": 0.10250546038150787, + "p95": 0.14243635535240173 + }, + { + "label": "current_group_partial", + "mean": 0.37420469522476196, + "median": 0.36897724866867065, + "p05": 0.30640295147895813, + "p95": 0.4604528248310089 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.472255825996399, + "group": 6, + "latest_source_mass": 0.5438836812973022, + "layer": 23, + "max_source_mass": 0.5438836812973022, + "mixer_index": 45, + "normalized_entropy": 0.7565898285237576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10688216984272003, + "median": 0.0909440815448761, + "p05": 0.04993019625544548, + "p95": 0.22096660733222961 + }, + { + "label": "completed_group_1", + "mean": 0.07465942203998566, + "median": 0.07390505075454712, + "p05": 0.05582176893949509, + "p95": 0.09690531343221664 + }, + { + "label": "completed_group_2", + "mean": 0.08327433466911316, + "median": 0.08253603428602219, + "p05": 0.05906296521425247, + "p95": 0.10942579060792923 + }, + { + "label": "completed_group_3", + "mean": 0.054589346051216125, + "median": 0.052618082612752914, + "p05": 0.039012178778648376, + "p95": 0.07247936725616455 + }, + { + "label": "completed_group_4", + "mean": 0.0637175589799881, + "median": 0.062460802495479584, + "p05": 0.04167243465781212, + "p95": 0.09087979793548584 + }, + { + "label": "completed_group_5", + "mean": 0.07299347221851349, + "median": 0.07232271879911423, + "p05": 0.05079510435461998, + "p95": 0.09688537567853928 + }, + { + "label": "current_group_partial", + "mean": 0.5438836812973022, + "median": 0.5471436977386475, + "p05": 0.42356419563293457, + "p95": 0.6585544347763062 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7924723625183105, + "group": 6, + "latest_source_mass": 0.3345995247364044, + "layer": 24, + "max_source_mass": 0.3345995247364044, + "mixer_index": 46, + "normalized_entropy": 0.9211485758417507, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15522946417331696, + "median": 0.16076582670211792, + "p05": 0.0863914042711258, + "p95": 0.2195497751235962 + }, + { + "label": "completed_group_1", + "mean": 0.13724806904792786, + "median": 0.13687172532081604, + "p05": 0.1200050488114357, + "p95": 0.1560649424791336 + }, + { + "label": "completed_group_2", + "mean": 0.06889298558235168, + "median": 0.06847138702869415, + "p05": 0.058477431535720825, + "p95": 0.08076588809490204 + }, + { + "label": "completed_group_3", + "mean": 0.08198929578065872, + "median": 0.08025878667831421, + "p05": 0.06727518886327744, + "p95": 0.10007832199335098 + }, + { + "label": "completed_group_4", + "mean": 0.09816981852054596, + "median": 0.09709671884775162, + "p05": 0.07776231318712234, + "p95": 0.12178758531808853 + }, + { + "label": "completed_group_5", + "mean": 0.123870849609375, + "median": 0.12361469119787216, + "p05": 0.1038728728890419, + "p95": 0.14525073766708374 + }, + { + "label": "current_group_partial", + "mean": 0.3345995247364044, + "median": 0.3317973017692566, + "p05": 0.265953928232193, + "p95": 0.412749707698822 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5755163431167603, + "group": 6, + "latest_source_mass": 0.48653730750083923, + "layer": 24, + "max_source_mass": 0.48653730750083923, + "mixer_index": 47, + "normalized_entropy": 0.8096552371041545, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12274101376533508, + "median": 0.10600537061691284, + "p05": 0.06405961513519287, + "p95": 0.25983333587646484 + }, + { + "label": "completed_group_1", + "mean": 0.08900541067123413, + "median": 0.08780360221862793, + "p05": 0.06876012682914734, + "p95": 0.11320354044437408 + }, + { + "label": "completed_group_2", + "mean": 0.08551642298698425, + "median": 0.08451470732688904, + "p05": 0.06253526359796524, + "p95": 0.1114322617650032 + }, + { + "label": "completed_group_3", + "mean": 0.06285787373781204, + "median": 0.061114855110645294, + "p05": 0.045210886746644974, + "p95": 0.08213447034358978 + }, + { + "label": "completed_group_4", + "mean": 0.07429315149784088, + "median": 0.07252945005893707, + "p05": 0.049904488027095795, + "p95": 0.10469712316989899 + }, + { + "label": "completed_group_5", + "mean": 0.07904879748821259, + "median": 0.07821422815322876, + "p05": 0.05642634630203247, + "p95": 0.10449491441249847 + }, + { + "label": "current_group_partial", + "mean": 0.48653730750083923, + "median": 0.49373340606689453, + "p05": 0.3549031913280487, + "p95": 0.5985680222511292 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8390922546386719, + "group": 7, + "latest_source_mass": 0.29349833726882935, + "layer": 25, + "max_source_mass": 0.29349833726882935, + "mixer_index": 48, + "normalized_entropy": 0.945106461123861, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15884169936180115, + "median": 0.15805873274803162, + "p05": 0.09690988063812256, + "p95": 0.2504432201385498 + }, + { + "label": "completed_group_1", + "mean": 0.14726480841636658, + "median": 0.14679506421089172, + "p05": 0.12764519453048706, + "p95": 0.16903650760650635 + }, + { + "label": "completed_group_2", + "mean": 0.08053630590438843, + "median": 0.08051620423793793, + "p05": 0.06807594001293182, + "p95": 0.09329371899366379 + }, + { + "label": "completed_group_3", + "mean": 0.0909707322716713, + "median": 0.08902134001255035, + "p05": 0.07499626278877258, + "p95": 0.11060230433940887 + }, + { + "label": "completed_group_4", + "mean": 0.10738551616668701, + "median": 0.10607586801052094, + "p05": 0.08672098070383072, + "p95": 0.1325560212135315 + }, + { + "label": "completed_group_5", + "mean": 0.12150255590677261, + "median": 0.12205702066421509, + "p05": 0.0993877649307251, + "p95": 0.14230164885520935 + }, + { + "label": "completed_group_6", + "mean": 0.29349833726882935, + "median": 0.29394760727882385, + "p05": 0.22660650312900543, + "p95": 0.3548165559768677 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.6550344228744507, + "group": 7, + "latest_source_mass": 0.3589511513710022, + "layer": 25, + "max_source_mass": 0.3589511513710022, + "mixer_index": 49, + "normalized_entropy": 0.795903318127166, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0844201073050499, + "median": 0.0718768984079361, + "p05": 0.02839931659400463, + "p95": 0.21728233993053436 + }, + { + "label": "completed_group_1", + "mean": 0.06695197522640228, + "median": 0.06580687314271927, + "p05": 0.04618899151682854, + "p95": 0.09124172478914261 + }, + { + "label": "completed_group_2", + "mean": 0.061310961842536926, + "median": 0.06096746027469635, + "p05": 0.040395740419626236, + "p95": 0.08348730951547623 + }, + { + "label": "completed_group_3", + "mean": 0.0447731651365757, + "median": 0.042184896767139435, + "p05": 0.02882835641503334, + "p95": 0.06167382374405861 + }, + { + "label": "completed_group_4", + "mean": 0.04748315364122391, + "median": 0.04516277089715004, + "p05": 0.0278231929987669, + "p95": 0.0743657797574997 + }, + { + "label": "completed_group_5", + "mean": 0.045562684535980225, + "median": 0.044303566217422485, + "p05": 0.03103901632130146, + "p95": 0.06420586258172989 + }, + { + "label": "completed_group_6", + "mean": 0.2905467748641968, + "median": 0.2895723581314087, + "p05": 0.1524062305688858, + "p95": 0.43639296293258667 + }, + { + "label": "current_group_partial", + "mean": 0.3589511513710022, + "median": 0.3606260418891907, + "p05": 0.16963356733322144, + "p95": 0.546379804611206 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8679088354110718, + "group": 7, + "latest_source_mass": 0.32435470819473267, + "layer": 26, + "max_source_mass": 0.32435470819473267, + "mixer_index": 50, + "normalized_entropy": 0.8982742712267442, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11674155294895172, + "median": 0.12052157521247864, + "p05": 0.05360257253050804, + "p95": 0.17403334379196167 + }, + { + "label": "completed_group_1", + "mean": 0.1088850274682045, + "median": 0.1090676486492157, + "p05": 0.08014046400785446, + "p95": 0.13653376698493958 + }, + { + "label": "completed_group_2", + "mean": 0.05779466778039932, + "median": 0.05755443871021271, + "p05": 0.04364454001188278, + "p95": 0.07319164276123047 + }, + { + "label": "completed_group_3", + "mean": 0.06273135542869568, + "median": 0.06075543537735939, + "p05": 0.04851231724023819, + "p95": 0.07996035367250443 + }, + { + "label": "completed_group_4", + "mean": 0.07066236436367035, + "median": 0.06902407109737396, + "p05": 0.052817195653915405, + "p95": 0.09365298599004745 + }, + { + "label": "completed_group_5", + "mean": 0.08129343390464783, + "median": 0.08064926415681839, + "p05": 0.06491829454898834, + "p95": 0.09953147917985916 + }, + { + "label": "completed_group_6", + "mean": 0.17753681540489197, + "median": 0.17456364631652832, + "p05": 0.11964619904756546, + "p95": 0.24453631043434143 + }, + { + "label": "current_group_partial", + "mean": 0.32435470819473267, + "median": 0.32046452164649963, + "p05": 0.19449321925640106, + "p95": 0.4612511992454529 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5923268795013428, + "group": 7, + "latest_source_mass": 0.4404740333557129, + "layer": 26, + "max_source_mass": 0.4404740333557129, + "mixer_index": 51, + "normalized_entropy": 0.7657473641769285, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08656083047389984, + "median": 0.0795329213142395, + "p05": 0.050312869250774384, + "p95": 0.14976806938648224 + }, + { + "label": "completed_group_1", + "mean": 0.05770102143287659, + "median": 0.056408509612083435, + "p05": 0.04085284471511841, + "p95": 0.07934759557247162 + }, + { + "label": "completed_group_2", + "mean": 0.05105586349964142, + "median": 0.049396127462387085, + "p05": 0.034691184759140015, + "p95": 0.07344280183315277 + }, + { + "label": "completed_group_3", + "mean": 0.039375390857458115, + "median": 0.036708764731884, + "p05": 0.024915706366300583, + "p95": 0.05899723991751671 + }, + { + "label": "completed_group_4", + "mean": 0.04410157352685928, + "median": 0.04195208102464676, + "p05": 0.027429571375250816, + "p95": 0.06846393644809723 + }, + { + "label": "completed_group_5", + "mean": 0.04500962793827057, + "median": 0.043660376220941544, + "p05": 0.030392050743103027, + "p95": 0.06387433409690857 + }, + { + "label": "completed_group_6", + "mean": 0.2357216626405716, + "median": 0.2248685657978058, + "p05": 0.11950507014989853, + "p95": 0.3818710744380951 + }, + { + "label": "current_group_partial", + "mean": 0.4404740333557129, + "median": 0.4484519362449646, + "p05": 0.24098733067512512, + "p95": 0.6138508915901184 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8469831943511963, + "group": 7, + "latest_source_mass": 0.33607518672943115, + "layer": 27, + "max_source_mass": 0.33607518672943115, + "mixer_index": 52, + "normalized_entropy": 0.8882111650319092, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10824646055698395, + "median": 0.11040513217449188, + "p05": 0.05085912346839905, + "p95": 0.16382771730422974 + }, + { + "label": "completed_group_1", + "mean": 0.09116974472999573, + "median": 0.09135521948337555, + "p05": 0.056257300078868866, + "p95": 0.1266864538192749 + }, + { + "label": "completed_group_2", + "mean": 0.0701109915971756, + "median": 0.06960364431142807, + "p05": 0.05100060999393463, + "p95": 0.0908425822854042 + }, + { + "label": "completed_group_3", + "mean": 0.0629766583442688, + "median": 0.06064727157354355, + "p05": 0.04317091405391693, + "p95": 0.08672259747982025 + }, + { + "label": "completed_group_4", + "mean": 0.07049155235290527, + "median": 0.06874367594718933, + "p05": 0.050685517489910126, + "p95": 0.09697030484676361 + }, + { + "label": "completed_group_5", + "mean": 0.07297641038894653, + "median": 0.07156282663345337, + "p05": 0.051110126078128815, + "p95": 0.09974705427885056 + }, + { + "label": "completed_group_6", + "mean": 0.18795298039913177, + "median": 0.1818247139453888, + "p05": 0.1166272684931755, + "p95": 0.27661576867103577 + }, + { + "label": "current_group_partial", + "mean": 0.33607518672943115, + "median": 0.3355332612991333, + "p05": 0.20117856562137604, + "p95": 0.4717060327529907 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6590619087219238, + "group": 7, + "latest_source_mass": 0.4179539084434509, + "layer": 27, + "max_source_mass": 0.4179539084434509, + "mixer_index": 53, + "normalized_entropy": 0.7978401294136326, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09444728493690491, + "median": 0.09060665965080261, + "p05": 0.047914937138557434, + "p95": 0.15662360191345215 + }, + { + "label": "completed_group_1", + "mean": 0.0654030293226242, + "median": 0.06396090984344482, + "p05": 0.04467916116118431, + "p95": 0.09097769111394882 + }, + { + "label": "completed_group_2", + "mean": 0.060137998312711716, + "median": 0.05839194357395172, + "p05": 0.04035535454750061, + "p95": 0.08539871126413345 + }, + { + "label": "completed_group_3", + "mean": 0.0449567511677742, + "median": 0.04218417406082153, + "p05": 0.02788582444190979, + "p95": 0.06733109056949615 + }, + { + "label": "completed_group_4", + "mean": 0.044405773282051086, + "median": 0.04241851717233658, + "p05": 0.02826966717839241, + "p95": 0.06812064349651337 + }, + { + "label": "completed_group_5", + "mean": 0.05244143307209015, + "median": 0.05142951011657715, + "p05": 0.035461653023958206, + "p95": 0.07271309196949005 + }, + { + "label": "completed_group_6", + "mean": 0.2202538251876831, + "median": 0.21207934617996216, + "p05": 0.11892802268266678, + "p95": 0.34954455494880676 + }, + { + "label": "current_group_partial", + "mean": 0.4179539084434509, + "median": 0.4249612092971802, + "p05": 0.2265779674053192, + "p95": 0.586824893951416 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.884438157081604, + "group": 7, + "latest_source_mass": 0.2829248011112213, + "layer": 28, + "max_source_mass": 0.2829248011112213, + "mixer_index": 54, + "normalized_entropy": 0.9062231946945226, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11613816022872925, + "median": 0.12147872895002365, + "p05": 0.04899268597364426, + "p95": 0.1770799458026886 + }, + { + "label": "completed_group_1", + "mean": 0.09775452315807343, + "median": 0.09778773784637451, + "p05": 0.05219751223921776, + "p95": 0.14079101383686066 + }, + { + "label": "completed_group_2", + "mean": 0.08394932001829147, + "median": 0.08419600129127502, + "p05": 0.05889316648244858, + "p95": 0.10900868475437164 + }, + { + "label": "completed_group_3", + "mean": 0.07057419419288635, + "median": 0.06822949647903442, + "p05": 0.05265102535486221, + "p95": 0.0926743820309639 + }, + { + "label": "completed_group_4", + "mean": 0.06920696794986725, + "median": 0.06799854338169098, + "p05": 0.05105053633451462, + "p95": 0.09136901795864105 + }, + { + "label": "completed_group_5", + "mean": 0.06340071558952332, + "median": 0.0628691017627716, + "p05": 0.0437660738825798, + "p95": 0.0860220268368721 + }, + { + "label": "completed_group_6", + "mean": 0.21605131030082703, + "median": 0.21297892928123474, + "p05": 0.14316481351852417, + "p95": 0.2999357581138611 + }, + { + "label": "current_group_partial", + "mean": 0.2829248011112213, + "median": 0.2790382504463196, + "p05": 0.14648588001728058, + "p95": 0.4251174330711365 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7424441576004028, + "group": 7, + "latest_source_mass": 0.278835266828537, + "layer": 28, + "max_source_mass": 0.3030073344707489, + "mixer_index": 55, + "normalized_entropy": 0.8379385150653496, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08763164281845093, + "median": 0.05824008211493492, + "p05": 0.027796253561973572, + "p95": 0.2880856990814209 + }, + { + "label": "completed_group_1", + "mean": 0.07771545648574829, + "median": 0.07654327154159546, + "p05": 0.046035416424274445, + "p95": 0.11438068747520447 + }, + { + "label": "completed_group_2", + "mean": 0.07502467930316925, + "median": 0.07301308214664459, + "p05": 0.04458038881421089, + "p95": 0.11157117038965225 + }, + { + "label": "completed_group_3", + "mean": 0.06006643548607826, + "median": 0.05717916786670685, + "p05": 0.036717794835567474, + "p95": 0.08720126748085022 + }, + { + "label": "completed_group_4", + "mean": 0.06314835697412491, + "median": 0.061049528419971466, + "p05": 0.03865773230791092, + "p95": 0.09540070593357086 + }, + { + "label": "completed_group_5", + "mean": 0.054570816457271576, + "median": 0.05328378826379776, + "p05": 0.03375566378235817, + "p95": 0.07989827543497086 + }, + { + "label": "completed_group_6", + "mean": 0.3030073344707489, + "median": 0.30074357986450195, + "p05": 0.14776623249053955, + "p95": 0.469026654958725 + }, + { + "label": "current_group_partial", + "mean": 0.278835266828537, + "median": 0.26309943199157715, + "p05": 0.10582277178764343, + "p95": 0.5049530863761902 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8953055143356323, + "group": 8, + "latest_source_mass": 0.18233531713485718, + "layer": 29, + "max_source_mass": 0.2779633104801178, + "mixer_index": 56, + "normalized_entropy": 0.911449288833841, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1135263666510582, + "median": 0.0953834131360054, + "p05": 0.02703639306128025, + "p95": 0.31321749091148376 + }, + { + "label": "completed_group_1", + "mean": 0.10274903476238251, + "median": 0.10363410413265228, + "p05": 0.06506763398647308, + "p95": 0.13818706572055817 + }, + { + "label": "completed_group_2", + "mean": 0.0858360230922699, + "median": 0.08501487225294113, + "p05": 0.059830889105796814, + "p95": 0.11487221717834473 + }, + { + "label": "completed_group_3", + "mean": 0.08475606143474579, + "median": 0.08265408128499985, + "p05": 0.05679357051849365, + "p95": 0.11555421352386475 + }, + { + "label": "completed_group_4", + "mean": 0.08078759908676147, + "median": 0.07859720289707184, + "p05": 0.055059194564819336, + "p95": 0.11340422928333282 + }, + { + "label": "completed_group_5", + "mean": 0.07204629480838776, + "median": 0.07071015238761902, + "p05": 0.0498509518802166, + "p95": 0.09922727942466736 + }, + { + "label": "completed_group_6", + "mean": 0.2779633104801178, + "median": 0.2749147415161133, + "p05": 0.15705733001232147, + "p95": 0.39374127984046936 + }, + { + "label": "completed_group_7", + "mean": 0.18233531713485718, + "median": 0.16221198439598083, + "p05": 0.07201667875051498, + "p95": 0.35018864274024963 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.801485300064087, + "group": 8, + "latest_source_mass": 0.3215197026729584, + "layer": 29, + "max_source_mass": 0.3215197026729584, + "mixer_index": 57, + "normalized_entropy": 0.8198912931549752, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07824120670557022, + "median": 0.06902759522199631, + "p05": 0.017845120280981064, + "p95": 0.17217300832271576 + }, + { + "label": "completed_group_1", + "mean": 0.060231711715459824, + "median": 0.05764210224151611, + "p05": 0.03765189275145531, + "p95": 0.0922410786151886 + }, + { + "label": "completed_group_2", + "mean": 0.054100219160318375, + "median": 0.0524706169962883, + "p05": 0.034336626529693604, + "p95": 0.07949143648147583 + }, + { + "label": "completed_group_3", + "mean": 0.039871927350759506, + "median": 0.037151023745536804, + "p05": 0.0235070139169693, + "p95": 0.060298819094896317 + }, + { + "label": "completed_group_4", + "mean": 0.04207881912589073, + "median": 0.040273066610097885, + "p05": 0.021681729704141617, + "p95": 0.06873302906751633 + }, + { + "label": "completed_group_5", + "mean": 0.045095667243003845, + "median": 0.044285234063863754, + "p05": 0.025416042655706406, + "p95": 0.06805071234703064 + }, + { + "label": "completed_group_6", + "mean": 0.2047339677810669, + "median": 0.1937887817621231, + "p05": 0.08607597649097443, + "p95": 0.3561563789844513 + }, + { + "label": "completed_group_7", + "mean": 0.15412676334381104, + "median": 0.13271009922027588, + "p05": 0.05149175971746445, + "p95": 0.32324373722076416 + }, + { + "label": "current_group_partial", + "mean": 0.3215197026729584, + "median": 0.31218427419662476, + "p05": 0.08214674144983292, + "p95": 0.5716250538825989 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.008213520050049, + "group": 8, + "latest_source_mass": 0.1996053159236908, + "layer": 30, + "max_source_mass": 0.21496367454528809, + "mixer_index": 58, + "normalized_entropy": 0.9139773606959576, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09438058733940125, + "median": 0.09028880298137665, + "p05": 0.02690170146524906, + "p95": 0.18525993824005127 + }, + { + "label": "completed_group_1", + "mean": 0.07153058052062988, + "median": 0.070882648229599, + "p05": 0.041449472308158875, + "p95": 0.10379751026630402 + }, + { + "label": "completed_group_2", + "mean": 0.07423336803913116, + "median": 0.0730072408914566, + "p05": 0.048166655004024506, + "p95": 0.10590565949678421 + }, + { + "label": "completed_group_3", + "mean": 0.06403130292892456, + "median": 0.06157837435603142, + "p05": 0.04371092468500137, + "p95": 0.08951176702976227 + }, + { + "label": "completed_group_4", + "mean": 0.0645662397146225, + "median": 0.0623253732919693, + "p05": 0.043447867035865784, + "p95": 0.09214113652706146 + }, + { + "label": "completed_group_5", + "mean": 0.05577655881643295, + "median": 0.055099695920944214, + "p05": 0.03416105732321739, + "p95": 0.079859159886837 + }, + { + "label": "completed_group_6", + "mean": 0.21496367454528809, + "median": 0.2118406593799591, + "p05": 0.13403329253196716, + "p95": 0.31010040640830994 + }, + { + "label": "completed_group_7", + "mean": 0.1609123945236206, + "median": 0.15419161319732666, + "p05": 0.07973575592041016, + "p95": 0.27241089940071106 + }, + { + "label": "current_group_partial", + "mean": 0.1996053159236908, + "median": 0.19533705711364746, + "p05": 0.0843900740146637, + "p95": 0.33647382259368896 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.8966764211654663, + "group": 8, + "latest_source_mass": 0.21615305542945862, + "layer": 30, + "max_source_mass": 0.21615305542945862, + "mixer_index": 59, + "normalized_entropy": 0.8632146393815058, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09610478579998016, + "median": 0.0941460132598877, + "p05": 0.030834276229143143, + "p95": 0.17860566079616547 + }, + { + "label": "completed_group_1", + "mean": 0.07267358899116516, + "median": 0.06876465678215027, + "p05": 0.03731950744986534, + "p95": 0.11969097703695297 + }, + { + "label": "completed_group_2", + "mean": 0.060228411108255386, + "median": 0.05793646723031998, + "p05": 0.0349978432059288, + "p95": 0.09366682916879654 + }, + { + "label": "completed_group_3", + "mean": 0.046177469193935394, + "median": 0.04286886006593704, + "p05": 0.02664034441113472, + "p95": 0.07394801080226898 + }, + { + "label": "completed_group_4", + "mean": 0.04206448048353195, + "median": 0.03912021219730377, + "p05": 0.022682487964630127, + "p95": 0.07153408974409103 + }, + { + "label": "completed_group_5", + "mean": 0.04096413776278496, + "median": 0.039336949586868286, + "p05": 0.023794004693627357, + "p95": 0.06405855715274811 + }, + { + "label": "completed_group_6", + "mean": 0.21562576293945312, + "median": 0.20660808682441711, + "p05": 0.10648336261510849, + "p95": 0.3508927822113037 + }, + { + "label": "completed_group_7", + "mean": 0.21000832319259644, + "median": 0.204543337225914, + "p05": 0.08861437439918518, + "p95": 0.3495482802391052 + }, + { + "label": "current_group_partial", + "mean": 0.21615305542945862, + "median": 0.19288693368434906, + "p05": 0.07421350479125977, + "p95": 0.43364205956459045 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.981720209121704, + "group": 8, + "latest_source_mass": 0.17098912596702576, + "layer": 31, + "max_source_mass": 0.23084521293640137, + "mixer_index": 60, + "normalized_entropy": 0.9019197352708571, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10501759499311447, + "median": 0.10529834032058716, + "p05": 0.012508269399404526, + "p95": 0.22564347088336945 + }, + { + "label": "completed_group_1", + "mean": 0.06946458667516708, + "median": 0.06936883926391602, + "p05": 0.025395508855581284, + "p95": 0.11417273432016373 + }, + { + "label": "completed_group_2", + "mean": 0.07816962152719498, + "median": 0.07627303898334503, + "p05": 0.042986176908016205, + "p95": 0.11851722747087479 + }, + { + "label": "completed_group_3", + "mean": 0.07385894656181335, + "median": 0.07061562687158585, + "p05": 0.04739787429571152, + "p95": 0.1065390408039093 + }, + { + "label": "completed_group_4", + "mean": 0.06364607810974121, + "median": 0.06080539897084236, + "p05": 0.0418400801718235, + "p95": 0.09455618262290955 + }, + { + "label": "completed_group_5", + "mean": 0.05328303202986717, + "median": 0.05173730105161667, + "p05": 0.03466875106096268, + "p95": 0.07786844670772552 + }, + { + "label": "completed_group_6", + "mean": 0.23084521293640137, + "median": 0.2265024036169052, + "p05": 0.1294252872467041, + "p95": 0.3483029305934906 + }, + { + "label": "completed_group_7", + "mean": 0.1547257900238037, + "median": 0.15095551311969757, + "p05": 0.057022616267204285, + "p95": 0.2705056667327881 + }, + { + "label": "current_group_partial", + "mean": 0.17098912596702576, + "median": 0.14655888080596924, + "p05": 0.04728491231799126, + "p95": 0.3703039288520813 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0028607845306396, + "group": 8, + "latest_source_mass": 0.12469469010829926, + "layer": 31, + "max_source_mass": 0.2620389461517334, + "mixer_index": 61, + "normalized_entropy": 0.911541225776195, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0842779129743576, + "median": 0.06333573907613754, + "p05": 0.03144961595535278, + "p95": 0.24860140681266785 + }, + { + "label": "completed_group_1", + "mean": 0.10298711061477661, + "median": 0.10017631947994232, + "p05": 0.06670551002025604, + "p95": 0.14914412796497345 + }, + { + "label": "completed_group_2", + "mean": 0.09207960218191147, + "median": 0.09137493371963501, + "p05": 0.056820835918188095, + "p95": 0.13043460249900818 + }, + { + "label": "completed_group_3", + "mean": 0.07771934568881989, + "median": 0.0746275931596756, + "p05": 0.04656440392136574, + "p95": 0.11502407491207123 + }, + { + "label": "completed_group_4", + "mean": 0.06265503168106079, + "median": 0.0610337033867836, + "p05": 0.038839153945446014, + "p95": 0.09315080940723419 + }, + { + "label": "completed_group_5", + "mean": 0.055270493030548096, + "median": 0.0542943961918354, + "p05": 0.035113874822854996, + "p95": 0.07895103842020035 + }, + { + "label": "completed_group_6", + "mean": 0.2620389461517334, + "median": 0.25993645191192627, + "p05": 0.13434287905693054, + "p95": 0.3996090292930603 + }, + { + "label": "completed_group_7", + "mean": 0.1382768601179123, + "median": 0.12482025474309921, + "p05": 0.058253247290849686, + "p95": 0.2677440643310547 + }, + { + "label": "current_group_partial", + "mean": 0.12469469010829926, + "median": 0.11286871135234833, + "p05": 0.04615473002195358, + "p95": 0.24487172067165375 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0655057430267334, + "group": 8, + "latest_source_mass": 0.09924688190221786, + "layer": 32, + "max_source_mass": 0.21871674060821533, + "mixer_index": 62, + "normalized_entropy": 0.9400521750629723, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10472239553928375, + "median": 0.08473113924264908, + "p05": 0.030345402657985687, + "p95": 0.2588862180709839 + }, + { + "label": "completed_group_1", + "mean": 0.11792458593845367, + "median": 0.11655837297439575, + "p05": 0.07099610567092896, + "p95": 0.17131033539772034 + }, + { + "label": "completed_group_2", + "mean": 0.0928182601928711, + "median": 0.09008213877677917, + "p05": 0.06447342038154602, + "p95": 0.12974750995635986 + }, + { + "label": "completed_group_3", + "mean": 0.10183011740446091, + "median": 0.10004840791225433, + "p05": 0.0707390084862709, + "p95": 0.1374078094959259 + }, + { + "label": "completed_group_4", + "mean": 0.08287280797958374, + "median": 0.08090724050998688, + "p05": 0.05679214745759964, + "p95": 0.11534547060728073 + }, + { + "label": "completed_group_5", + "mean": 0.06570011377334595, + "median": 0.06424097716808319, + "p05": 0.04414675012230873, + "p95": 0.09294299781322479 + }, + { + "label": "completed_group_6", + "mean": 0.21871674060821533, + "median": 0.21457019448280334, + "p05": 0.13963721692562103, + "p95": 0.3112483620643616 + }, + { + "label": "completed_group_7", + "mean": 0.11616809666156769, + "median": 0.10701116174459457, + "p05": 0.05174187570810318, + "p95": 0.21264401078224182 + }, + { + "label": "current_group_partial", + "mean": 0.09924688190221786, + "median": 0.08725139498710632, + "p05": 0.032173603773117065, + "p95": 0.20341788232326508 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0258567333221436, + "group": 8, + "latest_source_mass": 0.12343063950538635, + "layer": 32, + "max_source_mass": 0.22868779301643372, + "mixer_index": 63, + "normalized_entropy": 0.9220071330979595, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09674200415611267, + "median": 0.07686861604452133, + "p05": 0.04638649895787239, + "p95": 0.2122267782688141 + }, + { + "label": "completed_group_1", + "mean": 0.09482249617576599, + "median": 0.09244489669799805, + "p05": 0.06637479364871979, + "p95": 0.13019198179244995 + }, + { + "label": "completed_group_2", + "mean": 0.08996410667896271, + "median": 0.08937586098909378, + "p05": 0.06193297728896141, + "p95": 0.12055040895938873 + }, + { + "label": "completed_group_3", + "mean": 0.06836152076721191, + "median": 0.06606560945510864, + "p05": 0.04674140736460686, + "p95": 0.09358662366867065 + }, + { + "label": "completed_group_4", + "mean": 0.06293784081935883, + "median": 0.061281464993953705, + "p05": 0.04231695458292961, + "p95": 0.08961650729179382 + }, + { + "label": "completed_group_5", + "mean": 0.05016915872693062, + "median": 0.049648597836494446, + "p05": 0.03570414334535599, + "p95": 0.06624776124954224 + }, + { + "label": "completed_group_6", + "mean": 0.22868779301643372, + "median": 0.2242993861436844, + "p05": 0.1197199895977974, + "p95": 0.3530009686946869 + }, + { + "label": "completed_group_7", + "mean": 0.1848844289779663, + "median": 0.17580115795135498, + "p05": 0.10029035806655884, + "p95": 0.3003215491771698 + }, + { + "label": "current_group_partial", + "mean": 0.12343063950538635, + "median": 0.10480014234781265, + "p05": 0.05131315439939499, + "p95": 0.2450288087129593 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.7394249439239502, + "group": null, + "latest_source_mass": 0.2556654214859009, + "layer": null, + "max_source_mass": 0.3314756751060486, + "mixer_index": null, + "normalized_entropy": 0.7916464078663832, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10602210462093353, + "median": 0.10817556083202362, + "p05": 0.03750653192400932, + "p95": 0.16905617713928223 + }, + { + "label": "completed_group_1", + "mean": 0.05902107432484627, + "median": 0.05735805630683899, + "p05": 0.039041999727487564, + "p95": 0.08384283632040024 + }, + { + "label": "completed_group_2", + "mean": 0.049884356558322906, + "median": 0.04808305203914642, + "p05": 0.03180687129497528, + "p95": 0.07381068915128708 + }, + { + "label": "completed_group_3", + "mean": 0.034865088760852814, + "median": 0.03287554159760475, + "p05": 0.02153801918029785, + "p95": 0.05384448915719986 + }, + { + "label": "completed_group_4", + "mean": 0.03282830864191055, + "median": 0.0315137580037117, + "p05": 0.020427687093615532, + "p95": 0.050021201372146606 + }, + { + "label": "completed_group_5", + "mean": 0.043302543461322784, + "median": 0.041625380516052246, + "p05": 0.027595922350883484, + "p95": 0.06415434926748276 + }, + { + "label": "completed_group_6", + "mean": 0.08693542331457138, + "median": 0.07748770713806152, + "p05": 0.03693632781505585, + "p95": 0.17101936042308807 + }, + { + "label": "completed_group_7", + "mean": 0.3314756751060486, + "median": 0.32931262254714966, + "p05": 0.0907490998506546, + "p95": 0.5930418968200684 + }, + { + "label": "completed_group_8", + "mean": 0.2556654214859009, + "median": 0.23854607343673706, + "p05": 0.04562881588935852, + "p95": 0.5189820528030396 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", + "optimizer_state_before": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.08528144657611847, + 0.2063225358724594, + 0.07479097694158554, + 0.053203217685222626, + 0.1070452407002449, + 0.03943018242716789, + 0.041509829461574554, + 0.06257687509059906, + 0.061002109199762344, + 0.04113767296075821, + 0.040399398654699326, + 0.03907898813486099, + 0.06714949756860733, + 0.04437125846743584, + 0.0412614606320858, + 0.046277787536382675, + 0.10073095560073853, + 0.05776162073016167, + 0.05185067281126976, + 0.07827401161193848, + 0.04170985147356987, + 0.041845910251140594, + 0.038240060210227966, + 0.046402644366025925, + 0.030182205140590668, + 0.025695007294416428, + 0.024306239560246468, + 0.017157377675175667, + 0.030125634744763374, + 0.01829664409160614, + 0.024464696645736694, + 0.032671861350536346 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1464898420854297e-05, + "first_to_last_ratio": 0.8455323660117368, + "imbalance_abs_log_ratio": 0.16778883105478307, + "last_quartile_mean": 1.355938445612992e-05, + "mean": 1.3058880313110421e-05, + "normalized": [ + 0.8914129879074366, + 1.077571175585955, + 0.9537440677326601, + 0.8574453930361242, + 0.9048950001587052, + 0.8425968627438281, + 0.7839582579962869, + 0.7118870894358409, + 0.5788084225246481, + 0.530693141917732, + 0.5023627734883869, + 0.4844869540829288, + 0.6659476097626051, + 0.6212422516778212, + 0.5836177358524666, + 0.5530522631862488, + 0.6705030645582619, + 0.6390296210597277, + 0.6539820627182751, + 0.682372291616315, + 3.0705747439001825, + 2.5957154007666214, + 2.1136981524200404, + 1.7237888584415353, + 1.9773452676094638, + 1.7509357725203292, + 1.3561771885891878, + 0.9354730786944212, + 0.8232148160475435, + 0.5856284416831322, + 0.45203530207589593, + 0.4258039502093924 + ], + "peak_layer": 21, + "peak_normalized": 3.0705747439001825, + "population_cv": 0.6442856671110848, + "rest_mean": 9.924206164281781e-06, + "spike_contrast": 3.0215132797934157, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.998612071678508e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.1640855518635362e-05, + 1.4071873010834679e-05, + 1.2454829629859887e-05, + 1.119727676268667e-05, + 1.1816915503004566e-05, + 1.100337158277398e-05, + 1.0237617061648052e-05, + 9.29644829739118e-06, + 7.558589913969627e-06, + 6.930258223292185e-06, + 6.560295332747046e-06, + 6.326857146632392e-06, + 8.696530130691826e-06, + 8.112728210107889e-06, + 7.6213941611058544e-06, + 7.222243311844068e-06, + 8.756019269640092e-06, + 8.345011337951291e-06, + 8.540273483959027e-06, + 8.91101808520034e-06, + 4.009826807305217e-05, + 3.389713674550876e-05, + 2.7602531190495938e-05, + 2.2510752387461253e-05, + 2.5821915187407285e-05, + 2.2865260689286515e-05, + 1.7710155589156784e-05, + 1.2216230970807374e-05, + 1.0750263754744083e-05, + 7.64765172789339e-06, + 5.90307490710984e-06, + 5.560522822634084e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.568297117657494e-05, + "first_to_last_ratio": 0.8467277881503474, + "imbalance_abs_log_ratio": 0.16637601951453293, + "last_quartile_mean": 5.3952370308252284e-05, + "mean": 5.2044935273443116e-05, + "normalized": [ + 0.8954256553649687, + 1.0662758237422765, + 0.9521993752570108, + 0.8636540973713456, + 0.9057610370429903, + 0.8375327549345261, + 0.7860647798171227, + 0.7151678277570597, + 0.5850341337918961, + 0.5346127123576437, + 0.503507273341017, + 0.4847402912324663, + 0.6665718638058081, + 0.6225508325294531, + 0.5847778078263528, + 0.5518762696320233, + 0.6703631394314256, + 0.6385628522390465, + 0.6481962908175356, + 0.6867108224315941, + 3.0618207183884687, + 2.6065455407648237, + 2.120445260989786, + 1.7184046595544948, + 1.970749079693583, + 1.7371379145623163, + 1.3426067482179225, + 0.9360274925849964, + 0.8279776750708923, + 0.5898304420688857, + 0.45715187233946364, + 0.4317169550408049 + ], + "peak_layer": 21, + "peak_normalized": 3.0618207183884687, + "population_cv": 0.6425897668413451, + "rest_mean": 3.955807295386438e-05, + "spike_contrast": 3.0202176920627024, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011947399179916829, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.6602370275650173e-05, + 5.549425623030402e-05, + 4.95571548526641e-05, + 4.494882159633562e-05, + 4.714027454610914e-05, + 4.35893380199559e-05, + 4.091069058631547e-05, + 3.7220863305265084e-05, + 3.044806362595409e-05, + 2.7823884011013433e-05, + 2.620500345074106e-05, + 2.5228277081623673e-05, + 3.4691689506871626e-05, + 3.240061778342351e-05, + 3.0434723157668486e-05, + 2.872236473194789e-05, + 3.4889006201410666e-05, + 3.32339623128064e-05, + 3.373533400008455e-05, + 3.5739820305025205e-05, + 0.00015935226110741496, + 0.00013565749395638704, + 0.00011035843635909259, + 8.943425928009674e-05, + 0.00010256750829285011, + 9.04092303244397e-05, + 6.987588130868971e-05, + 4.8715490265749395e-05, + 4.30920445069205e-05, + 3.0697687179781497e-05, + 2.3792439606040716e-05, + 2.246868098154664e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.040512115228921175, + "first_to_last_ratio": 0.8467277882597376, + "imbalance_abs_log_ratio": 0.16637601938534127, + "last_quartile_mean": 0.047845500987023115, + "mean": 0.046153969364240766, + "normalized": [ + 0.8954256214236297, + 1.0662758259809275, + 0.9521993286286553, + 0.8636540898502394, + 0.9057610206102777, + 0.8375327282419698, + 0.7860647826245731, + 0.7151678297957529, + 0.5850341696121162, + 0.5346127266472548, + 0.5035072907230911, + 0.484740303083307, + 0.666571864231768, + 0.6225508316512409, + 0.5847778206533832, + 0.5518762470155281, + 0.6703631006740365, + 0.6385628347715187, + 0.6481963011119384, + 0.6867108353729822, + 3.0618206630891383, + 2.6065457195235173, + 2.120445297956706, + 1.7184047348201537, + 1.9707489675025585, + 1.7371379031231275, + 1.3426067663321783, + 0.9360274706240039, + 0.8279776759485932, + 0.5898304215013819, + 0.45715187640138183, + 0.4317169504730694 + ], + "peak_layer": 21, + "peak_normalized": 3.0618206630891383, + "population_cv": 0.6425897745941355, + "rest_mean": 0.03508049470406992, + "spike_contrast": 3.0202177427352903, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10595073252916336, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.041327446699142456, + 0.04921286180615425, + 0.04394777864217758, + 0.03986106440424919, + 0.041804466396570206, + 0.03865545988082886, + 0.036280009895563126, + 0.03300783410668373, + 0.027001649141311646, + 0.02467449940741062, + 0.023238860070705414, + 0.022372689098119736, + 0.03076493740081787, + 0.02873319201171398, + 0.02698981761932373, + 0.02547127939760685, + 0.030939918011426926, + 0.02947220951318741, + 0.029916832223534584, + 0.031694430857896805, + 0.14131517708301544, + 0.12030243128538132, + 0.0978669673204422, + 0.07931119948625565, + 0.0909578874707222, + 0.08017580956220627, + 0.061966631561517715, + 0.04320138320326805, + 0.03821445629000664, + 0.027223015204072, + 0.021099373698234558, + 0.019925450906157494 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8178302667593016e-06, + "first_to_last_ratio": 0.5015852279301855, + "imbalance_abs_log_ratio": 0.6899817399970322, + "last_quartile_mean": 3.6241702616734983e-06, + "mean": 2.789410469716813e-06, + "normalized": [ + 0.8037687132877044, + 0.7183737285046393, + 0.691319672118338, + 0.6807254875963729, + 0.6052756387601809, + 0.5818941580160492, + 0.565889251881305, + 0.5662715078678539, + 0.4369805018739323, + 0.41687613565271625, + 0.40783237228271835, + 0.41545507589336067, + 0.5056083509177715, + 0.5034210266522742, + 0.5059866535158252, + 0.503281516870639, + 0.5409761795262856, + 0.5584199152903219, + 0.570327765976957, + 0.6037643031707576, + 2.871505340690978, + 2.8594035701851728, + 2.496246607238993, + 2.1963141894218743, + 2.393616152828075, + 2.14842944375899, + 1.7201180299928378, + 1.2992827591669334, + 1.010592310992817, + 0.7253855724579202, + 0.5639386418465228, + 0.532719425762883 + ], + "peak_layer": 21, + "peak_normalized": 2.871505340690978, + "population_cv": 0.7722487160005334, + "rest_mean": 1.9818156126213445e-06, + "spike_contrast": 3.6080161305089784, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.150422698032344e-06, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.242040864075534e-06, + 2.0038391994603444e-06, + 1.9283743313280866e-06, + 1.8988228021044051e-06, + 1.6883622038221802e-06, + 1.6231416566370171e-06, + 1.5784974038979271e-06, + 1.5795636727489182e-06, + 1.2189179869892541e-06, + 1.162838657364773e-06, + 1.1376118891348597e-06, + 1.1588747383939335e-06, + 1.4103492276262841e-06, + 1.4042478824194404e-06, + 1.4114044688540162e-06, + 1.4038587323739193e-06, + 1.5090046190380235e-06, + 1.5576623582091997e-06, + 1.5908782415863243e-06, + 1.6841464685057872e-06, + 8.009807061171159e-06, + 7.976050255820155e-06, + 6.96305642122752e-06, + 6.126421794760972e-06, + 6.676777957181912e-06, + 5.9928515838691965e-06, + 4.798115242010681e-06, + 3.6242329315427924e-06, + 2.818956772898673e-06, + 2.0233981103956467e-06, + 1.5730563518445706e-06, + 1.4859731436445145e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.489204118523048e-05, + "first_to_last_ratio": 1.0102007473277157, + "imbalance_abs_log_ratio": 0.01014907083348142, + "last_quartile_mean": 3.45397103274081e-05, + "mean": 3.719139070312849e-05, + "normalized": [ + 0.978746778311648, + 1.126838378619792, + 1.0180419362177582, + 0.9373857091569082, + 0.9573174487244173, + 0.8927610480088312, + 0.8376961987226147, + 0.7566142687378048, + 0.6421710022561828, + 0.5898929811647327, + 0.5545314669590201, + 0.532480114188549, + 0.7191190555706168, + 0.6685869838104541, + 0.6276517153974418, + 0.5907065315366481, + 0.7404063048506205, + 0.7009976783342637, + 0.7076013069062036, + 0.7352296418703005, + 3.079471174742761, + 2.569668642671924, + 2.031249327096867, + 1.5752201563285502, + 1.8426406293315767, + 1.5558607485184148, + 1.2001391008391664, + 0.8699295985429139, + 0.6815919538112388, + 0.4881760018644189, + 0.4044368033178356, + 0.3868393135895246 + ], + "peak_layer": 21, + "peak_normalized": 3.079471174742761, + "population_cv": 0.6121845820849332, + "rest_mean": 2.8791301971338084e-05, + "spike_contrast": 2.867250322231956, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.255186985479668e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.640095383161679e-05, + 4.1908686398528516e-05, + 3.786239540204406e-05, + 3.486267814878374e-05, + 3.560396726243198e-05, + 3.320302494103089e-05, + 3.115508661721833e-05, + 2.8139536880189553e-05, + 2.3883232643129304e-05, + 2.1938940335530788e-05, + 2.0623796444851905e-05, + 1.98036759684328e-05, + 2.674503775779158e-05, + 2.4865679733920842e-05, + 2.3343240172835067e-05, + 2.196919740526937e-05, + 2.7536740162759088e-05, + 2.6071078536915593e-05, + 2.631667666719295e-05, + 2.7344212867319584e-05, + 0.00011452981561888009, + 9.55695504671894e-05, + 7.554498733952641e-05, + 5.858462827745825e-05, + 6.853036757092923e-05, + 5.786462497781031e-05, + 4.463484219741076e-05, + 3.235389158362523e-05, + 2.5349352654302493e-05, + 1.8155944417230785e-05, + 1.5041567166917957e-05, + 1.4387092051038053e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6185193746641744e-05, + "first_to_last_ratio": 1.1813045118653667, + "imbalance_abs_log_ratio": 0.16661934637193965, + "last_quartile_mean": 2.2166336862028402e-05, + "mean": 2.6416844121968097e-05, + "normalized": [ + 1.0582483794330777, + 1.1839113566742743, + 1.0890809034246702, + 1.013584100454415, + 1.0023428903115454, + 0.935888205711016, + 0.8664359614724408, + 0.7803558787188484, + 0.6948576391403528, + 0.6454578452882075, + 0.6088646607200737, + 0.5814486923097771, + 0.764934624577467, + 0.7118430529936276, + 0.6645388585723574, + 0.629085742341511, + 0.8117172258565646, + 0.7682825984070001, + 0.7655811246450389, + 0.7857710139637165, + 3.1099202343035985, + 2.532060860240954, + 1.9118190653037337, + 1.3711803024378004, + 1.6503311681253898, + 1.3613514907810353, + 1.0865408308937656, + 0.8606461743085547, + 0.5744935662233659, + 0.4301536296331042, + 0.37929000098103194, + 0.3699819217516835 + ], + "peak_layer": 21, + "peak_normalized": 3.1099202343035985, + "population_cv": 0.583027433434022, + "rest_mean": 2.096195011189492e-05, + "spike_contrast": 2.6654615376008266, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5873271776363254e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.795558248180896e-05, + 3.127520176349208e-05, + 2.8770080461981706e-05, + 2.6775693186209537e-05, + 2.6478735890123062e-05, + 2.4723212845856324e-05, + 2.2888503735885024e-05, + 2.061453960777726e-05, + 1.835594594012946e-05, + 1.7050959286279976e-05, + 1.608428283361718e-05, + 1.5360039469669573e-05, + 2.020715874095913e-05, + 1.8804646970238537e-05, + 1.755501943989657e-05, + 1.661845999478828e-05, + 2.1443007426569238e-05, + 2.0295601643738337e-05, + 2.0224237232469022e-05, + 2.0757590391440317e-05, + 8.215427806135267e-05, + 6.688905705232173e-05, + 5.050422623753548e-05, + 3.6222256312612444e-05, + 4.3596541217993945e-05, + 3.59626101271715e-05, + 2.8702979761874303e-05, + 2.2735555830877274e-05, + 1.5176306987996213e-05, + 1.136330138251651e-05, + 1.0019644832937047e-05, + 9.77375475486042e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.068558847502572e-05, + "first_to_last_ratio": 0.7914793598068353, + "imbalance_abs_log_ratio": 0.2338514773139948, + "last_quartile_mean": 0.00011457732580311131, + "mean": 0.00010754580421235005, + "normalized": [ + 0.8538738720411836, + 1.0095472825344882, + 0.8946236658075398, + 0.8160443171140711, + 0.8628188117624934, + 0.8243645351821149, + 0.7801447421092808, + 0.7044034320375209, + 0.5665887799168225, + 0.525608047514618, + 0.4958157795364288, + 0.4792984745903937, + 0.6722545548598279, + 0.6294814179611099, + 0.5936561952518512, + 0.5594680423070397, + 0.6646720414600984, + 0.6282112723775689, + 0.6412680496471301, + 0.6738175767739937, + 3.007412745002923, + 2.616093975094252, + 2.1689004360837822, + 1.8085787931443134, + 2.061415837562002, + 1.8302883103495597, + 1.4233838379602504, + 0.9902275928882596, + 0.8088632655175125, + 0.5559016015107706, + 0.43871981288186035, + 0.4142529012189394 + ], + "peak_layer": 21, + "peak_normalized": 3.007412745002923, + "population_cv": 0.6573728454605334, + "rest_mean": 8.100827242877265e-05, + "spike_contrast": 3.096578513808316, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000250848475843668, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.183055226458237e-05, + 0.00010857257439056411, + 9.621302160667256e-05, + 8.776214235695079e-05, + 9.279254300054163e-05, + 8.865694690030068e-05, + 8.390129369217902e-05, + 7.575563358841464e-05, + 6.093424599384889e-05, + 5.6526940170442685e-05, + 5.3322906751418486e-05, + 5.1546539907576516e-05, + 7.229815673781559e-05, + 6.769808533135802e-05, + 6.384523294400424e-05, + 6.016844054101966e-05, + 7.148268923629075e-05, + 6.756148650310934e-05, + 6.896568811498582e-05, + 7.246625318657607e-05, + 0.00032343462225981057, + 0.00028134993044659495, + 0.00023325614165514708, + 0.00019450506079010665, + 0.00022169662406668067, + 0.00019683982827700675, + 0.00015307895955629647, + 0.00010649482283042744, + 8.698985038790852e-05, + 5.978488479740918e-05, + 4.7182475100271404e-05, + 4.455116140889004e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13266965746879578, + 0.045500293374061584, + 0.018729008734226227, + 0.03815881907939911, + 0.02209169790148735, + 0.01640710048377514, + 0.02340921014547348, + 0.05481670796871185, + 0.037687551230192184, + 0.04149388521909714, + 0.048693906515836716, + 0.06190986558794975, + 0.04065176844596863, + 0.049332331866025925, + 0.0589165985584259, + 0.08173932880163193, + 0.043543290346860886, + 0.05228494107723236, + 0.04403071105480194, + 0.0687810555100441, + 0.030229199677705765, + 0.04699210822582245, + 0.06082639843225479, + 0.07667754590511322, + 0.050203632563352585, + 0.05539015680551529, + 0.06946423649787903, + 0.07630576938390732, + 0.07239200174808502, + 0.09628339111804962, + 0.12468792498111725, + 0.15056027472019196 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1373777851986233e-05, + "first_to_last_ratio": 1.0318498984683846, + "imbalance_abs_log_ratio": 0.03135320926034069, + "last_quartile_mean": 1.1022705791674525e-05, + "mean": 1.1832125267119409e-05, + "normalized": [ + 0.9800012201604696, + 1.1859002007221142, + 1.0527348906308314, + 0.9435939146843411, + 0.9942658078795703, + 0.9213667974504651, + 0.8458121244297144, + 0.7664247312681599, + 0.6009152433386855, + 0.5652338271700392, + 0.5379286363125223, + 0.5220099956402732, + 0.7166619388130793, + 0.6623381156132455, + 0.6162294438595727, + 0.5904010132472348, + 0.7324356475968063, + 0.6882143187892091, + 0.7092129547489621, + 0.8089361635707597, + 3.115397457088153, + 2.4321274256844663, + 1.9642577727784198, + 1.5948693957730091, + 1.9522870367789025, + 1.5037975945274742, + 1.0368577963312928, + 0.8807302172788337, + 0.6474760441659657, + 0.5016437023911682, + 0.4711656138967313, + 0.45877295737952767 + ], + "peak_layer": 21, + "peak_normalized": 3.115397457088153, + "population_cv": 0.603271891680543, + "rest_mean": 9.176935405034951e-06, + "spike_contrast": 2.851730928390011, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.617015052237548e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1595497198868543e-05, + 1.4031719729246106e-05, + 1.2456091099011246e-05, + 1.1164721399836708e-05, + 1.1764277587644756e-05, + 1.0901727364398539e-05, + 1.000775500870077e-05, + 9.068433428183198e-06, + 7.110104434104869e-06, + 6.687917448289227e-06, + 6.364839009620482e-06, + 6.176487659104168e-06, + 8.479633834213018e-06, + 7.836867553123739e-06, + 7.291303973033791e-06, + 6.985698746575508e-06, + 8.66627033246914e-06, + 8.143038030539174e-06, + 8.391496521653607e-06, + 9.571434020472225e-06, + 3.686177296913229e-05, + 2.8777236366295256e-05, + 2.3241344024427235e-05, + 1.8870694475481287e-05, + 2.309970477654133e-05, + 1.7793121514841914e-05, + 1.226823133038124e-05, + 1.0420910257380456e-05, + 7.661017662030645e-06, + 5.9355111261538696e-06, + 5.574890565185342e-06, + 5.428259100881405e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.52927956757776e-05, + "first_to_last_ratio": 1.0314200550157464, + "imbalance_abs_log_ratio": 0.03093654690539193, + "last_quartile_mean": 4.3913045374210924e-05, + "mean": 4.717359485084671e-05, + "normalized": [ + 0.9832135136979314, + 1.173164387917132, + 1.0505394497504255, + 0.9501840008496486, + 0.9944561477628384, + 0.9147691984159712, + 0.8464742625690872, + 0.7682410828301822, + 0.6067449473404415, + 0.5681699639617173, + 0.5386389915171271, + 0.5211401199814687, + 0.7167243803511351, + 0.6632392162921593, + 0.6172595012574109, + 0.588538933455825, + 0.7318717018485573, + 0.6893632656633977, + 0.7026754642712614, + 0.8079854981931587, + 3.1150142181403937, + 2.4538347722826446, + 1.966137677039432, + 1.5845641436792013, + 1.9375330190421414, + 1.4892879811658426, + 1.035008152926599, + 0.8840423036226741, + 0.6517018335432763, + 0.5065002027138268, + 0.477654085237329, + 0.4653275826797635 + ], + "peak_layer": 21, + "peak_normalized": 3.1150142181403937, + "population_cv": 0.6027267551758898, + "rest_mean": 3.659083860704278e-05, + "spike_contrast": 2.8509999371074524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010432047856738791, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.638171594706364e-05, + 5.534238152904436e-05, + 4.9557722377358004e-05, + 4.482359508983791e-05, + 4.691207141149789e-05, + 4.315295154810883e-05, + 3.993123391410336e-05, + 3.6240693589206785e-05, + 2.862234032363631e-05, + 2.6802619686350226e-05, + 2.5409537556697614e-05, + 2.458405288052745e-05, + 3.381046553840861e-05, + 3.1287378078559414e-05, + 2.9118349630152807e-05, + 2.7763497200794518e-05, + 3.4525019145803526e-05, + 3.251974339946173e-05, + 3.3147727663163096e-05, + 3.8115580537123606e-05, + 0.00014694641868118197, + 0.00011575620737858117, + 9.274978219764307e-05, + 7.47495869291015e-05, + 9.140039765043184e-05, + 7.025506783975288e-05, + 4.8825055273482576e-05, + 4.1703453462105244e-05, + 3.074311825912446e-05, + 2.3893435354693793e-05, + 2.253266029583756e-05, + 2.1951174858259037e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.0401661042124033, + "first_to_last_ratio": 1.0314200458773342, + "imbalance_abs_log_ratio": 0.030936538045362262, + "last_quartile_mean": 0.03894252819009125, + "mean": 0.041834015981294215, + "normalized": [ + 0.9832134785727225, + 1.1731643857154097, + 1.0505394772834944, + 0.9501839314909589, + 0.9944561321819699, + 0.9147691986147233, + 0.8464742543512297, + 0.7682411398349651, + 0.6067449159473433, + 0.5681700011232179, + 0.5386389788463947, + 0.5211401213699254, + 0.7167244214988735, + 0.663239179094363, + 0.6172595023900571, + 0.5885389178924195, + 0.7318717092957638, + 0.6893632192579286, + 0.7026754233788964, + 0.8079854502108175, + 3.1150143563768617, + 2.4538347987960454, + 1.9661377249833973, + 1.5845640989337733, + 1.937532949114683, + 1.4892880148885772, + 1.0350081198083678, + 0.8840423776654689, + 0.6517018613150984, + 0.5065001920223166, + 0.4776540925208594, + 0.46532757522307666 + ], + "peak_layer": 21, + "peak_normalized": 3.1150143563768617, + "population_cv": 0.6027267719407451, + "rest_mean": 0.03244912169045872, + "spike_contrast": 2.850999975725323, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09251244515180587, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04113176837563515, + 0.04907817766070366, + 0.04394828528165817, + 0.03975000977516174, + 0.04160209372639656, + 0.03826846927404404, + 0.03541141748428345, + 0.03213861212134361, + 0.025382576510310173, + 0.02376883290708065, + 0.02253343164920807, + 0.021801384165883064, + 0.029983460903167725, + 0.027745958417654037, + 0.025822443887591362, + 0.024620946496725082, + 0.030617132782936096, + 0.028838831931352615, + 0.029395734891295433, + 0.03380127623677254, + 0.13031356036663055, + 0.10265376418828964, + 0.08225143700838089, + 0.06628867983818054, + 0.08105478435754776, + 0.06230289861559868, + 0.043298546224832535, + 0.03698304295539856, + 0.027263306081295013, + 0.02118893712759018, + 0.019982188940048218, + 0.019466521218419075 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.809132328389751e-06, + "first_to_last_ratio": 0.5997744339081645, + "imbalance_abs_log_ratio": 0.5112016379368517, + "last_quartile_mean": 3.016354526153009e-06, + "mean": 2.487152045205221e-06, + "normalized": [ + 0.890408531440174, + 0.809673961899388, + 0.7756270411999202, + 0.7664197016433177, + 0.6818557608603003, + 0.6497146124011205, + 0.6267527377763467, + 0.6186766662305586, + 0.4670749596929491, + 0.4522141597899267, + 0.449235216497964, + 0.45731293359098185, + 0.5550856819872789, + 0.5467983404610478, + 0.5410652082029204, + 0.536255547908205, + 0.6011915384635866, + 0.6092812314836655, + 0.6241321581033094, + 0.6880744209704973, + 2.915841858020288, + 2.6264204515100484, + 2.3560883070579246, + 2.052603139729255, + 2.421562802944245, + 1.9231629319277221, + 1.4565088666484294, + 1.2845058483948424, + 0.8068415180205192, + 0.6295503966151642, + 0.5969673730151487, + 0.5830960955129549 + ], + "peak_layer": 21, + "peak_normalized": 2.915841858020288, + "population_cv": 0.7106082711854734, + "rest_mean": 1.8080198363653228e-06, + "spike_contrast": 3.4039814437617255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.154465972940671e-06, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.214581400039606e-06, + 2.013782250287477e-06, + 1.9291023818368558e-06, + 1.906202328427753e-06, + 1.695878950158658e-06, + 1.6159390270331642e-06, + 1.5588293535984121e-06, + 1.5387429357360816e-06, + 1.1616864412644645e-06, + 1.1247253723922768e-06, + 1.1173162874911213e-06, + 1.1374067980796099e-06, + 1.3805824892187957e-06, + 1.3599706107925158e-06, + 1.3457114391712821e-06, + 1.3337490827325382e-06, + 1.4952547644497827e-06, + 1.5153750609897543e-06, + 1.5523115735049942e-06, + 1.7113457033701707e-06, + 7.252142040670151e-06, + 6.532306997542037e-06, + 5.859949851583224e-06, + 5.105136096972274e-06, + 6.022794877935667e-06, + 4.783198619406903e-06, + 3.6225590065441793e-06, + 3.1947613479132997e-06, + 2.0067375317012193e-06, + 1.5657875565011636e-06, + 1.484748622715415e-06, + 1.4502486465062248e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.464051883383945e-05, + "first_to_last_ratio": 1.2085017018903634, + "imbalance_abs_log_ratio": 0.18938132942715577, + "last_quartile_mean": 2.866402155632386e-05, + "mean": 3.421727353725146e-05, + "normalized": [ + 1.059950361009556, + 1.2224231923999602, + 1.1066505484293667, + 1.0166204096806029, + 1.0363170309532028, + 0.9614969179125695, + 0.8913049122896795, + 0.8041913955656627, + 0.6595104635459774, + 0.6198402043713972, + 0.5858567866753434, + 0.5655998592669403, + 0.7633893605247403, + 0.7040501232645258, + 0.6548491311439005, + 0.6226536428847976, + 0.7974262571555777, + 0.7481683838617732, + 0.7573132817097481, + 0.8440618552058439, + 3.1217920974447857, + 2.4268766121530727, + 1.8727793487823996, + 1.45522837174708, + 1.7605498171411489, + 1.332061600542984, + 0.9562588529424351, + 0.830045802534728, + 0.533682078716892, + 0.4444300159412478, + 0.423319578555294, + 0.4213017056467665 + ], + "peak_layer": 21, + "peak_normalized": 3.1217920974447857, + "population_cv": 0.5796313993689626, + "rest_mean": 2.7073180481912225e-05, + "spike_contrast": 2.6888372455803045, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.279537603608333e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.626861143857241e-05, + 4.182798875262961e-05, + 3.7866564525756985e-05, + 3.478597864159383e-05, + 3.5459943319438025e-05, + 3.28998030454386e-05, + 3.0498023988911882e-05, + 2.751723695837427e-05, + 2.2566649931832217e-05, + 2.1209241822361946e-05, + 2.0046421923325397e-05, + 1.9353285097167827e-05, + 2.6121102564502507e-05, + 2.4090675651677884e-05, + 2.2407151845982298e-05, + 2.1305510017555207e-05, + 2.7285752366879024e-05, + 2.5600282242521644e-05, + 2.5913195713656023e-05, + 2.8881495381938294e-05, + 0.00010681921412469819, + 8.30411008791998e-05, + 6.408140325220302e-05, + 4.979394725523889e-05, + 6.0241214669076726e-05, + 4.557951615424827e-05, + 3.2720570743549615e-05, + 2.84019042737782e-05, + 1.826114566938486e-05, + 1.5207183423626702e-05, + 1.4484841813100502e-05, + 1.4415795703826007e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6040052716780338e-05, + "first_to_last_ratio": 1.3601380691640932, + "imbalance_abs_log_ratio": 0.3075862160391937, + "last_quartile_mean": 1.9145153942190518e-05, + "mean": 2.476174321941471e-05, + "normalized": [ + 1.1242415044070515, + 1.2621466755459967, + 1.1626111778912407, + 1.0807522798319469, + 1.0668131611583063, + 0.9898612219145286, + 0.9055963728518246, + 0.8209725998576762, + 0.7059309419194623, + 0.6698532100359812, + 0.6332217391849956, + 0.605864404334209, + 0.7980357739095889, + 0.7372559068278083, + 0.6856394720285257, + 0.6565553942525388, + 0.8578727127097449, + 0.8064680621295278, + 0.8069038983319957, + 0.8750777012450611, + 3.1375211320361442, + 2.3884634889966896, + 1.7586693778594196, + 1.27827401077098, + 1.559779974845964, + 1.1976668654320728, + 0.9315461470291763, + 0.8389979859505794, + 0.4605019751312927, + 0.40913221789275306, + 0.3951692615572719, + 0.3926033521296457 + ], + "peak_layer": 21, + "peak_normalized": 3.1375211320361442, + "population_cv": 0.5588199044745877, + "rest_mean": 2.0063699526805e-05, + "spike_contrast": 2.4986009729926506, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0131179159507154e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.78381794487359e-05, + 3.1252951885107905e-05, + 2.8788279450964183e-05, + 2.67613104369957e-05, + 2.6416153559694067e-05, + 2.451068939990364e-05, + 2.2424144844990224e-05, + 2.0328712707851082e-05, + 1.7480080714449286e-05, + 1.6586733181611635e-05, + 1.5679674106650054e-05, + 1.5002258805907331e-05, + 1.9760756913456134e-05, + 1.8255741451866925e-05, + 1.697762854746543e-05, + 1.6257456081802957e-05, + 2.124242382706143e-05, + 1.9969555069110356e-05, + 1.9980347133241594e-05, + 2.1668449335265905e-05, + 7.769049261696637e-05, + 5.914251960348338e-05, + 4.3547719542402774e-05, + 3.165229281876236e-05, + 3.8622871215920895e-05, + 2.96563193842303e-05, + 2.3066706489771605e-05, + 2.0775052689714357e-05, + 1.140283166023437e-05, + 1.013082692224998e-05, + 9.785079782886896e-06, + 9.721543392515741e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.001170565170469e-05, + "first_to_last_ratio": 0.9626745767731016, + "imbalance_abs_log_ratio": 0.038039850801722765, + "last_quartile_mean": 9.350169602839742e-05, + "mean": 9.76294164729552e-05, + "normalized": [ + 0.9359776868957951, + 1.1142596124196091, + 0.9861551159867753, + 0.8961521822246734, + 0.949684039356357, + 0.8995149572091795, + 0.846411402631371, + 0.747630614092772, + 0.5888085074137973, + 0.5541267695979373, + 0.5294131458395803, + 0.5184084456985816, + 0.7260183643779575, + 0.6675529417749002, + 0.6222711070136198, + 0.5862023232639773, + 0.7253041044725564, + 0.6816202250286443, + 0.6942528027900181, + 0.79595263681847, + 3.0863426882796725, + 2.474889704855176, + 2.017495070776267, + 1.693791348848989, + 2.023091547543398, + 1.5725916380980733, + 1.090726894893622, + 0.9642815025941395, + 0.6146236296489728, + 0.48585096781736253, + 0.45630461264981964, + 0.45429340908793464 + ], + "peak_layer": 21, + "peak_normalized": 3.0863426882796725, + "population_cv": 0.6173495462344238, + "rest_mean": 7.486509181304697e-05, + "spike_contrast": 2.9460562232025813, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022055676963645966, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.137895540334284e-05, + 0.00010878451575990766, + 9.627774852560833e-05, + 8.749081462156028e-05, + 9.271709859604016e-05, + 8.781912038102746e-05, + 8.263465133495629e-05, + 7.299074059119448e-05, + 5.7485030993120745e-05, + 5.409907316789031e-05, + 5.168629650142975e-05, + 5.06119140482042e-05, + 7.088074926286936e-05, + 6.517280417028815e-05, + 6.075196506571956e-05, + 5.723059075535275e-05, + 7.081101648509502e-05, + 6.654618482571095e-05, + 6.777949602110311e-05, + 7.770839147269726e-05, + 0.0003013178356923163, + 0.00024162203771993518, + 0.0001969668664969504, + 0.0001653638610150665, + 0.00019751324725802988, + 0.00015353120397776365, + 0.00010648703027982265, + 9.414224041393027e-05, + 6.0005346313118935e-05, + 4.743334648082964e-05, + 4.4548753066919744e-05, + 4.435240043676458e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.08528144657611847, + 0.3133305609226227, + 0.3510810136795044, + 0.36267346143722534, + 0.1070452407002449, + 0.1260577291250229, + 0.14502130448818207, + 0.17329032719135284, + 0.061002109199762344, + 0.0987050011754036, + 0.13416865468025208, + 0.17569874227046967, + 0.06714949756860733, + 0.09735818952322006, + 0.13312570750713348, + 0.17941679060459137, + 0.10073095560073853, + 0.13780468702316284, + 0.17965665459632874, + 0.24453195929527283, + 0.04170985147356987, + 0.07352267950773239, + 0.10462787002325058, + 0.14256280660629272, + 0.030182205140590668, + 0.06915032118558884, + 0.10093247890472412, + 0.1392749696969986, + 0.030125634744763374, + 0.08267941325902939, + 0.15948866307735443, + 0.2670789062976837 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1464836006780388e-05, + "first_to_last_ratio": 0.8455234102400129, + "imbalance_abs_log_ratio": 0.16779942298421527, + "last_quartile_mean": 1.355945425984828e-05, + "mean": 1.3058895916628899e-05, + "normalized": [ + 0.8914048885899613, + 1.0775847922014457, + 0.9537205719054163, + 0.8574486865386339, + 0.9048781790315702, + 0.8425943237574262, + 0.7839480584125434, + 0.7118847066289794, + 0.5788136508075115, + 0.5307043127450849, + 0.5023599445771596, + 0.4844905191035793, + 0.6659490427095206, + 0.6212508418929359, + 0.5836098650152179, + 0.5530587758649329, + 0.6705037956060123, + 0.6390296236121042, + 0.6539787044155942, + 0.6823628402237571, + 3.070552131396758, + 2.5957343072679153, + 2.1136920052807127, + 1.723798777803649, + 1.977372713282607, + 1.75094231645673, + 1.356176403898088, + 0.9354566389044705, + 0.8232143895880623, + 0.5856320599681389, + 0.4520406818345609, + 0.4258114506789209 + ], + "peak_layer": 21, + "peak_normalized": 3.070552131396758, + "population_cv": 0.6442860381062553, + "rest_mean": 9.924204715827256e-06, + "spike_contrast": 3.0215245714485643, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.998622840095777e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.1640763659670483e-05, + 1.4072067642700858e-05, + 1.2454537682060618e-05, + 1.119733315135818e-05, + 1.1816709957201965e-05, + 1.100335157389054e-05, + 1.0237496098852716e-05, + 9.296428288507741e-06, + 7.558667221019277e-06, + 6.9304123826441355e-06, + 6.560266228916589e-06, + 6.3269112615671474e-06, + 8.696559234522283e-06, + 8.112850082397927e-06, + 7.621300483151572e-06, + 7.22233698979835e-06, + 8.75603927852353e-06, + 8.345021342393011e-06, + 8.540239832655061e-06, + 8.91090530785732e-06, + 4.0098020690493286e-05, + 3.389742414583452e-05, + 2.7602483896771446e-05, + 2.2510908820549957e-05, + 2.5822304451139644e-05, + 2.2865373466629535e-05, + 1.7710166503093205e-05, + 1.2216030881972983e-05, + 1.0750271030701697e-05, + 7.6477081165649e-06, + 5.9031522141594905e-06, + 5.560627414524788e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.568271015159553e-05, + "first_to_last_ratio": 0.8467191676221869, + "imbalance_abs_log_ratio": 0.16638620055756023, + "last_quartile_mean": 5.3952611324348254e-05, + "mean": 5.204496915212076e-05, + "normalized": [ + 0.8954358371924881, + 1.066255627359817, + 0.9521818394578381, + 0.863653814778354, + 0.9057514302498563, + 0.8375363338827018, + 0.7860525947143857, + 0.7151691796362102, + 0.5850370033455559, + 0.5346182010583443, + 0.5035057922216662, + 0.4847392766839134, + 0.6665845013282112, + 0.6225484700608501, + 0.5847766932080282, + 0.5518779375078734, + 0.6703635418666527, + 0.6385663510046552, + 0.6481965678809122, + 0.6867100259137117, + 3.061813692446919, + 2.6065600609918396, + 2.1204217920704185, + 1.7184136066569389, + 1.970772681476859, + 1.7371423758267366, + 1.3426072722615594, + 0.9360215708260244, + 0.8279779050062409, + 0.5898318056357381, + 0.4571555590952484, + 0.4317206583534514 + ], + "peak_layer": 21, + "peak_normalized": 3.061813692446919, + "population_cv": 0.6425899958232327, + "rest_mean": 3.9558066755826415e-05, + "spike_contrast": 3.0202244925053914, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011947424209211022, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.6602930524386466e-05, + 5.549324123421684e-05, + 4.955627446179278e-05, + 4.494883614825085e-05, + 4.7139805246843025e-05, + 4.358955266070552e-05, + 4.0910083043854684e-05, + 3.722095789271407e-05, + 3.044823279196862e-05, + 2.7824187782243825e-05, + 2.6204943424090743e-05, + 2.5228240701835603e-05, + 3.469236980890855e-05, + 3.2400515920016915e-05, + 3.0434684958891012e-05, + 2.8722470233333297e-05, + 3.488904985715635e-05, + 3.3234166039619595e-05, + 3.373537037987262e-05, + 3.573980211513117e-05, + 0.00015935199917294085, + 0.0001356583379674703, + 0.00011035728675778955, + 8.943478314904496e-05, + 0.00010256880341330543, + 9.040952136274427e-05, + 6.987595406826586e-05, + 4.8715213779360056e-05, + 4.309208452468738e-05, + 3.0697778129251674e-05, + 2.379264697083272e-05, + 2.2468888346338645e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04051188472658396, + "first_to_last_ratio": 0.846719171620572, + "imbalance_abs_log_ratio": 0.16638619583535103, + "last_quartile_mean": 0.047845715656876564, + "mean": 0.04615399963222444, + "normalized": [ + 0.8954358499233408, + 1.0662556745533454, + 0.9521818348722626, + 0.8636537656053828, + 0.9057513866007281, + 0.8375363761309916, + 0.7860525635386049, + 0.7151692172158937, + 0.5850370145184794, + 0.53461818747956, + 0.5035057498056084, + 0.48473925875942064, + 0.666584502816651, + 0.6225484458781422, + 0.5847766703673021, + 0.551877902951673, + 0.670363548905047, + 0.6385663306455772, + 0.6481965620941408, + 0.6867099814534051, + 3.0618138122732117, + 2.606560153011009, + 2.1204217916267223, + 1.7184136164632142, + 1.9707726965272079, + 1.7371424139057434, + 1.3426073387034982, + 0.9360215296259088, + 0.8279778593866045, + 0.5898318104039583, + 0.45715553160267397, + 0.43172062235469105 + ], + "peak_layer": 21, + "peak_normalized": 3.0618138122732117, + "population_cv": 0.6425900249685824, + "rest_mean": 0.03508048918512133, + "spike_contrast": 3.020224589442675, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10595095604658127, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.04132794588804245, + 0.04921196401119232, + 0.0439470000565052, + 0.03986107558012009, + 0.04180404916405678, + 0.03865565359592438, + 0.03627946972846985, + 0.033007919788360596, + 0.027001798152923584, + 0.02467476762831211, + 0.023238804191350937, + 0.02237265557050705, + 0.030765540897846222, + 0.02873310074210167, + 0.026989782229065895, + 0.02547137252986431, + 0.03093995898962021, + 0.029472390189766884, + 0.02991686388850212, + 0.03169441223144531, + 0.14131495356559753, + 0.12030317634344101, + 0.09786594659090042, + 0.07931166142225266, + 0.09095904231071472, + 0.08017607033252716, + 0.06196669861674309, + 0.04320113733410835, + 0.038214489817619324, + 0.027223097160458565, + 0.021099556237459183, + 0.019925633445382118 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8178741925112263e-06, + "first_to_last_ratio": 0.5015983374707057, + "imbalance_abs_log_ratio": 0.68995560412126, + "last_quartile_mean": 3.6241631136135766e-06, + "mean": 2.789416313930815e-06, + "normalized": [ + 0.8037911571270303, + 0.7184217017994549, + 0.6913125177981712, + 0.6806867284307945, + 0.605325927592527, + 0.5818942838314957, + 0.5658827679202331, + 0.5663181288215361, + 0.4369798716346615, + 0.41689441779300684, + 0.4078213694495809, + 0.41540374891403675, + 0.5056054983122913, + 0.5034202979673874, + 0.5059732849415142, + 0.5032640783142611, + 0.5409407291356678, + 0.5584123873096231, + 0.5703010167375573, + 0.6038059140356934, + 2.8715612743645713, + 2.85957495161325, + 2.4960596032794165, + 2.196308283631167, + 2.3935907596217825, + 2.1483375605717328, + 1.7201321959397329, + 1.2992909597305788, + 1.010652795641223, + 0.7253891064801847, + 0.563940476297644, + 0.5327062049621921 + ], + "peak_layer": 21, + "peak_normalized": 2.8715612743645713, + "population_cv": 0.7722499515786573, + "rest_mean": 1.981818833748413e-06, + "spike_contrast": 3.608020362482595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.150442706915783e-06, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2421081666834652e-06, + 2.0039772152813384e-06, + 1.9283584151708055e-06, + 1.8987186649610521e-06, + 1.688506017671898e-06, + 1.623145408302662e-06, + 1.5784826246090233e-06, + 1.5796970274095656e-06, + 1.2189187827971182e-06, + 1.1628920901785023e-06, + 1.137583581112267e-06, + 1.1587339940888342e-06, + 1.4103442254054244e-06, + 1.4042487919141422e-06, + 1.4113701354290242e-06, + 1.4038130302651552e-06, + 1.508908894720662e-06, + 1.5576446230625152e-06, + 1.5908069599390728e-06, + 1.6842660670590703e-06, + 8.009979865164496e-06, + 7.97654502093792e-06, + 6.962549377931282e-06, + 6.126418156782165e-06, + 6.676721113763051e-06, + 5.992607839289121e-06, + 4.798164809471928e-06, + 3.6242633996153018e-06, + 2.8191313958814135e-06, + 2.023412207563524e-06, + 1.573064764670562e-06, + 1.4859393786537112e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.4892004123321385e-05, + "first_to_last_ratio": 1.010196136459353, + "imbalance_abs_log_ratio": 0.010144506514065088, + "last_quartile_mean": 3.453983129020344e-05, + "mean": 3.719143947478187e-05, + "normalized": [ + 0.9787509726025058, + 1.1268290755091765, + 1.0180314063323372, + 0.9373900555049763, + 0.9573129653471193, + 0.8927633987058169, + 0.8376905027658148, + 0.756615575252793, + 0.6421721653954596, + 0.5898966583006985, + 0.5545320603024807, + 0.5324801984524263, + 0.7191238348729887, + 0.6685886503056807, + 0.6276517726739922, + 0.5907083001617862, + 0.7404052849968692, + 0.7009964656156153, + 0.7076026287863885, + 0.7352288733491559, + 3.079466745153095, + 2.56966898996169, + 2.031234533994799, + 1.57522845930543, + 1.8426476034454873, + 1.5558698594237106, + 1.200138700827154, + 0.8699239581339129, + 0.6815952172432139, + 0.48817942111819174, + 0.40443641967892363, + 0.3868392464803102 + ], + "peak_layer": 21, + "peak_normalized": 3.079466745153095, + "population_cv": 0.6121837817284164, + "rest_mean": 2.879134468390408e-05, + "spike_contrast": 2.8672488989954323, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.255195134552196e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.6401157558429986e-05, + 4.190839536022395e-05, + 3.786205343203619e-05, + 3.486288551357575e-05, + 3.5603847209131345e-05, + 3.3203155908267945e-05, + 3.11549156322144e-05, + 2.813962237269152e-05, + 2.388330722169485e-05, + 2.193910586356651e-05, + 2.06238455575658e-05, + 1.9803705072263256e-05, + 2.6745250579551794e-05, + 2.4865774321369827e-05, + 2.334327291464433e-05, + 2.1969291992718354e-05, + 2.7536738343769684e-05, + 2.607106762297917e-05, + 2.6316760340705514e-05, + 2.7344220143277198e-05, + 0.00011452980106696486, + 9.556968871038407e-05, + 7.554453623015434e-05, + 5.85850139032118e-05, + 6.853071681689471e-05, + 5.786503970739432e-05, + 4.463488585315645e-05, + 3.23537242366001e-05, + 2.5349507268401794e-05, + 1.815609539335128e-05, + 1.5041572623886168e-05, + 1.4387108421942685e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6184012767771492e-05, + "first_to_last_ratio": 1.1812258678199312, + "imbalance_abs_log_ratio": 0.1665527702583449, + "last_quartile_mean": 2.2166812868817942e-05, + "mean": 2.6416474952384306e-05, + "normalized": [ + 1.0583395321083173, + 1.18417524025499, + 1.0887520391639638, + 1.0135132254608268, + 1.0023744568550392, + 0.9356357677270415, + 0.8663869238627898, + 0.7804236609988654, + 0.6948391179201333, + 0.645410883875558, + 0.609013089334734, + 0.5813567671594309, + 0.7649466228246298, + 0.711734220686414, + 0.6645112375219416, + 0.6291608441906736, + 0.8117379342921259, + 0.7682870001881678, + 0.7656778963007645, + 0.7856293365916892, + 3.109981047613895, + 2.532125441581352, + 1.911742908862577, + 1.3712180563471432, + 1.6504233650379605, + 1.361126344660198, + 1.0867081917704957, + 0.8610084831897756, + 0.5742988763844462, + 0.4302061202657988, + 0.3792452416796221, + 0.3700101252886388 + ], + "peak_layer": 21, + "peak_normalized": 3.109981047613895, + "population_cv": 0.5830394716350688, + "rest_mean": 2.0961481856863463e-05, + "spike_contrast": 2.6655289950267567, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5873437668196856e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7957599741057493e-05, + 3.1281735573429614e-05, + 2.8760990971932188e-05, + 2.6773446734296158e-05, + 2.6479199732420966e-05, + 2.471619882271625e-05, + 2.2886888473294675e-05, + 2.061604209302459e-05, + 1.8355200154474005e-05, + 1.7049480447894894e-05, + 1.6087979020085186e-05, + 1.535739647806622e-05, + 2.0207193301757798e-05, + 1.880150921351742e-05, + 1.7554044461576268e-05, + 1.6620211681583896e-05, + 2.144325480912812e-05, + 2.029543429671321e-05, + 2.0226510969223455e-05, + 2.0753557691932656e-05, + 8.215473644668236e-05, + 6.688982830382884e-05, + 5.050150866736658e-05, + 3.62227474397514e-05, + 4.3598367483355105e-05, + 3.595615999074653e-05, + 2.870699972845614e-05, + 2.2744809029973112e-05, + 1.5170951883192174e-05, + 1.1364529200363904e-05, + 1.0018322427640669e-05, + 9.774363206815906e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.068014514923561e-05, + "first_to_last_ratio": 0.7913424091198598, + "imbalance_abs_log_ratio": 0.23402452356595546, + "last_quartile_mean": 0.00011459027609816985, + "mean": 0.00010754564084436424, + "normalized": [ + 0.8540109519123018, + 1.0092398375236236, + 0.8947367225413311, + 0.8160956211381821, + 0.862539829414268, + 0.8245016378814637, + 0.7800932919128317, + 0.7042081007547356, + 0.5666342926354664, + 0.5256616841881218, + 0.49571278437559146, + 0.4792103383514405, + 0.6722886591537548, + 0.6295888625242309, + 0.5934294393111791, + 0.5594389550092885, + 0.6645012760984752, + 0.6282095204823469, + 0.6412757892302924, + 0.6736811938431846, + 3.0068371075614557, + 2.616557188534586, + 2.1686834473810053, + 1.8088340274566967, + 2.0615653735311414, + 1.8302172118360363, + 1.4236627074872454, + 0.9903997896635313, + 0.8090570392266916, + 0.5559293048344423, + 0.43884492996422475, + 0.4143530842408338 + ], + "peak_layer": 21, + "peak_normalized": 3.0068371075614557, + "population_cv": 0.6573647361404942, + "rest_mean": 8.100784921057143e-05, + "spike_contrast": 3.0966099965793172, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00025084971566684543, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.184515511151403e-05, + 0.00010853934509214014, + 9.622503421269357e-05, + 8.776752656558529e-05, + 9.276239870814607e-05, + 8.867155702318996e-05, + 8.389563299715519e-05, + 7.573451148346066e-05, + 6.093904812587425e-05, + 5.653262269333936e-05, + 5.3311749070417136e-05, + 5.1536982937250286e-05, + 7.230171468108892e-05, + 6.770953768864274e-05, + 6.382074934663251e-05, + 6.016522092977539e-05, + 7.146421557990834e-05, + 6.756119546480477e-05, + 6.896641571074724e-05, + 7.245147571666166e-05, + 0.00032337222364731133, + 0.00028139931964688003, + 0.0002332324511371553, + 0.0001945322146639228, + 0.00022171236923895776, + 0.00019683188293129206, + 0.00015310871822293848, + 0.00010651318007148802, + 8.701055776327848e-05, + 5.978777335258201e-05, + 4.7195859224302694e-05, + 4.456186798051931e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16101159155368805, + 0.3334391713142395, + 0.3570481836795807, + 0.37708213925361633, + 0.11129721254110336, + 0.12943366169929504, + 0.1512180119752884, + 0.1945808082818985, + 0.07549218833446503, + 0.11765842139720917, + 0.16134388744831085, + 0.21493762731552124, + 0.08329613506793976, + 0.12312135100364685, + 0.16809259355068207, + 0.23101076483726501, + 0.11538279056549072, + 0.16158254444599152, + 0.20080390572547913, + 0.27125677466392517, + 0.0543946772813797, + 0.09335283935070038, + 0.12897421419620514, + 0.17114612460136414, + 0.06173322722315788, + 0.09693212062120438, + 0.13771499693393707, + 0.18387214839458466, + 0.07983565330505371, + 0.15526357293128967, + 0.26127612590789795, + 0.39550676941871643 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1373772622391698e-05, + "first_to_last_ratio": 1.0318485034659548, + "imbalance_abs_log_ratio": 0.03135185731624994, + "last_quartile_mean": 1.1022715625585988e-05, + "mean": 1.1832133452571725e-05, + "normalized": [ + 0.9800044623886516, + 1.1858619463319722, + 1.052769444074511, + 0.9435868051213633, + 0.9942662730454839, + 0.9213689272443336, + 0.8458140758926892, + 0.7664188972680795, + 0.6009172873544569, + 0.5652322062780974, + 0.5379371038219452, + 0.5220047919249511, + 0.7166699752090641, + 0.6623360432121947, + 0.616224251829878, + 0.5903966461838739, + 0.7324373700274538, + 0.6882103836910076, + 0.7092295284797906, + 0.8089552049077647, + 3.1154122124911208, + 2.432133583525468, + 1.964217211992137, + 1.5948629117896413, + 1.9522993684268721, + 1.5038120812358047, + 1.036848162520458, + 0.8807155414212365, + 0.6474761343085111, + 0.5016554233955102, + 0.4711561792646531, + 0.4587695653410252 + ], + "peak_layer": 21, + "peak_normalized": 3.1154122124911208, + "population_cv": 0.6032722735217898, + "rest_mean": 9.176947801110887e-06, + "spike_contrast": 2.851725490613808, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.617013597046025e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1595543583098333e-05, + 1.4031276805326343e-05, + 1.245650855707936e-05, + 1.1164645002281759e-05, + 1.1764291230065282e-05, + 1.0901760106207803e-05, + 1.0007785022025928e-05, + 9.068370673048776e-06, + 7.110133537935326e-06, + 6.687902896373998e-06, + 6.364943601511186e-06, + 6.176430360937957e-06, + 8.479734788124915e-06, + 7.836848453735001e-06, + 7.2912475843622815e-06, + 6.985651907598367e-06, + 8.66629670781549e-06, + 8.142997103277594e-06, + 8.391698429477401e-06, + 9.571665941621177e-06, + 3.686197305796668e-05, + 2.8777329134754837e-05, + 2.324088018212933e-05, + 1.8870630810852163e-05, + 2.3099866666598246e-05, + 1.7793305232771672e-05, + 1.2268125828995835e-05, + 1.0420743819850031e-05, + 7.661024028493557e-06, + 5.935653916822048e-06, + 5.574782790063182e-06, + 5.428222721093334e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.529294619715074e-05, + "first_to_last_ratio": 1.0314184626645753, + "imbalance_abs_log_ratio": 0.03093500306067748, + "last_quartile_mean": 4.391325910546584e-05, + "mean": 4.717383586694268e-05, + "normalized": [ + 0.9832121149293677, + 1.1731445898725168, + 1.0505704052806353, + 0.9501685038967237, + 0.9944626347672784, + 0.9147662213723499, + 0.8464650022543867, + 0.7682388544141363, + 0.6067490194436969, + 0.5681705700165066, + 0.53864641920515, + 0.5211305553096096, + 0.7167263481864653, + 0.6632364446817158, + 0.6172546895643319, + 0.588531029519173, + 0.7318735922946267, + 0.6893643707481226, + 0.7026758072754231, + 0.8079900073900919, + 3.115049509935762, + 2.4538624912763685, + 1.9660898437352061, + 1.5845676157597377, + 1.937541165727475, + 1.4892951790023068, + 1.0350080319051866, + 0.8840334683174624, + 0.6516976556256888, + 0.50650640646803, + 0.47764736477161285, + 0.4653240870528551 + ], + "peak_layer": 21, + "peak_normalized": 3.115049509935762, + "population_cv": 0.6027314599912318, + "rest_mean": 3.6590978736596835e-05, + "spike_contrast": 2.8510104941924466, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010432126437081024, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.638188693206757e-05, + 5.534173033083789e-05, + 4.9559435865376145e-05, + 4.482309304876253e-05, + 4.6912617108318955e-05, + 4.315303158364259e-05, + 3.9931001083459705e-05, + 3.624077362474054e-05, + 2.8622678655665368e-05, + 2.680278521438595e-05, + 2.541001776990015e-05, + 2.4583727281424217e-05, + 3.3810731110861525e-05, + 3.128740718238987e-05, + 2.9118271413608454e-05, + 2.7763266189140268e-05, + 3.4525284718256444e-05, + 3.251996167819016e-05, + 3.314791320008226e-05, + 3.811598799075e-05, + 0.00014694883429910988, + 0.00011575810640351847, + 9.27479995880276e-05, + 7.475013262592256e-05, + 9.14012489374727e-05, + 7.025576633168384e-05, + 4.882529901806265e-05, + 4.170324973529205e-05, + 3.074307824135758e-05, + 2.38938500842778e-05, + 2.2532458388013765e-05, + 2.1951122107566334e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.040166237857192755, + "first_to_last_ratio": 1.0314184457192797, + "imbalance_abs_log_ratio": 0.030934986631559365, + "last_quartile_mean": 0.038942718179896474, + "mean": 0.0418342292541638, + "normalized": [ + 0.9832121171072725, + 1.1731446022953238, + 1.0505704535174525, + 0.9501684905945749, + 0.9944626387492155, + 0.9147662270065798, + 0.8464650412985084, + 0.7682388261849044, + 0.6067490356906232, + 0.5681705774712352, + 0.5386464289336567, + 0.5211305632866429, + 0.7167263331576038, + 0.6632364212118128, + 0.6172547081682093, + 0.5885310198061827, + 0.7318735882573365, + 0.6893643353923572, + 0.7026757592562258, + 0.8079899688020947, + 3.1150494118332737, + 2.453862539114119, + 1.9660897667076618, + 1.5845675971028321, + 1.9375412374473495, + 1.4892952935873054, + 1.0350080081219681, + 0.8840335073818288, + 0.6516976484197683, + 0.5065064256973429, + 0.4776473385482515, + 0.46532408985048596 + ], + "peak_layer": 21, + "peak_normalized": 3.1150494118332737, + "population_cv": 0.6027314563638121, + "rest_mean": 0.03244924586680201, + "spike_contrast": 2.851010464947826, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09251313954591751, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.041131921112537384, + 0.0490776002407074, + 0.043949805200099945, + 0.039749566465616226, + 0.04160257801413536, + 0.03826854005455971, + 0.03541121259331703, + 0.032138679176568985, + 0.02538287825882435, + 0.02376897819340229, + 0.022533858194947243, + 0.021801095455884933, + 0.02998369373381138, + 0.027745984494686127, + 0.02582237496972084, + 0.024620741605758667, + 0.0306173674762249, + 0.028839025646448135, + 0.029395898804068565, + 0.03380163758993149, + 0.13031569123268127, + 0.10265544801950455, + 0.08224985003471375, + 0.06628916412591934, + 0.08105554431676865, + 0.06230352073907852, + 0.043298762291669846, + 0.036982860416173935, + 0.02726326882839203, + 0.021189305931329727, + 0.019982008263468742, + 0.019466474652290344 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8091637201678168e-06, + "first_to_last_ratio": 0.5997838633923696, + "imbalance_abs_log_ratio": 0.511185916342951, + "last_quartile_mean": 3.0163594431087404e-06, + "mean": 2.4871596693287756e-06, + "normalized": [ + 0.8903650291088842, + 0.8096671832364504, + 0.7756357252979299, + 0.7664205976400669, + 0.6819345308185697, + 0.6497344242040057, + 0.6267720257428505, + 0.6186826317778965, + 0.4671058445444584, + 0.45225062104576175, + 0.4492165155127133, + 0.4573049495780591, + 0.555081192151318, + 0.5467853740610928, + 0.5411086649066317, + 0.5362598920190076, + 0.601197329062199, + 0.609347333830986, + 0.6241580819811688, + 0.6880678322200046, + 2.9157464374333077, + 2.626463595143142, + 2.355941396471949, + 2.0525708846936164, + 2.4216180933419804, + 1.9231831825131547, + 1.4564932487560838, + 1.2844496192048258, + 0.8068213094456221, + 0.6295223209584038, + 0.59698327836313, + 0.5831108549347289 + ], + "peak_layer": 21, + "peak_normalized": 2.9157464374333077, + "population_cv": 0.7105915967606766, + "rest_mean": 1.8080416052894347e-06, + "spike_contrast": 3.4039024307496555, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.154397215141216e-06, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.214479991380358e-06, + 2.013771563724731e-06, + 1.9291298940515844e-06, + 1.9062104001932312e-06, + 1.6960800621745875e-06, + 1.6159932556547574e-06, + 1.558882104291115e-06, + 1.5387624898721697e-06, + 1.1617668178587337e-06, + 1.1248195050939103e-06, + 1.1172732001796248e-06, + 1.137390427174978e-06, + 1.3805755543216947e-06, + 1.3599425301435986e-06, + 1.3458236480801133e-06, + 1.3337639757082798e-06, + 1.4952737501516822e-06, + 1.5155441133174463e-06, + 1.5523808087891666e-06, + 1.711334562060074e-06, + 7.251926945173182e-06, + 6.532434326800285e-06, + 5.8596024246071465e-06, + 5.105071522848448e-06, + 6.022950856277021e-06, + 4.78326364827808e-06, + 3.622531266955775e-06, + 3.194631290170946e-06, + 2.0066934212081833e-06, + 1.5657225276299869e-06, + 1.484792733208451e-06, + 1.45028980114148e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.464059318503132e-05, + "first_to_last_ratio": 1.2085006098527633, + "imbalance_abs_log_ratio": 0.1893804257974035, + "last_quartile_mean": 2.8664108981502068e-05, + "mean": 3.421737719122575e-05, + "normalized": [ + 1.0599522534665642, + 1.2224211904533802, + 1.1066526183773722, + 1.0166167984558396, + 1.0363163370114612, + 0.9614978327769692, + 0.8912999264066335, + 0.8041906605597857, + 0.6595162801975746, + 0.6198396025381588, + 0.5858589457780455, + 0.5655951157979022, + 0.7633864100877904, + 0.7040498510939207, + 0.65484714742333, + 0.6226490455428274, + 0.7974248515656103, + 0.7481679248848441, + 0.7573152935416273, + 0.844065677484853, + 3.1218032666740525, + 2.4268896738358103, + 1.8727607046170862, + 1.4552330006230305, + 1.7605509694422596, + 1.3320662835665362, + 0.9562601026332771, + 0.8300407364235332, + 0.5336839174330044, + 0.4444323110860877, + 0.4233154787321452, + 0.421299791488687 + ], + "peak_layer": 21, + "peak_normalized": 3.1218032666740525, + "population_cv": 0.5796332934591588, + "rest_mean": 2.7073248087685056e-05, + "spike_contrast": 2.6888415499527167, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.279567435034551e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.626878606155515e-05, + 4.182804696029052e-05, + 3.786675006267615e-05, + 3.478596045169979e-05, + 3.546002699295059e-05, + 3.289993401267566e-05, + 3.049794577236753e-05, + 2.7517295166035183e-05, + 2.2566917323274538e-05, + 2.120928547810763e-05, + 2.004655652854126e-05, + 1.9353181414771825e-05, + 2.6121080736629665e-05, + 2.4090739316307008e-05, + 2.2407151845982298e-05, + 2.1305417249095626e-05, + 2.728578692767769e-05, + 2.5600344088161364e-05, + 2.591334305179771e-05, + 2.888171366066672e-05, + 0.00010681991989258677, + 8.304179937113076e-05, + 6.408095941878855e-05, + 4.9794256483437493e-05, + 6.024143658578396e-05, + 4.557981446851045e-05, + 3.272071262472309e-05, + 2.840181696228683e-05, + 1.826126390369609e-05, + 1.5207308024400845e-05, + 1.4484745406662114e-05, + 1.4415773875953164e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6039962676804862e-05, + "first_to_last_ratio": 1.3600902622782072, + "imbalance_abs_log_ratio": 0.3075510668678258, + "last_quartile_mean": 1.914576068884344e-05, + "mean": 2.476154466535263e-05, + "normalized": [ + 1.123958441324931, + 1.2626354632798173, + 1.1622790571900958, + 1.0807265666022514, + 1.0670121245383055, + 0.9899364488555507, + 0.9055408259899391, + 0.8209444362453999, + 0.7058751897560187, + 0.6697662418103106, + 0.6333373009906147, + 0.6058288961358885, + 0.7980579670309706, + 0.7373130204202144, + 0.6858704929138739, + 0.6566823825816065, + 0.8578675442054349, + 0.8063397293421843, + 0.8071587377333899, + 0.875107050101149, + 3.1369738883948477, + 2.3886515998281816, + 1.7583385106457243, + 1.2781546769066958, + 1.5599158953991201, + 1.1978164843461745, + 0.9315129197863214, + 0.8390098557963587, + 0.46047650401210477, + 0.40917670978266013, + 0.3951608602894278, + 0.39257417776443665 + ], + "peak_layer": 21, + "peak_normalized": 3.1369738883948477, + "population_cv": 0.558754889865697, + "rest_mean": 2.0064156227885246e-05, + "spike_contrast": 2.4983578506037145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.012744222767651e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7830947146867402e-05, + 3.126480442006141e-05, + 2.87798247882165e-05, + 2.676045914995484e-05, + 2.6420868380228058e-05, + 2.4512355594197288e-05, + 2.2422589609050192e-05, + 2.0327852325863205e-05, + 1.7478560039307922e-05, + 1.6584446711931378e-05, + 1.5682409866712987e-05, + 1.5001259271230083e-05, + 1.9761147996177897e-05, + 1.8257009287481196e-05, + 1.6983212844934314e-05, + 1.6260470147244632e-05, + 2.1242125512799248e-05, + 1.996621722355485e-05, + 1.9986497136414982e-05, + 2.1669002308044583e-05, + 7.767631905153394e-05, + 5.9146703279111534e-05, + 4.353917756816372e-05, + 3.164908412145451e-05, + 3.862592711811885e-05, + 2.965978637803346e-05, + 2.306569876964204e-05, + 2.0775180018972605e-05, + 1.1402109521441162e-05, + 1.013184737530537e-05, + 9.784793292055838e-06, + 9.720743037178181e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.000182035379112e-05, + "first_to_last_ratio": 0.9625877483741323, + "imbalance_abs_log_ratio": 0.03813004983371345, + "last_quartile_mean": 9.349986066808924e-05, + "mean": 9.762683816916251e-05, + "normalized": [ + 0.9359774389006617, + 1.1139547805652443, + 0.9858638933617162, + 0.896277058777463, + 0.9496551618529471, + 0.8993751236921529, + 0.8464771316171845, + 0.7475897673874267, + 0.588915541149348, + 0.5540229040457487, + 0.5295015812507878, + 0.5183640792456576, + 0.7262152137199526, + 0.6676039603614504, + 0.6223324443316169, + 0.5861986137162843, + 0.7254235746253427, + 0.6817432367794596, + 0.6942948378413707, + 0.7958728955594262, + 3.0862176056110457, + 2.4751904263802063, + 2.0171676620749306, + 1.6939489172879052, + 2.0229993487511715, + 1.572447147387479, + 1.0908649591739474, + 0.9643206077313163, + 0.6147279541208738, + 0.4857837184299343, + 0.45630950884426186, + 0.45436290542568586 + ], + "peak_layer": 21, + "peak_normalized": 3.0862176056110457, + "population_cv": 0.617326253069026, + "rest_mean": 7.486342710089284e-05, + "spike_contrast": 2.946021394940767, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022054925793781876, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.137651795754209e-05, + 0.00010875188309000805, + 9.624677477404475e-05, + 8.750069537200034e-05, + 9.271183080272749e-05, + 8.780314965406433e-05, + 8.263888594228774e-05, + 7.298482523765415e-05, + 5.7493962231092155e-05, + 5.408750439528376e-05, + 5.169356518308632e-05, + 5.060624607722275e-05, + 7.089809514582157e-05, + 6.51760637992993e-05, + 6.0756348830182105e-05, + 5.72287171962671e-05, + 7.082080992404372e-05, + 6.655643664998934e-05, + 6.778180977562442e-05, + 7.769855437800288e-05, + 0.0003012976667378098, + 0.00024164501519408077, + 0.00019692970090545714, + 0.00016537487681489438, + 0.00019749903003685176, + 0.00015351304318755865, + 0.00010649769683368504, + 9.414357191417366e-05, + 6.00139464950189e-05, + 4.74255284643732e-05, + 4.454805457498878e-05, + 4.4358013838063926e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059412047266960144, + 0.08935670554637909, + 0.17747226357460022, + 0.17946027219295502, + 0.17260213196277618, + 0.12674333155155182, + 0.12412907183170319, + 0.11767174303531647, + 0.13582420349121094, + 0.10855290293693542, + 0.10545514523983002, + 0.11005345731973648, + 0.13049271702766418, + 0.09666567295789719, + 0.10358717292547226, + 0.11041814833879471, + 0.13209649920463562, + 0.10239993035793304, + 0.11153820157051086, + 0.11912636458873749, + 0.127323716878891, + 0.07468672841787338, + 0.07975661009550095, + 0.08746611326932907, + 0.09389390796422958, + 0.06575146317481995, + 0.0681111291050911, + 0.07568862289190292, + 0.08424623310565948, + 0.0719028189778328, + 0.08439797908067703, + 0.08961156010627747 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.588972493977053e-06, + "first_to_last_ratio": 1.465723686175851, + "imbalance_abs_log_ratio": 0.38234910423701196, + "last_quartile_mean": 5.1776283385152055e-06, + "mean": 8.926610817638903e-06, + "normalized": [ + 0.8001707694815087, + 1.2301759322409134, + 0.8021990606376289, + 0.7996309276786352, + 0.4189560575118035, + 0.770181199703876, + 0.9964730820564364, + 0.983425792900568, + 0.6138765442017692, + 0.5043130598626571, + 0.39700601441149197, + 0.37391212513577055, + 0.38881218172106297, + 0.7038080142822467, + 0.6284832461563316, + 0.48736568495191596, + 0.7748207724466236, + 1.0896235823984757, + 1.2330955711393075, + 1.3976034073521157, + 2.3882743999212943, + 4.5696632038041, + 2.815663441519544, + 2.192292008694111, + 1.06869084333539, + 1.167599927936825, + 0.6186038915435874, + 0.511201557417111, + 0.4679157405813128, + 0.21996309432798952, + 0.26258255269417496, + 0.3236163119534213 + ], + "peak_layer": 22, + "peak_normalized": 4.5696632038041, + "population_cv": 0.8757958168839719, + "rest_mean": 6.270255131244927e-06, + "spike_contrast": 3.711321315811602, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.3270931524166372e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 7.14281304681208e-06, + 1.0981301784340758e-05, + 7.160918812587624e-06, + 7.137994089134736e-06, + 3.739857675100211e-06, + 6.875107828818727e-06, + 8.895127393770963e-06, + 8.778659321251325e-06, + 5.479837000166299e-06, + 4.50180641564657e-06, + 3.5439181829133304e-06, + 3.3377680210833205e-06, + 3.4707750273810234e-06, + 6.2826202338328585e-06, + 5.610225343843922e-06, + 4.350523795437766e-06, + 6.916523489053361e-06, + 9.726645657792687e-06, + 1.1007364264514763e-05, + 1.2475861694838386e-05, + 2.1319196093827486e-05, + 4.079160498804413e-05, + 2.513433173589874e-05, + 1.956973756023217e-05, + 9.539787242829334e-06, + 1.0422710147395264e-05, + 5.52203619008651e-06, + 4.5632973524334375e-06, + 4.176901711616665e-06, + 1.9635249373095576e-06, + 2.3439722554030595e-06, + 2.8887968710478162e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.9898558636887174e-05, + "first_to_last_ratio": 1.4621096880940556, + "imbalance_abs_log_ratio": 0.37988038456939854, + "last_quartile_mean": 2.0448916302484577e-05, + "mean": 3.561926479278554e-05, + "normalized": [ + 0.7085385219770569, + 1.1946020815752356, + 0.7754024698062456, + 0.7950922364389753, + 0.4162410374634061, + 0.8241424849247687, + 0.9939701500428968, + 1.0071543674888268, + 0.5806040892681411, + 0.5114068129628899, + 0.4225456612843993, + 0.35601883603809836, + 0.39596565343777096, + 0.700750155357531, + 0.608338530669077, + 0.5057982656951122, + 0.7834207937068051, + 1.0599792777683275, + 1.2280812674610306, + 1.4767524648757513, + 2.409304768407655, + 4.508238395616571, + 2.902287830939038, + 2.2425871184827204, + 1.0734427327031963, + 1.1671335490266965, + 0.5965647485206973, + 0.46999739473988966, + 0.46865462397650987, + 0.22456393251833243, + 0.25925607539704565, + 0.3331636714293014 + ], + "peak_layer": 22, + "peak_normalized": 4.508238395616571, + "population_cv": 0.8793156123901573, + "rest_mean": 2.4886176578109843e-05, + "spike_contrast": 3.760237771291333, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.357794115203433e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.523762123018969e-05, + 4.255084786564112e-05, + 2.761926589300856e-05, + 2.832060090440791e-05, + 1.482619973103283e-05, + 2.9355349397519603e-05, + 3.540448597050272e-05, + 3.587409810279496e-05, + 2.0680690795416012e-05, + 1.8215934687759727e-05, + 1.5050765796331689e-05, + 1.2681129192060325e-05, + 1.4104005458648317e-05, + 2.4960205337265506e-05, + 2.1668571207555942e-05, + 1.80161623575259e-05, + 2.790487269521691e-05, + 3.775568256969564e-05, + 4.374335185275413e-05, + 5.2600837079808116e-05, + 8.581766451243311e-05, + 0.00016058013716246933, + 0.00010337735875509679, + 7.987930439412594e-05, + 3.8235240936046466e-05, + 4.157243893132545e-05, + 2.1249197743600234e-05, + 1.674096165515948e-05, + 1.6693133147782646e-05, + 7.998802175279707e-06, + 9.234510798705742e-06, + 1.1867045031976886e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.026514341006986797, + "first_to_last_ratio": 1.4621096829473716, + "imbalance_abs_log_ratio": 0.379880381049359, + "last_quartile_mean": 0.01813430368201807, + "mean": 0.03158751998853404, + "normalized": [ + 0.7085385035567081, + 1.1946021626253802, + 0.7754024540661398, + 0.7950922593029921, + 0.4162410309209195, + 0.8241424880807221, + 0.9939701540557254, + 1.0071544006568143, + 0.5806040665606002, + 0.5114068209416861, + 0.4225456864143744, + 0.3560188250781175, + 0.39596564639005655, + 0.7007501603721394, + 0.6083384888465566, + 0.5057982797995526, + 0.7834208304641355, + 1.059979249016567, + 1.2280813415682141, + 1.4767524934129506, + 2.409304781935525, + 4.508238208880784, + 2.902287717411148, + 2.2425871343428208, + 1.0734427669067585, + 1.167133566980318, + 0.5965647509522538, + 0.46999742239503073, + 0.4686546086713296, + 0.22456393938088035, + 0.25925609979218267, + 0.3331636602206165 + ], + "peak_layer": 22, + "peak_normalized": 4.508238208880784, + "population_cv": 0.8793155841158318, + "rest_mean": 0.022069310372764314, + "spike_contrast": 3.7602376563657023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08298585191369057, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.022380974143743515, + 0.03773451969027519, + 0.024493040516972542, + 0.025114992633461952, + 0.013148021884262562, + 0.026032617315649986, + 0.031397052109241486, + 0.03181350976228714, + 0.018339842557907104, + 0.016154073178768158, + 0.013347170315682888, + 0.01124575175344944, + 0.012507572770118713, + 0.02213495969772339, + 0.0192159041762352, + 0.015976913273334503, + 0.024746321141719818, + 0.03348211571574211, + 0.038792043924331665, + 0.04664694890379906, + 0.07610396295785904, + 0.14240406453609467, + 0.09167607128620148, + 0.07083776593208313, + 0.03390739485621452, + 0.03686685487627983, + 0.01884400099515915, + 0.01484605297446251, + 0.014803636819124222, + 0.007093417923897505, + 0.008189257234334946, + 0.010523813776671886 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.343415339112198e-06, + "first_to_last_ratio": 0.7161840718970601, + "imbalance_abs_log_ratio": 0.33381806140671977, + "last_quartile_mean": 3.272085251637691e-06, + "mean": 2.6718059498165303e-06, + "normalized": [ + 3.352170512510883, + 1.252135055595148, + 0.26582849245093404, + 0.3779079423767161, + 0.26494682358528443, + 0.3763245935115975, + 0.4665297795225812, + 0.6608799701010516, + 0.3129183029559072, + 0.27411212897478776, + 0.2550957946560799, + 0.3550642916181046, + 0.2737856172573793, + 0.47262961496261435, + 0.48685762845780345, + 0.5118283391293013, + 0.3284988072562945, + 0.6155436681481607, + 0.5758322948420541, + 0.7857703722429643, + 1.46149784943323, + 3.1539420937482103, + 2.83862849215308, + 2.483897584167724, + 1.672423695519606, + 1.9844503515036886, + 1.6695842115910071, + 1.4607872550915708, + 1.1268636557445397, + 0.7047297682619745, + 0.577264376468574, + 0.6012706361611467 + ], + "peak_layer": 1, + "peak_normalized": 3.352170512510883, + "population_cv": 0.8823003320995952, + "rest_mean": 2.0176697064387014e-06, + "spike_contrast": 3.074904502088927, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.204141664056806e-06, + "top_five_layers": [ + 1, + 22, + 23, + 24, + 26 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.956349120126106e-06, + 3.3454618915129686e-06, + 7.102421477611642e-07, + 1.0096966889250325e-06, + 7.078864996401535e-07, + 1.0054662880065734e-06, + 1.2464770406950265e-06, + 1.7657430362305604e-06, + 8.360569836440845e-07, + 7.323744171117141e-07, + 6.815664619352901e-07, + 9.486628869126434e-07, + 7.315020411624573e-07, + 1.2627746173166088e-06, + 1.300789108427125e-06, + 1.36750600177038e-06, + 8.776850677350012e-07, + 1.6446132349301479e-06, + 1.5385121514555067e-06, + 2.099425955748302e-06, + 3.904838649759768e-06, + 8.426721251453273e-06, + 7.5842644946533255e-06, + 6.636492344114231e-06, + 4.468391580303432e-06, + 5.3020662562630605e-06, + 4.4608050302485935e-06, + 3.902940079569817e-06, + 3.0107610200502677e-06, + 1.8829011878551682e-06, + 1.5423383956658654e-06, + 1.6064784631453222e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.9795748357864795e-05, + "first_to_last_ratio": 1.5829781482320753, + "imbalance_abs_log_ratio": 0.4593079767809046, + "last_quartile_mean": 1.2505383210736909e-05, + "mean": 2.345933930314459e-05, + "normalized": [ + 0.6329672888423138, + 1.1769775978635326, + 0.8122053642936898, + 0.8103566266208889, + 0.46133266366701603, + 0.8656603467447103, + 1.0301469021057994, + 0.9610116498331323, + 0.61408958600413, + 0.5510707221672605, + 0.4548082316962977, + 0.3885266091083219, + 0.4407469545604743, + 0.7576792724096253, + 0.6251552614433971, + 0.5326228510318479, + 0.8273444966654151, + 1.114509292339196, + 1.2367312992724153, + 1.4148882628934067, + 2.3782100012844283, + 4.280445457492481, + 3.0878808679778498, + 2.2801019909661657, + 1.0813850741694784, + 1.1060747126282833, + 0.5844412396792442, + 0.4054991266359997, + 0.4197995696214123, + 0.21544640490341943, + 0.19403464509867754, + 0.2578496299796893 + ], + "peak_layer": 22, + "peak_normalized": 4.280445457492481, + "population_cv": 0.8646936695537649, + "rest_mean": 1.6414566272470876e-05, + "spike_contrast": 3.746740099488776, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.150111366878263e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4848994396743365e-05, + 2.7611116820480675e-05, + 1.9053801224799827e-05, + 1.9010431060451083e-05, + 1.0822559488588013e-05, + 2.0307819795561954e-05, + 2.416656570858322e-05, + 2.2544698367710225e-05, + 1.4406135960598476e-05, + 1.2927755051350687e-05, + 1.0669500625226647e-05, + 9.11457755137235e-06, + 1.0339632353861816e-05, + 1.7774655134417117e-05, + 1.4665729395346716e-05, + 1.2494980182964355e-05, + 1.9408955267863348e-05, + 2.6145651645492762e-05, + 2.9012899176450446e-05, + 3.319234383525327e-05, + 5.5791235354263335e-05, + 0.00010041642235592008, + 7.2439645009581e-05, + 5.34896862518508e-05, + 2.5368579372297972e-05, + 2.5947781978175044e-05, + 1.371060534438584e-05, + 9.512741598882712e-06, + 9.84822054306278e-06, + 5.05423031427199e-06, + 4.551924575935118e-06, + 6.0489819588838145e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.3026621559220075e-05, + "first_to_last_ratio": 1.6815341811376112, + "imbalance_abs_log_ratio": 0.5197065797570352, + "last_quartile_mean": 7.74686694171578e-06, + "mean": 1.525919709166601e-05, + "normalized": [ + 0.5774672585272599, + 1.1989902313733303, + 0.8415468606033967, + 0.8210487753497788, + 0.5264205831559432, + 0.9142648476833113, + 1.0733362714129566, + 0.8764439105095523, + 0.6569979339342769, + 0.6167902521881127, + 0.4964350034575567, + 0.41696619601743, + 0.504954663560933, + 0.8317766688879529, + 0.6542358689500138, + 0.5551252336862755, + 0.852113468285689, + 1.140364193630489, + 1.2382881917527824, + 1.3141660941820785, + 2.300727833390314, + 4.0533116788495525, + 3.2452755011871197, + 2.231471710715914, + 1.0423588969466941, + 1.097025384808554, + 0.5832116217690613, + 0.3693409741830814, + 0.3982639506890586, + 0.21137018039235425, + 0.16048903759429983, + 0.19942072232487634 + ], + "peak_layer": 22, + "peak_normalized": 4.0533116788495525, + "population_cv": 0.8414682188802379, + "rest_mean": 1.0809645952273664e-05, + "spike_contrast": 3.6344181315319797, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.9286773244384676e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.811686711851507e-06, + 1.8295628251507878e-05, + 1.2841329407820012e-05, + 1.2528545084933285e-05, + 8.032755431486294e-06, + 1.3950947504781652e-05, + 1.6378249711124226e-05, + 1.3373830370255746e-05, + 1.0025260962720495e-05, + 9.411724022356793e-06, + 7.575199560960755e-06, + 6.362569365592208e-06, + 7.705202733632177e-06, + 1.2692244126810692e-05, + 9.983114068745635e-06, + 8.47076535137603e-06, + 1.3002567357034422e-05, + 1.7401041986886412e-05, + 1.889528357423842e-05, + 2.0053119442309253e-05, + 3.510725946398452e-05, + 6.185028178151697e-05, + 4.952029848936945e-05, + 3.4050466638291255e-05, + 1.5905559848761186e-05, + 1.6739726561354473e-05, + 8.899341082724277e-06, + 5.635846719087567e-06, + 6.077188118069898e-06, + 3.225339241907932e-06, + 2.448933855703217e-06, + 3.043000106117688e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 5.830217355651257e-05, + "first_to_last_ratio": 1.5342432776570591, + "imbalance_abs_log_ratio": 0.42803728076015524, + "last_quartile_mean": 3.8000605513843766e-05, + "mean": 7.032809671159157e-05, + "normalized": [ + 0.6819589535091998, + 1.1712319271696168, + 0.7869167841712673, + 0.779993936454377, + 0.40965657804504296, + 0.8112730725890709, + 0.9714853978881673, + 1.0195039829060222, + 0.5575391424076114, + 0.48579339303929814, + 0.4244288891192081, + 0.3806909807485322, + 0.3969007539297048, + 0.6868963514684149, + 0.5698714639132343, + 0.5145191503639089, + 0.8281057882616928, + 1.0694789461698069, + 1.2364533982124963, + 1.5208781026521505, + 2.5059295507917216, + 4.611853308457518, + 2.9277607037675337, + 2.3282138476317833, + 1.1223656127309714, + 1.0704287879823866, + 0.5800732884744664, + 0.412100266213389, + 0.4197925520111647, + 0.21365215237778729, + 0.20903494194495364, + 0.2952179945975007 + ], + "peak_layer": 22, + "peak_normalized": 4.611853308457518, + "population_cv": 0.9090106345148538, + "rest_mean": 4.819786850189471e-05, + "spike_contrast": 3.9385834881161212, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001898313290439546, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.796087523573078e-05, + 8.237051224568859e-05, + 5.534235970117152e-05, + 5.4855488997418433e-05, + 2.881036743929144e-05, + 5.705529110855423e-05, + 6.832271901657805e-05, + 7.169977470766753e-05, + 3.921066672774032e-05, + 3.4164924727519974e-05, + 2.984927596116904e-05, + 2.6773272111313418e-05, + 2.7913274607271887e-05, + 4.830811303691007e-05, + 4.007797542726621e-05, + 3.6185152566758916e-05, + 5.82391039642971e-05, + 7.521441875724122e-05, + 8.695741416886449e-05, + 0.00010696046228986233, + 0.0001762372558005154, + 0.00032434286549687386, + 0.00020590383792296052, + 0.00016373884864151478, + 7.893383735790849e-05, + 7.528121932409704e-05, + 4.0795450331643224e-05, + 2.8982227377127856e-05, + 2.9523211196647026e-05, + 1.5025749235064723e-05, + 1.4701029613206629e-05, + 2.0762119675055146e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.056020382791757584, + 0.0742802768945694, + 0.1285802572965622, + 0.10598119348287582, + 0.10133238881826401, + 0.10985831171274185, + 0.12611564993858337, + 0.1125042662024498, + 0.07848236709833145, + 0.0899009183049202, + 0.09790628403425217, + 0.09713682532310486, + 0.09147685021162033, + 0.10080859810113907, + 0.09896469116210938, + 0.10007023066282272, + 0.10974904894828796, + 0.10291008651256561, + 0.10674569010734558, + 0.12148815393447876, + 0.055056143552064896, + 0.05981758236885071, + 0.07101839780807495, + 0.08339047431945801, + 0.06143829599022865, + 0.06018613278865814, + 0.06816941499710083, + 0.08121132105588913, + 0.05916989967226982, + 0.07458337396383286, + 0.08333029597997665, + 0.09104464948177338 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.630958900657788e-06, + "first_to_last_ratio": 0.20421556767006768, + "imbalance_abs_log_ratio": 1.5885791387918327, + "last_quartile_mean": 1.778002990704408e-05, + "mean": 1.0223539980813712e-05, + "normalized": [ + 0.8429681862296823, + 0.35291594683302396, + 0.22484209501334745, + 0.36790271245902445, + 0.21868611610463093, + 0.18545641795638437, + 0.23387248761479024, + 0.41460977766546225, + 0.4054488572719161, + 0.3330083345972086, + 0.3085788428775866, + 0.3192217174198833, + 0.3816324391793382, + 0.4112131193630629, + 0.4344687728023616, + 0.46821459950254946, + 0.37185344111151275, + 0.4758162142971942, + 0.41846445085124506, + 0.6781708994429974, + 2.4552099328163637, + 2.8496156807280086, + 2.603123697672816, + 2.3316927886492507, + 2.616234924154862, + 2.65437990886044, + 2.4006671523655148, + 1.672837600494882, + 1.585060117848551, + 1.2276893902450918, + 0.8940161400914827, + 0.8621272374795348 + ], + "peak_layer": 22, + "peak_normalized": 2.8496156807280086, + "population_cv": 0.8996576825534632, + "rest_mean": 7.248915060834541e-06, + "spike_contrast": 3.626268804103626, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6286514548701233e-05, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.618118954473175e-06, + 3.6080502923141466e-06, + 2.298682147738873e-06, + 3.7612680898746476e-06, + 2.2357462512445636e-06, + 1.8960211036755936e-06, + 2.3910047275421675e-06, + 4.238779638399137e-06, + 4.145122602494666e-06, + 3.4045240226987517e-06, + 3.1547681373922387e-06, + 3.263575990786194e-06, + 3.9016344999254216e-06, + 4.204053766443394e-06, + 4.441808869160013e-06, + 4.786810677614994e-06, + 3.801658522206708e-06, + 4.864526090386789e-06, + 4.278188043826958e-06, + 6.933307304279879e-06, + 2.5100936909439042e-05, + 2.9133159841876477e-05, + 2.661313919816166e-05, + 2.383815444773063e-05, + 2.674718234629836e-05, + 2.7137159122503363e-05, + 2.4543316612835042e-05, + 1.71023220900679e-05, + 1.6204925486817956e-05, + 1.2551331565191504e-05, + 9.140009751718026e-06, + 8.813992280920502e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.470679973181177e-05, + "first_to_last_ratio": 0.2069929445185519, + "imbalance_abs_log_ratio": 1.5750705707490962, + "last_quartile_mean": 7.104976339178393e-05, + "mean": 4.0840292868438155e-05, + "normalized": [ + 0.8622316727455452, + 0.3538749471728574, + 0.22575416255506364, + 0.3708511987720451, + 0.22255638964420116, + 0.1850393452789397, + 0.23623438814969275, + 0.4242990843945505, + 0.41091535564437, + 0.3361491695289819, + 0.31238992658037873, + 0.3212003149521768, + 0.3849834068983422, + 0.4107834308653127, + 0.4382023170687432, + 0.4728175618985744, + 0.3691518544033243, + 0.4681213159064821, + 0.4170992962583633, + 0.6272493229024287, + 2.4438193245013227, + 2.8483751664462695, + 2.6052355716893176, + 2.3350839107599644, + 2.618275725913621, + 2.653345803911492, + 2.3878946201876037, + 1.6698710725380324, + 1.59563349859018, + 1.224673847138525, + 0.9002294831629131, + 0.8676575135403847 + ], + "peak_layer": 22, + "peak_normalized": 2.8483751664462695, + "population_cv": 0.8985289334793173, + "rest_mean": 2.8965161365906373e-05, + "spike_contrast": 3.6238708169484144, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010496600298210978, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.521379403537139e-05, + 1.4452356481342576e-05, + 9.219866115017794e-06, + 1.5145671568461694e-05, + 9.089268132811412e-06, + 7.557061053375946e-06, + 9.647881597629748e-06, + 1.73284988704836e-05, + 1.678190346865449e-05, + 1.3728430531045888e-05, + 1.2758096090692561e-05, + 1.3117914932081476e-05, + 1.572283508721739e-05, + 1.6776515622041188e-05, + 1.7896310964715667e-05, + 1.9310007701278664e-05, + 1.5076269846758805e-05, + 1.9118211639579386e-05, + 1.703445741441101e-05, + 2.5617046048864722e-05, + 9.980629693018273e-05, + 0.00011632847599685192, + 0.00010639858373906463, + 9.536551078781486e-05, + 0.00010693114745663479, + 0.0001083634197129868, + 9.752231562742963e-05, + 6.819802365498617e-05, + 6.516613939311355e-05, + 5.0016038585454226e-05, + 3.6765635741176084e-05, + 3.5435386962490156e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.013042137026786804, + "first_to_last_ratio": 0.20699294542918592, + "imbalance_abs_log_ratio": 1.5750705663497482, + "last_quartile_mean": 0.06300764018669724, + "mean": 0.03621758034569211, + "normalized": [ + 0.8622316598933797, + 0.3538749471508598, + 0.22575414844580807, + 0.37085122221171446, + 0.22255637683921708, + 0.1850393384194697, + 0.236234387422702, + 0.42429908068863703, + 0.41091534861415707, + 0.3361491670478898, + 0.3123899060706743, + 0.3212003125118064, + 0.3849834050274649, + 0.41078345814992373, + 0.43820230021845763, + 0.4728175684224875, + 0.36915184353764025, + 0.46812130102476074, + 0.41709928593336826, + 0.6272492932959962, + 2.4438193692237777, + 2.8483752780000278, + 2.605235605762955, + 2.3350840258688303, + 2.6182758157645756, + 2.6533456676353473, + 2.3878945209130196, + 1.6698711077439734, + 1.5956335166208298, + 1.2246738042807082, + 0.9002294337837535, + 0.8676575034757876 + ], + "peak_layer": 22, + "peak_normalized": 2.8483752780000278, + "population_cv": 0.8985289457588168, + "rest_mean": 0.025686594381652498, + "spike_contrast": 3.623871003234085, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09308490455150605, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.031227944418787956, + 0.012816494330763817, + 0.008176269009709358, + 0.013431333936750889, + 0.008060453459620476, + 0.006701677106320858, + 0.008555837906897068, + 0.015367086045444012, + 0.014882359653711319, + 0.012174509465694427, + 0.01131400652229786, + 0.011633098125457764, + 0.01394316740334034, + 0.014877582900226116, + 0.015870627015829086, + 0.01712430827319622, + 0.01336978655308485, + 0.016954220831394196, + 0.015106326900422573, + 0.02271745167672634, + 0.0885092243552208, + 0.10316126048564911, + 0.09435532987117767, + 0.08457109332084656, + 0.09482761472463608, + 0.09609775990247726, + 0.08648376166820526, + 0.06047869101166725, + 0.057789985090494156, + 0.044354721903800964, + 0.03260413184762001, + 0.03142445534467697 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 8.568965839117482e-07, + "first_to_last_ratio": 0.18051220552416403, + "imbalance_abs_log_ratio": 1.7119568828628857, + "last_quartile_mean": 4.747028498286454e-06, + "mean": 2.6146394631609837e-06, + "normalized": [ + 0.8589787963589357, + 0.3082289930915967, + 0.22695141403248437, + 0.38502719312209793, + 0.20717193772164363, + 0.14969905615852777, + 0.1917600510841699, + 0.2940249836979971, + 0.3501028028074662, + 0.30509132547078094, + 0.2840337285464057, + 0.28520738633381276, + 0.33814434197742116, + 0.362621645282935, + 0.4284643797973123, + 0.4847838221845296, + 0.3528575777552836, + 0.44423254880618596, + 0.4021648360838168, + 0.39838852122464097, + 2.445175567674409, + 2.9178538331950583, + 2.621993890930227, + 2.432580720612022, + 2.6636416217847234, + 2.7243600687846947, + 2.580677484419162, + 1.8055086156001192, + 1.6378764119490026, + 1.2592445178301535, + 0.939779150965119, + 0.9133727747172655 + ], + "peak_layer": 22, + "peak_normalized": 2.9178538331950583, + "population_cv": 0.9456027936919144, + "rest_mean": 1.8320637688399681e-06, + "spike_contrast": 3.7337937296941295, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.840548212494468e-06, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 27 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.2459198589785956e-06, + 8.059076890276629e-07, + 5.933961233495211e-07, + 1.0067072935271426e-06, + 5.416799240265391e-07, + 3.9140905983003904e-07, + 5.013833970224368e-07, + 7.687693255320482e-07, + 9.153926043836691e-07, + 7.977038194439956e-07, + 7.426457955261867e-07, + 7.457144874933874e-07, + 8.841255407787685e-07, + 9.481248639531259e-07, + 1.1202798759768484e-06, + 1.2675349125856883e-06, + 9.225953476743598e-07, + 1.1615079529292416e-06, + 1.0515160511204158e-06, + 1.0416423492642934e-06, + 6.39325253359857e-06, + 7.629135780007346e-06, + 6.855568699393189e-06, + 6.360321549436776e-06, + 6.964462500036461e-06, + 7.123219347704435e-06, + 6.747541192453355e-06, + 4.7207540774252266e-06, + 4.2824563024623785e-06, + 3.2924704100878444e-06, + 2.4571836547693238e-06, + 2.388140501352609e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 9.912464463468496e-06, + "first_to_last_ratio": 0.22715332429513255, + "imbalance_abs_log_ratio": 1.4821300520824365, + "last_quartile_mean": 4.363776974969369e-05, + "mean": 2.6559063869058264e-05, + "normalized": [ + 0.9366358613935047, + 0.3792603305202499, + 0.18134819814305192, + 0.34760749803128715, + 0.23747281376634713, + 0.19753839084684555, + 0.24598063870488243, + 0.4599435726461178, + 0.4623870697047853, + 0.36990754631923695, + 0.34903068824582323, + 0.37085556359303906, + 0.4380455173785828, + 0.45529019266842574, + 0.48514444968944176, + 0.5201735235841253, + 0.4152101632813542, + 0.5283175891854636, + 0.46450504110947355, + 0.6293184811026621, + 2.502663692008064, + 2.9461029153808944, + 2.6595225397265994, + 2.273367886792679, + 2.65101572499286, + 2.525147631813147, + 2.234627662919172, + 1.5898700335733806, + 1.426452730852729, + 1.112160691183748, + 0.8183556047810898, + 0.7867397560609359 + ], + "peak_layer": 22, + "peak_normalized": 2.9461029153808944, + "population_cv": 0.880175635609515, + "rest_mean": 1.8657572430432385e-05, + "spike_contrast": 3.7104032636486823, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.9227117637638e-05, + "top_five_layers": [ + 22, + 23, + 25, + 26, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.4876171664800495e-05, + 1.0072799341287464e-05, + 4.816438377019949e-06, + 9.2321297415765e-06, + 6.307055627985392e-06, + 5.246434739092365e-06, + 6.533015493914718e-06, + 1.2215670722071081e-05, + 1.2280567716516089e-05, + 9.824398148339242e-06, + 9.269928341382183e-06, + 9.849576599663123e-06, + 1.1634078873612452e-05, + 1.2092081306036562e-05, + 1.2884982425021008e-05, + 1.381532183586387e-05, + 1.1027593245671596e-05, + 1.4031620594323613e-05, + 1.2336819054326043e-05, + 1.6714109733584337e-05, + 6.646840483881533e-05, + 7.824573549441993e-05, + 7.063442899379879e-05, + 6.037852290319279e-05, + 7.040849595796317e-05, + 6.70655572321266e-05, + 5.934961882303469e-05, + 4.222545976517722e-05, + 3.788524918491021e-05, + 2.953794682980515e-05, + 2.1734758774982765e-05, + 2.0895071429549716e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 6.6299761840582505e-06, + "first_to_last_ratio": 0.25432294789989096, + "imbalance_abs_log_ratio": 1.369150371144901, + "last_quartile_mean": 2.6069122895933106e-05, + "mean": 1.693132426083821e-05, + "normalized": [ + 1.009812755573757, + 0.39968507734198944, + 0.16094253648199763, + 0.32854895069647194, + 0.25477694605768825, + 0.2155789505292242, + 0.25860705350981517, + 0.5046916184590391, + 0.515329188356828, + 0.40705935830374973, + 0.38337515923971865, + 0.42269040487394594, + 0.4945745095736154, + 0.5045501286967757, + 0.5247552857381682, + 0.5730846754557868, + 0.46495437450577576, + 0.5946150769636939, + 0.5125942030699491, + 0.6193454891464502, + 2.628840032502442, + 3.0610378618268497, + 2.7106344368298605, + 2.1323334410119363, + 2.565140847337925, + 2.330500972416668, + 2.0157738095858844, + 1.503546921244241, + 1.3029013314823592, + 1.0664512040978233, + 0.7882888849242707, + 0.7449785141652996 + ], + "peak_layer": 22, + "peak_normalized": 3.0610378618268497, + "population_cv": 0.8541047749122238, + "rest_mean": 1.1853189545473907e-05, + "spike_contrast": 3.7418832756911033, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.435325172380544e-05, + "top_five_layers": [ + 22, + 23, + 21, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.7097467207349837e-05, + 6.767197646695422e-06, + 2.724970272538485e-06, + 5.562768819800112e-06, + 4.313711087888805e-06, + 3.650037115221494e-06, + 4.3785598791146185e-06, + 8.54509744385723e-06, + 8.725205589144025e-06, + 6.892053988849511e-06, + 6.4910491346381605e-06, + 7.156708306865767e-06, + 8.373801392735913e-06, + 8.54270183481276e-06, + 8.884801900421735e-06, + 9.703082469059154e-06, + 7.872293281252496e-06, + 1.0067620678455569e-05, + 8.678898666403256e-06, + 1.0486339306226e-05, + 4.45097430201713e-05, + 5.182742461329326e-05, + 4.589463060256094e-05, + 3.610322892200202e-05, + 4.34312314609997e-05, + 3.945846765418537e-05, + 3.412972000660375e-05, + 2.5457040464971215e-05, + 2.2059844923205674e-05, + 1.8056431144941598e-05, + 1.3346774721867405e-05, + 1.2613472790690139e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.929049958311225e-05, + "first_to_last_ratio": 0.19264517754015853, + "imbalance_abs_log_ratio": 1.6469052404570816, + "last_quartile_mean": 0.00015204377268673852, + "mean": 8.56582414883178e-05, + "normalized": [ + 0.8435321808032064, + 0.34370726972583865, + 0.1808664069876229, + 0.33993318208269363, + 0.2101403998193939, + 0.17675988933980996, + 0.23091552849720237, + 0.4097137534640246, + 0.40325567762898656, + 0.3332485242329118, + 0.315649429844806, + 0.31114263500046885, + 0.3744856227345994, + 0.39208745649289284, + 0.44367077395553656, + 0.4702768538142674, + 0.35548322757045653, + 0.45205464826790226, + 0.41818583974351736, + 0.6083319800365703, + 2.3949709614831773, + 2.7856906015999057, + 2.6171687947428244, + 2.3886915605377412, + 2.7341447434319206, + 2.7222211356577324, + 2.584598405216518, + 1.8271047030330585, + 1.5687480327138839, + 1.1190385826225775, + 0.8424439925404574, + 0.8017372063774949 + ], + "peak_layer": 22, + "peak_normalized": 2.7856906015999057, + "population_cv": 0.9245587442879203, + "rest_mean": 6.052970898222325e-05, + "spike_contrast": 3.6569202255083786, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022135231702122838, + "top_five_layers": [ + 22, + 25, + 26, + 23, + 27 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.22554832464084e-05, + 2.944136031146627e-05, + 1.5492698366870172e-05, + 2.911807860073168e-05, + 1.8000257114181295e-05, + 1.5140941286517773e-05, + 1.977981810341589e-05, + 3.509535963530652e-05, + 3.4542172215878963e-05, + 2.8545482564368285e-05, + 2.7037975087296218e-05, + 2.665193096618168e-05, + 3.207777990610339e-05, + 3.358552203280851e-05, + 3.800405829679221e-05, + 4.028308831038885e-05, + 3.04500681522768e-05, + 3.8722206227248535e-05, + 3.582106364774518e-05, + 5.2108647651039064e-05, + 0.00020514900097623467, + 0.000238617358263582, + 0.0002241820766357705, + 0.00020461111853364855, + 0.00023420203069690615, + 0.00023318067542277277, + 0.00022139215434435755, + 0.0001565065758768469, + 0.00013437619782052934, + 9.58548771450296e-05, + 7.21622709534131e-05, + 6.867539923405275e-05 + ] + } + } + } + } + } + }, + "step": 4000 + }, + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 1.8740129107275643, + "forward": { + "activation_sha256": { + "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" + }, + "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", + "loss_nats": 1.2989667654037476, + "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" + }, + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22886337339878082, + "group": 1, + "latest_source_mass": 0.06350618600845337, + "layer": 1, + "max_source_mass": 0.9364938139915466, + "mixer_index": 1, + "normalized_entropy": 0.3301800538435402, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9364938139915466, + "median": 0.9452826976776123, + "p05": 0.8725647330284119, + "p95": 0.9712384939193726 + }, + { + "label": "current_group_partial", + "mean": 0.06350618600845337, + "median": 0.0547172985970974, + "p05": 0.028761589899659157, + "p95": 0.12743529677391052 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.603729784488678, + "group": 1, + "latest_source_mass": 0.4133155345916748, + "layer": 2, + "max_source_mass": 0.5866844654083252, + "mixer_index": 2, + "normalized_entropy": 0.8709979661187783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5866844654083252, + "median": 0.5734395384788513, + "p05": 0.29813364148139954, + "p95": 0.8843642473220825 + }, + { + "label": "current_group_partial", + "mean": 0.4133155345916748, + "median": 0.4265604615211487, + "p05": 0.11563576757907867, + "p95": 0.7018663287162781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3967982828617096, + "group": 1, + "latest_source_mass": 0.14110636711120605, + "layer": 2, + "max_source_mass": 0.858893632888794, + "mixer_index": 3, + "normalized_entropy": 0.5724589149178446, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.858893632888794, + "median": 0.8705973029136658, + "p05": 0.7599387168884277, + "p95": 0.9270997643470764 + }, + { + "label": "current_group_partial", + "mean": 0.14110636711120605, + "median": 0.12940272688865662, + "p05": 0.07290027290582657, + "p95": 0.2400612235069275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6852496862411499, + "group": 1, + "latest_source_mass": 0.518975019454956, + "layer": 3, + "max_source_mass": 0.518975019454956, + "mixer_index": 4, + "normalized_entropy": 0.9886063241108252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48102498054504395, + "median": 0.48102718591690063, + "p05": 0.38786420226097107, + "p95": 0.580143928527832 + }, + { + "label": "current_group_partial", + "mean": 0.518975019454956, + "median": 0.5189728140830994, + "p05": 0.41985607147216797, + "p95": 0.6121357679367065 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5890734195709229, + "group": 1, + "latest_source_mass": 0.3123423159122467, + "layer": 3, + "max_source_mass": 0.6876576542854309, + "mixer_index": 5, + "normalized_entropy": 0.8498533011344741, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6876576542854309, + "median": 0.6909762620925903, + "p05": 0.48803940415382385, + "p95": 0.8900076746940613 + }, + { + "label": "current_group_partial", + "mean": 0.3123423159122467, + "median": 0.3090237081050873, + "p05": 0.10999235510826111, + "p95": 0.5119606256484985 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6873323321342468, + "group": 1, + "latest_source_mass": 0.4831485152244568, + "layer": 4, + "max_source_mass": 0.5168514251708984, + "mixer_index": 6, + "normalized_entropy": 0.9916109470127238, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5168514251708984, + "median": 0.5219584703445435, + "p05": 0.4251725673675537, + "p95": 0.5947866439819336 + }, + { + "label": "current_group_partial", + "mean": 0.4831485152244568, + "median": 0.47804147005081177, + "p05": 0.405213326215744, + "p95": 0.5748274326324463 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.487301766872406, + "group": 1, + "latest_source_mass": 0.2060384750366211, + "layer": 4, + "max_source_mass": 0.7939615249633789, + "mixer_index": 7, + "normalized_entropy": 0.7030278424832499, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7939615249633789, + "median": 0.7953404188156128, + "p05": 0.6590759754180908, + "p95": 0.9329437017440796 + }, + { + "label": "current_group_partial", + "mean": 0.2060384750366211, + "median": 0.20465964078903198, + "p05": 0.06705628335475922, + "p95": 0.3409239649772644 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674697756767273, + "group": 2, + "latest_source_mass": 0.42952626943588257, + "layer": 5, + "max_source_mass": 0.5704737305641174, + "mixer_index": 8, + "normalized_entropy": 0.9733831077870527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5704737305641174, + "median": 0.5696132779121399, + "p05": 0.4695202708244324, + "p95": 0.6750621199607849 + }, + { + "label": "completed_group_1", + "mean": 0.42952626943588257, + "median": 0.4303867816925049, + "p05": 0.3249378800392151, + "p95": 0.5304797887802124 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9691112041473389, + "group": 2, + "latest_source_mass": 0.2681158185005188, + "layer": 5, + "max_source_mass": 0.5506307482719421, + "mixer_index": 9, + "normalized_entropy": 0.8821230329784768, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5506307482719421, + "median": 0.543993353843689, + "p05": 0.41262108087539673, + "p95": 0.7161091566085815 + }, + { + "label": "completed_group_1", + "mean": 0.18125343322753906, + "median": 0.1794438362121582, + "p05": 0.11575750261545181, + "p95": 0.25031501054763794 + }, + { + "label": "current_group_partial", + "mean": 0.2681158185005188, + "median": 0.2664111256599426, + "p05": 0.1564716249704361, + "p95": 0.3817881643772125 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0745112895965576, + "group": 2, + "latest_source_mass": 0.412794291973114, + "layer": 6, + "max_source_mass": 0.412794291973114, + "mixer_index": 10, + "normalized_entropy": 0.9780623252441762, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34262996912002563, + "median": 0.3412924110889435, + "p05": 0.29556700587272644, + "p95": 0.39125555753707886 + }, + { + "label": "completed_group_1", + "mean": 0.24457570910453796, + "median": 0.24382859468460083, + "p05": 0.22120657563209534, + "p95": 0.2734297513961792 + }, + { + "label": "current_group_partial", + "mean": 0.412794291973114, + "median": 0.4151499271392822, + "p05": 0.37735897302627563, + "p95": 0.43749260902404785 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074045658111572, + "group": 2, + "latest_source_mass": 0.3775783181190491, + "layer": 6, + "max_source_mass": 0.4399106502532959, + "mixer_index": 11, + "normalized_entropy": 0.9169791528842925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4399106502532959, + "median": 0.44856905937194824, + "p05": 0.22063353657722473, + "p95": 0.6359403133392334 + }, + { + "label": "completed_group_1", + "mean": 0.18251106142997742, + "median": 0.18562838435173035, + "p05": 0.1196163073182106, + "p95": 0.23045453429222107 + }, + { + "label": "current_group_partial", + "mean": 0.3775783181190491, + "median": 0.3675365447998047, + "p05": 0.23817461729049683, + "p95": 0.5557451844215393 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.060497522354126, + "group": 2, + "latest_source_mass": 0.4025024175643921, + "layer": 7, + "max_source_mass": 0.4025024175643921, + "mixer_index": 12, + "normalized_entropy": 0.9653064445872968, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3786153197288513, + "median": 0.3907395303249359, + "p05": 0.28039106726646423, + "p95": 0.4436405599117279 + }, + { + "label": "completed_group_1", + "mean": 0.2188822478055954, + "median": 0.21226292848587036, + "p05": 0.1885872334241867, + "p95": 0.2672030031681061 + }, + { + "label": "current_group_partial", + "mean": 0.4025024175643921, + "median": 0.39815032482147217, + "p05": 0.3533596992492676, + "p95": 0.460100919008255 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9941019415855408, + "group": 2, + "latest_source_mass": 0.4801652431488037, + "layer": 7, + "max_source_mass": 0.4801652431488037, + "mixer_index": 13, + "normalized_entropy": 0.9048705824970601, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32296955585479736, + "median": 0.2803739905357361, + "p05": 0.1784922033548355, + "p95": 0.6440245509147644 + }, + { + "label": "completed_group_1", + "mean": 0.19686520099639893, + "median": 0.2053825855255127, + "p05": 0.09813719242811203, + "p95": 0.2504696547985077 + }, + { + "label": "current_group_partial", + "mean": 0.4801652431488037, + "median": 0.5057140588760376, + "p05": 0.2529003322124481, + "p95": 0.598870575428009 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0503522157669067, + "group": 2, + "latest_source_mass": 0.4001518487930298, + "layer": 8, + "max_source_mass": 0.40130966901779175, + "mixer_index": 14, + "normalized_entropy": 0.9560717885654542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40130966901779175, + "median": 0.396759569644928, + "p05": 0.3479611873626709, + "p95": 0.4733521044254303 + }, + { + "label": "completed_group_1", + "mean": 0.19853851199150085, + "median": 0.19769755005836487, + "p05": 0.17136667668819427, + "p95": 0.22788912057876587 + }, + { + "label": "current_group_partial", + "mean": 0.4001518487930298, + "median": 0.40291696786880493, + "p05": 0.3506356179714203, + "p95": 0.43941178917884827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0129402875900269, + "group": 2, + "latest_source_mass": 0.42161279916763306, + "layer": 8, + "max_source_mass": 0.42161279916763306, + "mixer_index": 15, + "normalized_entropy": 0.9220179839951123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4093267023563385, + "median": 0.4046899676322937, + "p05": 0.2895161807537079, + "p95": 0.5762131810188293 + }, + { + "label": "completed_group_1", + "mean": 0.16906049847602844, + "median": 0.16888955235481262, + "p05": 0.11233551055192947, + "p95": 0.22477403283119202 + }, + { + "label": "current_group_partial", + "mean": 0.42161279916763306, + "median": 0.4227708578109741, + "p05": 0.3131049573421478, + "p95": 0.519075334072113 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0774019956588745, + "group": 3, + "latest_source_mass": 0.3560434877872467, + "layer": 9, + "max_source_mass": 0.38510820269584656, + "mixer_index": 16, + "normalized_entropy": 0.9806935592947451, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38510820269584656, + "median": 0.3892287313938141, + "p05": 0.2934730052947998, + "p95": 0.4718902111053467 + }, + { + "label": "completed_group_1", + "mean": 0.25884830951690674, + "median": 0.25734007358551025, + "p05": 0.22217319905757904, + "p95": 0.29551801085472107 + }, + { + "label": "completed_group_2", + "mean": 0.3560434877872467, + "median": 0.35412120819091797, + "p05": 0.29444968700408936, + "p95": 0.41908568143844604 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2740377187728882, + "group": 3, + "latest_source_mass": 0.3633861243724823, + "layer": 9, + "max_source_mass": 0.3633861243724823, + "mixer_index": 17, + "normalized_entropy": 0.9190239493895668, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29550763964653015, + "median": 0.30360478162765503, + "p05": 0.16904757916927338, + "p95": 0.45146965980529785 + }, + { + "label": "completed_group_1", + "mean": 0.09541244804859161, + "median": 0.09486871212720871, + "p05": 0.061360105872154236, + "p95": 0.13037073612213135 + }, + { + "label": "completed_group_2", + "mean": 0.24569380283355713, + "median": 0.24333015084266663, + "p05": 0.1698218435049057, + "p95": 0.31589236855506897 + }, + { + "label": "current_group_partial", + "mean": 0.3633861243724823, + "median": 0.36472851037979126, + "p05": 0.2795379161834717, + "p95": 0.4399876892566681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.360302448272705, + "group": 3, + "latest_source_mass": 0.28569501638412476, + "layer": 10, + "max_source_mass": 0.3109527826309204, + "mixer_index": 18, + "normalized_entropy": 0.9812507981160736, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3109527826309204, + "median": 0.32172977924346924, + "p05": 0.23520725965499878, + "p95": 0.37052470445632935 + }, + { + "label": "completed_group_1", + "mean": 0.20984819531440735, + "median": 0.20958667993545532, + "p05": 0.1891367882490158, + "p95": 0.23184512555599213 + }, + { + "label": "completed_group_2", + "mean": 0.19350400567054749, + "median": 0.19153150916099548, + "p05": 0.16607457399368286, + "p95": 0.22877082228660583 + }, + { + "label": "current_group_partial", + "mean": 0.28569501638412476, + "median": 0.28060320019721985, + "p05": 0.23824527859687805, + "p95": 0.3439165949821472 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2722704410552979, + "group": 3, + "latest_source_mass": 0.29912811517715454, + "layer": 10, + "max_source_mass": 0.35036465525627136, + "mixer_index": 19, + "normalized_entropy": 0.9177491279900463, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35036465525627136, + "median": 0.36210548877716064, + "p05": 0.12807878851890564, + "p95": 0.5797606706619263 + }, + { + "label": "completed_group_1", + "mean": 0.12631990015506744, + "median": 0.1264108121395111, + "p05": 0.06977735459804535, + "p95": 0.1755298376083374 + }, + { + "label": "completed_group_2", + "mean": 0.22418735921382904, + "median": 0.22563719749450684, + "p05": 0.1176326721906662, + "p95": 0.32001155614852905 + }, + { + "label": "current_group_partial", + "mean": 0.29912811517715454, + "median": 0.28589946031570435, + "p05": 0.193109393119812, + "p95": 0.44119665026664734 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3558036088943481, + "group": 3, + "latest_source_mass": 0.314197838306427, + "layer": 11, + "max_source_mass": 0.314197838306427, + "mixer_index": 20, + "normalized_entropy": 0.978005571485618, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298976868391037, + "median": 0.2961939573287964, + "p05": 0.24950163066387177, + "p95": 0.35835936665534973 + }, + { + "label": "completed_group_1", + "mean": 0.1941717267036438, + "median": 0.1918032318353653, + "p05": 0.16809208691120148, + "p95": 0.22895419597625732 + }, + { + "label": "completed_group_2", + "mean": 0.1926535815000534, + "median": 0.19306811690330505, + "p05": 0.16983944177627563, + "p95": 0.2155935913324356 + }, + { + "label": "current_group_partial", + "mean": 0.314197838306427, + "median": 0.31541913747787476, + "p05": 0.2673022747039795, + "p95": 0.35941022634506226 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2696614265441895, + "group": 3, + "latest_source_mass": 0.2862822115421295, + "layer": 11, + "max_source_mass": 0.3667730689048767, + "mixer_index": 21, + "normalized_entropy": 0.9158671218416545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3667730689048767, + "median": 0.4067659378051758, + "p05": 0.10593366622924805, + "p95": 0.5672274827957153 + }, + { + "label": "completed_group_1", + "mean": 0.15310388803482056, + "median": 0.15050218999385834, + "p05": 0.08651046454906464, + "p95": 0.22459912300109863 + }, + { + "label": "completed_group_2", + "mean": 0.19384080171585083, + "median": 0.18732509016990662, + "p05": 0.1267009675502777, + "p95": 0.281107634305954 + }, + { + "label": "current_group_partial", + "mean": 0.2862822115421295, + "median": 0.26271265745162964, + "p05": 0.17024393379688263, + "p95": 0.475044310092926 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3606936931610107, + "group": 3, + "latest_source_mass": 0.3028184771537781, + "layer": 12, + "max_source_mass": 0.3028184771537781, + "mixer_index": 22, + "normalized_entropy": 0.9815330216461395, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298227459192276, + "median": 0.2954302132129669, + "p05": 0.25287362933158875, + "p95": 0.35405808687210083 + }, + { + "label": "completed_group_1", + "mean": 0.18611131608486176, + "median": 0.1850002110004425, + "p05": 0.15901757776737213, + "p95": 0.21725836396217346 + }, + { + "label": "completed_group_2", + "mean": 0.21284276247024536, + "median": 0.21402311325073242, + "p05": 0.18819917738437653, + "p95": 0.23418807983398438 + }, + { + "label": "current_group_partial", + "mean": 0.3028184771537781, + "median": 0.3041122555732727, + "p05": 0.25661808252334595, + "p95": 0.34458184242248535 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.269316554069519, + "group": 3, + "latest_source_mass": 0.2653314769268036, + "layer": 12, + "max_source_mass": 0.4081852436065674, + "mixer_index": 23, + "normalized_entropy": 0.9156183489371815, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4081852436065674, + "median": 0.4254342317581177, + "p05": 0.20765496790409088, + "p95": 0.587238073348999 + }, + { + "label": "completed_group_1", + "mean": 0.14420127868652344, + "median": 0.1438048928976059, + "p05": 0.1064920499920845, + "p95": 0.1871720254421234 + }, + { + "label": "completed_group_2", + "mean": 0.18228203058242798, + "median": 0.17507661879062653, + "p05": 0.12183455377817154, + "p95": 0.26505547761917114 + }, + { + "label": "current_group_partial", + "mean": 0.2653314769268036, + "median": 0.24924281239509583, + "p05": 0.15609969198703766, + "p95": 0.4061311185359955 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3636136054992676, + "group": 4, + "latest_source_mass": 0.3136237859725952, + "layer": 13, + "max_source_mass": 0.3136237859725952, + "mixer_index": 24, + "normalized_entropy": 0.9836392931712563, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27427661418914795, + "median": 0.27403637766838074, + "p05": 0.22928935289382935, + "p95": 0.319773405790329 + }, + { + "label": "completed_group_1", + "mean": 0.21341991424560547, + "median": 0.20697325468063354, + "p05": 0.18552586436271667, + "p95": 0.26688331365585327 + }, + { + "label": "completed_group_2", + "mean": 0.19867971539497375, + "median": 0.197545126080513, + "p05": 0.1794913411140442, + "p95": 0.222100168466568 + }, + { + "label": "completed_group_3", + "mean": 0.3136237859725952, + "median": 0.31875866651535034, + "p05": 0.24156072735786438, + "p95": 0.3695686459541321 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.570773959159851, + "group": 4, + "latest_source_mass": 0.2294243425130844, + "layer": 13, + "max_source_mass": 0.2658740282058716, + "mixer_index": 25, + "normalized_entropy": 0.9759767351225285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2658740282058716, + "median": 0.2747163474559784, + "p05": 0.17927880585193634, + "p95": 0.3512483835220337 + }, + { + "label": "completed_group_1", + "mean": 0.13419361412525177, + "median": 0.13374894857406616, + "p05": 0.11072897166013718, + "p95": 0.1586270034313202 + }, + { + "label": "completed_group_2", + "mean": 0.1907704919576645, + "median": 0.19074805080890656, + "p05": 0.14699944853782654, + "p95": 0.23316305875778198 + }, + { + "label": "completed_group_3", + "mean": 0.17973749339580536, + "median": 0.17616114020347595, + "p05": 0.1363743096590042, + "p95": 0.23202598094940186 + }, + { + "label": "current_group_partial", + "mean": 0.2294243425130844, + "median": 0.22995705902576447, + "p05": 0.17975632846355438, + "p95": 0.2789157032966614 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5560038089752197, + "group": 4, + "latest_source_mass": 0.32468360662460327, + "layer": 14, + "max_source_mass": 0.32468360662460327, + "mixer_index": 26, + "normalized_entropy": 0.9667995248241249, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20477107167243958, + "median": 0.206403911113739, + "p05": 0.16619175672531128, + "p95": 0.2431035190820694 + }, + { + "label": "completed_group_1", + "mean": 0.15721669793128967, + "median": 0.15658551454544067, + "p05": 0.13878680765628815, + "p95": 0.17778633534908295 + }, + { + "label": "completed_group_2", + "mean": 0.13811248540878296, + "median": 0.13743364810943604, + "p05": 0.12352626770734787, + "p95": 0.15553249418735504 + }, + { + "label": "completed_group_3", + "mean": 0.1752161681652069, + "median": 0.17309701442718506, + "p05": 0.15223677456378937, + "p95": 0.20474471151828766 + }, + { + "label": "current_group_partial", + "mean": 0.32468360662460327, + "median": 0.32497870922088623, + "p05": 0.2946181893348694, + "p95": 0.35203880071640015 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5659973621368408, + "group": 4, + "latest_source_mass": 0.26703086495399475, + "layer": 14, + "max_source_mass": 0.26703086495399475, + "mixer_index": 27, + "normalized_entropy": 0.9730088685238187, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20531430840492249, + "median": 0.20229148864746094, + "p05": 0.10698921978473663, + "p95": 0.3393132984638214 + }, + { + "label": "completed_group_1", + "mean": 0.15986156463623047, + "median": 0.15952220559120178, + "p05": 0.11969029903411865, + "p95": 0.20180393755435944 + }, + { + "label": "completed_group_2", + "mean": 0.21557100117206573, + "median": 0.21654027700424194, + "p05": 0.1673085242509842, + "p95": 0.2610440254211426 + }, + { + "label": "completed_group_3", + "mean": 0.15222224593162537, + "median": 0.1510905921459198, + "p05": 0.11800260841846466, + "p95": 0.18893754482269287 + }, + { + "label": "current_group_partial", + "mean": 0.26703086495399475, + "median": 0.258539617061615, + "p05": 0.19330690801143646, + "p95": 0.36521556973457336 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5552496910095215, + "group": 4, + "latest_source_mass": 0.31774693727493286, + "layer": 15, + "max_source_mass": 0.31774693727493286, + "mixer_index": 28, + "normalized_entropy": 0.9663309649872576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2162160575389862, + "median": 0.2145705670118332, + "p05": 0.17727568745613098, + "p95": 0.26878347992897034 + }, + { + "label": "completed_group_1", + "mean": 0.17982497811317444, + "median": 0.18024154007434845, + "p05": 0.15453973412513733, + "p95": 0.2030555009841919 + }, + { + "label": "completed_group_2", + "mean": 0.1307673156261444, + "median": 0.13107559084892273, + "p05": 0.11463287472724915, + "p95": 0.14630235731601715 + }, + { + "label": "completed_group_3", + "mean": 0.1554446816444397, + "median": 0.15516987442970276, + "p05": 0.13627474009990692, + "p95": 0.1754152774810791 + }, + { + "label": "current_group_partial", + "mean": 0.31774693727493286, + "median": 0.319039523601532, + "p05": 0.27606821060180664, + "p95": 0.35650834441185 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5068919658660889, + "group": 4, + "latest_source_mass": 0.3298327624797821, + "layer": 15, + "max_source_mass": 0.3298327624797821, + "mixer_index": 29, + "normalized_entropy": 0.9362846209998111, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20646697282791138, + "median": 0.18775638937950134, + "p05": 0.0806753933429718, + "p95": 0.4274584949016571 + }, + { + "label": "completed_group_1", + "mean": 0.1197461187839508, + "median": 0.11989086866378784, + "p05": 0.0809398666024208, + "p95": 0.15743419528007507 + }, + { + "label": "completed_group_2", + "mean": 0.19215428829193115, + "median": 0.19620640575885773, + "p05": 0.1148109883069992, + "p95": 0.2491212636232376 + }, + { + "label": "completed_group_3", + "mean": 0.15179988741874695, + "median": 0.15212899446487427, + "p05": 0.10697861760854721, + "p95": 0.19542087614536285 + }, + { + "label": "current_group_partial", + "mean": 0.3298327624797821, + "median": 0.32837340235710144, + "p05": 0.22746729850769043, + "p95": 0.4339301884174347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5435223579406738, + "group": 4, + "latest_source_mass": 0.3224271535873413, + "layer": 16, + "max_source_mass": 0.3224271535873413, + "mixer_index": 30, + "normalized_entropy": 0.9590443632623663, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22410975396633148, + "median": 0.22760415077209473, + "p05": 0.14593973755836487, + "p95": 0.30093055963516235 + }, + { + "label": "completed_group_1", + "mean": 0.16986176371574402, + "median": 0.16963240504264832, + "p05": 0.13569067418575287, + "p95": 0.20651353895664215 + }, + { + "label": "completed_group_2", + "mean": 0.13080403208732605, + "median": 0.13075244426727295, + "p05": 0.10945598781108856, + "p95": 0.15187400579452515 + }, + { + "label": "completed_group_3", + "mean": 0.15279728174209595, + "median": 0.15036612749099731, + "p05": 0.12810495495796204, + "p95": 0.18468116223812103 + }, + { + "label": "current_group_partial", + "mean": 0.3224271535873413, + "median": 0.3164973258972168, + "p05": 0.2600978910923004, + "p95": 0.40530598163604736 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4949920177459717, + "group": 4, + "latest_source_mass": 0.31847020983695984, + "layer": 16, + "max_source_mass": 0.31847020983695984, + "mixer_index": 31, + "normalized_entropy": 0.9288907675133354, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27174079418182373, + "median": 0.2585451304912567, + "p05": 0.13639721274375916, + "p95": 0.4698348045349121 + }, + { + "label": "completed_group_1", + "mean": 0.12178683280944824, + "median": 0.11714950203895569, + "p05": 0.08594125509262085, + "p95": 0.1723237782716751 + }, + { + "label": "completed_group_2", + "mean": 0.1529826819896698, + "median": 0.15223106741905212, + "p05": 0.10371473431587219, + "p95": 0.203575000166893 + }, + { + "label": "completed_group_3", + "mean": 0.1350194811820984, + "median": 0.13363957405090332, + "p05": 0.08964549005031586, + "p95": 0.18468207120895386 + }, + { + "label": "current_group_partial", + "mean": 0.31847020983695984, + "median": 0.3185216188430786, + "p05": 0.21444711089134216, + "p95": 0.4301546812057495 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.544264793395996, + "group": 5, + "latest_source_mass": 0.2861037850379944, + "layer": 17, + "max_source_mass": 0.2861037850379944, + "mixer_index": 32, + "normalized_entropy": 0.9595056643474137, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24016115069389343, + "median": 0.23868846893310547, + "p05": 0.18435582518577576, + "p95": 0.2983453571796417 + }, + { + "label": "completed_group_1", + "mean": 0.20730282366275787, + "median": 0.1998077780008316, + "p05": 0.16255050897598267, + "p95": 0.28107208013534546 + }, + { + "label": "completed_group_2", + "mean": 0.09692573547363281, + "median": 0.0967404842376709, + "p05": 0.08098706603050232, + "p95": 0.11221462488174438 + }, + { + "label": "completed_group_3", + "mean": 0.1695065200328827, + "median": 0.1696585714817047, + "p05": 0.136396586894989, + "p95": 0.20130634307861328 + }, + { + "label": "completed_group_4", + "mean": 0.2861037850379944, + "median": 0.28883200883865356, + "p05": 0.22401469945907593, + "p95": 0.3409370183944702 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.686682939529419, + "group": 5, + "latest_source_mass": 0.12281867861747742, + "layer": 17, + "max_source_mass": 0.2885604202747345, + "mixer_index": 33, + "normalized_entropy": 0.9413556721740635, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16755425930023193, + "median": 0.1327776312828064, + "p05": 0.0790192037820816, + "p95": 0.3667412996292114 + }, + { + "label": "completed_group_1", + "mean": 0.10962115228176117, + "median": 0.10525335371494293, + "p05": 0.07840172946453094, + "p95": 0.15405339002609253 + }, + { + "label": "completed_group_2", + "mean": 0.1783454418182373, + "median": 0.17896884679794312, + "p05": 0.10802385956048965, + "p95": 0.24048911035060883 + }, + { + "label": "completed_group_3", + "mean": 0.13310004770755768, + "median": 0.13033035397529602, + "p05": 0.0904831737279892, + "p95": 0.17934636771678925 + }, + { + "label": "completed_group_4", + "mean": 0.2885604202747345, + "median": 0.2831825315952301, + "p05": 0.16967125236988068, + "p95": 0.4189499616622925 + }, + { + "label": "current_group_partial", + "mean": 0.12281867861747742, + "median": 0.12062528729438782, + "p05": 0.07461707293987274, + "p95": 0.1782219111919403 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7070834636688232, + "group": 5, + "latest_source_mass": 0.2988377809524536, + "layer": 18, + "max_source_mass": 0.2988377809524536, + "mixer_index": 34, + "normalized_entropy": 0.9527414214834803, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.181177020072937, + "median": 0.18419909477233887, + "p05": 0.13624237477779388, + "p95": 0.22192169725894928 + }, + { + "label": "completed_group_1", + "mean": 0.14557164907455444, + "median": 0.144476979970932, + "p05": 0.1293315887451172, + "p95": 0.16486899554729462 + }, + { + "label": "completed_group_2", + "mean": 0.08318156749010086, + "median": 0.0820261612534523, + "p05": 0.07088497281074524, + "p95": 0.0990293100476265 + }, + { + "label": "completed_group_3", + "mean": 0.11424209177494049, + "median": 0.11267061531543732, + "p05": 0.0979941189289093, + "p95": 0.1353992223739624 + }, + { + "label": "completed_group_4", + "mean": 0.17698991298675537, + "median": 0.17597860097885132, + "p05": 0.14837387204170227, + "p95": 0.21050144731998444 + }, + { + "label": "current_group_partial", + "mean": 0.2988377809524536, + "median": 0.2989804744720459, + "p05": 0.2721370458602905, + "p95": 0.32572072744369507 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7419235706329346, + "group": 5, + "latest_source_mass": 0.26016175746917725, + "layer": 18, + "max_source_mass": 0.26016175746917725, + "mixer_index": 35, + "normalized_entropy": 0.972186055410333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1532188355922699, + "median": 0.1451343297958374, + "p05": 0.11514341086149216, + "p95": 0.22103679180145264 + }, + { + "label": "completed_group_1", + "mean": 0.11420352011919022, + "median": 0.11261284351348877, + "p05": 0.0928458571434021, + "p95": 0.14123988151550293 + }, + { + "label": "completed_group_2", + "mean": 0.16597424447536469, + "median": 0.1655328869819641, + "p05": 0.13871800899505615, + "p95": 0.19611573219299316 + }, + { + "label": "completed_group_3", + "mean": 0.1250365972518921, + "median": 0.12369821965694427, + "p05": 0.10375828295946121, + "p95": 0.14912742376327515 + }, + { + "label": "completed_group_4", + "mean": 0.18140508234500885, + "median": 0.1796584278345108, + "p05": 0.14511024951934814, + "p95": 0.22286373376846313 + }, + { + "label": "current_group_partial", + "mean": 0.26016175746917725, + "median": 0.26266711950302124, + "p05": 0.1889389455318451, + "p95": 0.3234381675720215 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7127439975738525, + "group": 5, + "latest_source_mass": 0.2907913327217102, + "layer": 19, + "max_source_mass": 0.2907913327217102, + "mixer_index": 36, + "normalized_entropy": 0.9559006256078307, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18731768429279327, + "median": 0.19042012095451355, + "p05": 0.13879702985286713, + "p95": 0.23792000114917755 + }, + { + "label": "completed_group_1", + "mean": 0.16715484857559204, + "median": 0.1667211651802063, + "p05": 0.14574480056762695, + "p95": 0.19026139378547668 + }, + { + "label": "completed_group_2", + "mean": 0.08496320247650146, + "median": 0.08452782034873962, + "p05": 0.07410907745361328, + "p95": 0.097386933863163 + }, + { + "label": "completed_group_3", + "mean": 0.11362075805664062, + "median": 0.11248823255300522, + "p05": 0.0977015346288681, + "p95": 0.13219426572322845 + }, + { + "label": "completed_group_4", + "mean": 0.15615218877792358, + "median": 0.15540224313735962, + "p05": 0.1289147436618805, + "p95": 0.18598109483718872 + }, + { + "label": "current_group_partial", + "mean": 0.2907913327217102, + "median": 0.2900415062904358, + "p05": 0.25822532176971436, + "p95": 0.32475030422210693 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076834201812744, + "group": 5, + "latest_source_mass": 0.3738824725151062, + "layer": 19, + "max_source_mass": 0.3738824725151062, + "mixer_index": 37, + "normalized_entropy": 0.8972652009334232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15032559633255005, + "median": 0.12544110417366028, + "p05": 0.06976518034934998, + "p95": 0.35840943455696106 + }, + { + "label": "completed_group_1", + "mean": 0.08624927699565887, + "median": 0.08602266013622284, + "p05": 0.061998721212148666, + "p95": 0.11138717085123062 + }, + { + "label": "completed_group_2", + "mean": 0.15736976265907288, + "median": 0.15802210569381714, + "p05": 0.10121746361255646, + "p95": 0.20687000453472137 + }, + { + "label": "completed_group_3", + "mean": 0.08584291487932205, + "median": 0.0827617198228836, + "p05": 0.060561805963516235, + "p95": 0.11351694166660309 + }, + { + "label": "completed_group_4", + "mean": 0.14632999897003174, + "median": 0.1426895558834076, + "p05": 0.10106660425662994, + "p95": 0.20179018378257751 + }, + { + "label": "current_group_partial", + "mean": 0.3738824725151062, + "median": 0.3799295425415039, + "p05": 0.23631010949611664, + "p95": 0.4884527325630188 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.709290862083435, + "group": 5, + "latest_source_mass": 0.2901708781719208, + "layer": 20, + "max_source_mass": 0.2901708781719208, + "mixer_index": 38, + "normalized_entropy": 0.9539733939957076, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18991519510746002, + "median": 0.1924753338098526, + "p05": 0.14201581478118896, + "p95": 0.23709386587142944 + }, + { + "label": "completed_group_1", + "mean": 0.17091524600982666, + "median": 0.17006167769432068, + "p05": 0.15297070145606995, + "p95": 0.19153237342834473 + }, + { + "label": "completed_group_2", + "mean": 0.07822999358177185, + "median": 0.07778562605381012, + "p05": 0.06707640737295151, + "p95": 0.09089364111423492 + }, + { + "label": "completed_group_3", + "mean": 0.116765096783638, + "median": 0.11516137421131134, + "p05": 0.10162825882434845, + "p95": 0.13653048872947693 + }, + { + "label": "completed_group_4", + "mean": 0.1540035903453827, + "median": 0.15287795662879944, + "p05": 0.12593276798725128, + "p95": 0.18590238690376282 + }, + { + "label": "current_group_partial", + "mean": 0.2901708781719208, + "median": 0.29104921221733093, + "p05": 0.25643545389175415, + "p95": 0.3205650746822357 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6690887212753296, + "group": 5, + "latest_source_mass": 0.3342471718788147, + "layer": 20, + "max_source_mass": 0.3342471718788147, + "mixer_index": 39, + "normalized_entropy": 0.9315361520005944, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19029226899147034, + "median": 0.19644704461097717, + "p05": 0.09456793963909149, + "p95": 0.30468082427978516 + }, + { + "label": "completed_group_1", + "mean": 0.1100241169333458, + "median": 0.10808776319026947, + "p05": 0.09035106748342514, + "p95": 0.1359245330095291 + }, + { + "label": "completed_group_2", + "mean": 0.11471512168645859, + "median": 0.11494918912649155, + "p05": 0.08799737691879272, + "p95": 0.14024731516838074 + }, + { + "label": "completed_group_3", + "mean": 0.09999909996986389, + "median": 0.09682698547840118, + "p05": 0.07921088486909866, + "p95": 0.1264391839504242 + }, + { + "label": "completed_group_4", + "mean": 0.1507222205400467, + "median": 0.1476232260465622, + "p05": 0.11397356539964676, + "p95": 0.19991227984428406 + }, + { + "label": "current_group_partial", + "mean": 0.3342471718788147, + "median": 0.33802270889282227, + "p05": 0.2548266053199768, + "p95": 0.40130624175071716 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7458693981170654, + "group": 6, + "latest_source_mass": 0.2361394464969635, + "layer": 21, + "max_source_mass": 0.2361394464969635, + "mixer_index": 40, + "normalized_entropy": 0.9743882636597644, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20528709888458252, + "median": 0.2140858769416809, + "p05": 0.13168928027153015, + "p95": 0.27098023891448975 + }, + { + "label": "completed_group_1", + "mean": 0.1722964197397232, + "median": 0.17238952219486237, + "p05": 0.15053534507751465, + "p95": 0.19365043938159943 + }, + { + "label": "completed_group_2", + "mean": 0.10435444861650467, + "median": 0.10361101478338242, + "p05": 0.08888213336467743, + "p95": 0.12212536484003067 + }, + { + "label": "completed_group_3", + "mean": 0.12075909972190857, + "median": 0.1180175170302391, + "p05": 0.10358668118715286, + "p95": 0.14458929002285004 + }, + { + "label": "completed_group_4", + "mean": 0.16116347908973694, + "median": 0.15841388702392578, + "p05": 0.1337081342935562, + "p95": 0.19791515171527863 + }, + { + "label": "completed_group_5", + "mean": 0.2361394464969635, + "median": 0.23709020018577576, + "p05": 0.20446214079856873, + "p95": 0.26509594917297363 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4612125158309937, + "group": 6, + "latest_source_mass": 0.5543455481529236, + "layer": 21, + "max_source_mass": 0.5543455481529236, + "mixer_index": 41, + "normalized_entropy": 0.7509146897354808, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08934466540813446, + "median": 0.07911523431539536, + "p05": 0.04488275945186615, + "p95": 0.1737232506275177 + }, + { + "label": "completed_group_1", + "mean": 0.054379627108573914, + "median": 0.05344422906637192, + "p05": 0.04221750423312187, + "p95": 0.06924191862344742 + }, + { + "label": "completed_group_2", + "mean": 0.07567165046930313, + "median": 0.07438549399375916, + "p05": 0.05811355262994766, + "p95": 0.09632256627082825 + }, + { + "label": "completed_group_3", + "mean": 0.05758609622716904, + "median": 0.05612001568078995, + "p05": 0.044792283326387405, + "p95": 0.07347827404737473 + }, + { + "label": "completed_group_4", + "mean": 0.07884950935840607, + "median": 0.07757881283760071, + "p05": 0.05758974328637123, + "p95": 0.10452556610107422 + }, + { + "label": "completed_group_5", + "mean": 0.08982288837432861, + "median": 0.08912864327430725, + "p05": 0.0663854256272316, + "p95": 0.11604885756969452 + }, + { + "label": "current_group_partial", + "mean": 0.5543455481529236, + "median": 0.5596089363098145, + "p05": 0.45455610752105713, + "p95": 0.6367484331130981 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7241814136505127, + "group": 6, + "latest_source_mass": 0.39484071731567383, + "layer": 22, + "max_source_mass": 0.39484071731567383, + "mixer_index": 42, + "normalized_entropy": 0.886053970419732, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13360010087490082, + "median": 0.14072942733764648, + "p05": 0.07699814438819885, + "p95": 0.17506597936153412 + }, + { + "label": "completed_group_1", + "mean": 0.10609637200832367, + "median": 0.10586297512054443, + "p05": 0.08648421615362167, + "p95": 0.12602579593658447 + }, + { + "label": "completed_group_2", + "mean": 0.06363911926746368, + "median": 0.06324908882379532, + "p05": 0.054980404675006866, + "p95": 0.07316673547029495 + }, + { + "label": "completed_group_3", + "mean": 0.07426095008850098, + "median": 0.07362888753414154, + "p05": 0.06318790465593338, + "p95": 0.08665015548467636 + }, + { + "label": "completed_group_4", + "mean": 0.09383076429367065, + "median": 0.09294568747282028, + "p05": 0.07638302445411682, + "p95": 0.11445100605487823 + }, + { + "label": "completed_group_5", + "mean": 0.13373197615146637, + "median": 0.13327741622924805, + "p05": 0.11399447172880173, + "p95": 0.15641780197620392 + }, + { + "label": "current_group_partial", + "mean": 0.39484071731567383, + "median": 0.3879151940345764, + "p05": 0.3302172124385834, + "p95": 0.4852263331413269 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3979300260543823, + "group": 6, + "latest_source_mass": 0.5830056071281433, + "layer": 22, + "max_source_mass": 0.5830056071281433, + "mixer_index": 43, + "normalized_entropy": 0.7183939231382495, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08959861099720001, + "median": 0.07544538378715515, + "p05": 0.04165133833885193, + "p95": 0.20099139213562012 + }, + { + "label": "completed_group_1", + "mean": 0.05414731800556183, + "median": 0.05336771905422211, + "p05": 0.04069708660244942, + "p95": 0.07046990841627121 + }, + { + "label": "completed_group_2", + "mean": 0.07668881118297577, + "median": 0.07607124000787735, + "p05": 0.0555989071726799, + "p95": 0.09980960935354233 + }, + { + "label": "completed_group_3", + "mean": 0.05603286623954773, + "median": 0.054573021829128265, + "p05": 0.04162490367889404, + "p95": 0.0730118378996849 + }, + { + "label": "completed_group_4", + "mean": 0.06979228556156158, + "median": 0.06884932518005371, + "p05": 0.04576495662331581, + "p95": 0.09806846082210541 + }, + { + "label": "completed_group_5", + "mean": 0.07073447108268738, + "median": 0.07023533433675766, + "p05": 0.04956750199198723, + "p95": 0.09379896521568298 + }, + { + "label": "current_group_partial", + "mean": 0.5830056071281433, + "median": 0.5862616896629333, + "p05": 0.4658474922180176, + "p95": 0.6849814057350159 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7459560632705688, + "group": 6, + "latest_source_mass": 0.37691548466682434, + "layer": 23, + "max_source_mass": 0.37691548466682434, + "mixer_index": 44, + "normalized_entropy": 0.897243926765161, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14318783581256866, + "median": 0.1493408977985382, + "p05": 0.08292607218027115, + "p95": 0.19464610517024994 + }, + { + "label": "completed_group_1", + "mean": 0.1094653457403183, + "median": 0.10858799517154694, + "p05": 0.09432064741849899, + "p95": 0.1279931515455246 + }, + { + "label": "completed_group_2", + "mean": 0.06349228322505951, + "median": 0.06325754523277283, + "p05": 0.05439673364162445, + "p95": 0.0737837627530098 + }, + { + "label": "completed_group_3", + "mean": 0.07859592139720917, + "median": 0.0775163471698761, + "p05": 0.06603655964136124, + "p95": 0.09357087314128876 + }, + { + "label": "completed_group_4", + "mean": 0.09567482024431229, + "median": 0.09468592703342438, + "p05": 0.07671141624450684, + "p95": 0.11774884164333344 + }, + { + "label": "completed_group_5", + "mean": 0.13266831636428833, + "median": 0.13199329376220703, + "p05": 0.11244359612464905, + "p95": 0.15504513680934906 + }, + { + "label": "current_group_partial", + "mean": 0.37691548466682434, + "median": 0.37180638313293457, + "p05": 0.31207239627838135, + "p95": 0.4595233201980591 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4055393934249878, + "group": 6, + "latest_source_mass": 0.5772827863693237, + "layer": 23, + "max_source_mass": 0.5772827863693237, + "mixer_index": 45, + "normalized_entropy": 0.7223043644164862, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10154955089092255, + "median": 0.08766824007034302, + "p05": 0.04901552200317383, + "p95": 0.20287738740444183 + }, + { + "label": "completed_group_1", + "mean": 0.05619276314973831, + "median": 0.055477917194366455, + "p05": 0.041608311235904694, + "p95": 0.07371112704277039 + }, + { + "label": "completed_group_2", + "mean": 0.07791627198457718, + "median": 0.07736755907535553, + "p05": 0.055920060724020004, + "p95": 0.10158214718103409 + }, + { + "label": "completed_group_3", + "mean": 0.053915880620479584, + "median": 0.05215863138437271, + "p05": 0.03798118978738785, + "p95": 0.07156119495630264 + }, + { + "label": "completed_group_4", + "mean": 0.06398355215787888, + "median": 0.06229078769683838, + "p05": 0.040616974234580994, + "p95": 0.09270931780338287 + }, + { + "label": "completed_group_5", + "mean": 0.06915920227766037, + "median": 0.0684593990445137, + "p05": 0.046986356377601624, + "p95": 0.09408729523420334 + }, + { + "label": "current_group_partial", + "mean": 0.5772827863693237, + "median": 0.5824689269065857, + "p05": 0.4518115520477295, + "p95": 0.6926202178001404 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7853620052337646, + "group": 6, + "latest_source_mass": 0.3407441973686218, + "layer": 24, + "max_source_mass": 0.3407441973686218, + "mixer_index": 46, + "normalized_entropy": 0.9174945750195659, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15284113585948944, + "median": 0.1593906283378601, + "p05": 0.08678612112998962, + "p95": 0.211342453956604 + }, + { + "label": "completed_group_1", + "mean": 0.12206703424453735, + "median": 0.12153112143278122, + "p05": 0.1060115247964859, + "p95": 0.1399766057729721 + }, + { + "label": "completed_group_2", + "mean": 0.06508444249629974, + "median": 0.06482555717229843, + "p05": 0.055350616574287415, + "p95": 0.07557765394449234 + }, + { + "label": "completed_group_3", + "mean": 0.08347313106060028, + "median": 0.08190013468265533, + "p05": 0.06966532766819, + "p95": 0.10050469636917114 + }, + { + "label": "completed_group_4", + "mean": 0.09811970591545105, + "median": 0.09664031863212585, + "p05": 0.07688035070896149, + "p95": 0.12369741499423981 + }, + { + "label": "completed_group_5", + "mean": 0.13767032325267792, + "median": 0.1373325139284134, + "p05": 0.11642646044492722, + "p95": 0.1606694608926773 + }, + { + "label": "current_group_partial", + "mean": 0.3407441973686218, + "median": 0.33805108070373535, + "p05": 0.271301805973053, + "p95": 0.41781002283096313 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.509842872619629, + "group": 6, + "latest_source_mass": 0.5227814316749573, + "layer": 24, + "max_source_mass": 0.5227814316749573, + "mixer_index": 47, + "normalized_entropy": 0.7759057494780099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11788781732320786, + "median": 0.10097194463014603, + "p05": 0.06265684217214584, + "p95": 0.2408325970172882 + }, + { + "label": "completed_group_1", + "mean": 0.06822571158409119, + "median": 0.0673636943101883, + "p05": 0.05209728330373764, + "p95": 0.08648915588855743 + }, + { + "label": "completed_group_2", + "mean": 0.08556114137172699, + "median": 0.08463940769433975, + "p05": 0.06234566867351532, + "p95": 0.1125786080956459 + }, + { + "label": "completed_group_3", + "mean": 0.06067090481519699, + "median": 0.058974385261535645, + "p05": 0.043267179280519485, + "p95": 0.07997901737689972 + }, + { + "label": "completed_group_4", + "mean": 0.07228855788707733, + "median": 0.06972275674343109, + "p05": 0.04689513146877289, + "p95": 0.106512151658535 + }, + { + "label": "completed_group_5", + "mean": 0.07258446514606476, + "median": 0.07082539796829224, + "p05": 0.050949547439813614, + "p95": 0.09970614314079285 + }, + { + "label": "current_group_partial", + "mean": 0.5227814316749573, + "median": 0.5290273427963257, + "p05": 0.38773614168167114, + "p95": 0.6392589807510376 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.839367389678955, + "group": 7, + "latest_source_mass": 0.29583215713500977, + "layer": 25, + "max_source_mass": 0.29583215713500977, + "mixer_index": 48, + "normalized_entropy": 0.9452478525649903, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15906387567520142, + "median": 0.15846745669841766, + "p05": 0.09519976377487183, + "p95": 0.24517810344696045 + }, + { + "label": "completed_group_1", + "mean": 0.13139380514621735, + "median": 0.1308426856994629, + "p05": 0.11478274315595627, + "p95": 0.15000829100608826 + }, + { + "label": "completed_group_2", + "mean": 0.07717745751142502, + "median": 0.07715500891208649, + "p05": 0.06616044044494629, + "p95": 0.08791667222976685 + }, + { + "label": "completed_group_3", + "mean": 0.09607303142547607, + "median": 0.09436552226543427, + "p05": 0.08152313530445099, + "p95": 0.11384639143943787 + }, + { + "label": "completed_group_4", + "mean": 0.1107850894331932, + "median": 0.1089654266834259, + "p05": 0.08814431726932526, + "p95": 0.139452263712883 + }, + { + "label": "completed_group_5", + "mean": 0.12967455387115479, + "median": 0.1299237608909607, + "p05": 0.10832332074642181, + "p95": 0.15138982236385345 + }, + { + "label": "completed_group_6", + "mean": 0.29583215713500977, + "median": 0.29647016525268555, + "p05": 0.2276897132396698, + "p95": 0.3604946434497833 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5841834545135498, + "group": 7, + "latest_source_mass": 0.40767741203308105, + "layer": 25, + "max_source_mass": 0.40767741203308105, + "mixer_index": 49, + "normalized_entropy": 0.7618312045616817, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07896879315376282, + "median": 0.06809917092323303, + "p05": 0.025932887569069862, + "p95": 0.19368338584899902 + }, + { + "label": "completed_group_1", + "mean": 0.048752427101135254, + "median": 0.04767637699842453, + "p05": 0.034443892538547516, + "p95": 0.0667153000831604 + }, + { + "label": "completed_group_2", + "mean": 0.055707164108753204, + "median": 0.0549345538020134, + "p05": 0.038696639239788055, + "p95": 0.0750277042388916 + }, + { + "label": "completed_group_3", + "mean": 0.04139826446771622, + "median": 0.03878612071275711, + "p05": 0.026673054322600365, + "p95": 0.05746251344680786 + }, + { + "label": "completed_group_4", + "mean": 0.04449018836021423, + "median": 0.04196460545063019, + "p05": 0.026162121444940567, + "p95": 0.07164917886257172 + }, + { + "label": "completed_group_5", + "mean": 0.04285799711942673, + "median": 0.04109475016593933, + "p05": 0.029393533244729042, + "p95": 0.06277710199356079 + }, + { + "label": "completed_group_6", + "mean": 0.2801477909088135, + "median": 0.2763082981109619, + "p05": 0.1455708146095276, + "p95": 0.4254714846611023 + }, + { + "label": "current_group_partial", + "mean": 0.40767741203308105, + "median": 0.4120897948741913, + "p05": 0.21399420499801636, + "p95": 0.5853031277656555 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8125314712524414, + "group": 7, + "latest_source_mass": 0.3700707256793976, + "layer": 26, + "max_source_mass": 0.3700707256793976, + "mixer_index": 50, + "normalized_entropy": 0.8716433883436914, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10889513790607452, + "median": 0.11333495378494263, + "p05": 0.04850132763385773, + "p95": 0.1620434820652008 + }, + { + "label": "completed_group_1", + "mean": 0.08656036853790283, + "median": 0.08640051633119583, + "p05": 0.0650845319032669, + "p95": 0.1084962785243988 + }, + { + "label": "completed_group_2", + "mean": 0.048123255372047424, + "median": 0.04800213500857353, + "p05": 0.03576180338859558, + "p95": 0.06107594817876816 + }, + { + "label": "completed_group_3", + "mean": 0.06561170518398285, + "median": 0.06379580497741699, + "p05": 0.050968848168849945, + "p95": 0.08332888036966324 + }, + { + "label": "completed_group_4", + "mean": 0.06917259097099304, + "median": 0.06724178791046143, + "p05": 0.050207968801259995, + "p95": 0.09373342245817184 + }, + { + "label": "completed_group_5", + "mean": 0.08286689221858978, + "median": 0.08183546364307404, + "p05": 0.06535022705793381, + "p95": 0.10340933501720428 + }, + { + "label": "completed_group_6", + "mean": 0.16869932413101196, + "median": 0.16509854793548584, + "p05": 0.11059704422950745, + "p95": 0.23654375970363617 + }, + { + "label": "current_group_partial", + "mean": 0.3700707256793976, + "median": 0.3694133758544922, + "p05": 0.23441238701343536, + "p95": 0.5025027990341187 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.50089693069458, + "group": 7, + "latest_source_mass": 0.49096518754959106, + "layer": 26, + "max_source_mass": 0.49096518754959106, + "mixer_index": 51, + "normalized_entropy": 0.7217788529328457, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07974998652935028, + "median": 0.07402504235506058, + "p05": 0.04579866677522659, + "p95": 0.1359037309885025 + }, + { + "label": "completed_group_1", + "mean": 0.04166398197412491, + "median": 0.04038450866937637, + "p05": 0.027992289513349533, + "p95": 0.05969971418380737 + }, + { + "label": "completed_group_2", + "mean": 0.04607272148132324, + "median": 0.044438350945711136, + "p05": 0.030038591474294662, + "p95": 0.06808309257030487 + }, + { + "label": "completed_group_3", + "mean": 0.03586764633655548, + "median": 0.03317485377192497, + "p05": 0.02203454077243805, + "p95": 0.05572832003235817 + }, + { + "label": "completed_group_4", + "mean": 0.04223340004682541, + "median": 0.03930673003196716, + "p05": 0.02465122565627098, + "p95": 0.06968094408512115 + }, + { + "label": "completed_group_5", + "mean": 0.04095568507909775, + "median": 0.03919431194663048, + "p05": 0.026239857077598572, + "p95": 0.06058088690042496 + }, + { + "label": "completed_group_6", + "mean": 0.22249138355255127, + "median": 0.2125401496887207, + "p05": 0.10647299885749817, + "p95": 0.3711881935596466 + }, + { + "label": "current_group_partial", + "mean": 0.49096518754959106, + "median": 0.4988659620285034, + "p05": 0.2878011167049408, + "p95": 0.6676807403564453 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8059929609298706, + "group": 7, + "latest_source_mass": 0.37139976024627686, + "layer": 27, + "max_source_mass": 0.37139976024627686, + "mixer_index": 52, + "normalized_entropy": 0.8684990295379666, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10711923241615295, + "median": 0.10972198843955994, + "p05": 0.04896187782287598, + "p95": 0.16201043128967285 + }, + { + "label": "completed_group_1", + "mean": 0.07651707530021667, + "median": 0.07666032761335373, + "p05": 0.05141128972172737, + "p95": 0.10152198374271393 + }, + { + "label": "completed_group_2", + "mean": 0.058284737169742584, + "median": 0.05812416970729828, + "p05": 0.039273619651794434, + "p95": 0.07745087146759033 + }, + { + "label": "completed_group_3", + "mean": 0.06609587371349335, + "median": 0.06383530795574188, + "p05": 0.04481930285692215, + "p95": 0.09336265176534653 + }, + { + "label": "completed_group_4", + "mean": 0.07139429450035095, + "median": 0.0694514587521553, + "p05": 0.0485839769244194, + "p95": 0.10034126043319702 + }, + { + "label": "completed_group_5", + "mean": 0.07493151724338531, + "median": 0.07319007813930511, + "p05": 0.051876723766326904, + "p95": 0.10402432829141617 + }, + { + "label": "completed_group_6", + "mean": 0.17425751686096191, + "median": 0.16937679052352905, + "p05": 0.10449051856994629, + "p95": 0.2624811828136444 + }, + { + "label": "current_group_partial", + "mean": 0.37139976024627686, + "median": 0.3732905089855194, + "p05": 0.23830299079418182, + "p95": 0.5014542937278748 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5622732639312744, + "group": 7, + "latest_source_mass": 0.4768889844417572, + "layer": 27, + "max_source_mass": 0.4768889844417572, + "mixer_index": 53, + "normalized_entropy": 0.7512946301290215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08775827288627625, + "median": 0.08504067361354828, + "p05": 0.0428270623087883, + "p95": 0.144285649061203 + }, + { + "label": "completed_group_1", + "mean": 0.047765426337718964, + "median": 0.04646281152963638, + "p05": 0.031814731657505035, + "p95": 0.06807165592908859 + }, + { + "label": "completed_group_2", + "mean": 0.053928617388010025, + "median": 0.052262187004089355, + "p05": 0.035437777638435364, + "p95": 0.0784272700548172 + }, + { + "label": "completed_group_3", + "mean": 0.04293380305171013, + "median": 0.03996016085147858, + "p05": 0.02556057646870613, + "p95": 0.06623870134353638 + }, + { + "label": "completed_group_4", + "mean": 0.04174989461898804, + "median": 0.03903091698884964, + "p05": 0.025013400241732597, + "p95": 0.06869364529848099 + }, + { + "label": "completed_group_5", + "mean": 0.04734005779027939, + "median": 0.04609068110585213, + "p05": 0.029884062707424164, + "p95": 0.06911227852106094 + }, + { + "label": "completed_group_6", + "mean": 0.20163494348526, + "median": 0.19222964346408844, + "p05": 0.09963390231132507, + "p95": 0.3307305574417114 + }, + { + "label": "current_group_partial", + "mean": 0.4768889844417572, + "median": 0.48383551836013794, + "p05": 0.276212215423584, + "p95": 0.6487575173377991 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8757455348968506, + "group": 7, + "latest_source_mass": 0.3109493851661682, + "layer": 28, + "max_source_mass": 0.3109493851661682, + "mixer_index": 54, + "normalized_entropy": 0.9020429270551009, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11563669890165329, + "median": 0.12087491154670715, + "p05": 0.05733206868171692, + "p95": 0.1688953936100006 + }, + { + "label": "completed_group_1", + "mean": 0.08108697831630707, + "median": 0.08185897022485733, + "p05": 0.05175650864839554, + "p95": 0.10753051191568375 + }, + { + "label": "completed_group_2", + "mean": 0.07280858606100082, + "median": 0.07357072830200195, + "p05": 0.04870978742837906, + "p95": 0.0948689877986908 + }, + { + "label": "completed_group_3", + "mean": 0.07553227245807648, + "median": 0.07362619787454605, + "p05": 0.056317005306482315, + "p95": 0.09756095707416534 + }, + { + "label": "completed_group_4", + "mean": 0.07606014609336853, + "median": 0.07456053793430328, + "p05": 0.05528895929455757, + "p95": 0.1022825762629509 + }, + { + "label": "completed_group_5", + "mean": 0.06935927271842957, + "median": 0.06843917071819305, + "p05": 0.04978285729885101, + "p95": 0.09167985618114471 + }, + { + "label": "completed_group_6", + "mean": 0.19856664538383484, + "median": 0.19508783519268036, + "p05": 0.1281973123550415, + "p95": 0.27796900272369385 + }, + { + "label": "current_group_partial", + "mean": 0.3109493851661682, + "median": 0.3105033040046692, + "p05": 0.18172834813594818, + "p95": 0.43916669487953186 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6924852132797241, + "group": 7, + "latest_source_mass": 0.3315581679344177, + "layer": 28, + "max_source_mass": 0.3315581679344177, + "mixer_index": 55, + "normalized_entropy": 0.8139133413255193, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08638577908277512, + "median": 0.058608733117580414, + "p05": 0.026544082909822464, + "p95": 0.2740764319896698 + }, + { + "label": "completed_group_1", + "mean": 0.059912845492362976, + "median": 0.05901291221380234, + "p05": 0.03629481792449951, + "p95": 0.08707985281944275 + }, + { + "label": "completed_group_2", + "mean": 0.06609142571687698, + "median": 0.0647939145565033, + "p05": 0.038691695779561996, + "p95": 0.09703366458415985 + }, + { + "label": "completed_group_3", + "mean": 0.056745611131191254, + "median": 0.05418796092271805, + "p05": 0.032174739986658096, + "p95": 0.08370660990476608 + }, + { + "label": "completed_group_4", + "mean": 0.06277773529291153, + "median": 0.0600711889564991, + "p05": 0.03501531109213829, + "p95": 0.10094858705997467 + }, + { + "label": "completed_group_5", + "mean": 0.05352145433425903, + "median": 0.05179818347096443, + "p05": 0.032271772623062134, + "p95": 0.08084158599376678 + }, + { + "label": "completed_group_6", + "mean": 0.2830069661140442, + "median": 0.2759752869606018, + "p05": 0.12981083989143372, + "p95": 0.46204638481140137 + }, + { + "label": "current_group_partial", + "mean": 0.3315581679344177, + "median": 0.3204721510410309, + "p05": 0.13446375727653503, + "p95": 0.5667697787284851 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8980522155761719, + "group": 8, + "latest_source_mass": 0.20123262703418732, + "layer": 29, + "max_source_mass": 0.26602888107299805, + "mixer_index": 56, + "normalized_entropy": 0.9127701729200177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11603894829750061, + "median": 0.09991352260112762, + "p05": 0.030035395175218582, + "p95": 0.2941743731498718 + }, + { + "label": "completed_group_1", + "mean": 0.08804173022508621, + "median": 0.08880250155925751, + "p05": 0.05664174258708954, + "p95": 0.11779069900512695 + }, + { + "label": "completed_group_2", + "mean": 0.07975024729967117, + "median": 0.07984748482704163, + "p05": 0.05428915470838547, + "p95": 0.10499341040849686 + }, + { + "label": "completed_group_3", + "mean": 0.09027000516653061, + "median": 0.08898856490850449, + "p05": 0.06189651042222977, + "p95": 0.11996553838253021 + }, + { + "label": "completed_group_4", + "mean": 0.08430293202400208, + "median": 0.0819794088602066, + "p05": 0.05464346706867218, + "p95": 0.12159879505634308 + }, + { + "label": "completed_group_5", + "mean": 0.07433462142944336, + "median": 0.07287181913852692, + "p05": 0.052520088851451874, + "p95": 0.1010315865278244 + }, + { + "label": "completed_group_6", + "mean": 0.26602888107299805, + "median": 0.2639757990837097, + "p05": 0.15474572777748108, + "p95": 0.3846370279788971 + }, + { + "label": "completed_group_7", + "mean": 0.20123262703418732, + "median": 0.18140269815921783, + "p05": 0.08419936150312424, + "p95": 0.3782624900341034 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7512003183364868, + "group": 8, + "latest_source_mass": 0.33482813835144043, + "layer": 29, + "max_source_mass": 0.33482813835144043, + "mixer_index": 57, + "normalized_entropy": 0.7970056117156376, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07744461297988892, + "median": 0.06970229744911194, + "p05": 0.014401955530047417, + "p95": 0.17144526541233063 + }, + { + "label": "completed_group_1", + "mean": 0.046086348593235016, + "median": 0.04394831880927086, + "p05": 0.028066720813512802, + "p95": 0.0718703418970108 + }, + { + "label": "completed_group_2", + "mean": 0.05025433003902435, + "median": 0.04861236736178398, + "p05": 0.029644077643752098, + "p95": 0.07761462777853012 + }, + { + "label": "completed_group_3", + "mean": 0.03748488053679466, + "median": 0.03434944897890091, + "p05": 0.020774798467755318, + "p95": 0.05938909202814102 + }, + { + "label": "completed_group_4", + "mean": 0.04005061462521553, + "median": 0.037498839199543, + "p05": 0.019541850313544273, + "p95": 0.06922487914562225 + }, + { + "label": "completed_group_5", + "mean": 0.041623204946517944, + "median": 0.039552345871925354, + "p05": 0.022081367671489716, + "p95": 0.06821337342262268 + }, + { + "label": "completed_group_6", + "mean": 0.198423832654953, + "median": 0.18318486213684082, + "p05": 0.07548617571592331, + "p95": 0.3721975088119507 + }, + { + "label": "completed_group_7", + "mean": 0.17380405962467194, + "median": 0.1514919549226761, + "p05": 0.05777408182621002, + "p95": 0.36326974630355835 + }, + { + "label": "current_group_partial", + "mean": 0.33482813835144043, + "median": 0.33602219820022583, + "p05": 0.0795319601893425, + "p95": 0.5979493856430054 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0160694122314453, + "group": 8, + "latest_source_mass": 0.21711516380310059, + "layer": 30, + "max_source_mass": 0.21711516380310059, + "mixer_index": 58, + "normalized_entropy": 0.9175527313077866, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09385271370410919, + "median": 0.09211117774248123, + "p05": 0.032377734780311584, + "p95": 0.16496515274047852 + }, + { + "label": "completed_group_1", + "mean": 0.06208505108952522, + "median": 0.06140740588307381, + "p05": 0.0407925546169281, + "p95": 0.0863042101264 + }, + { + "label": "completed_group_2", + "mean": 0.06560768187046051, + "median": 0.06516212224960327, + "p05": 0.041897669434547424, + "p95": 0.09085705131292343 + }, + { + "label": "completed_group_3", + "mean": 0.07029052078723907, + "median": 0.06780634075403214, + "p05": 0.04846132919192314, + "p95": 0.09917552024126053 + }, + { + "label": "completed_group_4", + "mean": 0.06935097277164459, + "median": 0.06708167493343353, + "p05": 0.04668639227747917, + "p95": 0.09815650433301926 + }, + { + "label": "completed_group_5", + "mean": 0.056832700967788696, + "median": 0.05595892667770386, + "p05": 0.038822539150714874, + "p95": 0.0771557092666626 + }, + { + "label": "completed_group_6", + "mean": 0.19425815343856812, + "median": 0.19145764410495758, + "p05": 0.12249615788459778, + "p95": 0.27496030926704407 + }, + { + "label": "completed_group_7", + "mean": 0.17060703039169312, + "median": 0.16317546367645264, + "p05": 0.09239593893289566, + "p95": 0.2726203501224518 + }, + { + "label": "current_group_partial", + "mean": 0.21711516380310059, + "median": 0.21198657155036926, + "p05": 0.11019522696733475, + "p95": 0.3391069769859314 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7837426662445068, + "group": 8, + "latest_source_mass": 0.29026859998703003, + "layer": 30, + "max_source_mass": 0.29026859998703003, + "mixer_index": 59, + "normalized_entropy": 0.8118162725118464, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08836327493190765, + "median": 0.07904411852359772, + "p05": 0.02631576545536518, + "p95": 0.17534467577934265 + }, + { + "label": "completed_group_1", + "mean": 0.04953558370471001, + "median": 0.04658038914203644, + "p05": 0.025181813165545464, + "p95": 0.08318150043487549 + }, + { + "label": "completed_group_2", + "mean": 0.04813014343380928, + "median": 0.045740917325019836, + "p05": 0.026714935898780823, + "p95": 0.07843323051929474 + }, + { + "label": "completed_group_3", + "mean": 0.0373263955116272, + "median": 0.03401809185743332, + "p05": 0.01990434154868126, + "p95": 0.06290259957313538 + }, + { + "label": "completed_group_4", + "mean": 0.037015400826931, + "median": 0.03405017405748367, + "p05": 0.018646355718374252, + "p95": 0.06544196605682373 + }, + { + "label": "completed_group_5", + "mean": 0.03390160948038101, + "median": 0.03198609501123428, + "p05": 0.018189728260040283, + "p95": 0.05570102855563164 + }, + { + "label": "completed_group_6", + "mean": 0.1840541511774063, + "median": 0.17327937483787537, + "p05": 0.07966600358486176, + "p95": 0.32982927560806274 + }, + { + "label": "completed_group_7", + "mean": 0.23140482604503632, + "median": 0.22406929731369019, + "p05": 0.08831030130386353, + "p95": 0.40350067615509033 + }, + { + "label": "current_group_partial", + "mean": 0.29026859998703003, + "median": 0.2745037376880646, + "p05": 0.10280418395996094, + "p95": 0.5399085879325867 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9973840713500977, + "group": 8, + "latest_source_mass": 0.19101998209953308, + "layer": 31, + "max_source_mass": 0.20831164717674255, + "mixer_index": 60, + "normalized_entropy": 0.9090486661912383, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10261696577072144, + "median": 0.10389091819524765, + "p05": 0.01371697522699833, + "p95": 0.21163305640220642 + }, + { + "label": "completed_group_1", + "mean": 0.06039813905954361, + "median": 0.06085003539919853, + "p05": 0.026107093319296837, + "p95": 0.09348798543214798 + }, + { + "label": "completed_group_2", + "mean": 0.06671924144029617, + "median": 0.06656201928853989, + "p05": 0.034684643149375916, + "p95": 0.09819839894771576 + }, + { + "label": "completed_group_3", + "mean": 0.08377155661582947, + "median": 0.0804532840847969, + "p05": 0.05448959022760391, + "p95": 0.12039553374052048 + }, + { + "label": "completed_group_4", + "mean": 0.07002593576908112, + "median": 0.06737451255321503, + "p05": 0.044841229915618896, + "p95": 0.10655929893255234 + }, + { + "label": "completed_group_5", + "mean": 0.05768897384405136, + "median": 0.056077782064676285, + "p05": 0.039495617151260376, + "p95": 0.08242568373680115 + }, + { + "label": "completed_group_6", + "mean": 0.20831164717674255, + "median": 0.2021285891532898, + "p05": 0.11652229726314545, + "p95": 0.31706833839416504 + }, + { + "label": "completed_group_7", + "mean": 0.1594475358724594, + "median": 0.1546279788017273, + "p05": 0.06593266129493713, + "p95": 0.2735573351383209 + }, + { + "label": "current_group_partial", + "mean": 0.19101998209953308, + "median": 0.1713225245475769, + "p05": 0.06408411264419556, + "p95": 0.3850628137588501 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.969987154006958, + "group": 8, + "latest_source_mass": 0.16864418983459473, + "layer": 31, + "max_source_mass": 0.24503237009048462, + "mixer_index": 61, + "normalized_entropy": 0.8965797917640489, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08120635151863098, + "median": 0.061164144426584244, + "p05": 0.0259944386780262, + "p95": 0.23622018098831177 + }, + { + "label": "completed_group_1", + "mean": 0.07894632965326309, + "median": 0.07658770680427551, + "p05": 0.0496862456202507, + "p95": 0.11596276611089706 + }, + { + "label": "completed_group_2", + "mean": 0.08615730702877045, + "median": 0.08467075228691101, + "p05": 0.050409119576215744, + "p95": 0.12811316549777985 + }, + { + "label": "completed_group_3", + "mean": 0.07609779387712479, + "median": 0.0722871869802475, + "p05": 0.04099677503108978, + "p95": 0.11950768530368805 + }, + { + "label": "completed_group_4", + "mean": 0.058728985488414764, + "median": 0.056080836802721024, + "p05": 0.03323862701654434, + "p95": 0.09344407916069031 + }, + { + "label": "completed_group_5", + "mean": 0.05076971277594566, + "median": 0.04946017265319824, + "p05": 0.03047419898211956, + "p95": 0.07563400268554688 + }, + { + "label": "completed_group_6", + "mean": 0.24503237009048462, + "median": 0.24110376834869385, + "p05": 0.10613486170768738, + "p95": 0.3991503119468689 + }, + { + "label": "completed_group_7", + "mean": 0.1544169783592224, + "median": 0.1400495171546936, + "p05": 0.06199025735259056, + "p95": 0.2995939254760742 + }, + { + "label": "current_group_partial", + "mean": 0.16864418983459473, + "median": 0.15222924947738647, + "p05": 0.0543956533074379, + "p95": 0.3329208791255951 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.071648359298706, + "group": 8, + "latest_source_mass": 0.11520501971244812, + "layer": 32, + "max_source_mass": 0.20955611765384674, + "mixer_index": 62, + "normalized_entropy": 0.9428478002054053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10569609701633453, + "median": 0.08689188957214355, + "p05": 0.03173588588833809, + "p95": 0.24571751058101654 + }, + { + "label": "completed_group_1", + "mean": 0.09922641515731812, + "median": 0.09795533865690231, + "p05": 0.06407676637172699, + "p95": 0.14120742678642273 + }, + { + "label": "completed_group_2", + "mean": 0.08180510997772217, + "median": 0.080217644572258, + "p05": 0.054607540369033813, + "p95": 0.11442367732524872 + }, + { + "label": "completed_group_3", + "mean": 0.10646799206733704, + "median": 0.10510502755641937, + "p05": 0.07415887713432312, + "p95": 0.14216329157352448 + }, + { + "label": "completed_group_4", + "mean": 0.0826798751950264, + "median": 0.08022409677505493, + "p05": 0.05491747707128525, + "p95": 0.11860857903957367 + }, + { + "label": "completed_group_5", + "mean": 0.06615597754716873, + "median": 0.06485182791948318, + "p05": 0.04477996751666069, + "p95": 0.09238749742507935 + }, + { + "label": "completed_group_6", + "mean": 0.20955611765384674, + "median": 0.206287682056427, + "p05": 0.129981130361557, + "p95": 0.2996031939983368 + }, + { + "label": "completed_group_7", + "mean": 0.13320739567279816, + "median": 0.12465103715658188, + "p05": 0.06303662061691284, + "p95": 0.23065781593322754 + }, + { + "label": "current_group_partial", + "mean": 0.11520501971244812, + "median": 0.10386130958795547, + "p05": 0.03912060335278511, + "p95": 0.22735844552516937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781701564788818, + "group": 8, + "latest_source_mass": 0.16112206876277924, + "layer": 32, + "max_source_mass": 0.22251944243907928, + "mixer_index": 63, + "normalized_entropy": 0.9003040366848136, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09446848928928375, + "median": 0.074150450527668, + "p05": 0.042592622339725494, + "p95": 0.19443637132644653 + }, + { + "label": "completed_group_1", + "mean": 0.07056106626987457, + "median": 0.06868506222963333, + "p05": 0.04818308353424072, + "p95": 0.0982733890414238 + }, + { + "label": "completed_group_2", + "mean": 0.08033448457717896, + "median": 0.08047192543745041, + "p05": 0.048296794295310974, + "p95": 0.11253741383552551 + }, + { + "label": "completed_group_3", + "mean": 0.0658978745341301, + "median": 0.06256154924631119, + "p05": 0.04223886504769325, + "p95": 0.0967884361743927 + }, + { + "label": "completed_group_4", + "mean": 0.058234501630067825, + "median": 0.0563921295106411, + "p05": 0.03529864549636841, + "p95": 0.08846980333328247 + }, + { + "label": "completed_group_5", + "mean": 0.04316519945859909, + "median": 0.042268507182598114, + "p05": 0.029443470761179924, + "p95": 0.05996086448431015 + }, + { + "label": "completed_group_6", + "mean": 0.2036968618631363, + "median": 0.19645483791828156, + "p05": 0.09398645162582397, + "p95": 0.3395216464996338 + }, + { + "label": "completed_group_7", + "mean": 0.22251944243907928, + "median": 0.2130982130765915, + "p05": 0.11867167800664902, + "p95": 0.35465875267982483 + }, + { + "label": "current_group_partial", + "mean": 0.16112206876277924, + "median": 0.14023703336715698, + "p05": 0.06008770316839218, + "p95": 0.3247341811656952 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.6036605834960938, + "group": null, + "latest_source_mass": 0.31639498472213745, + "layer": null, + "max_source_mass": 0.3412485122680664, + "mixer_index": null, + "normalized_entropy": 0.7298573846467136, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09759747982025146, + "median": 0.09915056824684143, + "p05": 0.025461740791797638, + "p95": 0.16398483514785767 + }, + { + "label": "completed_group_1", + "mean": 0.04215940088033676, + "median": 0.04050712287425995, + "p05": 0.025720715522766113, + "p95": 0.06398092210292816 + }, + { + "label": "completed_group_2", + "mean": 0.041145481169223785, + "median": 0.03881259262561798, + "p05": 0.023510465398430824, + "p95": 0.0662468671798706 + }, + { + "label": "completed_group_3", + "mean": 0.029554028064012527, + "median": 0.027420252561569214, + "p05": 0.01627083122730255, + "p95": 0.04924973472952843 + }, + { + "label": "completed_group_4", + "mean": 0.0289996936917305, + "median": 0.027226522564888, + "p05": 0.01560397818684578, + "p95": 0.048351090401411057 + }, + { + "label": "completed_group_5", + "mean": 0.03647579252719879, + "median": 0.0347352996468544, + "p05": 0.020744025707244873, + "p95": 0.058257561177015305 + }, + { + "label": "completed_group_6", + "mean": 0.06642461568117142, + "median": 0.057809896767139435, + "p05": 0.022837381809949875, + "p95": 0.14078694581985474 + }, + { + "label": "completed_group_7", + "mean": 0.3412485122680664, + "median": 0.3336990475654602, + "p05": 0.0819513276219368, + "p95": 0.6423859596252441 + }, + { + "label": "completed_group_8", + "mean": 0.31639498472213745, + "median": 0.30713334679603577, + "p05": 0.05841828137636185, + "p95": 0.6156357526779175 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.24537226557731628, + 0.07698935270309448, + 0.0523640513420105, + 0.09380833059549332, + 0.03647654876112938, + 0.039120711386203766, + 0.06371714919805527, + 0.05156813934445381, + 0.03728010877966881, + 0.03983496502041817, + 0.040284499526023865, + 0.046871740370988846, + 0.04314620420336723, + 0.04033593460917473, + 0.04471700266003609, + 0.08397205173969269, + 0.0528092160820961, + 0.04443390294909477, + 0.07560808956623077, + 0.03708760067820549, + 0.03599191829562187, + 0.033304885029792786, + 0.04061548039317131, + 0.025487789884209633, + 0.021565141156315804, + 0.02342291921377182, + 0.017244853079319, + 0.024166326969861984, + 0.016819356009364128, + 0.021590016782283783, + 0.026201345026493073 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4657019391961512e-05, + "first_to_last_ratio": 0.6094218601945491, + "imbalance_abs_log_ratio": 0.4952445414057865, + "last_quartile_mean": 2.4050695174082648e-05, + "mean": 2.0123495630741672e-05, + "normalized": [ + 0.6757922451357102, + 0.7862872769192628, + 0.6956720089134658, + 0.6457028717597475, + 0.8486081172555344, + 0.7866590564328257, + 0.7355608412572463, + 0.6525459469054529, + 0.5676581895263636, + 0.5065196225058339, + 0.47654698952417385, + 0.4567643060278592, + 0.6388028952758977, + 0.6012584522984605, + 0.5677051478189228, + 0.5320958926863361, + 0.6601484899512206, + 0.6199678744151763, + 0.6252065487643861, + 0.6703348689792878, + 3.1476439082057013, + 2.6843722856032843, + 2.147025515739066, + 1.709881143629556, + 2.310023308278431, + 2.0253173950794507, + 1.5260326037153789, + 0.9852764539113186, + 1.0403760218043532, + 0.7424001651630263, + 0.4909889920627068, + 0.44082456445456286 + ], + "peak_layer": 21, + "peak_normalized": 3.1476439082057013, + "population_cv": 0.7023423019155828, + "rest_mean": 1.4907078501117346e-05, + "spike_contrast": 3.2395447657361793, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.829214813071303e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.359930229227757e-05, + 1.5822848581592552e-05, + 1.3999352631799411e-05, + 1.299379891861463e-05, + 1.7076961739803664e-05, + 1.5830330085009336e-05, + 1.4802055375184864e-05, + 1.3131505511410069e-05, + 1.1423267096688505e-05, + 1.0192945410381071e-05, + 9.58979126153281e-06, + 9.191694516630378e-06, + 1.2854947271989658e-05, + 1.2099421837774571e-05, + 1.1424212061683647e-05, + 1.0707629371609073e-05, + 1.32844952531741e-05, + 1.2475920811994001e-05, + 1.2581341252371203e-05, + 1.348948080703849e-05, + 6.334159843390808e-05, + 5.401895396062173e-05, + 4.3205658585065976e-05, + 3.4408785722916946e-05, + 4.648574395105243e-05, + 4.075646575074643e-05, + 3.0709110433235765e-05, + 1.9827206415357068e-05, + 2.0936002329108305e-05, + 1.4939686479920056e-05, + 9.880414836516138e-06, + 8.870931196724996e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.829688234371133e-05, + "first_to_last_ratio": 0.6072503732166264, + "imbalance_abs_log_ratio": 0.4988140964987654, + "last_quartile_mean": 9.600139401300112e-05, + "mean": 8.040016996346822e-05, + "normalized": [ + 0.6783743745296411, + 0.7863508052165757, + 0.7000215696280339, + 0.647124428880743, + 0.8353484843612504, + 0.7743411566859596, + 0.7293747929726763, + 0.6497369355766927, + 0.5707307458024328, + 0.510545581500116, + 0.4799754930157162, + 0.4590012320770852, + 0.6477982227600472, + 0.6029757489491335, + 0.5684984161378385, + 0.5312301622013348, + 0.6575876681964574, + 0.6204772859169876, + 0.6292313575338936, + 0.6709435017853759, + 3.152621969125565, + 2.693161601564409, + 2.150493956812164, + 1.7016971884685128, + 2.3039790532500866, + 2.0188035652467615, + 1.519776732418517, + 0.9811964658615112, + 1.039948884922119, + 0.7451655332994308, + 0.496455864160715, + 0.4470312211422162 + ], + "peak_layer": 21, + "peak_normalized": 3.152621969125565, + "population_cv": 0.7020159094649734, + "rest_mean": 5.954986355155362e-05, + "spike_contrast": 3.24084411076324, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019299182458780705, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.454141501104459e-05, + 6.322273839032277e-05, + 5.6281853176187724e-05, + 5.2028914069524035e-05, + 6.716216012137011e-05, + 6.225716060725972e-05, + 5.864185732207261e-05, + 5.223896005190909e-05, + 4.588684896589257e-05, + 4.104795152670704e-05, + 3.8590111216763034e-05, + 3.690377707243897e-05, + 5.208308721194044e-05, + 4.847935269935988e-05, + 4.57073692814447e-05, + 4.27109953307081e-05, + 5.287016028887592e-05, + 4.988647924619727e-05, + 5.059030809206888e-05, + 5.394397157942876e-05, + 0.0002534713421482593, + 0.00021653065050486475, + 0.00017290007963310927, + 0.00013681674317922443, + 0.00018524030747357756, + 0.00016231214976869524, + 0.0001221903075929731, + 7.888836262281984e-05, + 8.361206710105762e-05, + 5.991143552819267e-05, + 3.991513585788198e-05, + 3.594138615881093e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05169825907796621, + "first_to_last_ratio": 0.607250394007124, + "imbalance_abs_log_ratio": 0.4988140622616565, + "last_quartile_mean": 0.08513499470427632, + "mean": 0.07129967457149178, + "normalized": [ + 0.6783743842436116, + 0.7863508228047997, + 0.700021554566655, + 0.6471244375634602, + 0.8353485410067275, + 0.7743411217427667, + 0.7293748339206622, + 0.6497369595226863, + 0.5707307154188002, + 0.5105455806753446, + 0.47997548951564134, + 0.4590012256825957, + 0.6477982322753953, + 0.6029757319104283, + 0.5684984046970705, + 0.5312301796601451, + 0.6575876919843755, + 0.6204773092105361, + 0.6292313633824181, + 0.6709435195164998, + 3.152621923151655, + 2.693161643472544, + 2.150493986671358, + 1.7016971770874902, + 2.30397915541657, + 2.0188034853391237, + 1.5197766802362542, + 0.9811964117969386, + 1.0399488445555949, + 0.7451654867762114, + 0.49645587282964704, + 0.44703123336599304 + ], + "peak_layer": 21, + "peak_normalized": 3.152621923151655, + "population_cv": 0.7020159090531873, + "rest_mean": 0.052809414074376775, + "spike_contrast": 3.2408441611351573, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.17114708125591277, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.04836787283420563, + 0.05606655776500702, + 0.04991130903363228, + 0.04613976180553436, + 0.059560079127550125, + 0.05521026998758316, + 0.05200418829917908, + 0.046326033771038055, + 0.04069291427731514, + 0.03640173375606537, + 0.03422209620475769, + 0.03272663801908493, + 0.04618780314922333, + 0.04299197345972061, + 0.040533751249313354, + 0.037876538932323456, + 0.046885788440704346, + 0.0442398302257061, + 0.0448639914393425, + 0.04783805459737778, + 0.22478091716766357, + 0.19202154874801636, + 0.1533295214176178, + 0.12133045494556427, + 0.1642729640007019, + 0.1439400315284729, + 0.10835958272218704, + 0.06995898485183716, + 0.0741480141878128, + 0.05313005670905113, + 0.03539714217185974, + 0.0318731814622879 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.317203296797743e-06, + "first_to_last_ratio": 0.35664923571220175, + "imbalance_abs_log_ratio": 1.0310025132235103, + "last_quartile_mean": 6.49714920086808e-06, + "mean": 4.492464356076198e-06, + "normalized": [ + 0.5694178936246215, + 0.527096091999763, + 0.5069565208451643, + 0.5168980803173514, + 0.5167362729922896, + 0.5011287746943238, + 0.4906181312174465, + 0.49753075069099545, + 0.4121289001968579, + 0.3863295396034041, + 0.3787441822616284, + 0.3878491467448858, + 0.49538413383836966, + 0.4937248622259753, + 0.4965317157240282, + 0.4922453146709141, + 0.5531637227610273, + 0.5731353625097597, + 0.5949767701689571, + 0.6269395119699129, + 2.991445440442326, + 2.8607213177085975, + 2.466076440453951, + 2.094358907038209, + 2.7670265254785567, + 2.47746135973099, + 1.9032918373610526, + 1.2284996183916332, + 1.2122250507961028, + 0.87482980171944, + 0.5899517341136686, + 0.5165762877077978 + ], + "peak_layer": 21, + "peak_normalized": 2.991445440442326, + "population_cv": 0.8381751118691627, + "rest_mean": 3.131475834918631e-06, + "spike_contrast": 3.7815403964740346, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.184180237032706e-05, + "top_five_layers": [ + 21, + 22, + 25, + 26, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.5580895908206003e-06, + 2.367960405535996e-06, + 2.2774840999773005e-06, + 2.322146201549913e-06, + 2.321419287909521e-06, + 2.2513031581183895e-06, + 2.2040844669390935e-06, + 2.23513916353113e-06, + 1.8514743942432688e-06, + 1.7355716863676207e-06, + 1.7014947388815926e-06, + 1.742398467285966e-06, + 2.2254955638345564e-06, + 2.2180413452588255e-06, + 2.2306510345515562e-06, + 2.2113945306045935e-06, + 2.485068307578331e-06, + 2.5747901872819057e-06, + 2.67291193267738e-06, + 2.8165034109406406e-06, + 1.343896201433381e-05, + 1.2851688552473206e-05, + 1.107876050809864e-05, + 9.408832738699857e-06, + 1.2430768038029782e-05, + 1.1129906852147542e-05, + 8.550470738555305e-06, + 5.518990747077623e-06, + 5.44587783224415e-06, + 3.930141701857792e-06, + 2.6503371373109985e-06, + 2.3207005597214447e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.480986444832524e-05, + "first_to_last_ratio": 0.7542988096897616, + "imbalance_abs_log_ratio": 0.28196669015672604, + "last_quartile_mean": 5.940598589404544e-05, + "mean": 5.624602624720865e-05, + "normalized": [ + 0.7593253658919704, + 0.8584935198542845, + 0.7758342247870141, + 0.7257052209284447, + 0.9115911301096014, + 0.8478599752338415, + 0.7912991525483561, + 0.7033001562001767, + 0.6342038809025199, + 0.5708934677878957, + 0.5360030772522035, + 0.5119164360939763, + 0.7122797103032084, + 0.65763851786042, + 0.6170495828015774, + 0.5742094044317299, + 0.7380398429228872, + 0.6958734981683372, + 0.7020379955965178, + 0.7406705623157649, + 3.198556354491257, + 2.6580706472996987, + 2.0667008648707084, + 1.5629991771988188, + 2.1587737798302467, + 1.7912836314300635, + 1.3377158632775958, + 0.8975280712077295, + 0.8407131217921131, + 0.5976858891866252, + 0.431587196610723, + 0.394160680813692 + ], + "peak_layer": 21, + "peak_normalized": 3.198556354491257, + "population_cv": 0.6612253154131089, + "rest_mean": 4.240304419740655e-05, + "spike_contrast": 3.089356714727384, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013099812931613997, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.270903446013108e-05, + 4.828684905078262e-05, + 4.363759217085317e-05, + 4.081803490407765e-05, + 5.127337863086723e-05, + 4.768875442096032e-05, + 4.4507432903628796e-05, + 3.955783904530108e-05, + 3.567144813132472e-05, + 3.211048897355795e-05, + 3.0148043151712045e-05, + 2.8793265300919302e-05, + 4.006290328106843e-05, + 3.698955333675258e-05, + 3.4706587030086666e-05, + 3.229699723306112e-05, + 4.151180837652646e-05, + 3.9140119042713195e-05, + 3.948684752685949e-05, + 4.16597758885473e-05, + 0.00017990608466789126, + 0.00014950591139495373, + 0.00011624371109064668, + 8.791249274509028e-05, + 0.00012142244668211788, + 0.00010075258614961058, + 7.524120155721903e-05, + 5.0482387450756505e-05, + 4.7286772314691916e-05, + 3.3617456210777164e-05, + 2.4275064788525924e-05, + 2.216997199866455e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.384813680895604e-05, + "first_to_last_ratio": 0.9740578591651334, + "imbalance_abs_log_ratio": 0.026284573443628244, + "last_quartile_mean": 3.474961624760908e-05, + "mean": 3.869269224310301e-05, + "normalized": [ + 0.8541900771208318, + 0.9413918806879924, + 0.859570413488977, + 0.8162196112220202, + 0.9909206111274242, + 0.9266644039643532, + 0.8522558488291101, + 0.7571395238293102, + 0.7081640210424396, + 0.6442698940442247, + 0.603596002806512, + 0.575557073332959, + 0.7887193829441419, + 0.7247641884135786, + 0.6750858213648586, + 0.6336813762759561, + 0.8435498470464944, + 0.7939958245882761, + 0.7970984689300282, + 0.8220042438012486, + 3.299009695161972, + 2.613224785716232, + 1.9509379481481457, + 1.3432491513722262, + 1.9254306181237832, + 1.4942554786145408, + 1.1183073895840108, + 0.8225722329666332, + 0.6567052933181462, + 0.46462952270788027, + 0.36224409286636666, + 0.34059527655932637 + ], + "peak_layer": 21, + "peak_normalized": 3.299009695161972, + "population_cv": 0.6253906355726564, + "rest_mean": 2.9905363724672633e-05, + "spike_contrast": 2.880562397960604, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.614426624262706e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.305091377114877e-05, + 3.642498631961644e-05, + 3.325909347040579e-05, + 3.158173421979882e-05, + 3.834138624370098e-05, + 3.5855140595231205e-05, + 3.297607327112928e-05, + 2.9295766580617055e-05, + 2.740077252383344e-05, + 2.4928536731749773e-05, + 2.3354754375759512e-05, + 2.2269852706813253e-05, + 3.051767635042779e-05, + 2.804307769110892e-05, + 2.612088792375289e-05, + 2.4518838472431526e-05, + 3.263921462348662e-05, + 3.072183608310297e-05, + 3.084188574575819e-05, + 3.180555722792633e-05, + 0.00012764756684191525, + 0.00010111270239576697, + 7.548704161308706e-05, + 5.1973926019854844e-05, + 7.450009434251115e-05, + 5.781676736660302e-05, + 4.327032365836203e-05, + 3.182753425789997e-05, + 2.5409695808775723e-05, + 1.7977767129195854e-05, + 1.4016199202160351e-05, + 1.3178548215364572e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001188372980323038, + "first_to_last_ratio": 0.585909807801185, + "imbalance_abs_log_ratio": 0.534589412852804, + "last_quartile_mean": 0.00020282524110371014, + "mean": 0.00016756316449573205, + "normalized": [ + 0.6676704870164084, + 0.7483084683488639, + 0.6769676096236417, + 0.6368287558643128, + 0.8100066941936953, + 0.7564679298832396, + 0.7242231413041909, + 0.6531986131482728, + 0.5690738743162227, + 0.5079946750459964, + 0.47515152884923867, + 0.45873382032473325, + 0.6302192489862594, + 0.5851397763517093, + 0.5571686419883745, + 0.5132471085839729, + 0.6453685522588855, + 0.614712668030142, + 0.6206621143750464, + 0.6587669430337698, + 3.1211444947467313, + 2.6936917077524942, + 2.215351775191127, + 1.776377347392615, + 2.3967757154563434, + 2.05632794722869, + 1.5725614581941492, + 1.0240120317645556, + 1.032250695352248, + 0.7057034519699571, + 0.4689146276836353, + 0.42697809574047824 + ], + "peak_layer": 21, + "peak_normalized": 3.1211444947467313, + "population_cv": 0.718404168086637, + "rest_mean": 0.00012285891932185255, + "spike_contrast": 3.3287456107546904, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004089660884346813, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011187697964487597, + 0.00012538893497548997, + 0.00011343483492964879, + 0.00010670904157450423, + 0.0001357272849418223, + 0.00012675616017077118, + 0.00012135312135796994, + 0.00010945202666334808, + 9.535581921227276e-05, + 8.512119529768825e-05, + 7.961789378896356e-05, + 7.686689059482887e-05, + 0.0001056015316862613, + 9.804787259781733e-05, + 9.336094080936164e-05, + 8.60013096826151e-05, + 0.00010813999688252807, + 0.00010300319991074502, + 0.00010400010796729475, + 0.00011038507363991812, + 0.000522988848388195, + 0.0004513635067269206, + 0.0003712113539222628, + 0.0002976554096676409, + 0.0004016113234683871, + 0.000344564818078652, + 0.0002635033742990345, + 0.00017158669652417302, + 0.0001729671930661425, + 0.00011824990360764787, + 7.857281889300793e-05, + 7.154580089263618e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13583384454250336, + 0.0415157787501812, + 0.018140379339456558, + 0.039518751204013824, + 0.022877270355820656, + 0.015532293356955051, + 0.02314678393304348, + 0.05432315170764923, + 0.03743875026702881, + 0.04006272554397583, + 0.045583780854940414, + 0.05995568633079529, + 0.038651175796985626, + 0.04559725895524025, + 0.05632574483752251, + 0.07952121645212173, + 0.042337141931056976, + 0.05044656619429588, + 0.04418695345520973, + 0.06310147047042847, + 0.029010163620114326, + 0.04586305841803551, + 0.06033914536237717, + 0.07656749337911606, + 0.045884523540735245, + 0.051604680716991425, + 0.06502768397331238, + 0.06792232394218445, + 0.060306549072265625, + 0.08025261014699936, + 0.09663844108581543, + 0.12230555713176727 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4509995708067436e-05, + "first_to_last_ratio": 0.7743031141222962, + "imbalance_abs_log_ratio": 0.2557918617425859, + "last_quartile_mean": 1.8739425740932347e-05, + "mean": 1.7722179109114222e-05, + "normalized": [ + 0.7655666384169513, + 0.8917483357831588, + 0.7902102983812471, + 0.7316333655654478, + 0.9592978291511814, + 0.8828085684394261, + 0.8112146819967133, + 0.7175029311175319, + 0.593548092754179, + 0.5514020456598471, + 0.522638501525885, + 0.5037614169375741, + 0.7045039393419045, + 0.6588959250307225, + 0.6114268576480177, + 0.579302599818883, + 0.7435448955446501, + 0.6887296875838754, + 0.6921332531242588, + 0.8093892960298869, + 3.2325455196933226, + 2.5123032009763855, + 1.9971252826810495, + 1.5895697479726458, + 2.322878133149869, + 1.7452609194969326, + 1.1261946298991334, + 0.8913309023592738, + 0.8429252014113704, + 0.5609036083922285, + 0.5024807882604625, + 0.46722290585598436 + ], + "peak_layer": 21, + "peak_normalized": 3.2325455196933226, + "population_cv": 0.6537085087930902, + "rest_mean": 1.335436960895701e-05, + "spike_contrast": 3.09324600258607, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.130835040996317e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.3567509085987695e-05, + 1.580372372700367e-05, + 1.4004248441779055e-05, + 1.2966137546754908e-05, + 1.700084794720169e-05, + 1.564529156894423e-05, + 1.437649189028889e-05, + 1.2715715456579346e-05, + 1.05189656096627e-05, + 9.772045814315788e-06, + 9.2622931333608e-06, + 8.927750059228856e-06, + 1.2485344996093772e-05, + 1.167707159765996e-05, + 1.0835816283361055e-05, + 1.0266504432365764e-05, + 1.3177235814509913e-05, + 1.2205790881125722e-05, + 1.2266109479242004e-05, + 1.4344142073241528e-05, + 5.7287750678369775e-05, + 4.452348730410449e-05, + 3.539341196301393e-05, + 2.8170639780000784e-05, + 4.116646232432686e-05, + 3.092982660746202e-05, + 1.9958622942795046e-05, + 1.579632589709945e-05, + 1.4938471394998487e-05, + 9.940434210875537e-06, + 8.905054528440814e-06, + 8.280208021460567e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.7723796544451034e-05, + "first_to_last_ratio": 0.7709532556520535, + "imbalance_abs_log_ratio": 0.2601275354642249, + "last_quartile_mean": 7.487327684430056e-05, + "mean": 7.085550987540046e-05, + "normalized": [ + 0.767459364760998, + 0.8911005669273344, + 0.7949447373790853, + 0.7325731165004719, + 0.9435545502986998, + 0.8683878924182112, + 0.8054854811328411, + 0.7138473259026528, + 0.5978031833741329, + 0.5556350739964981, + 0.526183189154655, + 0.5057617239608265, + 0.7151088387962502, + 0.660810541031278, + 0.6122576977156078, + 0.5785452674352782, + 0.739958743085955, + 0.6892518806239643, + 0.6970577547891826, + 0.8113719765230756, + 3.2405178430845596, + 2.524955075289844, + 1.989505542525185, + 1.5842933275736053, + 2.315101352895563, + 1.736925913599177, + 1.1174905309838439, + 0.8878674587909959, + 0.8457916652531124, + 0.5666462793963278, + 0.5093376021450956, + 0.4744685026556913 + ], + "peak_layer": 21, + "peak_normalized": 3.2405178430845596, + "population_cv": 0.6532804064480114, + "rest_mean": 5.3392583881591066e-05, + "spike_contrast": 3.093225654863168, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016515531024197116, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.437872459879145e-05, + 6.313938501989469e-05, + 5.63262146897614e-05, + 5.190684169065207e-05, + 6.685603875666857e-05, + 6.153006688691676e-05, + 5.7073084462899715e-05, + 5.058001625002362e-05, + 4.2357649363111705e-05, + 3.936980647267774e-05, + 3.728297815541737e-05, + 3.58360048267059e-05, + 5.066940138931386e-05, + 4.682206781581044e-05, + 4.33818313467782e-05, + 4.099311991012655e-05, + 5.2430154028115794e-05, + 4.883729343418963e-05, + 4.93903826281894e-05, + 5.749017509515397e-05, + 0.0002296085440320894, + 0.00017890697927214205, + 0.0001409674296155572, + 0.00011225591151742265, + 0.00016403768677264452, + 0.00012307077122386545, + 7.918036135379225e-05, + 6.291030149441212e-05, + 5.992899968987331e-05, + 4.015001104562543e-05, + 3.608937549870461e-05, + 3.36187076754868e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05119003914296627, + "first_to_last_ratio": 0.7709532409156976, + "imbalance_abs_log_ratio": 0.260127554578686, + "last_quartile_mean": 0.06639837077818811, + "mean": 0.06283537502167746, + "normalized": [ + 0.7674593499219178, + 0.8911005434486201, + 0.7949446960721035, + 0.7325731497404293, + 0.9435544639718605, + 0.8683878801940695, + 0.8054854824220893, + 0.7138473266473999, + 0.5978031662245904, + 0.5556351000440413, + 0.5261832202468875, + 0.5057617435862689, + 0.715108825077915, + 0.6608105011251547, + 0.6122576924651686, + 0.5785453059609885, + 0.7399587078512944, + 0.6892519015892086, + 0.6970578010795431, + 0.8113719810435487, + 3.2405178171891893, + 2.52495521313829, + 1.9895055108588098, + 1.5842933381515236, + 2.315101350530748, + 1.7369258738398763, + 1.1174905410660712, + 0.8878674720175045, + 0.8457916556516879, + 0.5666462655907423, + 0.5093376096374259, + 0.4744685136150315 + ], + "peak_layer": 21, + "peak_normalized": 3.2405178171891893, + "population_cv": 0.6532804095990112, + "rest_mean": 0.04734907733897368, + "spike_contrast": 3.0932256918071666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1464613825082779, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.048223596066236496, + 0.05599263682961464, + 0.049950648099184036, + 0.04603150859475136, + 0.05928859859704971, + 0.05456547811627388, + 0.050612982362508774, + 0.04485486447811127, + 0.03756318613886833, + 0.03491353988647461, + 0.03306291997432709, + 0.03177972882986069, + 0.04493413120508194, + 0.041522275656461716, + 0.038471441715955734, + 0.036353111267089844, + 0.04649558290839195, + 0.04330940172076225, + 0.04379988834261894, + 0.05098286271095276, + 0.20361915230751038, + 0.158656507730484, + 0.12501132488250732, + 0.09954966604709625, + 0.1454702615737915, + 0.10914038866758347, + 0.07021793723106384, + 0.055789485573768616, + 0.0531456358730793, + 0.035605430603027344, + 0.03200441971421242, + 0.029813406988978386 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.293479383297381e-06, + "first_to_last_ratio": 0.445969741543177, + "imbalance_abs_log_ratio": 0.8075041733371805, + "last_quartile_mean": 5.142679356140434e-06, + "mean": 3.873292421019414e-06, + "normalized": [ + 0.6548345314656568, + 0.6098593868708077, + 0.5890922218088536, + 0.6012559418227537, + 0.5982935563608919, + 0.5743111722490878, + 0.5597275993993273, + 0.5496382066817245, + 0.4494881925876058, + 0.4317042818847807, + 0.42858151997472205, + 0.44091762118925537, + 0.5606916778798313, + 0.5514610505577893, + 0.5418155695170769, + 0.543856847054833, + 0.6362195338062625, + 0.6499443412604623, + 0.6710047275371834, + 0.7226720079645516, + 3.0150009122712964, + 2.639960940380615, + 2.3465929682489137, + 2.0112500416813845, + 2.8998499180306, + 2.237539331592885, + 1.506545124394083, + 1.1285810413622663, + 1.0054446853908898, + 0.6799751247000538, + 0.607691428375801, + 0.5561984956977537 + ], + "peak_layer": 21, + "peak_normalized": 3.0150009122712964, + "population_cv": 0.7767874334961873, + "rest_mean": 2.7381803546530946e-06, + "spike_contrast": 3.653118598414905, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.000289757939754e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.5363656277477276e-06, + 2.362163741054246e-06, + 2.2817264380137203e-06, + 2.3288400825549616e-06, + 2.3173658973973943e-06, + 2.224475110779167e-06, + 2.167988668588805e-06, + 2.128909500243026e-06, + 1.7409992096872884e-06, + 1.6721169231459498e-06, + 1.6600215531070717e-06, + 1.707802880446252e-06, + 2.1717228264606092e-06, + 2.135969907612889e-06, + 2.0986101390008116e-06, + 2.106516603816999e-06, + 2.4642642983963015e-06, + 2.5174244910886046e-06, + 2.5989975256379694e-06, + 2.799120011331979e-06, + 1.1677980182867032e-05, + 1.0225340702163521e-05, + 9.089040759135969e-06, + 7.790159543219488e-06, + 1.1231966709601693e-05, + 8.666644134791568e-06, + 5.835289812239353e-06, + 4.371324394014664e-06, + 3.8943812796787824e-06, + 2.6337424969824497e-06, + 2.353766603846452e-06, + 2.1543194179685088e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.442123326953151e-05, + "first_to_last_ratio": 0.9390502622041328, + "imbalance_abs_log_ratio": 0.06288627383246957, + "last_quartile_mean": 4.730442560685333e-05, + "mean": 5.047756872045284e-05, + "normalized": [ + 0.8442457511608136, + 0.9554189601166778, + 0.8651399119276457, + 0.8073037852085534, + 1.011773735449473, + 0.9346086902233326, + 0.859731545150837, + 0.7619317954676181, + 0.657244201472419, + 0.6123980431859172, + 0.5782989976765822, + 0.5555348776283999, + 0.7735150062029652, + 0.7093353168527977, + 0.6548347172091762, + 0.6154299343077795, + 0.8164721063408712, + 0.7613444152108153, + 0.7654949233837004, + 0.8721362954625393, + 3.2652345954629065, + 2.4905637834711873, + 1.8895133502771215, + 1.4453947777525853, + 2.0912180007544503, + 1.5360217136102785, + 1.0169235828420913, + 0.8253134404517026, + 0.6717824033622655, + 0.48834596354787496, + 0.4437661443704546, + 0.4237292344581673 + ], + "peak_layer": 21, + "peak_normalized": 3.2652345954629065, + "population_cv": 0.6182834768826335, + "rest_mean": 3.892021615886026e-05, + "spike_contrast": 2.900479074737969, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011288727255305275, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.2615472921170294e-05, + 4.8227226216113195e-05, + 4.367015935713425e-05, + 4.075073229614645e-05, + 5.107187826070003e-05, + 4.717677438748069e-05, + 4.3397158151492476e-05, + 3.846046456601471e-05, + 3.317608934594318e-05, + 3.091236430918798e-05, + 2.9191127396188676e-05, + 2.8042049962095916e-05, + 3.904515688191168e-05, + 3.580552220228128e-05, + 3.305446443846449e-05, + 3.106540680164471e-05, + 4.1213526856154203e-05, + 3.843081503873691e-05, + 3.8640322600258514e-05, + 4.402331978781149e-05, + 0.00016482110368087888, + 0.00012571760453283787, + 9.537803998682648e-05, + 7.296001422218978e-05, + 0.00010555960034253076, + 7.753464160487056e-05, + 5.133183003636077e-05, + 4.165981590631418e-05, + 3.390994243090972e-05, + 2.4650516934343614e-05, + 2.2400236048270017e-05, + 2.138882155122701e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.35970580636058e-05, + "first_to_last_ratio": 1.183082469649485, + "imbalance_abs_log_ratio": 0.16812329486526428, + "last_quartile_mean": 2.8397900337040483e-05, + "mean": 3.5426135951865945e-05, + "normalized": [ + 0.9325726638234952, + 1.0264228324198814, + 0.9395008785893599, + 0.8896754656866502, + 1.0823546164520876, + 1.002292895319797, + 0.9051263930494292, + 0.8090082728462467, + 0.7240290452324956, + 0.6814307833477395, + 0.6400266234002182, + 0.6142285583444715, + 0.8434981841324323, + 0.7676858556869983, + 0.7064344235164509, + 0.6656552980925776, + 0.9160791051424548, + 0.8560305037244728, + 0.8526061894702537, + 0.9377555262910077, + 3.3385841431029775, + 2.450559245049383, + 1.764118224393788, + 1.2414543227163002, + 1.7869665660194411, + 1.2750853185739486, + 0.921819584871679, + 0.7867281257526472, + 0.5090598376007598, + 0.4052896271109379, + 0.3729632040504296, + 0.3549576861891871 + ], + "peak_layer": 21, + "peak_normalized": 3.3385841431029775, + "population_cv": 0.5922680449883523, + "rest_mean": 2.8102526947030264e-05, + "spike_contrast": 2.6678606062470647, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.497362457797863e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.303744597360492e-05, + 3.6362194805406034e-05, + 3.328288585180417e-05, + 3.151776400045492e-05, + 3.834364179056138e-05, + 3.5507364373188466e-05, + 3.206513065379113e-05, + 2.8660037060035393e-05, + 2.5649551389506087e-05, + 2.414045957266353e-05, + 2.2673670173389837e-05, + 2.175974441342987e-05, + 2.98818813462276e-05, + 2.7196143491892144e-05, + 2.5026241928571835e-05, + 2.3581595087307505e-05, + 3.24531429214403e-05, + 3.032585300388746e-05, + 3.0204542781575583e-05, + 3.3221054763998836e-05, + 0.00011827313574030995, + 8.681384497322142e-05, + 6.249589205253869e-05, + 4.3979929614579305e-05, + 6.330532050924376e-05, + 4.5171345846029e-05, + 3.265650593675673e-05, + 2.7870737540069968e-05, + 1.8034023014479317e-05, + 1.435784542991314e-05, + 1.3212645171734039e-05, + 1.2574779248097911e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001175618963316083, + "first_to_last_ratio": 0.7472280359888956, + "imbalance_abs_log_ratio": 0.2913848713729993, + "last_quartile_mean": 0.00015733068175904918, + "mean": 0.0001475783419664367, + "normalized": [ + 0.7539335055517485, + 0.8485908925894718, + 0.7671380034067055, + 0.7221187547562506, + 0.9161421981299216, + 0.8510423035463365, + 0.8040139845348296, + 0.7098739762546249, + 0.5921519684486223, + 0.5526890935329054, + 0.5267158319052243, + 0.5064890538358937, + 0.699155203785276, + 0.6460998215976334, + 0.6066243745851786, + 0.5643569853524535, + 0.729020737260662, + 0.6850322033333521, + 0.6876316692176827, + 0.7819456161539314, + 3.243720072824733, + 2.5794511063238676, + 2.043978587180185, + 1.6534243806222502, + 2.3371192579790736, + 1.7969709048957057, + 1.167004731785933, + 0.9411888715768515, + 0.8017105792030909, + 0.5375092481175581, + 0.48493150700127435, + 0.4622245747107726 + ], + "peak_layer": 21, + "peak_normalized": 3.243720072824733, + "population_cv": 0.6705093936976806, + "rest_mean": 0.00011009511891407547, + "spike_contrast": 3.178957885701889, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003499877464491874, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011126425670227036, + 0.00012523363693617284, + 0.00011321295460220426, + 0.00010656908852979541, + 0.00013520274660550058, + 0.00012559541210066527, + 0.00011865505075547844, + 0.0001047620244207792, + 8.738880569580942e-05, + 8.156494004651904e-05, + 7.773184916004539e-05, + 7.474681478925049e-05, + 0.0001031801657518372, + 9.535034041618928e-05, + 8.952461939770728e-05, + 8.328686817549169e-05, + 0.00010758767166407779, + 0.00010109591676155105, + 0.00010147954162675887, + 0.00011539823753992096, + 0.00047870283015072346, + 0.00038067111745476723, + 0.0003016469709109515, + 0.00024400962865911424, + 0.00034490818507038057, + 0.0002651939867064357, + 0.00017222462338395417, + 0.00013889909314457327, + 0.00011831511801574379, + 7.932472362881526e-05, + 7.156538777053356e-05, + 6.821433635195717e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.35791492462158203, + 0.3940892517566681, + 0.4057151973247528, + 0.09380833059549332, + 0.11286329478025436, + 0.13231655955314636, + 0.1621972620487213, + 0.05156813934445381, + 0.08533531427383423, + 0.12003090232610703, + 0.16063299775123596, + 0.046871740370988846, + 0.079384446144104, + 0.11459959298372269, + 0.15625107288360596, + 0.08397205173969269, + 0.11654024571180344, + 0.15294702351093292, + 0.21572710573673248, + 0.03708760067820549, + 0.06553087383508682, + 0.09390194714069366, + 0.12852425873279572, + 0.025487789884209633, + 0.06059965491294861, + 0.08860157430171967, + 0.12100546807050705, + 0.024166326969861984, + 0.06877554953098297, + 0.12682415544986725, + 0.2025238573551178 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4657116139460413e-05, + "first_to_last_ratio": 0.6094207983868245, + "imbalance_abs_log_ratio": 0.49524628372702406, + "last_quartile_mean": 2.4050895831351227e-05, + "mean": 2.012364140568934e-05, + "normalized": [ + 0.6757810223737426, + 0.7862782366272546, + 0.6956742911341667, + 0.645696070130233, + 0.8486116417698539, + 0.7866567023571445, + 0.7355711504661757, + 0.6525555016113147, + 0.5676629357160193, + 0.5065196141176592, + 0.4765477406007062, + 0.4567702622869, + 0.6387850255773179, + 0.6012516562554105, + 0.5676978717094758, + 0.5320979135959467, + 0.660140815357474, + 0.6199672249932086, + 0.6251969579127784, + 0.6703296515344952, + 3.1476456930462797, + 2.6843935159061667, + 2.147025871519746, + 1.7098926204474763, + 2.310003501268429, + 2.02536057377974, + 1.5260208260637402, + 0.9852869427838804, + 1.0403527573808369, + 0.7424129105715245, + 0.4909974573147262, + 0.4408150437901761 + ], + "peak_layer": 21, + "peak_normalized": 3.1476456930462797, + "population_cv": 0.7023455047176551, + "rest_mean": 1.4907175278090614e-05, + "spike_contrast": 3.239551262652627, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.829255849472247e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.3599174963019323e-05, + 1.582278127898462e-05, + 1.3999499969941098e-05, + 1.2993756172363646e-05, + 1.7077156371669844e-05, + 1.5830397387617268e-05, + 1.4802370060351677e-05, + 1.3131792911735829e-05, + 1.1423445357650053e-05, + 1.0193019079451915e-05, + 9.589875844540074e-06, + 9.191880963044241e-06, + 1.2854680790042039e-05, + 1.2099372725060675e-05, + 1.1424148397054523e-05, + 1.0707747605920304e-05, + 1.3284437045513187e-05, + 1.2475998119043652e-05, + 1.2581239388964605e-05, + 1.3489473531080876e-05, + 6.334209319902584e-05, + 5.4019772505853325e-05, + 4.3205978727201e-05, + 3.440926593611948e-05, + 4.648568210541271e-05, + 4.07576299039647e-05, + 3.0709095881320536e-05, + 1.982756111829076e-05, + 2.0935685824952088e-05, + 1.4940051187295467e-05, + 9.88065676210681e-06, + 8.870803867466748e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.829710562466062e-05, + "first_to_last_ratio": 0.6072508868460227, + "imbalance_abs_log_ratio": 0.49881325067107773, + "last_quartile_mean": 9.600168050383218e-05, + "mean": 8.04003983603252e-05, + "normalized": [ + 0.6783620403407459, + 0.7863497478511254, + 0.7000374993610626, + 0.6471153960919661, + 0.8353499122055624, + 0.7743388664853165, + 0.7293878791763712, + 0.6497369450201427, + 0.570732789612889, + 0.5105438596817938, + 0.4799740390322282, + 0.4590044982477276, + 0.6477850704647679, + 0.6029712759057101, + 0.5684996518209374, + 0.5312286531130485, + 0.6575848951936205, + 0.6204730346458087, + 0.629222556567805, + 0.6709457586481146, + 3.1526166331987144, + 2.693185986758889, + 2.1505035942092414, + 1.701690725446724, + 2.3039587527644554, + 2.018829142146718, + 1.519765537386825, + 0.9812068005325093, + 1.0399315417453956, + 0.7451740498188497, + 0.49646309627653795, + 0.44702977024839613 + ], + "peak_layer": 21, + "peak_normalized": 3.1526166331987144, + "population_cv": 0.7020174312406612, + "rest_mean": 5.955002699112002e-05, + "spike_contrast": 3.2408449417296112, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019299240375403315, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.454057827591896e-05, + 6.322283297777176e-05, + 5.628329381579533e-05, + 5.202833563089371e-05, + 6.71624657115899e-05, + 6.22571533313021e-05, + 5.8643076044972986e-05, + 5.223910920904018e-05, + 4.588714364217594e-05, + 4.1047929698834196e-05, + 3.859010394080542e-05, + 3.6904144508298486e-05, + 5.2082177717238665e-05, + 4.8479130782652646e-05, + 4.5707598474109545e-05, + 4.27109953307081e-05, + 5.287008752929978e-05, + 4.988627915736288e-05, + 5.058974420535378e-05, + 5.3944306273479015e-05, + 0.00025347163318656385, + 0.00021653322619386017, + 0.00017290134564973414, + 0.00013681661221198738, + 0.0001852392015280202, + 0.00016231466725002974, + 0.00012218975462019444, + 7.88894176366739e-05, + 8.361091022379696e-05, + 5.9912290453212336e-05, + 3.991583071183413e-05, + 3.59413716068957e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05169845558702946, + "first_to_last_ratio": 0.6072508754048147, + "imbalance_abs_log_ratio": 0.49881326951206806, + "last_quartile_mean": 0.08513525081798434, + "mean": 0.0712998784147203, + "normalized": [ + 0.6783619951609938, + 0.786349724122647, + 0.7000374743682577, + 0.6471153772114631, + 0.8353498624040391, + 0.774338855687226, + 0.729387848399732, + 0.6497369828883814, + 0.5707327933468814, + 0.5105438598068986, + 0.4799740650385461, + 0.45900445901159087, + 0.6477850423892585, + 0.6029712911228355, + 0.5684996530360322, + 0.5312286608963362, + 0.6575848715023773, + 0.6204730273795, + 0.6292225631787356, + 0.6709457289261637, + 3.152616671814094, + 2.693186128732524, + 2.150503721956032, + 1.7016907445638214, + 2.303958774916342, + 2.0188290625883343, + 1.5197655429953278, + 0.981206773145755, + 1.039931555380646, + 0.7451740672617071, + 0.49646307443703724, + 0.4470297463304832 + ], + "peak_layer": 21, + "peak_normalized": 3.152616671814094, + "population_cv": 0.70201745433721, + "rest_mean": 0.052809559360698415, + "spike_contrast": 3.2408450927884243, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.17114760130643844, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.048367127776145935, + 0.056066639721393585, + 0.04991258680820465, + 0.046139247715473175, + 0.059560343623161316, + 0.05521026626229286, + 0.05200526490807533, + 0.0463261678814888, + 0.04069317877292633, + 0.036401715129613876, + 0.03422209247946739, + 0.0327269621193409, + 0.04618699476122856, + 0.04299177974462509, + 0.04053395614027977, + 0.037876538932323456, + 0.04688572138547897, + 0.044239651411771774, + 0.044863492250442505, + 0.047838348895311356, + 0.22478118538856506, + 0.1920238435268402, + 0.15333065390586853, + 0.12133034318685532, + 0.1642719805240631, + 0.14394226670265198, + 0.10835909843444824, + 0.06995992362499237, + 0.07414699345827103, + 0.05313082039356232, + 0.03539775684475899, + 0.03187316656112671 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.317259230721902e-06, + "first_to_last_ratio": 0.35665874650259244, + "imbalance_abs_log_ratio": 1.0309758465083512, + "last_quartile_mean": 6.497132773120029e-06, + "mean": 4.492468697492313e-06, + "normalized": [ + 0.5694283261976025, + 0.527162238879597, + 0.5069677223499952, + 0.5169219758743683, + 0.5167285360874581, + 0.5011532928366966, + 0.49061467097646116, + 0.4975013703289752, + 0.41211554520475835, + 0.3863198030086292, + 0.3787459166580058, + 0.38785476948100367, + 0.4953856795983567, + 0.493696852070983, + 0.4965364995551244, + 0.49228294995504224, + 0.553135351492583, + 0.5731144625451857, + 0.5949797886628256, + 0.6269543934401883, + 2.991422102255377, + 2.8607693678126824, + 2.46602465979906, + 2.0943619443179213, + 2.767059684919099, + 2.477440137836224, + 1.9032189385510152, + 1.2285251544343567, + 1.2121681046978803, + 0.874835232215944, + 0.589970244793492, + 0.5166042831631082 + ], + "peak_layer": 21, + "peak_normalized": 2.991422102255377, + "population_cv": 0.8381704476164953, + "rest_mean": 3.1314773423218867e-06, + "spike_contrast": 3.7815448495732364, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1841822015412617e-05, + "top_five_layers": [ + 21, + 22, + 25, + 26, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.5581389309081715e-06, + 2.3682598566665547e-06, + 2.277536623296328e-06, + 2.3222557956614764e-06, + 2.3213867734739324e-06, + 2.2514154807140585e-06, + 2.2040710518922424e-06, + 2.235009333162452e-06, + 1.8514161865823553e-06, + 1.7355296222376637e-06, + 1.7015041748891235e-06, + 1.742425411066506e-06, + 2.225504658781574e-06, + 2.2179176539793843e-06, + 2.2306746814138023e-06, + 2.211565742982202e-06, + 2.484943252056837e-06, + 2.5746987830643775e-06, + 2.6729280762083363e-06, + 2.8165729872853262e-06, + 1.3438870155368932e-05, + 1.2851916835643351e-05, + 1.1078538591391407e-05, + 9.408855476067401e-06, + 1.2430929018591996e-05, + 1.1129822269140277e-05, + 8.550151505914982e-06, + 5.519110800378257e-06, + 5.445627266453812e-06, + 3.930169896193547e-06, + 2.6504228571866406e-06, + 2.3208285711007193e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.480992902244907e-05, + "first_to_last_ratio": 0.7542989526231301, + "imbalance_abs_log_ratio": 0.28196650066503776, + "last_quartile_mean": 5.940606024523731e-05, + "mean": 5.624609735832564e-05, + "normalized": [ + 0.7593189081163986, + 0.8584899766461837, + 0.7758411348304551, + 0.7257032038756371, + 0.9115949579325926, + 0.8478585798985628, + 0.791306819195711, + 0.703296291763442, + 0.6342085768594776, + 0.5708926166563143, + 0.5360034021261918, + 0.5119200900833328, + 0.7122730532881045, + 0.6576365221836818, + 0.6170512605021473, + 0.5742091312248548, + 0.7380382630332617, + 0.6958730711433307, + 0.702031804287528, + 0.7406728598801368, + 3.1985536041966367, + 2.658074789581204, + 2.066704719937707, + 1.5629982359965144, + 2.1587687220561924, + 1.7912910686863504, + 1.3377069279004006, + 0.8975311406544447, + 0.8407126410078246, + 0.5976870739310991, + 0.4315898849446666, + 0.3941606675796141 + ], + "peak_layer": 21, + "peak_normalized": 3.1985536041966367, + "population_cv": 0.6612253265050619, + "rest_mean": 4.240309937341846e-05, + "spike_contrast": 3.089356401125345, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013099828647682442, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.2708725231932476e-05, + 4.828671080758795e-05, + 4.363803600426763e-05, + 4.081797305843793e-05, + 5.1273658755235374e-05, + 4.7688736231066287e-05, + 4.450792039278895e-05, + 3.9557671698275954e-05, + 3.5671757359523326e-05, + 3.2110481697600335e-05, + 3.0148099540383555e-05, + 2.8793507226509973e-05, + 4.00625795009546e-05, + 3.698948785313405e-05, + 3.4706725273281336e-05, + 3.229702269891277e-05, + 4.151177199673839e-05, + 3.9140144508564845e-05, + 3.948654921259731e-05, + 4.1659957787487656e-05, + 0.0001799061574274674, + 0.00014950633340049535, + 0.0001162440748885274, + 8.791255095275119e-05, + 0.00012142231571488082, + 0.00010075313184643164, + 7.524079410359263e-05, + 5.0482623919378966e-05, + 4.728680505650118e-05, + 3.3617565350141376e-05, + 2.427524668746628e-05, + 2.2169999283505604e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.384801061656617e-05, + "first_to_last_ratio": 0.9740047819331958, + "imbalance_abs_log_ratio": 0.02633906576931732, + "last_quartile_mean": 3.475138032627001e-05, + "mean": 3.8690275459885015e-05, + "normalized": [ + 0.8541538250770527, + 0.9415230863648691, + 0.8597053468534278, + 0.816066837065049, + 0.9910309334070734, + 0.9267294340872885, + 0.8523049476412223, + 0.7572490181408758, + 0.7081191176405685, + 0.6443517927963444, + 0.6037734323315225, + 0.5756116900407472, + 0.7887443909138365, + 0.7248706260931408, + 0.6750345264379841, + 0.633794207108887, + 0.8434851920289833, + 0.7938278400728305, + 0.7971348135467743, + 0.8220444008320537, + 3.298671908004925, + 2.612487605975051, + 1.950851070533051, + 1.3428804993059804, + 1.9255864325842058, + 1.4942946568254274, + 1.1186305566147283, + 0.822664893239064, + 0.6568214429021311, + 0.4646495190211241, + 0.3623372415841501, + 0.34056871492963064 + ], + "peak_layer": 21, + "peak_normalized": 3.298671908004925, + "population_cv": 0.6252780463659825, + "rest_mean": 2.9905465689134196e-05, + "spike_contrast": 2.8800169546676924, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.612824822193944e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.304744677734561e-05, + 3.64277875632979e-05, + 3.3262236684095114e-05, + 3.157385071972385e-05, + 3.834325980278663e-05, + 3.5855417081620544e-05, + 3.297591320006177e-05, + 2.9298173103597946e-05, + 2.7397323719924316e-05, + 2.4930148356361315e-05, + 2.336016041226685e-05, + 2.227057484560646e-05, + 3.0516737751895562e-05, + 2.8045444196322933e-05, + 2.611727177281864e-05, + 2.452167245792225e-05, + 3.263467442593537e-05, + 3.071341780014336e-05, + 3.084136551478878e-05, + 3.1805124308448285e-05, + 0.00012762652477249503, + 0.00010107786511071026, + 7.547896530013531e-05, + 5.195641642785631e-05, + 7.450146949850023e-05, + 5.781467189081013e-05, + 4.3280124373268336e-05, + 3.182913133059628e-05, + 2.541260255384259e-05, + 1.7977417883230373e-05, + 1.401892768626567e-05, + 1.3176697393646464e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011881819409609307, + "first_to_last_ratio": 0.5858660432005869, + "imbalance_abs_log_ratio": 0.5346641107578359, + "last_quartile_mean": 0.0002028077842624043, + "mean": 0.0001675532937497337, + "normalized": [ + 0.6676405577533395, + 0.7480364199301687, + 0.6769763115579581, + 0.6368334430352999, + 0.8101874659576359, + 0.7560101568775733, + 0.7242722329347805, + 0.6531372168922783, + 0.569150085590359, + 0.5080318969181871, + 0.47519098467081156, + 0.45888126162494614, + 0.6302390928396696, + 0.5851986956962459, + 0.5573344319393583, + 0.5133335361358992, + 0.6455392341889507, + 0.6148024241191441, + 0.6209170612517536, + 0.6589570869141987, + 3.121371094689797, + 2.6936365726462816, + 2.2146820529210776, + 1.7763796870270414, + 2.3963701078287363, + 2.056340004797266, + 1.572482658633456, + 1.024072444271845, + 1.0322195327669619, + 0.7057443743535575, + 0.46898398312387723, + 0.42704789011154415 + ], + "peak_layer": 21, + "peak_normalized": 3.121371094689797, + "population_cv": 0.7183385489551041, + "rest_mean": 0.00012285727657231123, + "spike_contrast": 3.328348128139876, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004089117865078151, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011186537449248135, + 0.00012533596600405872, + 0.0001134296107920818, + 0.0001067035409505479, + 0.00013574957847595215, + 0.0001266719918930903, + 0.00012135419819969684, + 0.00010943529196083546, + 9.53629714786075e-05, + 8.512241765856743e-05, + 7.96198146417737e-05, + 7.6887066825293e-05, + 0.00010559863585513085, + 9.805196896195412e-05, + 9.338321979157627e-05, + 8.601072477176785e-05, + 0.0001081622249330394, + 0.00010301217116648331, + 0.00010403669875813648, + 0.00011041043035220355, + 0.0005229960079304874, + 0.00045132767991162837, + 0.0003710772725753486, + 0.0002976382675115019, + 0.00040151970461010933, + 0.0003445465408731252, + 0.0002634746488183737, + 0.00017158671107608825, + 0.00017295178258791566, + 0.00011824979446828365, + 7.857981108827516e-05, + 7.155328057706356e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4510050277749542e-05, + "first_to_last_ratio": 0.7743096996079971, + "imbalance_abs_log_ratio": 0.255783356730045, + "last_quartile_mean": 1.873933683782525e-05, + "mean": 1.772222358908948e-05, + "normalized": [ + 0.7655738518273417, + 0.8917418894414295, + 0.7902270979958571, + 0.7316384060873433, + 0.9592931634098782, + 0.8828162060633833, + 0.8112185990320713, + 0.7174816289091832, + 0.5935539930427093, + 0.5514073845754808, + 0.522640268952773, + 0.50377149417381, + 0.7044978089956251, + 0.6589104369318574, + 0.6114200884798895, + 0.5793087411391302, + 0.7435486231843338, + 0.6887380175928803, + 0.692133312159999, + 0.8093969639662545, + 3.2325450017939272, + 2.512318244382783, + 1.9971229388240013, + 1.5895601132651613, + 2.322861628648705, + 1.745234574445335, + 1.1261863634676146, + 0.8913245597041289, + 0.8429168761495727, + 0.5609154923495704, + 0.5024744978075042, + 0.4672217332004658 + ], + "peak_layer": 21, + "peak_normalized": 3.2325450017939272, + "population_cv": 0.6537061924560114, + "rest_mean": 1.3354412287838016e-05, + "spike_contrast": 3.0932401760178796, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1308404615847397e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.356767097604461e-05, + 1.5803649148438126e-05, + 1.4004581316839904e-05, + 1.2966259419044945e-05, + 1.7000807929434814e-05, + 1.564546619192697e-05, + 1.4376597391674295e-05, + 1.2715369848592672e-05, + 1.0519096576899756e-05, + 9.77216495812172e-06, + 9.262347703042906e-06, + 8.927951057557948e-06, + 1.2485267689044122e-05, + 1.1677358088491019e-05, + 1.0835723514901474e-05, + 1.0266639037581626e-05, + 1.3177334949432407e-05, + 1.220596914208727e-05, + 1.2266141311556567e-05, + 1.4344313967740163e-05, + 5.728788528358564e-05, + 4.452386565390043e-05, + 3.5393459256738424e-05, + 2.817053973558359e-05, + 4.116627314942889e-05, + 3.092943734372966e-05, + 1.9958526536356658e-05, + 1.579625313752331e-05, + 1.4938361346139573e-05, + 9.940669770003296e-06, + 8.90496539796004e-06, + 8.280208021460567e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.772373378931661e-05, + "first_to_last_ratio": 0.7709541968266947, + "imbalance_abs_log_ratio": 0.26012631467157654, + "last_quartile_mean": 7.487310404030723e-05, + "mean": 7.085559036568156e-05, + "normalized": [ + 0.767456541889667, + 0.8910969874796977, + 0.7949526654342541, + 0.7325741840266811, + 0.943544133913626, + 0.868390294625501, + 0.8054913948148298, + 0.7138323441642012, + 0.5978064063949543, + 0.5556378314836203, + 0.5261854153191788, + 0.505768953649729, + 0.715108334510866, + 0.6608161569684796, + 0.612256437426044, + 0.5785464585899268, + 0.739960161627241, + 0.6892568994732284, + 0.697060043562065, + 0.8113764458982461, + 3.2405349047340315, + 2.5249733605503017, + 1.9894958890200054, + 1.5842775624032772, + 2.3150907133984013, + 1.7369124395330813, + 1.1174777605817472, + 0.8878712764898554, + 0.8457898829566253, + 0.5666580094993431, + 0.5093321459106411, + 0.474467963670652 + ], + "peak_layer": 21, + "peak_normalized": 3.2405349047340315, + "population_cv": 0.6532808754860918, + "rest_mean": 5.3392646400930564e-05, + "spike_contrast": 3.093225357948515, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016515548777533694, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.437858635559678e-05, + 6.313920312095433e-05, + 5.632684042211622e-05, + 5.1906976295867935e-05, + 6.685537664452568e-05, + 6.153030699351802e-05, + 5.707356831408106e-05, + 5.057901216787286e-05, + 4.2357925849501044e-05, + 3.9370046579279006e-05, + 3.728317824425176e-05, + 3.583655779948458e-05, + 5.0669423217186704e-05, + 4.682251892518252e-05, + 4.338179132901132e-05, + 4.099325087736361e-05, + 5.2430314099183306e-05, + 4.8837704525794834e-05, + 4.9390600906917825e-05, + 5.749055708292872e-05, + 0.00022961001377552748, + 0.00017890847811941057, + 0.00014096690574660897, + 0.00011225492198718712, + 0.0001640371192479506, + 0.00012306995631661266, + 7.917954644653946e-05, + 6.291064346441999e-05, + 5.9928941482212394e-05, + 4.015088779851794e-05, + 3.608902989071794e-05, + 3.36187076754868e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.051189984660595655, + "first_to_last_ratio": 0.7709541992191667, + "imbalance_abs_log_ratio": 0.26012631156831567, + "last_quartile_mean": 0.06639821757562459, + "mean": 0.06283544522011653, + "normalized": [ + 0.7674565953661515, + 0.8910969393261454, + 0.7949526416572095, + 0.7325742284914066, + 0.9435441611668938, + 0.8683903486633082, + 0.8054914004920491, + 0.7138323596913118, + 0.5978064112769625, + 0.5556377993412199, + 0.5261854188690636, + 0.5057690043708923, + 0.7151083226055974, + 0.6608161658178341, + 0.6122564748857159, + 0.5785464975021892, + 0.7399601340699639, + 0.6892568823568349, + 0.6970600459500164, + 0.8113764696625058, + 3.2405348286343636, + 2.524973261140758, + 1.9894956995598303, + 1.584277576610783, + 2.3150909383371494, + 1.736912431812052, + 1.1174776724854902, + 0.8878712823128355, + 0.8457898807407633, + 0.5666580234128831, + 0.5093321198410424, + 0.47446798354877795 + ], + "peak_layer": 21, + "peak_normalized": 3.2405348286343636, + "population_cv": 0.6532808615381831, + "rest_mean": 0.04734913239048587, + "spike_contrast": 3.0932253054239998, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14646153450012206, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.048223476856946945, + 0.05599247291684151, + 0.04995120316743851, + 0.04603162780404091, + 0.05928801745176315, + 0.05456569418311119, + 0.0506134107708931, + 0.044853974133729935, + 0.03756343200802803, + 0.03491374850273132, + 0.033063095062971115, + 0.031780220568180084, + 0.04493414983153343, + 0.04152267798781395, + 0.03847140818834305, + 0.036353226751089096, + 0.046495724469423294, + 0.043309763073921204, + 0.043800078332424164, + 0.05098320171236992, + 0.20362044870853424, + 0.15865781903266907, + 0.12501084804534912, + 0.09954878687858582, + 0.1454697698354721, + 0.10913966596126556, + 0.07021720707416534, + 0.05578978732228279, + 0.05314558371901512, + 0.03560620918869972, + 0.032004110515117645, + 0.029813406988978386 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.293416855536634e-06, + "first_to_last_ratio": 0.44595751640176395, + "imbalance_abs_log_ratio": 0.8075315862036556, + "last_quartile_mean": 5.1426801235265884e-06, + "mean": 3.873313932700739e-06, + "normalized": [ + 0.6547872198845468, + 0.6098622810029859, + 0.5891095547173398, + 0.6012100431571663, + 0.5982908205734621, + 0.5743244780654446, + 0.5596766481388143, + 0.5495961168488454, + 0.44946743969713066, + 0.4316777868522346, + 0.4285841881361952, + 0.44095385743945303, + 0.560667078741325, + 0.5514542308757108, + 0.5417913687261439, + 0.5438534743570708, + 0.636286678312979, + 0.6499450755816364, + 0.6710355180394164, + 0.7226628285450687, + 3.0150865448734856, + 2.639875835373709, + 2.3467426593605225, + 2.0112905298698336, + 2.8997859114535975, + 2.237494500860103, + 1.506591468158109, + 1.1286195048247083, + 1.0054186728222039, + 0.6799993493915261, + 0.607662928646794, + 0.5561954066724383 + ], + "peak_layer": 21, + "peak_normalized": 3.0150865448734856, + "population_cv": 0.7767952241449113, + "rest_mean": 2.738177386163443e-06, + "spike_contrast": 3.6531786927134653, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.000305128400214e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.536196461733198e-06, + 2.3621880700375186e-06, + 2.281806246173801e-06, + 2.328675236640265e-06, + 2.3173681711341487e-06, + 2.2245390027819667e-06, + 2.1678033590433188e-06, + 2.128758296748856e-06, + 1.7409284964742255e-06, + 1.6720235862521804e-06, + 1.6600411072431598e-06, + 1.707952719698369e-06, + 2.171639607695397e-06, + 2.1359553556976607e-06, + 2.0985280571039766e-06, + 2.1065152395749465e-06, + 2.464538056301535e-06, + 2.5174413167405874e-06, + 2.59913122135913e-06, + 2.79910000244854e-06, + 1.1678376722557005e-05, + 1.022506785375299e-05, + 9.089671038964298e-06, + 7.790359632053878e-06, + 1.1231781172682531e-05, + 8.666518624522723e-06, + 5.835501724504866e-06, + 4.371497652755352e-06, + 3.894302153639728e-06, + 2.633850954225636e-06, + 2.3536692879133625e-06, + 2.1543194179685088e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.4421208258427214e-05, + "first_to_last_ratio": 0.9390471381412793, + "imbalance_abs_log_ratio": 0.06288960067042051, + "last_quartile_mean": 4.730455634671671e-05, + "mean": 5.047763789889359e-05, + "normalized": [ + 0.8442417833686091, + 0.9554185876612655, + 0.8651397352686891, + 0.8073036157415787, + 1.0117688173530386, + 0.9346106525630644, + 0.8597336821790709, + 0.7619236882899906, + 0.6572438052320593, + 0.6123997263969396, + 0.5783007276195616, + 0.5555384405465414, + 0.7735135857629635, + 0.7093370113557148, + 0.654834684626451, + 0.6154293791596152, + 0.8164712035979397, + 0.7613467591474443, + 0.7654961084945793, + 0.8721378389215614, + 3.265225507983762, + 2.4905834329623366, + 1.889506148192275, + 1.4453841483405094, + 2.0912250805998966, + 1.5360203292388628, + 1.0169178649058275, + 0.8253190119907806, + 0.6717866718190295, + 0.48834958251132726, + 0.44376373442170963, + 0.42372865374700563 + ], + "peak_layer": 21, + "peak_normalized": 3.265225507983762, + "population_cv": 0.6182834965857263, + "rest_mean": 3.8920269852732655e-05, + "spike_contrast": 2.9004789991258666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011288742534816265, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.261533103999682e-05, + 4.822727350983769e-05, + 4.367021028883755e-05, + 4.0750779589870945e-05, + 5.1071699999738485e-05, + 4.717693809652701e-05, + 4.33973254985176e-05, + 3.846010804409161e-05, + 3.317611481179483e-05, + 3.091249163844623e-05, + 2.9191254725446925e-05, + 2.8042268240824342e-05, + 3.9045138692017645e-05, + 3.5805656807497144e-05, + 3.305450809421018e-05, + 3.106542135355994e-05, + 4.1213537770090625e-05, + 3.843098602374084e-05, + 3.8640435377601534e-05, + 4.402345803100616e-05, + 0.00016482087085023522, + 0.00012571876868605614, + 9.537780715618283e-05, + 7.295957766473293e-05, + 0.00010556010238360614, + 7.753467798465863e-05, + 5.1331611757632345e-05, + 4.166015423834324e-05, + 3.3910204365383834e-05, + 2.4650733394082636e-05, + 2.240014509879984e-05, + 2.138882155122701e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.359962329341215e-05, + "first_to_last_ratio": 1.183161693959634, + "imbalance_abs_log_ratio": 0.1681902569407737, + "last_quartile_mean": 2.839816693267494e-05, + "mean": 3.542719008464701e-05, + "normalized": [ + 0.9327605618775979, + 1.0266348424598963, + 0.9395266301163642, + 0.8894102418069995, + 1.082225986222851, + 1.0025232858328599, + 0.9050487327854893, + 0.8091772559917478, + 0.7238157816582237, + 0.6814271430375796, + 0.6400264742225962, + 0.6141081065852076, + 0.843581628077757, + 0.7675711067089205, + 0.7064606405143428, + 0.6655561644133133, + 0.9162025945827288, + 0.8560421033047051, + 0.8525829253737746, + 0.9377293691727334, + 3.3385973508618734, + 2.4504448425731913, + 1.7642485194416198, + 1.2415583752024149, + 1.7870101279275883, + 1.2751219306613844, + 0.9216663623251447, + 0.7867121103297002, + 0.5090691818722745, + 0.4053411321779632, + 0.37287136742526056, + 0.3549471244558967 + ], + "peak_layer": 21, + "peak_normalized": 3.3385973508618734, + "population_cv": 0.592278758078665, + "rest_mean": 2.8103131289417097e-05, + "spike_contrast": 2.6679271717711734, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.497710757888853e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.304508572909981e-05, + 3.6370787711348385e-05, + 3.3284788514720276e-05, + 3.150930569972843e-05, + 3.834022572846152e-05, + 3.5516583011485636e-05, + 3.2063333492260426e-05, + 2.8666876460192725e-05, + 2.5642759283073246e-05, + 2.414104892523028e-05, + 2.2674339561490342e-05, + 2.1756124624516815e-05, + 2.9885726689826697e-05, + 2.7192887500859797e-05, + 2.5027915398823097e-05, + 2.3578784748679027e-05, + 3.2458483474329114e-05, + 3.032716631423682e-05, + 3.0204617360141128e-05, + 3.322111660963856e-05, + 0.00011827712296508253, + 8.681237522978336e-05, + 6.250236765481532e-05, + 4.398492455948144e-05, + 6.330874748528004e-05, + 4.517398701864295e-05, + 3.265204941271804e-05, + 2.7870999474544078e-05, + 1.8034890672424808e-05, + 1.436009733879473e-05, + 1.3209784810896963e-05, + 1.2574779248097911e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011756234653148567, + "first_to_last_ratio": 0.7472422235887175, + "imbalance_abs_log_ratio": 0.29136588457853974, + "last_quartile_mean": 0.00015732829706394114, + "mean": 0.0001475833446420438, + "normalized": [ + 0.7536685945005097, + 0.8489437148344299, + 0.7668873856587688, + 0.7219862591209203, + 0.9160304874966925, + 0.8511301995015967, + 0.8040925298742102, + 0.7099228291391, + 0.592249280993652, + 0.5528729353184129, + 0.5267845496399454, + 0.5066519805725661, + 0.6991026141257565, + 0.6459439706463571, + 0.6067103504076782, + 0.5642338800686106, + 0.7290006596953322, + 0.6851008790441766, + 0.687775193830433, + 0.7817329016891325, + 3.243635755848753, + 2.5800278485304857, + 2.0438475774053395, + 1.653426311601236, + 2.3369623381128184, + 1.7968342666222143, + 1.1670758041878126, + 0.9412043950671267, + 0.8016088114740478, + 0.5374717514485083, + 0.4848750369936043, + 0.46220890654977165 + ], + "peak_layer": 21, + "peak_normalized": 3.243635755848753, + "population_cv": 0.6705219269582756, + "rest_mean": 0.0001100977226289817, + "spike_contrast": 3.179045807260368, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003500057035125792, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0001112289319280535, + 0.00012528995284810662, + 0.00011317980533931404, + 0.00010655314690666273, + 0.00013519084313884377, + 0.00012561264156829566, + 0.00011867066496051848, + 0.00010477278556209058, + 8.74061297508888e-05, + 8.159483695635572e-05, + 7.774462574161589e-05, + 7.47733938624151e-05, + 0.00010317590204067528, + 9.533057163935155e-05, + 8.954034274211153e-05, + 8.327152318088338e-05, + 0.00010758835560409352, + 0.00010110947914654389, + 0.00010150416346732527, + 0.00011537075624801219, + 0.00047870661364868283, + 0.0003807691391557455, + 0.00030163786141201854, + 0.00024401818518526852, + 0.0003448967181611806, + 0.0002651828108355403, + 0.0001722409506328404, + 0.00013890609261579812, + 0.00011830410949187353, + 7.932187872938812e-05, + 7.155947969295084e-05, + 6.821433635195717e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0626930519938469, + 0.09030644595623016, + 0.20229491591453552, + 0.20300470292568207, + 0.19049881398677826, + 0.12695962190628052, + 0.12132706493139267, + 0.11585728824138641, + 0.13427142798900604, + 0.10485932976007462, + 0.10419609397649765, + 0.10852954536676407, + 0.1286265105009079, + 0.09096142649650574, + 0.10080574452877045, + 0.10566592961549759, + 0.1287592649459839, + 0.09650235623121262, + 0.1065765842795372, + 0.11446042358875275, + 0.12395420670509338, + 0.07221997529268265, + 0.07677769660949707, + 0.08416415005922318, + 0.09094838052988052, + 0.059439923614263535, + 0.062182098627090454, + 0.06896103918552399, + 0.0776243805885315, + 0.06285123527050018, + 0.07231374084949493, + 0.08016443997621536 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 9.220272659149487e-06, + "first_to_last_ratio": 1.069429605014154, + "imbalance_abs_log_ratio": 0.06712542691043744, + "last_quartile_mean": 8.621673288189413e-06, + "mean": 1.2944014976312701e-05, + "normalized": [ + 0.6303729714265094, + 0.8682643012395536, + 0.6280898219637274, + 0.5502831812812704, + 0.33676078665107495, + 0.7338236582315528, + 0.9360409006501473, + 1.0149193793174633, + 0.6122272943195698, + 0.5131421981111545, + 0.43080038533307174, + 0.37792854873844645, + 0.39300605492101337, + 0.6863011395079349, + 0.6243318369244146, + 0.5414566873457767, + 0.793386920098201, + 1.135440497453951, + 1.3017177042809938, + 1.597588130028887, + 2.6171146314958285, + 4.210609056409907, + 2.860451118900051, + 2.2773498949645425, + 1.2385768946194602, + 1.256602631548737, + 0.8025963162001695, + 0.5822644883727114, + 0.47699344412607797, + 0.2914573895713198, + 0.32576029794875644, + 0.3543414380177253 + ], + "peak_layer": 22, + "peak_normalized": 4.210609056409907, + "population_cv": 0.8557638034672043, + "rest_mean": 9.010903349243799e-06, + "spike_contrast": 3.7934951067201736, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4182817762484775e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.159557182807475e-06, + 1.1238826118642464e-05, + 8.130004061968066e-06, + 7.122873739717761e-06, + 4.359036665846361e-06, + 9.498624422121793e-06, + 1.2116127436456736e-05, + 1.3137131645635236e-05, + 7.924679266579915e-06, + 6.642120297328802e-06, + 5.5762866395525634e-06, + 4.891912794846576e-06, + 5.087076260679169e-06, + 8.883492228051182e-06, + 8.081360647338443e-06, + 7.008623470028397e-06, + 1.0269612175761722e-05, + 1.4697158803755883e-05, + 1.684945345914457e-05, + 2.0679204681073315e-05, + 3.38759709848091e-05, + 5.450218668556772e-05, + 3.7025722122052684e-05, + 2.9478051146725193e-05, + 1.603215787326917e-05, + 1.6265483282040805e-05, + 1.0388818736828398e-05, + 7.536840257671429e-06, + 6.174210284370929e-06, + 3.7726288155681686e-06, + 4.216646175336791e-06, + 4.586600880429614e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.628570084401872e-05, + "first_to_last_ratio": 1.065353016422061, + "imbalance_abs_log_ratio": 0.0633062150559101, + "last_quartile_mean": 3.405979077797383e-05, + "mean": 5.163632124549622e-05, + "normalized": [ + 0.5687040132298282, + 0.8764593520729407, + 0.6051388108071001, + 0.5571670858837222, + 0.3412536542304683, + 0.7524665950392966, + 0.8942377527803768, + 1.0263056508948738, + 0.5946205407803693, + 0.5239558700278845, + 0.42939602856142756, + 0.3883393864768877, + 0.40120729316033693, + 0.7381389545660257, + 0.6270734263766993, + 0.5557805536677036, + 0.7656375942833815, + 1.1752771944325544, + 1.2670292024807646, + 1.589140645161745, + 2.652708081192408, + 4.256645166896803, + 2.872494573593182, + 2.263949247584013, + 1.2185560925718533, + 1.2799157794404965, + 0.7966972545457802, + 0.5211011847842488, + 0.47241308488259476, + 0.29979401599371164, + 0.32815719518376885, + 0.36023871841675353 + ], + "peak_layer": 22, + "peak_normalized": 4.256645166896803, + "population_cv": 0.8639609553478637, + "rest_mean": 3.5831106625092046e-05, + "spike_contrast": 3.823060270757878, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013698448019567878, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.9365783120738342e-05, + 4.5257136662257835e-05, + 3.124714203295298e-05, + 2.8770058634108864e-05, + 1.762108331604395e-05, + 3.885460682795383e-05, + 4.6175147872418165e-05, + 5.2994648285675794e-05, + 3.0704017262905836e-05, + 2.7055153623223305e-05, + 2.2172431272338144e-05, + 2.0052417312399484e-05, + 2.0716868675663136e-05, + 3.8114780181786045e-05, + 3.2379764888901263e-05, + 2.86984632111853e-05, + 3.953470877604559e-05, + 6.06869907642249e-05, + 6.542472692672163e-05, + 8.205737685784698e-05, + 0.00013697608665097505, + 0.0002197974972659722, + 0.00014832505257800221, + 0.00011690201063174754, + 6.292175385169685e-05, + 6.609014235436916e-05, + 4.113851537113078e-05, + 2.690774817892816e-05, + 2.4393673811573535e-05, + 1.5480260117328726e-05, + 1.6944830349530093e-05, + 1.860140218923334e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.032178521854802966, + "first_to_last_ratio": 1.0653530079886349, + "imbalance_abs_log_ratio": 0.06330620713982399, + "last_quartile_mean": 0.030204562819562852, + "mean": 0.04579160569119267, + "normalized": [ + 0.5687040352411169, + 0.8764592932397164, + 0.6051388058773625, + 0.5571671042879097, + 0.3412536282040628, + 0.7524665522800605, + 0.8942377987777075, + 1.0263056510911557, + 0.5946205787763836, + 0.523955869943047, + 0.42939603938950854, + 0.388339353567458, + 0.4012073030262001, + 0.7381389650225099, + 0.627073439274379, + 0.5557805622734189, + 0.7656375421356683, + 1.1752771374183286, + 1.2670292275109216, + 1.589140749744225, + 2.6527080853277174, + 4.256645179045396, + 2.872494621066323, + 2.2639491530094316, + 1.2185561059252055, + 1.2799157254825457, + 0.796697271015912, + 0.52110118872801, + 0.47241309150505434, + 0.299794020683871, + 0.32815719848905306, + 0.3602387226403389 + ], + "peak_layer": 22, + "peak_normalized": 4.256645179045396, + "population_cv": 0.8639609572615631, + "rest_mean": 0.031775383451194676, + "spike_contrast": 3.823060262160724, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12147920578718185, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.026041870936751366, + 0.040134478360414505, + 0.02771027758717537, + 0.025513576343655586, + 0.01562655158340931, + 0.03445665165781975, + 0.04094858467578888, + 0.046996183693408966, + 0.02722863107919693, + 0.023992780596017838, + 0.019662734121084213, + 0.017782682552933693, + 0.018371926620602608, + 0.03380056843161583, + 0.028714699670672417, + 0.02545008435845375, + 0.03505977243185043, + 0.05381782725453377, + 0.058019302785396576, + 0.07276930660009384, + 0.12147176265716553, + 0.19491861760616302, + 0.13153614103794098, + 0.10366986691951752, + 0.05579964071512222, + 0.05860939621925354, + 0.03648204728960991, + 0.02386206015944481, + 0.02163255400955677, + 0.013728049583733082, + 0.015026845037937164, + 0.01649590954184532 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.4475698694459425e-06, + "first_to_last_ratio": 0.4032796404419227, + "imbalance_abs_log_ratio": 0.9081250607923619, + "last_quartile_mean": 6.069162992616839e-06, + "mean": 4.194480464292383e-06, + "normalized": [ + 1.9615731362727682, + 0.7747288452238285, + 0.1785692992504359, + 0.2633308680375469, + 0.18148950204723951, + 0.349408887478932, + 0.383871487008665, + 0.575200617814069, + 0.3462103003815746, + 0.3364194451166286, + 0.2854708899046967, + 0.31108734109147806, + 0.30067908576983693, + 0.513261777831553, + 0.5156365685672093, + 0.5885731468966555, + 0.3799425036372034, + 0.7326093628890369, + 0.5694277551741158, + 0.8619428981160375, + 1.5998405056629517, + 3.0497744670188736, + 2.8846131894263896, + 2.4808153927254097, + 1.8266821603141778, + 2.5325708585606295, + 2.0509024328203003, + 1.5887038326529273, + 1.3182729046192574, + 0.967953515357782, + 0.6305408473131235, + 0.6598961750186662 + ], + "peak_layer": 22, + "peak_normalized": 3.0497744670188736, + "population_cv": 0.8409982900394547, + "rest_mean": 3.131610654876298e-06, + "spike_contrast": 3.1721623566680712, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.93397743513924e-06, + "top_five_layers": [ + 22, + 23, + 26, + 24, + 27 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.227780199376866e-06, + 3.2495850064151455e-06, + 7.490054372283339e-07, + 1.1045361816286459e-06, + 7.612541708112985e-07, + 1.4655887525805156e-06, + 1.6101414530567126e-06, + 2.4126677544700215e-06, + 1.4521723414873122e-06, + 1.411104790349782e-06, + 1.197402070829412e-06, + 1.3048497748968657e-06, + 1.2611925512828748e-06, + 2.1528665001824265e-06, + 2.162827513529919e-06, + 2.4687585664651124e-06, + 1.5936614090605872e-06, + 3.072915660595754e-06, + 2.3884535949036945e-06, + 3.615402647483279e-06, + 6.710499746986898e-06, + 1.279221942240838e-05, + 1.2099453670089133e-05, + 1.0405731700302567e-05, + 7.661982635909226e-06, + 1.0622818990668748e-05, + 8.60247018863447e-06, + 6.663787189609138e-06, + 5.529469945031451e-06, + 4.060062110511353e-06, + 2.6447912659932626e-06, + 2.7679216145770624e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.4055149197010905e-05, + "first_to_last_ratio": 1.0862063963884478, + "imbalance_abs_log_ratio": 0.08269125537722917, + "last_quartile_mean": 2.2146020569380198e-05, + "mean": 3.472467025744663e-05, + "normalized": [ + 0.4666666997881285, + 0.8663177887052829, + 0.6219999695375312, + 0.5679169854003736, + 0.3844149632749681, + 0.7928386669686874, + 0.9321622159542684, + 0.9095983613860097, + 0.6255219017157577, + 0.5547521439337422, + 0.4549635591797121, + 0.4053293755903786, + 0.4456534403712468, + 0.7958744306526087, + 0.6527431991239231, + 0.5563097060509357, + 0.7807692106312896, + 1.1853454205486673, + 1.2646459398653336, + 1.548550635017449, + 2.506691252154866, + 4.290967865992318, + 3.0302171410000085, + 2.2576657022736453, + 1.197190310100688, + 1.343054897090299, + 0.7711430639674054, + 0.472191241764576, + 0.47986836664550414, + 0.29867429600060663, + 0.25046027339059757, + 0.2895009759231917 + ], + "peak_layer": 22, + "peak_normalized": 4.290967865992318, + "population_cv": 0.8713516434982468, + "rest_mean": 2.4072257407102302e-05, + "spike_contrast": 3.832116701364666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.224769964930602e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.6204847270273603e-05, + 3.0082599550951272e-05, + 2.1598743842332624e-05, + 1.9720730051631108e-05, + 1.3348682841751724e-05, + 2.7531061277841218e-05, + 3.236902557546273e-05, + 3.1585503165842965e-05, + 2.1721041775890626e-05, + 1.926358527271077e-05, + 1.579845957166981e-05, + 1.4074928913032636e-05, + 1.54751687659882e-05, + 2.763647717074491e-05, + 2.2666292352369055e-05, + 1.9317671103635803e-05, + 2.7111953386338428e-05, + 4.116072886972688e-05, + 4.391441325424239e-05, + 5.37729101779405e-05, + 8.704402716830373e-05, + 0.0001490024442318827, + 0.00010522329102968797, + 7.839669706299901e-05, + 4.157203875365667e-05, + 4.663713843910955e-05, + 2.6777688617585227e-05, + 1.6396685168729164e-05, + 1.6663270798744634e-05, + 1.0371366442996077e-05, + 8.697150406078435e-06, + 1.0052825928141829e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.579084118930041e-05, + "first_to_last_ratio": 1.101662799461071, + "imbalance_abs_log_ratio": 0.09682067430659903, + "last_quartile_mean": 1.433364292324768e-05, + "mean": 2.2897426177337366e-05, + "normalized": [ + 0.39886794777854795, + 0.877049729540605, + 0.6305457825207785, + 0.5772129763101231, + 0.4367770087210306, + 0.8436499656131583, + 0.972408138490726, + 0.7805596072336106, + 0.6568359938107564, + 0.6066017752376727, + 0.49315422395705855, + 0.4249778682175203, + 0.4986390140344086, + 0.857095556009823, + 0.6846582999905741, + 0.5629018786115396, + 0.7936422327048538, + 1.202661838363613, + 1.2736141330113293, + 1.5255025761233132, + 2.344832382358406, + 4.19579128393446, + 3.181636935928761, + 2.172433813999188, + 1.1390303293883177, + 1.4232014401744595, + 0.759652938746426, + 0.4324960642291122, + 0.4879317473364259, + 0.30796555094004274, + 0.21544260190277736, + 0.24222836478058168 + ], + "peak_layer": 22, + "peak_normalized": 4.19579128393446, + "population_cv": 0.858855507029887, + "rest_mean": 1.6084403240610198e-05, + "spike_contrast": 3.7109085828539374, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.968775003566407e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.133049388765357e-06, + 2.0082181436009705e-05, + 1.4437875506700948e-05, + 1.3216691513662226e-05, + 1.0001069313148037e-05, + 1.93174128071405e-05, + 2.226564356533345e-05, + 1.7872805983643048e-05, + 1.5039853678899817e-05, + 1.3889619367546402e-05, + 1.1291962437098846e-05, + 9.73089936451288e-06, + 1.141755001299316e-05, + 1.9625282220658846e-05, + 1.5676912880735472e-05, + 1.2889004210592248e-05, + 1.8172364434576593e-05, + 2.7537860660231672e-05, + 2.9162485589040443e-05, + 3.493008262012154e-05, + 5.36906263732817e-05, + 9.607282117940485e-05, + 7.285129686351866e-05, + 4.9743142881197855e-05, + 2.6080862880917266e-05, + 3.2587649911874905e-05, + 1.7394097085343674e-05, + 9.903046702675056e-06, + 1.117238116421504e-05, + 7.051618467812659e-06, + 4.933081072522327e-06, + 5.546406100620516e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.034897362245829e-05, + "first_to_last_ratio": 1.0677221298454265, + "imbalance_abs_log_ratio": 0.06552752863846958, + "last_quartile_mean": 6.588696783182968e-05, + "mean": 0.00010326150248829435, + "normalized": [ + 0.5211538788724873, + 0.8562183017948943, + 0.6056449715109208, + 0.5367927676701035, + 0.33719464941795635, + 0.7400076028920698, + 0.8752614309891937, + 0.977887157814224, + 0.5891182329173178, + 0.4959165872409433, + 0.4220040517626212, + 0.3715438426401133, + 0.3950700830815582, + 0.7426266558595574, + 0.622546917450552, + 0.5356415681171627, + 0.7637846157174611, + 1.2023092054640232, + 1.2927045115765263, + 1.5352424858917262, + 2.711590866084263, + 4.42744668313384, + 2.973944015958163, + 2.3638740639156786, + 1.2453125473146385, + 1.2805717504661769, + 0.7715376325621908, + 0.48453681266643056, + 0.47249674362438154, + 0.27056316221994425, + 0.2589436600974457, + 0.3205125432754346 + ], + "peak_layer": 22, + "peak_normalized": 4.42744668313384, + "population_cv": 0.9059813522703718, + "rest_mean": 6.990356949380064e-05, + "spike_contrast": 4.054075388577877, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002833943406585604, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.381513255997561e-05, + 8.841438830131665e-05, + 6.25398097326979e-05, + 5.5430027714464813e-05, + 3.481922612991184e-05, + 7.641429692739621e-05, + 9.03808104339987e-05, + 0.00010097809717990458, + 6.083323387429118e-05, + 5.120909190736711e-05, + 4.357677244115621e-05, + 3.8366175431292504e-05, + 4.079553036717698e-05, + 7.66847442719154e-05, + 6.428513006540015e-05, + 5.5311153118964285e-05, + 7.886954699642956e-05, + 0.00012415225501172245, + 0.00013348661013878882, + 0.0001585314457770437, + 0.0002800029469653964, + 0.00045718479668721557, + 0.00030709392740391195, + 0.00024409718753304332, + 0.00012859284470323473, + 0.00013223376299720258, + 7.96701351646334e-05, + 5.003399928682484e-05, + 4.879072366748005e-05, + 2.7938758648815565e-05, + 2.6738911401480436e-05, + 3.309660678496584e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059514161199331284, + 0.08157797157764435, + 0.14292451739311218, + 0.10997853428125381, + 0.0991024449467659, + 0.1065012738108635, + 0.12155740708112717, + 0.1095810979604721, + 0.06964563578367233, + 0.08311934769153595, + 0.09363401681184769, + 0.0941167026758194, + 0.08564268052577972, + 0.09493856877088547, + 0.09063401818275452, + 0.09473779052495956, + 0.10505300015211105, + 0.09355910867452621, + 0.09618847072124481, + 0.11416846513748169, + 0.047205954790115356, + 0.053352247923612595, + 0.06512638926506042, + 0.07829347252845764, + 0.05368548259139061, + 0.05310077592730522, + 0.061235878616571426, + 0.07320034503936768, + 0.05304349213838577, + 0.0639304369688034, + 0.07404901832342148, + 0.08157160878181458 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.406091136388568e-06, + "first_to_last_ratio": 0.13952873923075965, + "imbalance_abs_log_ratio": 1.9694846829479884, + "last_quartile_mean": 3.15783770474809e-05, + "mean": 1.6862885999557875e-05, + "normalized": [ + 0.5744948983086973, + 0.20976707009476936, + 0.1380798037056023, + 0.25679297795080097, + 0.1911977573081938, + 0.15397727234373496, + 0.20783011320461262, + 0.35817424975363493, + 0.40845309421862647, + 0.30185320868879156, + 0.2734882217077138, + 0.2725610842103788, + 0.35283552344917235, + 0.37101154440150896, + 0.4205580365391262, + 0.41850822128869175, + 0.3521537888379847, + 0.4454210973675298, + 0.41314047811549354, + 0.5286573606706721, + 2.6023136902657806, + 2.9811111131589443, + 2.583962732113176, + 2.2024122663803394, + 2.8672118131663793, + 2.8994466418579012, + 2.457231935453014, + 1.624795250078144, + 1.8440638614795903, + 1.4598750180798175, + 0.9499358030010668, + 0.8786840728001112 + ], + "peak_layer": 22, + "peak_normalized": 2.9811111131589443, + "population_cv": 0.9711949879643771, + "rest_mean": 1.1718449412809503e-05, + "spike_contrast": 3.8096203683057026, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.464284356799908e-05, + "top_five_layers": [ + 22, + 26, + 25, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.687641977507155e-06, + 3.5372781894693617e-06, + 2.3284239887289004e-06, + 4.330270712671336e-06, + 3.2241459848592058e-06, + 2.596501190055278e-06, + 3.5046155062445905e-06, + 6.039851541572716e-06, + 6.88769796397537e-06, + 5.090116246719845e-06, + 4.611800704878988e-06, + 4.596166490955511e-06, + 5.949825208517723e-06, + 6.2563253777625505e-06, + 7.091822226357181e-06, + 7.05725642546895e-06, + 5.938329195487313e-06, + 7.511085186706623e-06, + 6.9667407842644025e-06, + 8.914688805816695e-06, + 4.388251909404062e-05, + 5.027013685321435e-05, + 4.357306897873059e-05, + 3.7139026971999556e-05, + 4.8349465942010283e-05, + 4.88930381834507e-05, + 4.1436022002017125e-05, + 2.739873707469087e-05, + 3.1096238672034815e-05, + 2.4617706003482454e-05, + 1.6018659152905457e-05, + 1.4817149349255487e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.77339601350468e-05, + "first_to_last_ratio": 0.14031202708197554, + "imbalance_abs_log_ratio": 1.9638865715137788, + "last_quartile_mean": 0.00012638945145226899, + "mean": 6.748338435613732e-05, + "normalized": [ + 0.581715983352747, + 0.21329408694826651, + 0.1383654282103214, + 0.26055913189981234, + 0.18941324497814624, + 0.15268445345034073, + 0.20588194209370914, + 0.3604059140126238, + 0.4112482421872158, + 0.3050474729826253, + 0.2735060049358427, + 0.27320166025083037, + 0.35640352937895586, + 0.3762475521803712, + 0.424911703962158, + 0.41981156521974744, + 0.35303414718819204, + 0.4487530978367584, + 0.4155766160707765, + 0.5338541629759799, + 2.603208152672671, + 2.9606390037219725, + 2.573692405652612, + 2.185365730719144, + 2.873997131282303, + 2.8916679373421434, + 2.4621472618371403, + 1.629468290031214, + 1.8469331471633454, + 1.452438714650874, + 0.9530063641283487, + 0.8735199206828114 + ], + "peak_layer": 22, + "peak_normalized": 2.9606390037219725, + "population_cv": 0.9681787165444765, + "rest_mean": 4.699617262236576e-05, + "spike_contrast": 3.7899751783986364, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017811432771850378, + "top_five_layers": [ + 22, + 26, + 25, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.9256163290701807e-05, + 1.4393806850421242e-05, + 9.337367373518646e-06, + 1.7583412045496516e-05, + 1.278224681300344e-05, + 1.03036636573961e-05, + 1.3893610230297782e-05, + 2.432141081953887e-05, + 2.775242319330573e-05, + 2.0585635866154917e-05, + 1.8457110854797065e-05, + 1.843657264544163e-05, + 2.405131635896396e-05, + 2.5390458176843822e-05, + 2.8674479835899547e-05, + 2.8330305212875828e-05, + 2.382393904554192e-05, + 3.028337778232526e-05, + 2.8044516511727124e-05, + 3.602628567023203e-05, + 0.00017567329632584006, + 0.00019979393982794136, + 0.00017368147382512689, + 0.00014747587556485087, + 0.0001939470530487597, + 0.00019513953884597868, + 0.00016615403001196682, + 0.00010996203491231427, + 0.00012463729945011437, + 9.801548003451899e-05, + 6.431209476431832e-05, + 5.894808055018075e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.015726652927696705, + "first_to_last_ratio": 0.14031202298106044, + "imbalance_abs_log_ratio": 1.9638866007408895, + "last_quartile_mean": 0.11208343086764216, + "mean": 0.05984494066797197, + "normalized": [ + 0.5817159394623853, + 0.21329409008400638, + 0.13836543030820245, + 0.26055913499009653, + 0.18941324063527437, + 0.1526844532232541, + 0.20588194088455028, + 0.36040590002189726, + 0.41124824181954295, + 0.3050474835118139, + 0.2735060178075916, + 0.273201651110741, + 0.35640353553867715, + 0.37624753453424215, + 0.42491172006038797, + 0.41981156257536634, + 0.35303415045149444, + 0.4487531036157151, + 0.4155766045421345, + 0.5338542063524581, + 2.6032081764056025, + 2.9606389456555853, + 2.5736924190660213, + 2.185365706690064, + 2.873996992351884, + 2.891668001746537, + 2.4621473625257404, + 1.6294681803958442, + 1.8469332209343858, + 1.4524387488575696, + 0.9530063959374594, + 0.8735199079034743 + ], + "peak_layer": 22, + "peak_normalized": 2.9606389456555853, + "population_cv": 0.9681787143090018, + "rest_mean": 0.04167667665967235, + "spike_contrast": 3.789975088527889, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15795356631278992, + "top_five_layers": [ + 22, + 26, + 25, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.03481275588274002, + 0.01276457216590643, + 0.008280470967292786, + 0.015593145973980427, + 0.011335424147546291, + 0.009137392044067383, + 0.012320992536842823, + 0.02156846970319748, + 0.024611126631498337, + 0.018255548551678658, + 0.016367951408028603, + 0.016349736601114273, + 0.021328948438167572, + 0.022516511380672455, + 0.02542881667613983, + 0.0251235980540514, + 0.021127307787537575, + 0.026855602860450745, + 0.02487015724182129, + 0.03194847330451012, + 0.1557888388633728, + 0.1771792620420456, + 0.1540224701166153, + 0.1307830810546875, + 0.1719941794872284, + 0.17305169999599457, + 0.14734706282615662, + 0.09751542657613754, + 0.11052960902452469, + 0.08692111074924469, + 0.05703261122107506, + 0.05227574706077576 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0061295867558329e-06, + "first_to_last_ratio": 0.11842320932790659, + "imbalance_abs_log_ratio": 2.1334905510136277, + "last_quartile_mean": 8.49605066832737e-06, + "mean": 4.365163350783519e-06, + "normalized": [ + 0.5368020226986673, + 0.16971060838105761, + 0.13410619991648343, + 0.2737672720355794, + 0.17254528945791953, + 0.13905042866169556, + 0.15902329960072473, + 0.2589205349972531, + 0.33151883176756186, + 0.2629812299966789, + 0.24045074835041885, + 0.24352968420582896, + 0.32204843448988535, + 0.36030617882440874, + 0.42811660272336727, + 0.41111875160131917, + 0.35123001138976506, + 0.43809006822116814, + 0.40879993942951587, + 0.3687203828633752, + 2.5744168736304367, + 2.993182569498261, + 2.6417671760629977, + 2.209152675699325, + 2.926027285435564, + 2.960732429505928, + 2.7232927720785614, + 1.7159123604866093, + 1.8688437451101783, + 1.456254776895987, + 1.0064626168064281, + 0.9131181991770502 + ], + "peak_layer": 22, + "peak_normalized": 2.993182569498261, + "population_cv": 1.010043630070163, + "rest_mean": 3.0160778355484246e-06, + "spike_contrast": 3.8627070547516635, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1650225133053027e-05, + "top_five_layers": [ + 22, + 26, + 25, + 27, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.3432285161106847e-06, + 7.408145279441669e-07, + 5.853954689882812e-07, + 1.1950388625336927e-06, + 7.531883738920442e-07, + 6.069778351047717e-07, + 6.941626793377509e-07, + 1.1302304301352706e-06, + 1.447133854526328e-06, + 1.147956027125474e-06, + 1.049606794367719e-06, + 1.0630468523231684e-06, + 1.4057940234124544e-06, + 1.5727953268651618e-06, + 1.8687989040699904e-06, + 1.7946005073099514e-06, + 1.5331763734138804e-06, + 1.9123347101412946e-06, + 1.784478513400245e-06, + 1.6095247019620729e-06, + 1.1237750186410267e-05, + 1.306573085457785e-05, + 1.1531745258253068e-05, + 9.643312296248041e-06, + 1.277258706977591e-05, + 1.2924080692755524e-05, + 1.188761780213099e-05, + 7.490237749152584e-06, + 8.157808224495966e-06, + 6.356789981509792e-06, + 4.393373728817096e-06, + 3.985910097981105e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.197550119513835e-05, + "first_to_last_ratio": 0.1587346511851631, + "imbalance_abs_log_ratio": 1.8405213313314666, + "last_quartile_mean": 7.544352229160722e-05, + "mean": 4.3035416283032646e-05, + "normalized": [ + 0.6336653627326162, + 0.22557503269458085, + 0.11384714614846705, + 0.24237874371328197, + 0.21264877275758737, + 0.17043176216342354, + 0.2212795583484355, + 0.40634029614109113, + 0.4665961306783005, + 0.344071525310092, + 0.31263674643247324, + 0.325989461953546, + 0.4165298351149085, + 0.4247779117423742, + 0.4717272077470243, + 0.4737094580054267, + 0.40283409959357847, + 0.521705164723695, + 0.4749365612827932, + 0.5725415698804192, + 2.7082874349482955, + 3.047902384226458, + 2.6438885206938516, + 2.1412461889756425, + 2.9349243481759872, + 2.7222959905080586, + 2.283569880862303, + 1.5378599969243498, + 1.615015848446693, + 1.3002882037476036, + 0.8505045545457767, + 0.7799943007808661 + ], + "peak_layer": 22, + "peak_normalized": 3.047902384226458, + "population_cv": 0.9449871736776022, + "rest_mean": 2.9525086692619544e-05, + "spike_contrast": 3.928564114944936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001159911960712634, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.727005266933702e-05, + 9.707715435069986e-06, + 4.899459327134537e-06, + 1.0430870133859571e-05, + 9.151428457698785e-06, + 7.334601832553744e-06, + 9.522857908450533e-06, + 1.748702379700262e-05, + 2.0080158719792962e-05, + 1.4807261322857812e-05, + 1.3454452528094407e-05, + 1.4029092199052684e-05, + 1.7925534848473035e-05, + 1.8280494259670377e-05, + 2.0300976757425815e-05, + 2.038628372247331e-05, + 1.733613316901028e-05, + 2.245179894089233e-05, + 2.0439092622837052e-05, + 2.4639564799144864e-05, + 0.00011655227717710659, + 0.00013116774789523333, + 0.00011378084309399128, + 9.214942110702395e-05, + 0.00012630569108296186, + 0.00011715514119714499, + 9.827438043430448e-05, + 6.61824451526627e-05, + 6.95028793415986e-05, + 5.5958444136194885e-05, + 3.6601817555492744e-05, + 3.3567379432497546e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 7.998359336625072e-06, + "first_to_last_ratio": 0.19394161321534342, + "imbalance_abs_log_ratio": 1.6401981280247118, + "last_quartile_mean": 4.124106840208697e-05, + "mean": 2.6285412580762113e-05, + "normalized": [ + 0.6933384670594045, + 0.2456346256669455, + 0.10397585233667712, + 0.2374486790180797, + 0.24369297059389394, + 0.19793868329883177, + 0.24449766268182652, + 0.46778427650330523, + 0.5456273930041772, + 0.3924025527768945, + 0.36245298346490396, + 0.39321573934626614, + 0.49513567631933314, + 0.492561727340712, + 0.5399874725375117, + 0.5520792544136082, + 0.47500593515532746, + 0.6277771942942818, + 0.5578424227128029, + 0.6222043300599132, + 2.928340543497061, + 3.220069339518848, + 2.7546600577445735, + 2.05455257890152, + 2.8717816218189083, + 2.429966999218037, + 1.946404647300736, + 1.3608722565402098, + 1.3761877892744612, + 1.13757832466242, + 0.7380055991332236, + 0.6909763438053044 + ], + "peak_layer": 22, + "peak_normalized": 3.220069339518848, + "population_cv": 0.9165336328191448, + "rest_mean": 1.7689689221461542e-05, + "spike_contrast": 4.109869755811259, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.27023187209852e-05, + "top_five_layers": [ + 22, + 21, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.822468766476959e-05, + 6.4566074797767214e-06, + 2.7330481771059567e-06, + 6.241436494747177e-06, + 6.405570275092032e-06, + 5.2028999562026e-06, + 6.426721938623814e-06, + 1.2295902706682682e-05, + 1.4342041140480433e-05, + 1.0314462997484952e-05, + 9.527226211503148e-06, + 1.033583794196602e-05, + 1.3014845535508357e-05, + 1.2947188224643469e-05, + 1.4193793504091445e-05, + 1.4511630979541223e-05, + 1.2485726983868517e-05, + 1.6501382560818456e-05, + 1.4663118236057926e-05, + 1.6354897525161505e-05, + 7.697263936279342e-05, + 8.464085112791508e-05, + 7.24073761375621e-05, + 5.4004762205295265e-05, + 7.548596477136016e-05, + 6.387268513208255e-05, + 5.116204920341261e-05, + 3.577108873287216e-05, + 3.617366382968612e-05, + 2.9901715606683865e-05, + 1.9398781660129316e-05, + 1.8162598280468956e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.53738644207624e-05, + "first_to_last_ratio": 0.131222819995581, + "imbalance_abs_log_ratio": 2.0308584847035727, + "last_quartile_mean": 0.0002695709817999159, + "mean": 0.00014080404997685037, + "normalized": [ + 0.577470254414939, + 0.20295174191935292, + 0.1151700578824124, + 0.23719265118106814, + 0.1794139596985285, + 0.14982370308789797, + 0.19985925174978317, + 0.3479392297690331, + 0.40530357048439714, + 0.29950920312948914, + 0.2746918219238327, + 0.2680798564391957, + 0.33955736615799725, + 0.36796996435571633, + 0.4082064803862017, + 0.40241709302556344, + 0.34764889743937344, + 0.4446227376348821, + 0.404675572121426, + 0.502974600996141, + 2.462986532813244, + 2.8953303082983743, + 2.595561209030766, + 2.254551431946411, + 3.008292708807131, + 2.942171245358548, + 2.6598889524780778, + 1.7951160354833269, + 1.7746811041390398, + 1.3931209151914112, + 0.9062455092896525, + 0.836576033366786 + ], + "peak_layer": 25, + "peak_normalized": 3.008292708807131, + "population_cv": 0.9883689549295365, + "rest_mean": 9.795413286526498e-05, + "spike_contrast": 3.799672269993553, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003721936023794115, + "top_five_layers": [ + 25, + 26, + 22, + 27, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.131015056278557e-05, + 2.8576427212101407e-05, + 1.6216410585911945e-05, + 3.339768591104075e-05, + 2.5262212147936225e-05, + 2.1095784177305177e-05, + 2.814099207171239e-05, + 4.8991252697305754e-05, + 5.706838419428095e-05, + 4.217210880597122e-05, + 3.867772102239542e-05, + 3.7746729503851384e-05, + 4.7811052354518324e-05, + 5.181166125112213e-05, + 5.7477125665172935e-05, + 5.666195647791028e-05, + 4.8950372729450464e-05, + 6.260468217078596e-05, + 5.697995948139578e-05, + 7.082086085574701e-05, + 0.0003467984788585454, + 0.0004076742334291339, + 0.00036546553019434214, + 0.00031744997249916196, + 0.0004235797969158739, + 0.00041426962707191706, + 0.00037452313699759543, + 0.00025275960797443986, + 0.00024988228688016534, + 0.000196157066966407, + 0.00012760303798131645, + 0.00011779329361161217 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 1.8740129107275643, + "forward": { + "activation_sha256": { + "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" + }, + "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", + "loss_nats": 1.2989667654037476, + "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" + }, + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22886337339878082, + "group": 1, + "latest_source_mass": 0.06350618600845337, + "layer": 1, + "max_source_mass": 0.9364938139915466, + "mixer_index": 1, + "normalized_entropy": 0.3301800538435402, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9364938139915466, + "median": 0.9452826976776123, + "p05": 0.8725647330284119, + "p95": 0.9712384939193726 + }, + { + "label": "current_group_partial", + "mean": 0.06350618600845337, + "median": 0.0547172985970974, + "p05": 0.028761589899659157, + "p95": 0.12743529677391052 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.603729784488678, + "group": 1, + "latest_source_mass": 0.4133155345916748, + "layer": 2, + "max_source_mass": 0.5866844654083252, + "mixer_index": 2, + "normalized_entropy": 0.8709979661187783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5866844654083252, + "median": 0.5734395384788513, + "p05": 0.29813364148139954, + "p95": 0.8843642473220825 + }, + { + "label": "current_group_partial", + "mean": 0.4133155345916748, + "median": 0.4265604615211487, + "p05": 0.11563576757907867, + "p95": 0.7018663287162781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3967982828617096, + "group": 1, + "latest_source_mass": 0.14110636711120605, + "layer": 2, + "max_source_mass": 0.858893632888794, + "mixer_index": 3, + "normalized_entropy": 0.5724589149178446, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.858893632888794, + "median": 0.8705973029136658, + "p05": 0.7599387168884277, + "p95": 0.9270997643470764 + }, + { + "label": "current_group_partial", + "mean": 0.14110636711120605, + "median": 0.12940272688865662, + "p05": 0.07290027290582657, + "p95": 0.2400612235069275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6852496862411499, + "group": 1, + "latest_source_mass": 0.518975019454956, + "layer": 3, + "max_source_mass": 0.518975019454956, + "mixer_index": 4, + "normalized_entropy": 0.9886063241108252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48102498054504395, + "median": 0.48102718591690063, + "p05": 0.38786420226097107, + "p95": 0.580143928527832 + }, + { + "label": "current_group_partial", + "mean": 0.518975019454956, + "median": 0.5189728140830994, + "p05": 0.41985607147216797, + "p95": 0.6121357679367065 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5890734195709229, + "group": 1, + "latest_source_mass": 0.3123423159122467, + "layer": 3, + "max_source_mass": 0.6876576542854309, + "mixer_index": 5, + "normalized_entropy": 0.8498533011344741, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6876576542854309, + "median": 0.6909762620925903, + "p05": 0.48803940415382385, + "p95": 0.8900076746940613 + }, + { + "label": "current_group_partial", + "mean": 0.3123423159122467, + "median": 0.3090237081050873, + "p05": 0.10999235510826111, + "p95": 0.5119606256484985 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6873323321342468, + "group": 1, + "latest_source_mass": 0.4831485152244568, + "layer": 4, + "max_source_mass": 0.5168514251708984, + "mixer_index": 6, + "normalized_entropy": 0.9916109470127238, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5168514251708984, + "median": 0.5219584703445435, + "p05": 0.4251725673675537, + "p95": 0.5947866439819336 + }, + { + "label": "current_group_partial", + "mean": 0.4831485152244568, + "median": 0.47804147005081177, + "p05": 0.405213326215744, + "p95": 0.5748274326324463 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.487301766872406, + "group": 1, + "latest_source_mass": 0.2060384750366211, + "layer": 4, + "max_source_mass": 0.7939615249633789, + "mixer_index": 7, + "normalized_entropy": 0.7030278424832499, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7939615249633789, + "median": 0.7953404188156128, + "p05": 0.6590759754180908, + "p95": 0.9329437017440796 + }, + { + "label": "current_group_partial", + "mean": 0.2060384750366211, + "median": 0.20465964078903198, + "p05": 0.06705628335475922, + "p95": 0.3409239649772644 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674697756767273, + "group": 2, + "latest_source_mass": 0.42952626943588257, + "layer": 5, + "max_source_mass": 0.5704737305641174, + "mixer_index": 8, + "normalized_entropy": 0.9733831077870527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5704737305641174, + "median": 0.5696132779121399, + "p05": 0.4695202708244324, + "p95": 0.6750621199607849 + }, + { + "label": "completed_group_1", + "mean": 0.42952626943588257, + "median": 0.4303867816925049, + "p05": 0.3249378800392151, + "p95": 0.5304797887802124 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9691112041473389, + "group": 2, + "latest_source_mass": 0.2681158185005188, + "layer": 5, + "max_source_mass": 0.5506307482719421, + "mixer_index": 9, + "normalized_entropy": 0.8821230329784768, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5506307482719421, + "median": 0.543993353843689, + "p05": 0.41262108087539673, + "p95": 0.7161091566085815 + }, + { + "label": "completed_group_1", + "mean": 0.18125343322753906, + "median": 0.1794438362121582, + "p05": 0.11575750261545181, + "p95": 0.25031501054763794 + }, + { + "label": "current_group_partial", + "mean": 0.2681158185005188, + "median": 0.2664111256599426, + "p05": 0.1564716249704361, + "p95": 0.3817881643772125 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0745112895965576, + "group": 2, + "latest_source_mass": 0.412794291973114, + "layer": 6, + "max_source_mass": 0.412794291973114, + "mixer_index": 10, + "normalized_entropy": 0.9780623252441762, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34262996912002563, + "median": 0.3412924110889435, + "p05": 0.29556700587272644, + "p95": 0.39125555753707886 + }, + { + "label": "completed_group_1", + "mean": 0.24457570910453796, + "median": 0.24382859468460083, + "p05": 0.22120657563209534, + "p95": 0.2734297513961792 + }, + { + "label": "current_group_partial", + "mean": 0.412794291973114, + "median": 0.4151499271392822, + "p05": 0.37735897302627563, + "p95": 0.43749260902404785 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074045658111572, + "group": 2, + "latest_source_mass": 0.3775783181190491, + "layer": 6, + "max_source_mass": 0.4399106502532959, + "mixer_index": 11, + "normalized_entropy": 0.9169791528842925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4399106502532959, + "median": 0.44856905937194824, + "p05": 0.22063353657722473, + "p95": 0.6359403133392334 + }, + { + "label": "completed_group_1", + "mean": 0.18251106142997742, + "median": 0.18562838435173035, + "p05": 0.1196163073182106, + "p95": 0.23045453429222107 + }, + { + "label": "current_group_partial", + "mean": 0.3775783181190491, + "median": 0.3675365447998047, + "p05": 0.23817461729049683, + "p95": 0.5557451844215393 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.060497522354126, + "group": 2, + "latest_source_mass": 0.4025024175643921, + "layer": 7, + "max_source_mass": 0.4025024175643921, + "mixer_index": 12, + "normalized_entropy": 0.9653064445872968, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3786153197288513, + "median": 0.3907395303249359, + "p05": 0.28039106726646423, + "p95": 0.4436405599117279 + }, + { + "label": "completed_group_1", + "mean": 0.2188822478055954, + "median": 0.21226292848587036, + "p05": 0.1885872334241867, + "p95": 0.2672030031681061 + }, + { + "label": "current_group_partial", + "mean": 0.4025024175643921, + "median": 0.39815032482147217, + "p05": 0.3533596992492676, + "p95": 0.460100919008255 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9941019415855408, + "group": 2, + "latest_source_mass": 0.4801652431488037, + "layer": 7, + "max_source_mass": 0.4801652431488037, + "mixer_index": 13, + "normalized_entropy": 0.9048705824970601, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32296955585479736, + "median": 0.2803739905357361, + "p05": 0.1784922033548355, + "p95": 0.6440245509147644 + }, + { + "label": "completed_group_1", + "mean": 0.19686520099639893, + "median": 0.2053825855255127, + "p05": 0.09813719242811203, + "p95": 0.2504696547985077 + }, + { + "label": "current_group_partial", + "mean": 0.4801652431488037, + "median": 0.5057140588760376, + "p05": 0.2529003322124481, + "p95": 0.598870575428009 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0503522157669067, + "group": 2, + "latest_source_mass": 0.4001518487930298, + "layer": 8, + "max_source_mass": 0.40130966901779175, + "mixer_index": 14, + "normalized_entropy": 0.9560717885654542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40130966901779175, + "median": 0.396759569644928, + "p05": 0.3479611873626709, + "p95": 0.4733521044254303 + }, + { + "label": "completed_group_1", + "mean": 0.19853851199150085, + "median": 0.19769755005836487, + "p05": 0.17136667668819427, + "p95": 0.22788912057876587 + }, + { + "label": "current_group_partial", + "mean": 0.4001518487930298, + "median": 0.40291696786880493, + "p05": 0.3506356179714203, + "p95": 0.43941178917884827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0129402875900269, + "group": 2, + "latest_source_mass": 0.42161279916763306, + "layer": 8, + "max_source_mass": 0.42161279916763306, + "mixer_index": 15, + "normalized_entropy": 0.9220179839951123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4093267023563385, + "median": 0.4046899676322937, + "p05": 0.2895161807537079, + "p95": 0.5762131810188293 + }, + { + "label": "completed_group_1", + "mean": 0.16906049847602844, + "median": 0.16888955235481262, + "p05": 0.11233551055192947, + "p95": 0.22477403283119202 + }, + { + "label": "current_group_partial", + "mean": 0.42161279916763306, + "median": 0.4227708578109741, + "p05": 0.3131049573421478, + "p95": 0.519075334072113 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0774019956588745, + "group": 3, + "latest_source_mass": 0.3560434877872467, + "layer": 9, + "max_source_mass": 0.38510820269584656, + "mixer_index": 16, + "normalized_entropy": 0.9806935592947451, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38510820269584656, + "median": 0.3892287313938141, + "p05": 0.2934730052947998, + "p95": 0.4718902111053467 + }, + { + "label": "completed_group_1", + "mean": 0.25884830951690674, + "median": 0.25734007358551025, + "p05": 0.22217319905757904, + "p95": 0.29551801085472107 + }, + { + "label": "completed_group_2", + "mean": 0.3560434877872467, + "median": 0.35412120819091797, + "p05": 0.29444968700408936, + "p95": 0.41908568143844604 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2740377187728882, + "group": 3, + "latest_source_mass": 0.3633861243724823, + "layer": 9, + "max_source_mass": 0.3633861243724823, + "mixer_index": 17, + "normalized_entropy": 0.9190239493895668, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29550763964653015, + "median": 0.30360478162765503, + "p05": 0.16904757916927338, + "p95": 0.45146965980529785 + }, + { + "label": "completed_group_1", + "mean": 0.09541244804859161, + "median": 0.09486871212720871, + "p05": 0.061360105872154236, + "p95": 0.13037073612213135 + }, + { + "label": "completed_group_2", + "mean": 0.24569380283355713, + "median": 0.24333015084266663, + "p05": 0.1698218435049057, + "p95": 0.31589236855506897 + }, + { + "label": "current_group_partial", + "mean": 0.3633861243724823, + "median": 0.36472851037979126, + "p05": 0.2795379161834717, + "p95": 0.4399876892566681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.360302448272705, + "group": 3, + "latest_source_mass": 0.28569501638412476, + "layer": 10, + "max_source_mass": 0.3109527826309204, + "mixer_index": 18, + "normalized_entropy": 0.9812507981160736, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3109527826309204, + "median": 0.32172977924346924, + "p05": 0.23520725965499878, + "p95": 0.37052470445632935 + }, + { + "label": "completed_group_1", + "mean": 0.20984819531440735, + "median": 0.20958667993545532, + "p05": 0.1891367882490158, + "p95": 0.23184512555599213 + }, + { + "label": "completed_group_2", + "mean": 0.19350400567054749, + "median": 0.19153150916099548, + "p05": 0.16607457399368286, + "p95": 0.22877082228660583 + }, + { + "label": "current_group_partial", + "mean": 0.28569501638412476, + "median": 0.28060320019721985, + "p05": 0.23824527859687805, + "p95": 0.3439165949821472 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2722704410552979, + "group": 3, + "latest_source_mass": 0.29912811517715454, + "layer": 10, + "max_source_mass": 0.35036465525627136, + "mixer_index": 19, + "normalized_entropy": 0.9177491279900463, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35036465525627136, + "median": 0.36210548877716064, + "p05": 0.12807878851890564, + "p95": 0.5797606706619263 + }, + { + "label": "completed_group_1", + "mean": 0.12631990015506744, + "median": 0.1264108121395111, + "p05": 0.06977735459804535, + "p95": 0.1755298376083374 + }, + { + "label": "completed_group_2", + "mean": 0.22418735921382904, + "median": 0.22563719749450684, + "p05": 0.1176326721906662, + "p95": 0.32001155614852905 + }, + { + "label": "current_group_partial", + "mean": 0.29912811517715454, + "median": 0.28589946031570435, + "p05": 0.193109393119812, + "p95": 0.44119665026664734 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3558036088943481, + "group": 3, + "latest_source_mass": 0.314197838306427, + "layer": 11, + "max_source_mass": 0.314197838306427, + "mixer_index": 20, + "normalized_entropy": 0.978005571485618, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298976868391037, + "median": 0.2961939573287964, + "p05": 0.24950163066387177, + "p95": 0.35835936665534973 + }, + { + "label": "completed_group_1", + "mean": 0.1941717267036438, + "median": 0.1918032318353653, + "p05": 0.16809208691120148, + "p95": 0.22895419597625732 + }, + { + "label": "completed_group_2", + "mean": 0.1926535815000534, + "median": 0.19306811690330505, + "p05": 0.16983944177627563, + "p95": 0.2155935913324356 + }, + { + "label": "current_group_partial", + "mean": 0.314197838306427, + "median": 0.31541913747787476, + "p05": 0.2673022747039795, + "p95": 0.35941022634506226 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2696614265441895, + "group": 3, + "latest_source_mass": 0.2862822115421295, + "layer": 11, + "max_source_mass": 0.3667730689048767, + "mixer_index": 21, + "normalized_entropy": 0.9158671218416545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3667730689048767, + "median": 0.4067659378051758, + "p05": 0.10593366622924805, + "p95": 0.5672274827957153 + }, + { + "label": "completed_group_1", + "mean": 0.15310388803482056, + "median": 0.15050218999385834, + "p05": 0.08651046454906464, + "p95": 0.22459912300109863 + }, + { + "label": "completed_group_2", + "mean": 0.19384080171585083, + "median": 0.18732509016990662, + "p05": 0.1267009675502777, + "p95": 0.281107634305954 + }, + { + "label": "current_group_partial", + "mean": 0.2862822115421295, + "median": 0.26271265745162964, + "p05": 0.17024393379688263, + "p95": 0.475044310092926 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3606936931610107, + "group": 3, + "latest_source_mass": 0.3028184771537781, + "layer": 12, + "max_source_mass": 0.3028184771537781, + "mixer_index": 22, + "normalized_entropy": 0.9815330216461395, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298227459192276, + "median": 0.2954302132129669, + "p05": 0.25287362933158875, + "p95": 0.35405808687210083 + }, + { + "label": "completed_group_1", + "mean": 0.18611131608486176, + "median": 0.1850002110004425, + "p05": 0.15901757776737213, + "p95": 0.21725836396217346 + }, + { + "label": "completed_group_2", + "mean": 0.21284276247024536, + "median": 0.21402311325073242, + "p05": 0.18819917738437653, + "p95": 0.23418807983398438 + }, + { + "label": "current_group_partial", + "mean": 0.3028184771537781, + "median": 0.3041122555732727, + "p05": 0.25661808252334595, + "p95": 0.34458184242248535 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.269316554069519, + "group": 3, + "latest_source_mass": 0.2653314769268036, + "layer": 12, + "max_source_mass": 0.4081852436065674, + "mixer_index": 23, + "normalized_entropy": 0.9156183489371815, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4081852436065674, + "median": 0.4254342317581177, + "p05": 0.20765496790409088, + "p95": 0.587238073348999 + }, + { + "label": "completed_group_1", + "mean": 0.14420127868652344, + "median": 0.1438048928976059, + "p05": 0.1064920499920845, + "p95": 0.1871720254421234 + }, + { + "label": "completed_group_2", + "mean": 0.18228203058242798, + "median": 0.17507661879062653, + "p05": 0.12183455377817154, + "p95": 0.26505547761917114 + }, + { + "label": "current_group_partial", + "mean": 0.2653314769268036, + "median": 0.24924281239509583, + "p05": 0.15609969198703766, + "p95": 0.4061311185359955 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3636136054992676, + "group": 4, + "latest_source_mass": 0.3136237859725952, + "layer": 13, + "max_source_mass": 0.3136237859725952, + "mixer_index": 24, + "normalized_entropy": 0.9836392931712563, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27427661418914795, + "median": 0.27403637766838074, + "p05": 0.22928935289382935, + "p95": 0.319773405790329 + }, + { + "label": "completed_group_1", + "mean": 0.21341991424560547, + "median": 0.20697325468063354, + "p05": 0.18552586436271667, + "p95": 0.26688331365585327 + }, + { + "label": "completed_group_2", + "mean": 0.19867971539497375, + "median": 0.197545126080513, + "p05": 0.1794913411140442, + "p95": 0.222100168466568 + }, + { + "label": "completed_group_3", + "mean": 0.3136237859725952, + "median": 0.31875866651535034, + "p05": 0.24156072735786438, + "p95": 0.3695686459541321 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.570773959159851, + "group": 4, + "latest_source_mass": 0.2294243425130844, + "layer": 13, + "max_source_mass": 0.2658740282058716, + "mixer_index": 25, + "normalized_entropy": 0.9759767351225285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2658740282058716, + "median": 0.2747163474559784, + "p05": 0.17927880585193634, + "p95": 0.3512483835220337 + }, + { + "label": "completed_group_1", + "mean": 0.13419361412525177, + "median": 0.13374894857406616, + "p05": 0.11072897166013718, + "p95": 0.1586270034313202 + }, + { + "label": "completed_group_2", + "mean": 0.1907704919576645, + "median": 0.19074805080890656, + "p05": 0.14699944853782654, + "p95": 0.23316305875778198 + }, + { + "label": "completed_group_3", + "mean": 0.17973749339580536, + "median": 0.17616114020347595, + "p05": 0.1363743096590042, + "p95": 0.23202598094940186 + }, + { + "label": "current_group_partial", + "mean": 0.2294243425130844, + "median": 0.22995705902576447, + "p05": 0.17975632846355438, + "p95": 0.2789157032966614 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5560038089752197, + "group": 4, + "latest_source_mass": 0.32468360662460327, + "layer": 14, + "max_source_mass": 0.32468360662460327, + "mixer_index": 26, + "normalized_entropy": 0.9667995248241249, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20477107167243958, + "median": 0.206403911113739, + "p05": 0.16619175672531128, + "p95": 0.2431035190820694 + }, + { + "label": "completed_group_1", + "mean": 0.15721669793128967, + "median": 0.15658551454544067, + "p05": 0.13878680765628815, + "p95": 0.17778633534908295 + }, + { + "label": "completed_group_2", + "mean": 0.13811248540878296, + "median": 0.13743364810943604, + "p05": 0.12352626770734787, + "p95": 0.15553249418735504 + }, + { + "label": "completed_group_3", + "mean": 0.1752161681652069, + "median": 0.17309701442718506, + "p05": 0.15223677456378937, + "p95": 0.20474471151828766 + }, + { + "label": "current_group_partial", + "mean": 0.32468360662460327, + "median": 0.32497870922088623, + "p05": 0.2946181893348694, + "p95": 0.35203880071640015 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5659973621368408, + "group": 4, + "latest_source_mass": 0.26703086495399475, + "layer": 14, + "max_source_mass": 0.26703086495399475, + "mixer_index": 27, + "normalized_entropy": 0.9730088685238187, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20531430840492249, + "median": 0.20229148864746094, + "p05": 0.10698921978473663, + "p95": 0.3393132984638214 + }, + { + "label": "completed_group_1", + "mean": 0.15986156463623047, + "median": 0.15952220559120178, + "p05": 0.11969029903411865, + "p95": 0.20180393755435944 + }, + { + "label": "completed_group_2", + "mean": 0.21557100117206573, + "median": 0.21654027700424194, + "p05": 0.1673085242509842, + "p95": 0.2610440254211426 + }, + { + "label": "completed_group_3", + "mean": 0.15222224593162537, + "median": 0.1510905921459198, + "p05": 0.11800260841846466, + "p95": 0.18893754482269287 + }, + { + "label": "current_group_partial", + "mean": 0.26703086495399475, + "median": 0.258539617061615, + "p05": 0.19330690801143646, + "p95": 0.36521556973457336 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5552496910095215, + "group": 4, + "latest_source_mass": 0.31774693727493286, + "layer": 15, + "max_source_mass": 0.31774693727493286, + "mixer_index": 28, + "normalized_entropy": 0.9663309649872576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2162160575389862, + "median": 0.2145705670118332, + "p05": 0.17727568745613098, + "p95": 0.26878347992897034 + }, + { + "label": "completed_group_1", + "mean": 0.17982497811317444, + "median": 0.18024154007434845, + "p05": 0.15453973412513733, + "p95": 0.2030555009841919 + }, + { + "label": "completed_group_2", + "mean": 0.1307673156261444, + "median": 0.13107559084892273, + "p05": 0.11463287472724915, + "p95": 0.14630235731601715 + }, + { + "label": "completed_group_3", + "mean": 0.1554446816444397, + "median": 0.15516987442970276, + "p05": 0.13627474009990692, + "p95": 0.1754152774810791 + }, + { + "label": "current_group_partial", + "mean": 0.31774693727493286, + "median": 0.319039523601532, + "p05": 0.27606821060180664, + "p95": 0.35650834441185 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5068919658660889, + "group": 4, + "latest_source_mass": 0.3298327624797821, + "layer": 15, + "max_source_mass": 0.3298327624797821, + "mixer_index": 29, + "normalized_entropy": 0.9362846209998111, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20646697282791138, + "median": 0.18775638937950134, + "p05": 0.0806753933429718, + "p95": 0.4274584949016571 + }, + { + "label": "completed_group_1", + "mean": 0.1197461187839508, + "median": 0.11989086866378784, + "p05": 0.0809398666024208, + "p95": 0.15743419528007507 + }, + { + "label": "completed_group_2", + "mean": 0.19215428829193115, + "median": 0.19620640575885773, + "p05": 0.1148109883069992, + "p95": 0.2491212636232376 + }, + { + "label": "completed_group_3", + "mean": 0.15179988741874695, + "median": 0.15212899446487427, + "p05": 0.10697861760854721, + "p95": 0.19542087614536285 + }, + { + "label": "current_group_partial", + "mean": 0.3298327624797821, + "median": 0.32837340235710144, + "p05": 0.22746729850769043, + "p95": 0.4339301884174347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5435223579406738, + "group": 4, + "latest_source_mass": 0.3224271535873413, + "layer": 16, + "max_source_mass": 0.3224271535873413, + "mixer_index": 30, + "normalized_entropy": 0.9590443632623663, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22410975396633148, + "median": 0.22760415077209473, + "p05": 0.14593973755836487, + "p95": 0.30093055963516235 + }, + { + "label": "completed_group_1", + "mean": 0.16986176371574402, + "median": 0.16963240504264832, + "p05": 0.13569067418575287, + "p95": 0.20651353895664215 + }, + { + "label": "completed_group_2", + "mean": 0.13080403208732605, + "median": 0.13075244426727295, + "p05": 0.10945598781108856, + "p95": 0.15187400579452515 + }, + { + "label": "completed_group_3", + "mean": 0.15279728174209595, + "median": 0.15036612749099731, + "p05": 0.12810495495796204, + "p95": 0.18468116223812103 + }, + { + "label": "current_group_partial", + "mean": 0.3224271535873413, + "median": 0.3164973258972168, + "p05": 0.2600978910923004, + "p95": 0.40530598163604736 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4949920177459717, + "group": 4, + "latest_source_mass": 0.31847020983695984, + "layer": 16, + "max_source_mass": 0.31847020983695984, + "mixer_index": 31, + "normalized_entropy": 0.9288907675133354, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27174079418182373, + "median": 0.2585451304912567, + "p05": 0.13639721274375916, + "p95": 0.4698348045349121 + }, + { + "label": "completed_group_1", + "mean": 0.12178683280944824, + "median": 0.11714950203895569, + "p05": 0.08594125509262085, + "p95": 0.1723237782716751 + }, + { + "label": "completed_group_2", + "mean": 0.1529826819896698, + "median": 0.15223106741905212, + "p05": 0.10371473431587219, + "p95": 0.203575000166893 + }, + { + "label": "completed_group_3", + "mean": 0.1350194811820984, + "median": 0.13363957405090332, + "p05": 0.08964549005031586, + "p95": 0.18468207120895386 + }, + { + "label": "current_group_partial", + "mean": 0.31847020983695984, + "median": 0.3185216188430786, + "p05": 0.21444711089134216, + "p95": 0.4301546812057495 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.544264793395996, + "group": 5, + "latest_source_mass": 0.2861037850379944, + "layer": 17, + "max_source_mass": 0.2861037850379944, + "mixer_index": 32, + "normalized_entropy": 0.9595056643474137, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24016115069389343, + "median": 0.23868846893310547, + "p05": 0.18435582518577576, + "p95": 0.2983453571796417 + }, + { + "label": "completed_group_1", + "mean": 0.20730282366275787, + "median": 0.1998077780008316, + "p05": 0.16255050897598267, + "p95": 0.28107208013534546 + }, + { + "label": "completed_group_2", + "mean": 0.09692573547363281, + "median": 0.0967404842376709, + "p05": 0.08098706603050232, + "p95": 0.11221462488174438 + }, + { + "label": "completed_group_3", + "mean": 0.1695065200328827, + "median": 0.1696585714817047, + "p05": 0.136396586894989, + "p95": 0.20130634307861328 + }, + { + "label": "completed_group_4", + "mean": 0.2861037850379944, + "median": 0.28883200883865356, + "p05": 0.22401469945907593, + "p95": 0.3409370183944702 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.686682939529419, + "group": 5, + "latest_source_mass": 0.12281867861747742, + "layer": 17, + "max_source_mass": 0.2885604202747345, + "mixer_index": 33, + "normalized_entropy": 0.9413556721740635, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16755425930023193, + "median": 0.1327776312828064, + "p05": 0.0790192037820816, + "p95": 0.3667412996292114 + }, + { + "label": "completed_group_1", + "mean": 0.10962115228176117, + "median": 0.10525335371494293, + "p05": 0.07840172946453094, + "p95": 0.15405339002609253 + }, + { + "label": "completed_group_2", + "mean": 0.1783454418182373, + "median": 0.17896884679794312, + "p05": 0.10802385956048965, + "p95": 0.24048911035060883 + }, + { + "label": "completed_group_3", + "mean": 0.13310004770755768, + "median": 0.13033035397529602, + "p05": 0.0904831737279892, + "p95": 0.17934636771678925 + }, + { + "label": "completed_group_4", + "mean": 0.2885604202747345, + "median": 0.2831825315952301, + "p05": 0.16967125236988068, + "p95": 0.4189499616622925 + }, + { + "label": "current_group_partial", + "mean": 0.12281867861747742, + "median": 0.12062528729438782, + "p05": 0.07461707293987274, + "p95": 0.1782219111919403 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7070834636688232, + "group": 5, + "latest_source_mass": 0.2988377809524536, + "layer": 18, + "max_source_mass": 0.2988377809524536, + "mixer_index": 34, + "normalized_entropy": 0.9527414214834803, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.181177020072937, + "median": 0.18419909477233887, + "p05": 0.13624237477779388, + "p95": 0.22192169725894928 + }, + { + "label": "completed_group_1", + "mean": 0.14557164907455444, + "median": 0.144476979970932, + "p05": 0.1293315887451172, + "p95": 0.16486899554729462 + }, + { + "label": "completed_group_2", + "mean": 0.08318156749010086, + "median": 0.0820261612534523, + "p05": 0.07088497281074524, + "p95": 0.0990293100476265 + }, + { + "label": "completed_group_3", + "mean": 0.11424209177494049, + "median": 0.11267061531543732, + "p05": 0.0979941189289093, + "p95": 0.1353992223739624 + }, + { + "label": "completed_group_4", + "mean": 0.17698991298675537, + "median": 0.17597860097885132, + "p05": 0.14837387204170227, + "p95": 0.21050144731998444 + }, + { + "label": "current_group_partial", + "mean": 0.2988377809524536, + "median": 0.2989804744720459, + "p05": 0.2721370458602905, + "p95": 0.32572072744369507 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7419235706329346, + "group": 5, + "latest_source_mass": 0.26016175746917725, + "layer": 18, + "max_source_mass": 0.26016175746917725, + "mixer_index": 35, + "normalized_entropy": 0.972186055410333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1532188355922699, + "median": 0.1451343297958374, + "p05": 0.11514341086149216, + "p95": 0.22103679180145264 + }, + { + "label": "completed_group_1", + "mean": 0.11420352011919022, + "median": 0.11261284351348877, + "p05": 0.0928458571434021, + "p95": 0.14123988151550293 + }, + { + "label": "completed_group_2", + "mean": 0.16597424447536469, + "median": 0.1655328869819641, + "p05": 0.13871800899505615, + "p95": 0.19611573219299316 + }, + { + "label": "completed_group_3", + "mean": 0.1250365972518921, + "median": 0.12369821965694427, + "p05": 0.10375828295946121, + "p95": 0.14912742376327515 + }, + { + "label": "completed_group_4", + "mean": 0.18140508234500885, + "median": 0.1796584278345108, + "p05": 0.14511024951934814, + "p95": 0.22286373376846313 + }, + { + "label": "current_group_partial", + "mean": 0.26016175746917725, + "median": 0.26266711950302124, + "p05": 0.1889389455318451, + "p95": 0.3234381675720215 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7127439975738525, + "group": 5, + "latest_source_mass": 0.2907913327217102, + "layer": 19, + "max_source_mass": 0.2907913327217102, + "mixer_index": 36, + "normalized_entropy": 0.9559006256078307, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18731768429279327, + "median": 0.19042012095451355, + "p05": 0.13879702985286713, + "p95": 0.23792000114917755 + }, + { + "label": "completed_group_1", + "mean": 0.16715484857559204, + "median": 0.1667211651802063, + "p05": 0.14574480056762695, + "p95": 0.19026139378547668 + }, + { + "label": "completed_group_2", + "mean": 0.08496320247650146, + "median": 0.08452782034873962, + "p05": 0.07410907745361328, + "p95": 0.097386933863163 + }, + { + "label": "completed_group_3", + "mean": 0.11362075805664062, + "median": 0.11248823255300522, + "p05": 0.0977015346288681, + "p95": 0.13219426572322845 + }, + { + "label": "completed_group_4", + "mean": 0.15615218877792358, + "median": 0.15540224313735962, + "p05": 0.1289147436618805, + "p95": 0.18598109483718872 + }, + { + "label": "current_group_partial", + "mean": 0.2907913327217102, + "median": 0.2900415062904358, + "p05": 0.25822532176971436, + "p95": 0.32475030422210693 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076834201812744, + "group": 5, + "latest_source_mass": 0.3738824725151062, + "layer": 19, + "max_source_mass": 0.3738824725151062, + "mixer_index": 37, + "normalized_entropy": 0.8972652009334232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15032559633255005, + "median": 0.12544110417366028, + "p05": 0.06976518034934998, + "p95": 0.35840943455696106 + }, + { + "label": "completed_group_1", + "mean": 0.08624927699565887, + "median": 0.08602266013622284, + "p05": 0.061998721212148666, + "p95": 0.11138717085123062 + }, + { + "label": "completed_group_2", + "mean": 0.15736976265907288, + "median": 0.15802210569381714, + "p05": 0.10121746361255646, + "p95": 0.20687000453472137 + }, + { + "label": "completed_group_3", + "mean": 0.08584291487932205, + "median": 0.0827617198228836, + "p05": 0.060561805963516235, + "p95": 0.11351694166660309 + }, + { + "label": "completed_group_4", + "mean": 0.14632999897003174, + "median": 0.1426895558834076, + "p05": 0.10106660425662994, + "p95": 0.20179018378257751 + }, + { + "label": "current_group_partial", + "mean": 0.3738824725151062, + "median": 0.3799295425415039, + "p05": 0.23631010949611664, + "p95": 0.4884527325630188 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.709290862083435, + "group": 5, + "latest_source_mass": 0.2901708781719208, + "layer": 20, + "max_source_mass": 0.2901708781719208, + "mixer_index": 38, + "normalized_entropy": 0.9539733939957076, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18991519510746002, + "median": 0.1924753338098526, + "p05": 0.14201581478118896, + "p95": 0.23709386587142944 + }, + { + "label": "completed_group_1", + "mean": 0.17091524600982666, + "median": 0.17006167769432068, + "p05": 0.15297070145606995, + "p95": 0.19153237342834473 + }, + { + "label": "completed_group_2", + "mean": 0.07822999358177185, + "median": 0.07778562605381012, + "p05": 0.06707640737295151, + "p95": 0.09089364111423492 + }, + { + "label": "completed_group_3", + "mean": 0.116765096783638, + "median": 0.11516137421131134, + "p05": 0.10162825882434845, + "p95": 0.13653048872947693 + }, + { + "label": "completed_group_4", + "mean": 0.1540035903453827, + "median": 0.15287795662879944, + "p05": 0.12593276798725128, + "p95": 0.18590238690376282 + }, + { + "label": "current_group_partial", + "mean": 0.2901708781719208, + "median": 0.29104921221733093, + "p05": 0.25643545389175415, + "p95": 0.3205650746822357 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6690887212753296, + "group": 5, + "latest_source_mass": 0.3342471718788147, + "layer": 20, + "max_source_mass": 0.3342471718788147, + "mixer_index": 39, + "normalized_entropy": 0.9315361520005944, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19029226899147034, + "median": 0.19644704461097717, + "p05": 0.09456793963909149, + "p95": 0.30468082427978516 + }, + { + "label": "completed_group_1", + "mean": 0.1100241169333458, + "median": 0.10808776319026947, + "p05": 0.09035106748342514, + "p95": 0.1359245330095291 + }, + { + "label": "completed_group_2", + "mean": 0.11471512168645859, + "median": 0.11494918912649155, + "p05": 0.08799737691879272, + "p95": 0.14024731516838074 + }, + { + "label": "completed_group_3", + "mean": 0.09999909996986389, + "median": 0.09682698547840118, + "p05": 0.07921088486909866, + "p95": 0.1264391839504242 + }, + { + "label": "completed_group_4", + "mean": 0.1507222205400467, + "median": 0.1476232260465622, + "p05": 0.11397356539964676, + "p95": 0.19991227984428406 + }, + { + "label": "current_group_partial", + "mean": 0.3342471718788147, + "median": 0.33802270889282227, + "p05": 0.2548266053199768, + "p95": 0.40130624175071716 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7458693981170654, + "group": 6, + "latest_source_mass": 0.2361394464969635, + "layer": 21, + "max_source_mass": 0.2361394464969635, + "mixer_index": 40, + "normalized_entropy": 0.9743882636597644, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20528709888458252, + "median": 0.2140858769416809, + "p05": 0.13168928027153015, + "p95": 0.27098023891448975 + }, + { + "label": "completed_group_1", + "mean": 0.1722964197397232, + "median": 0.17238952219486237, + "p05": 0.15053534507751465, + "p95": 0.19365043938159943 + }, + { + "label": "completed_group_2", + "mean": 0.10435444861650467, + "median": 0.10361101478338242, + "p05": 0.08888213336467743, + "p95": 0.12212536484003067 + }, + { + "label": "completed_group_3", + "mean": 0.12075909972190857, + "median": 0.1180175170302391, + "p05": 0.10358668118715286, + "p95": 0.14458929002285004 + }, + { + "label": "completed_group_4", + "mean": 0.16116347908973694, + "median": 0.15841388702392578, + "p05": 0.1337081342935562, + "p95": 0.19791515171527863 + }, + { + "label": "completed_group_5", + "mean": 0.2361394464969635, + "median": 0.23709020018577576, + "p05": 0.20446214079856873, + "p95": 0.26509594917297363 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4612125158309937, + "group": 6, + "latest_source_mass": 0.5543455481529236, + "layer": 21, + "max_source_mass": 0.5543455481529236, + "mixer_index": 41, + "normalized_entropy": 0.7509146897354808, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08934466540813446, + "median": 0.07911523431539536, + "p05": 0.04488275945186615, + "p95": 0.1737232506275177 + }, + { + "label": "completed_group_1", + "mean": 0.054379627108573914, + "median": 0.05344422906637192, + "p05": 0.04221750423312187, + "p95": 0.06924191862344742 + }, + { + "label": "completed_group_2", + "mean": 0.07567165046930313, + "median": 0.07438549399375916, + "p05": 0.05811355262994766, + "p95": 0.09632256627082825 + }, + { + "label": "completed_group_3", + "mean": 0.05758609622716904, + "median": 0.05612001568078995, + "p05": 0.044792283326387405, + "p95": 0.07347827404737473 + }, + { + "label": "completed_group_4", + "mean": 0.07884950935840607, + "median": 0.07757881283760071, + "p05": 0.05758974328637123, + "p95": 0.10452556610107422 + }, + { + "label": "completed_group_5", + "mean": 0.08982288837432861, + "median": 0.08912864327430725, + "p05": 0.0663854256272316, + "p95": 0.11604885756969452 + }, + { + "label": "current_group_partial", + "mean": 0.5543455481529236, + "median": 0.5596089363098145, + "p05": 0.45455610752105713, + "p95": 0.6367484331130981 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7241814136505127, + "group": 6, + "latest_source_mass": 0.39484071731567383, + "layer": 22, + "max_source_mass": 0.39484071731567383, + "mixer_index": 42, + "normalized_entropy": 0.886053970419732, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13360010087490082, + "median": 0.14072942733764648, + "p05": 0.07699814438819885, + "p95": 0.17506597936153412 + }, + { + "label": "completed_group_1", + "mean": 0.10609637200832367, + "median": 0.10586297512054443, + "p05": 0.08648421615362167, + "p95": 0.12602579593658447 + }, + { + "label": "completed_group_2", + "mean": 0.06363911926746368, + "median": 0.06324908882379532, + "p05": 0.054980404675006866, + "p95": 0.07316673547029495 + }, + { + "label": "completed_group_3", + "mean": 0.07426095008850098, + "median": 0.07362888753414154, + "p05": 0.06318790465593338, + "p95": 0.08665015548467636 + }, + { + "label": "completed_group_4", + "mean": 0.09383076429367065, + "median": 0.09294568747282028, + "p05": 0.07638302445411682, + "p95": 0.11445100605487823 + }, + { + "label": "completed_group_5", + "mean": 0.13373197615146637, + "median": 0.13327741622924805, + "p05": 0.11399447172880173, + "p95": 0.15641780197620392 + }, + { + "label": "current_group_partial", + "mean": 0.39484071731567383, + "median": 0.3879151940345764, + "p05": 0.3302172124385834, + "p95": 0.4852263331413269 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3979300260543823, + "group": 6, + "latest_source_mass": 0.5830056071281433, + "layer": 22, + "max_source_mass": 0.5830056071281433, + "mixer_index": 43, + "normalized_entropy": 0.7183939231382495, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08959861099720001, + "median": 0.07544538378715515, + "p05": 0.04165133833885193, + "p95": 0.20099139213562012 + }, + { + "label": "completed_group_1", + "mean": 0.05414731800556183, + "median": 0.05336771905422211, + "p05": 0.04069708660244942, + "p95": 0.07046990841627121 + }, + { + "label": "completed_group_2", + "mean": 0.07668881118297577, + "median": 0.07607124000787735, + "p05": 0.0555989071726799, + "p95": 0.09980960935354233 + }, + { + "label": "completed_group_3", + "mean": 0.05603286623954773, + "median": 0.054573021829128265, + "p05": 0.04162490367889404, + "p95": 0.0730118378996849 + }, + { + "label": "completed_group_4", + "mean": 0.06979228556156158, + "median": 0.06884932518005371, + "p05": 0.04576495662331581, + "p95": 0.09806846082210541 + }, + { + "label": "completed_group_5", + "mean": 0.07073447108268738, + "median": 0.07023533433675766, + "p05": 0.04956750199198723, + "p95": 0.09379896521568298 + }, + { + "label": "current_group_partial", + "mean": 0.5830056071281433, + "median": 0.5862616896629333, + "p05": 0.4658474922180176, + "p95": 0.6849814057350159 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7459560632705688, + "group": 6, + "latest_source_mass": 0.37691548466682434, + "layer": 23, + "max_source_mass": 0.37691548466682434, + "mixer_index": 44, + "normalized_entropy": 0.897243926765161, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14318783581256866, + "median": 0.1493408977985382, + "p05": 0.08292607218027115, + "p95": 0.19464610517024994 + }, + { + "label": "completed_group_1", + "mean": 0.1094653457403183, + "median": 0.10858799517154694, + "p05": 0.09432064741849899, + "p95": 0.1279931515455246 + }, + { + "label": "completed_group_2", + "mean": 0.06349228322505951, + "median": 0.06325754523277283, + "p05": 0.05439673364162445, + "p95": 0.0737837627530098 + }, + { + "label": "completed_group_3", + "mean": 0.07859592139720917, + "median": 0.0775163471698761, + "p05": 0.06603655964136124, + "p95": 0.09357087314128876 + }, + { + "label": "completed_group_4", + "mean": 0.09567482024431229, + "median": 0.09468592703342438, + "p05": 0.07671141624450684, + "p95": 0.11774884164333344 + }, + { + "label": "completed_group_5", + "mean": 0.13266831636428833, + "median": 0.13199329376220703, + "p05": 0.11244359612464905, + "p95": 0.15504513680934906 + }, + { + "label": "current_group_partial", + "mean": 0.37691548466682434, + "median": 0.37180638313293457, + "p05": 0.31207239627838135, + "p95": 0.4595233201980591 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4055393934249878, + "group": 6, + "latest_source_mass": 0.5772827863693237, + "layer": 23, + "max_source_mass": 0.5772827863693237, + "mixer_index": 45, + "normalized_entropy": 0.7223043644164862, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10154955089092255, + "median": 0.08766824007034302, + "p05": 0.04901552200317383, + "p95": 0.20287738740444183 + }, + { + "label": "completed_group_1", + "mean": 0.05619276314973831, + "median": 0.055477917194366455, + "p05": 0.041608311235904694, + "p95": 0.07371112704277039 + }, + { + "label": "completed_group_2", + "mean": 0.07791627198457718, + "median": 0.07736755907535553, + "p05": 0.055920060724020004, + "p95": 0.10158214718103409 + }, + { + "label": "completed_group_3", + "mean": 0.053915880620479584, + "median": 0.05215863138437271, + "p05": 0.03798118978738785, + "p95": 0.07156119495630264 + }, + { + "label": "completed_group_4", + "mean": 0.06398355215787888, + "median": 0.06229078769683838, + "p05": 0.040616974234580994, + "p95": 0.09270931780338287 + }, + { + "label": "completed_group_5", + "mean": 0.06915920227766037, + "median": 0.0684593990445137, + "p05": 0.046986356377601624, + "p95": 0.09408729523420334 + }, + { + "label": "current_group_partial", + "mean": 0.5772827863693237, + "median": 0.5824689269065857, + "p05": 0.4518115520477295, + "p95": 0.6926202178001404 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7853620052337646, + "group": 6, + "latest_source_mass": 0.3407441973686218, + "layer": 24, + "max_source_mass": 0.3407441973686218, + "mixer_index": 46, + "normalized_entropy": 0.9174945750195659, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15284113585948944, + "median": 0.1593906283378601, + "p05": 0.08678612112998962, + "p95": 0.211342453956604 + }, + { + "label": "completed_group_1", + "mean": 0.12206703424453735, + "median": 0.12153112143278122, + "p05": 0.1060115247964859, + "p95": 0.1399766057729721 + }, + { + "label": "completed_group_2", + "mean": 0.06508444249629974, + "median": 0.06482555717229843, + "p05": 0.055350616574287415, + "p95": 0.07557765394449234 + }, + { + "label": "completed_group_3", + "mean": 0.08347313106060028, + "median": 0.08190013468265533, + "p05": 0.06966532766819, + "p95": 0.10050469636917114 + }, + { + "label": "completed_group_4", + "mean": 0.09811970591545105, + "median": 0.09664031863212585, + "p05": 0.07688035070896149, + "p95": 0.12369741499423981 + }, + { + "label": "completed_group_5", + "mean": 0.13767032325267792, + "median": 0.1373325139284134, + "p05": 0.11642646044492722, + "p95": 0.1606694608926773 + }, + { + "label": "current_group_partial", + "mean": 0.3407441973686218, + "median": 0.33805108070373535, + "p05": 0.271301805973053, + "p95": 0.41781002283096313 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.509842872619629, + "group": 6, + "latest_source_mass": 0.5227814316749573, + "layer": 24, + "max_source_mass": 0.5227814316749573, + "mixer_index": 47, + "normalized_entropy": 0.7759057494780099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11788781732320786, + "median": 0.10097194463014603, + "p05": 0.06265684217214584, + "p95": 0.2408325970172882 + }, + { + "label": "completed_group_1", + "mean": 0.06822571158409119, + "median": 0.0673636943101883, + "p05": 0.05209728330373764, + "p95": 0.08648915588855743 + }, + { + "label": "completed_group_2", + "mean": 0.08556114137172699, + "median": 0.08463940769433975, + "p05": 0.06234566867351532, + "p95": 0.1125786080956459 + }, + { + "label": "completed_group_3", + "mean": 0.06067090481519699, + "median": 0.058974385261535645, + "p05": 0.043267179280519485, + "p95": 0.07997901737689972 + }, + { + "label": "completed_group_4", + "mean": 0.07228855788707733, + "median": 0.06972275674343109, + "p05": 0.04689513146877289, + "p95": 0.106512151658535 + }, + { + "label": "completed_group_5", + "mean": 0.07258446514606476, + "median": 0.07082539796829224, + "p05": 0.050949547439813614, + "p95": 0.09970614314079285 + }, + { + "label": "current_group_partial", + "mean": 0.5227814316749573, + "median": 0.5290273427963257, + "p05": 0.38773614168167114, + "p95": 0.6392589807510376 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.839367389678955, + "group": 7, + "latest_source_mass": 0.29583215713500977, + "layer": 25, + "max_source_mass": 0.29583215713500977, + "mixer_index": 48, + "normalized_entropy": 0.9452478525649903, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15906387567520142, + "median": 0.15846745669841766, + "p05": 0.09519976377487183, + "p95": 0.24517810344696045 + }, + { + "label": "completed_group_1", + "mean": 0.13139380514621735, + "median": 0.1308426856994629, + "p05": 0.11478274315595627, + "p95": 0.15000829100608826 + }, + { + "label": "completed_group_2", + "mean": 0.07717745751142502, + "median": 0.07715500891208649, + "p05": 0.06616044044494629, + "p95": 0.08791667222976685 + }, + { + "label": "completed_group_3", + "mean": 0.09607303142547607, + "median": 0.09436552226543427, + "p05": 0.08152313530445099, + "p95": 0.11384639143943787 + }, + { + "label": "completed_group_4", + "mean": 0.1107850894331932, + "median": 0.1089654266834259, + "p05": 0.08814431726932526, + "p95": 0.139452263712883 + }, + { + "label": "completed_group_5", + "mean": 0.12967455387115479, + "median": 0.1299237608909607, + "p05": 0.10832332074642181, + "p95": 0.15138982236385345 + }, + { + "label": "completed_group_6", + "mean": 0.29583215713500977, + "median": 0.29647016525268555, + "p05": 0.2276897132396698, + "p95": 0.3604946434497833 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5841834545135498, + "group": 7, + "latest_source_mass": 0.40767741203308105, + "layer": 25, + "max_source_mass": 0.40767741203308105, + "mixer_index": 49, + "normalized_entropy": 0.7618312045616817, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07896879315376282, + "median": 0.06809917092323303, + "p05": 0.025932887569069862, + "p95": 0.19368338584899902 + }, + { + "label": "completed_group_1", + "mean": 0.048752427101135254, + "median": 0.04767637699842453, + "p05": 0.034443892538547516, + "p95": 0.0667153000831604 + }, + { + "label": "completed_group_2", + "mean": 0.055707164108753204, + "median": 0.0549345538020134, + "p05": 0.038696639239788055, + "p95": 0.0750277042388916 + }, + { + "label": "completed_group_3", + "mean": 0.04139826446771622, + "median": 0.03878612071275711, + "p05": 0.026673054322600365, + "p95": 0.05746251344680786 + }, + { + "label": "completed_group_4", + "mean": 0.04449018836021423, + "median": 0.04196460545063019, + "p05": 0.026162121444940567, + "p95": 0.07164917886257172 + }, + { + "label": "completed_group_5", + "mean": 0.04285799711942673, + "median": 0.04109475016593933, + "p05": 0.029393533244729042, + "p95": 0.06277710199356079 + }, + { + "label": "completed_group_6", + "mean": 0.2801477909088135, + "median": 0.2763082981109619, + "p05": 0.1455708146095276, + "p95": 0.4254714846611023 + }, + { + "label": "current_group_partial", + "mean": 0.40767741203308105, + "median": 0.4120897948741913, + "p05": 0.21399420499801636, + "p95": 0.5853031277656555 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8125314712524414, + "group": 7, + "latest_source_mass": 0.3700707256793976, + "layer": 26, + "max_source_mass": 0.3700707256793976, + "mixer_index": 50, + "normalized_entropy": 0.8716433883436914, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10889513790607452, + "median": 0.11333495378494263, + "p05": 0.04850132763385773, + "p95": 0.1620434820652008 + }, + { + "label": "completed_group_1", + "mean": 0.08656036853790283, + "median": 0.08640051633119583, + "p05": 0.0650845319032669, + "p95": 0.1084962785243988 + }, + { + "label": "completed_group_2", + "mean": 0.048123255372047424, + "median": 0.04800213500857353, + "p05": 0.03576180338859558, + "p95": 0.06107594817876816 + }, + { + "label": "completed_group_3", + "mean": 0.06561170518398285, + "median": 0.06379580497741699, + "p05": 0.050968848168849945, + "p95": 0.08332888036966324 + }, + { + "label": "completed_group_4", + "mean": 0.06917259097099304, + "median": 0.06724178791046143, + "p05": 0.050207968801259995, + "p95": 0.09373342245817184 + }, + { + "label": "completed_group_5", + "mean": 0.08286689221858978, + "median": 0.08183546364307404, + "p05": 0.06535022705793381, + "p95": 0.10340933501720428 + }, + { + "label": "completed_group_6", + "mean": 0.16869932413101196, + "median": 0.16509854793548584, + "p05": 0.11059704422950745, + "p95": 0.23654375970363617 + }, + { + "label": "current_group_partial", + "mean": 0.3700707256793976, + "median": 0.3694133758544922, + "p05": 0.23441238701343536, + "p95": 0.5025027990341187 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.50089693069458, + "group": 7, + "latest_source_mass": 0.49096518754959106, + "layer": 26, + "max_source_mass": 0.49096518754959106, + "mixer_index": 51, + "normalized_entropy": 0.7217788529328457, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07974998652935028, + "median": 0.07402504235506058, + "p05": 0.04579866677522659, + "p95": 0.1359037309885025 + }, + { + "label": "completed_group_1", + "mean": 0.04166398197412491, + "median": 0.04038450866937637, + "p05": 0.027992289513349533, + "p95": 0.05969971418380737 + }, + { + "label": "completed_group_2", + "mean": 0.04607272148132324, + "median": 0.044438350945711136, + "p05": 0.030038591474294662, + "p95": 0.06808309257030487 + }, + { + "label": "completed_group_3", + "mean": 0.03586764633655548, + "median": 0.03317485377192497, + "p05": 0.02203454077243805, + "p95": 0.05572832003235817 + }, + { + "label": "completed_group_4", + "mean": 0.04223340004682541, + "median": 0.03930673003196716, + "p05": 0.02465122565627098, + "p95": 0.06968094408512115 + }, + { + "label": "completed_group_5", + "mean": 0.04095568507909775, + "median": 0.03919431194663048, + "p05": 0.026239857077598572, + "p95": 0.06058088690042496 + }, + { + "label": "completed_group_6", + "mean": 0.22249138355255127, + "median": 0.2125401496887207, + "p05": 0.10647299885749817, + "p95": 0.3711881935596466 + }, + { + "label": "current_group_partial", + "mean": 0.49096518754959106, + "median": 0.4988659620285034, + "p05": 0.2878011167049408, + "p95": 0.6676807403564453 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8059929609298706, + "group": 7, + "latest_source_mass": 0.37139976024627686, + "layer": 27, + "max_source_mass": 0.37139976024627686, + "mixer_index": 52, + "normalized_entropy": 0.8684990295379666, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10711923241615295, + "median": 0.10972198843955994, + "p05": 0.04896187782287598, + "p95": 0.16201043128967285 + }, + { + "label": "completed_group_1", + "mean": 0.07651707530021667, + "median": 0.07666032761335373, + "p05": 0.05141128972172737, + "p95": 0.10152198374271393 + }, + { + "label": "completed_group_2", + "mean": 0.058284737169742584, + "median": 0.05812416970729828, + "p05": 0.039273619651794434, + "p95": 0.07745087146759033 + }, + { + "label": "completed_group_3", + "mean": 0.06609587371349335, + "median": 0.06383530795574188, + "p05": 0.04481930285692215, + "p95": 0.09336265176534653 + }, + { + "label": "completed_group_4", + "mean": 0.07139429450035095, + "median": 0.0694514587521553, + "p05": 0.0485839769244194, + "p95": 0.10034126043319702 + }, + { + "label": "completed_group_5", + "mean": 0.07493151724338531, + "median": 0.07319007813930511, + "p05": 0.051876723766326904, + "p95": 0.10402432829141617 + }, + { + "label": "completed_group_6", + "mean": 0.17425751686096191, + "median": 0.16937679052352905, + "p05": 0.10449051856994629, + "p95": 0.2624811828136444 + }, + { + "label": "current_group_partial", + "mean": 0.37139976024627686, + "median": 0.3732905089855194, + "p05": 0.23830299079418182, + "p95": 0.5014542937278748 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5622732639312744, + "group": 7, + "latest_source_mass": 0.4768889844417572, + "layer": 27, + "max_source_mass": 0.4768889844417572, + "mixer_index": 53, + "normalized_entropy": 0.7512946301290215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08775827288627625, + "median": 0.08504067361354828, + "p05": 0.0428270623087883, + "p95": 0.144285649061203 + }, + { + "label": "completed_group_1", + "mean": 0.047765426337718964, + "median": 0.04646281152963638, + "p05": 0.031814731657505035, + "p95": 0.06807165592908859 + }, + { + "label": "completed_group_2", + "mean": 0.053928617388010025, + "median": 0.052262187004089355, + "p05": 0.035437777638435364, + "p95": 0.0784272700548172 + }, + { + "label": "completed_group_3", + "mean": 0.04293380305171013, + "median": 0.03996016085147858, + "p05": 0.02556057646870613, + "p95": 0.06623870134353638 + }, + { + "label": "completed_group_4", + "mean": 0.04174989461898804, + "median": 0.03903091698884964, + "p05": 0.025013400241732597, + "p95": 0.06869364529848099 + }, + { + "label": "completed_group_5", + "mean": 0.04734005779027939, + "median": 0.04609068110585213, + "p05": 0.029884062707424164, + "p95": 0.06911227852106094 + }, + { + "label": "completed_group_6", + "mean": 0.20163494348526, + "median": 0.19222964346408844, + "p05": 0.09963390231132507, + "p95": 0.3307305574417114 + }, + { + "label": "current_group_partial", + "mean": 0.4768889844417572, + "median": 0.48383551836013794, + "p05": 0.276212215423584, + "p95": 0.6487575173377991 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8757455348968506, + "group": 7, + "latest_source_mass": 0.3109493851661682, + "layer": 28, + "max_source_mass": 0.3109493851661682, + "mixer_index": 54, + "normalized_entropy": 0.9020429270551009, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11563669890165329, + "median": 0.12087491154670715, + "p05": 0.05733206868171692, + "p95": 0.1688953936100006 + }, + { + "label": "completed_group_1", + "mean": 0.08108697831630707, + "median": 0.08185897022485733, + "p05": 0.05175650864839554, + "p95": 0.10753051191568375 + }, + { + "label": "completed_group_2", + "mean": 0.07280858606100082, + "median": 0.07357072830200195, + "p05": 0.04870978742837906, + "p95": 0.0948689877986908 + }, + { + "label": "completed_group_3", + "mean": 0.07553227245807648, + "median": 0.07362619787454605, + "p05": 0.056317005306482315, + "p95": 0.09756095707416534 + }, + { + "label": "completed_group_4", + "mean": 0.07606014609336853, + "median": 0.07456053793430328, + "p05": 0.05528895929455757, + "p95": 0.1022825762629509 + }, + { + "label": "completed_group_5", + "mean": 0.06935927271842957, + "median": 0.06843917071819305, + "p05": 0.04978285729885101, + "p95": 0.09167985618114471 + }, + { + "label": "completed_group_6", + "mean": 0.19856664538383484, + "median": 0.19508783519268036, + "p05": 0.1281973123550415, + "p95": 0.27796900272369385 + }, + { + "label": "current_group_partial", + "mean": 0.3109493851661682, + "median": 0.3105033040046692, + "p05": 0.18172834813594818, + "p95": 0.43916669487953186 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6924852132797241, + "group": 7, + "latest_source_mass": 0.3315581679344177, + "layer": 28, + "max_source_mass": 0.3315581679344177, + "mixer_index": 55, + "normalized_entropy": 0.8139133413255193, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08638577908277512, + "median": 0.058608733117580414, + "p05": 0.026544082909822464, + "p95": 0.2740764319896698 + }, + { + "label": "completed_group_1", + "mean": 0.059912845492362976, + "median": 0.05901291221380234, + "p05": 0.03629481792449951, + "p95": 0.08707985281944275 + }, + { + "label": "completed_group_2", + "mean": 0.06609142571687698, + "median": 0.0647939145565033, + "p05": 0.038691695779561996, + "p95": 0.09703366458415985 + }, + { + "label": "completed_group_3", + "mean": 0.056745611131191254, + "median": 0.05418796092271805, + "p05": 0.032174739986658096, + "p95": 0.08370660990476608 + }, + { + "label": "completed_group_4", + "mean": 0.06277773529291153, + "median": 0.0600711889564991, + "p05": 0.03501531109213829, + "p95": 0.10094858705997467 + }, + { + "label": "completed_group_5", + "mean": 0.05352145433425903, + "median": 0.05179818347096443, + "p05": 0.032271772623062134, + "p95": 0.08084158599376678 + }, + { + "label": "completed_group_6", + "mean": 0.2830069661140442, + "median": 0.2759752869606018, + "p05": 0.12981083989143372, + "p95": 0.46204638481140137 + }, + { + "label": "current_group_partial", + "mean": 0.3315581679344177, + "median": 0.3204721510410309, + "p05": 0.13446375727653503, + "p95": 0.5667697787284851 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8980522155761719, + "group": 8, + "latest_source_mass": 0.20123262703418732, + "layer": 29, + "max_source_mass": 0.26602888107299805, + "mixer_index": 56, + "normalized_entropy": 0.9127701729200177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11603894829750061, + "median": 0.09991352260112762, + "p05": 0.030035395175218582, + "p95": 0.2941743731498718 + }, + { + "label": "completed_group_1", + "mean": 0.08804173022508621, + "median": 0.08880250155925751, + "p05": 0.05664174258708954, + "p95": 0.11779069900512695 + }, + { + "label": "completed_group_2", + "mean": 0.07975024729967117, + "median": 0.07984748482704163, + "p05": 0.05428915470838547, + "p95": 0.10499341040849686 + }, + { + "label": "completed_group_3", + "mean": 0.09027000516653061, + "median": 0.08898856490850449, + "p05": 0.06189651042222977, + "p95": 0.11996553838253021 + }, + { + "label": "completed_group_4", + "mean": 0.08430293202400208, + "median": 0.0819794088602066, + "p05": 0.05464346706867218, + "p95": 0.12159879505634308 + }, + { + "label": "completed_group_5", + "mean": 0.07433462142944336, + "median": 0.07287181913852692, + "p05": 0.052520088851451874, + "p95": 0.1010315865278244 + }, + { + "label": "completed_group_6", + "mean": 0.26602888107299805, + "median": 0.2639757990837097, + "p05": 0.15474572777748108, + "p95": 0.3846370279788971 + }, + { + "label": "completed_group_7", + "mean": 0.20123262703418732, + "median": 0.18140269815921783, + "p05": 0.08419936150312424, + "p95": 0.3782624900341034 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7512003183364868, + "group": 8, + "latest_source_mass": 0.33482813835144043, + "layer": 29, + "max_source_mass": 0.33482813835144043, + "mixer_index": 57, + "normalized_entropy": 0.7970056117156376, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07744461297988892, + "median": 0.06970229744911194, + "p05": 0.014401955530047417, + "p95": 0.17144526541233063 + }, + { + "label": "completed_group_1", + "mean": 0.046086348593235016, + "median": 0.04394831880927086, + "p05": 0.028066720813512802, + "p95": 0.0718703418970108 + }, + { + "label": "completed_group_2", + "mean": 0.05025433003902435, + "median": 0.04861236736178398, + "p05": 0.029644077643752098, + "p95": 0.07761462777853012 + }, + { + "label": "completed_group_3", + "mean": 0.03748488053679466, + "median": 0.03434944897890091, + "p05": 0.020774798467755318, + "p95": 0.05938909202814102 + }, + { + "label": "completed_group_4", + "mean": 0.04005061462521553, + "median": 0.037498839199543, + "p05": 0.019541850313544273, + "p95": 0.06922487914562225 + }, + { + "label": "completed_group_5", + "mean": 0.041623204946517944, + "median": 0.039552345871925354, + "p05": 0.022081367671489716, + "p95": 0.06821337342262268 + }, + { + "label": "completed_group_6", + "mean": 0.198423832654953, + "median": 0.18318486213684082, + "p05": 0.07548617571592331, + "p95": 0.3721975088119507 + }, + { + "label": "completed_group_7", + "mean": 0.17380405962467194, + "median": 0.1514919549226761, + "p05": 0.05777408182621002, + "p95": 0.36326974630355835 + }, + { + "label": "current_group_partial", + "mean": 0.33482813835144043, + "median": 0.33602219820022583, + "p05": 0.0795319601893425, + "p95": 0.5979493856430054 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0160694122314453, + "group": 8, + "latest_source_mass": 0.21711516380310059, + "layer": 30, + "max_source_mass": 0.21711516380310059, + "mixer_index": 58, + "normalized_entropy": 0.9175527313077866, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09385271370410919, + "median": 0.09211117774248123, + "p05": 0.032377734780311584, + "p95": 0.16496515274047852 + }, + { + "label": "completed_group_1", + "mean": 0.06208505108952522, + "median": 0.06140740588307381, + "p05": 0.0407925546169281, + "p95": 0.0863042101264 + }, + { + "label": "completed_group_2", + "mean": 0.06560768187046051, + "median": 0.06516212224960327, + "p05": 0.041897669434547424, + "p95": 0.09085705131292343 + }, + { + "label": "completed_group_3", + "mean": 0.07029052078723907, + "median": 0.06780634075403214, + "p05": 0.04846132919192314, + "p95": 0.09917552024126053 + }, + { + "label": "completed_group_4", + "mean": 0.06935097277164459, + "median": 0.06708167493343353, + "p05": 0.04668639227747917, + "p95": 0.09815650433301926 + }, + { + "label": "completed_group_5", + "mean": 0.056832700967788696, + "median": 0.05595892667770386, + "p05": 0.038822539150714874, + "p95": 0.0771557092666626 + }, + { + "label": "completed_group_6", + "mean": 0.19425815343856812, + "median": 0.19145764410495758, + "p05": 0.12249615788459778, + "p95": 0.27496030926704407 + }, + { + "label": "completed_group_7", + "mean": 0.17060703039169312, + "median": 0.16317546367645264, + "p05": 0.09239593893289566, + "p95": 0.2726203501224518 + }, + { + "label": "current_group_partial", + "mean": 0.21711516380310059, + "median": 0.21198657155036926, + "p05": 0.11019522696733475, + "p95": 0.3391069769859314 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7837426662445068, + "group": 8, + "latest_source_mass": 0.29026859998703003, + "layer": 30, + "max_source_mass": 0.29026859998703003, + "mixer_index": 59, + "normalized_entropy": 0.8118162725118464, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08836327493190765, + "median": 0.07904411852359772, + "p05": 0.02631576545536518, + "p95": 0.17534467577934265 + }, + { + "label": "completed_group_1", + "mean": 0.04953558370471001, + "median": 0.04658038914203644, + "p05": 0.025181813165545464, + "p95": 0.08318150043487549 + }, + { + "label": "completed_group_2", + "mean": 0.04813014343380928, + "median": 0.045740917325019836, + "p05": 0.026714935898780823, + "p95": 0.07843323051929474 + }, + { + "label": "completed_group_3", + "mean": 0.0373263955116272, + "median": 0.03401809185743332, + "p05": 0.01990434154868126, + "p95": 0.06290259957313538 + }, + { + "label": "completed_group_4", + "mean": 0.037015400826931, + "median": 0.03405017405748367, + "p05": 0.018646355718374252, + "p95": 0.06544196605682373 + }, + { + "label": "completed_group_5", + "mean": 0.03390160948038101, + "median": 0.03198609501123428, + "p05": 0.018189728260040283, + "p95": 0.05570102855563164 + }, + { + "label": "completed_group_6", + "mean": 0.1840541511774063, + "median": 0.17327937483787537, + "p05": 0.07966600358486176, + "p95": 0.32982927560806274 + }, + { + "label": "completed_group_7", + "mean": 0.23140482604503632, + "median": 0.22406929731369019, + "p05": 0.08831030130386353, + "p95": 0.40350067615509033 + }, + { + "label": "current_group_partial", + "mean": 0.29026859998703003, + "median": 0.2745037376880646, + "p05": 0.10280418395996094, + "p95": 0.5399085879325867 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9973840713500977, + "group": 8, + "latest_source_mass": 0.19101998209953308, + "layer": 31, + "max_source_mass": 0.20831164717674255, + "mixer_index": 60, + "normalized_entropy": 0.9090486661912383, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10261696577072144, + "median": 0.10389091819524765, + "p05": 0.01371697522699833, + "p95": 0.21163305640220642 + }, + { + "label": "completed_group_1", + "mean": 0.06039813905954361, + "median": 0.06085003539919853, + "p05": 0.026107093319296837, + "p95": 0.09348798543214798 + }, + { + "label": "completed_group_2", + "mean": 0.06671924144029617, + "median": 0.06656201928853989, + "p05": 0.034684643149375916, + "p95": 0.09819839894771576 + }, + { + "label": "completed_group_3", + "mean": 0.08377155661582947, + "median": 0.0804532840847969, + "p05": 0.05448959022760391, + "p95": 0.12039553374052048 + }, + { + "label": "completed_group_4", + "mean": 0.07002593576908112, + "median": 0.06737451255321503, + "p05": 0.044841229915618896, + "p95": 0.10655929893255234 + }, + { + "label": "completed_group_5", + "mean": 0.05768897384405136, + "median": 0.056077782064676285, + "p05": 0.039495617151260376, + "p95": 0.08242568373680115 + }, + { + "label": "completed_group_6", + "mean": 0.20831164717674255, + "median": 0.2021285891532898, + "p05": 0.11652229726314545, + "p95": 0.31706833839416504 + }, + { + "label": "completed_group_7", + "mean": 0.1594475358724594, + "median": 0.1546279788017273, + "p05": 0.06593266129493713, + "p95": 0.2735573351383209 + }, + { + "label": "current_group_partial", + "mean": 0.19101998209953308, + "median": 0.1713225245475769, + "p05": 0.06408411264419556, + "p95": 0.3850628137588501 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.969987154006958, + "group": 8, + "latest_source_mass": 0.16864418983459473, + "layer": 31, + "max_source_mass": 0.24503237009048462, + "mixer_index": 61, + "normalized_entropy": 0.8965797917640489, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08120635151863098, + "median": 0.061164144426584244, + "p05": 0.0259944386780262, + "p95": 0.23622018098831177 + }, + { + "label": "completed_group_1", + "mean": 0.07894632965326309, + "median": 0.07658770680427551, + "p05": 0.0496862456202507, + "p95": 0.11596276611089706 + }, + { + "label": "completed_group_2", + "mean": 0.08615730702877045, + "median": 0.08467075228691101, + "p05": 0.050409119576215744, + "p95": 0.12811316549777985 + }, + { + "label": "completed_group_3", + "mean": 0.07609779387712479, + "median": 0.0722871869802475, + "p05": 0.04099677503108978, + "p95": 0.11950768530368805 + }, + { + "label": "completed_group_4", + "mean": 0.058728985488414764, + "median": 0.056080836802721024, + "p05": 0.03323862701654434, + "p95": 0.09344407916069031 + }, + { + "label": "completed_group_5", + "mean": 0.05076971277594566, + "median": 0.04946017265319824, + "p05": 0.03047419898211956, + "p95": 0.07563400268554688 + }, + { + "label": "completed_group_6", + "mean": 0.24503237009048462, + "median": 0.24110376834869385, + "p05": 0.10613486170768738, + "p95": 0.3991503119468689 + }, + { + "label": "completed_group_7", + "mean": 0.1544169783592224, + "median": 0.1400495171546936, + "p05": 0.06199025735259056, + "p95": 0.2995939254760742 + }, + { + "label": "current_group_partial", + "mean": 0.16864418983459473, + "median": 0.15222924947738647, + "p05": 0.0543956533074379, + "p95": 0.3329208791255951 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.071648359298706, + "group": 8, + "latest_source_mass": 0.11520501971244812, + "layer": 32, + "max_source_mass": 0.20955611765384674, + "mixer_index": 62, + "normalized_entropy": 0.9428478002054053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10569609701633453, + "median": 0.08689188957214355, + "p05": 0.03173588588833809, + "p95": 0.24571751058101654 + }, + { + "label": "completed_group_1", + "mean": 0.09922641515731812, + "median": 0.09795533865690231, + "p05": 0.06407676637172699, + "p95": 0.14120742678642273 + }, + { + "label": "completed_group_2", + "mean": 0.08180510997772217, + "median": 0.080217644572258, + "p05": 0.054607540369033813, + "p95": 0.11442367732524872 + }, + { + "label": "completed_group_3", + "mean": 0.10646799206733704, + "median": 0.10510502755641937, + "p05": 0.07415887713432312, + "p95": 0.14216329157352448 + }, + { + "label": "completed_group_4", + "mean": 0.0826798751950264, + "median": 0.08022409677505493, + "p05": 0.05491747707128525, + "p95": 0.11860857903957367 + }, + { + "label": "completed_group_5", + "mean": 0.06615597754716873, + "median": 0.06485182791948318, + "p05": 0.04477996751666069, + "p95": 0.09238749742507935 + }, + { + "label": "completed_group_6", + "mean": 0.20955611765384674, + "median": 0.206287682056427, + "p05": 0.129981130361557, + "p95": 0.2996031939983368 + }, + { + "label": "completed_group_7", + "mean": 0.13320739567279816, + "median": 0.12465103715658188, + "p05": 0.06303662061691284, + "p95": 0.23065781593322754 + }, + { + "label": "current_group_partial", + "mean": 0.11520501971244812, + "median": 0.10386130958795547, + "p05": 0.03912060335278511, + "p95": 0.22735844552516937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781701564788818, + "group": 8, + "latest_source_mass": 0.16112206876277924, + "layer": 32, + "max_source_mass": 0.22251944243907928, + "mixer_index": 63, + "normalized_entropy": 0.9003040366848136, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09446848928928375, + "median": 0.074150450527668, + "p05": 0.042592622339725494, + "p95": 0.19443637132644653 + }, + { + "label": "completed_group_1", + "mean": 0.07056106626987457, + "median": 0.06868506222963333, + "p05": 0.04818308353424072, + "p95": 0.0982733890414238 + }, + { + "label": "completed_group_2", + "mean": 0.08033448457717896, + "median": 0.08047192543745041, + "p05": 0.048296794295310974, + "p95": 0.11253741383552551 + }, + { + "label": "completed_group_3", + "mean": 0.0658978745341301, + "median": 0.06256154924631119, + "p05": 0.04223886504769325, + "p95": 0.0967884361743927 + }, + { + "label": "completed_group_4", + "mean": 0.058234501630067825, + "median": 0.0563921295106411, + "p05": 0.03529864549636841, + "p95": 0.08846980333328247 + }, + { + "label": "completed_group_5", + "mean": 0.04316519945859909, + "median": 0.042268507182598114, + "p05": 0.029443470761179924, + "p95": 0.05996086448431015 + }, + { + "label": "completed_group_6", + "mean": 0.2036968618631363, + "median": 0.19645483791828156, + "p05": 0.09398645162582397, + "p95": 0.3395216464996338 + }, + { + "label": "completed_group_7", + "mean": 0.22251944243907928, + "median": 0.2130982130765915, + "p05": 0.11867167800664902, + "p95": 0.35465875267982483 + }, + { + "label": "current_group_partial", + "mean": 0.16112206876277924, + "median": 0.14023703336715698, + "p05": 0.06008770316839218, + "p95": 0.3247341811656952 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.6036605834960938, + "group": null, + "latest_source_mass": 0.31639498472213745, + "layer": null, + "max_source_mass": 0.3412485122680664, + "mixer_index": null, + "normalized_entropy": 0.7298573846467136, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09759747982025146, + "median": 0.09915056824684143, + "p05": 0.025461740791797638, + "p95": 0.16398483514785767 + }, + { + "label": "completed_group_1", + "mean": 0.04215940088033676, + "median": 0.04050712287425995, + "p05": 0.025720715522766113, + "p95": 0.06398092210292816 + }, + { + "label": "completed_group_2", + "mean": 0.041145481169223785, + "median": 0.03881259262561798, + "p05": 0.023510465398430824, + "p95": 0.0662468671798706 + }, + { + "label": "completed_group_3", + "mean": 0.029554028064012527, + "median": 0.027420252561569214, + "p05": 0.01627083122730255, + "p95": 0.04924973472952843 + }, + { + "label": "completed_group_4", + "mean": 0.0289996936917305, + "median": 0.027226522564888, + "p05": 0.01560397818684578, + "p95": 0.048351090401411057 + }, + { + "label": "completed_group_5", + "mean": 0.03647579252719879, + "median": 0.0347352996468544, + "p05": 0.020744025707244873, + "p95": 0.058257561177015305 + }, + { + "label": "completed_group_6", + "mean": 0.06642461568117142, + "median": 0.057809896767139435, + "p05": 0.022837381809949875, + "p95": 0.14078694581985474 + }, + { + "label": "completed_group_7", + "mean": 0.3412485122680664, + "median": 0.3336990475654602, + "p05": 0.0819513276219368, + "p95": 0.6423859596252441 + }, + { + "label": "completed_group_8", + "mean": 0.31639498472213745, + "median": 0.30713334679603577, + "p05": 0.05841828137636185, + "p95": 0.6156357526779175 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.24537226557731628, + 0.07698935270309448, + 0.0523640513420105, + 0.09380833059549332, + 0.03647654876112938, + 0.039120711386203766, + 0.06371714919805527, + 0.05156813934445381, + 0.03728010877966881, + 0.03983496502041817, + 0.040284499526023865, + 0.046871740370988846, + 0.04314620420336723, + 0.04033593460917473, + 0.04471700266003609, + 0.08397205173969269, + 0.0528092160820961, + 0.04443390294909477, + 0.07560808956623077, + 0.03708760067820549, + 0.03599191829562187, + 0.033304885029792786, + 0.04061548039317131, + 0.025487789884209633, + 0.021565141156315804, + 0.02342291921377182, + 0.017244853079319, + 0.024166326969861984, + 0.016819356009364128, + 0.021590016782283783, + 0.026201345026493073 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6127158914969186e-05, + "first_to_last_ratio": 0.6403133863364645, + "imbalance_abs_log_ratio": 0.4457975563250414, + "last_quartile_mean": 2.518635290016391e-05, + "mean": 2.1593947650444534e-05, + "normalized": [ + 0.6929919556166857, + 0.8032314127318161, + 0.7123629256452019, + 0.6640523991876173, + 0.8725736234454132, + 0.8071370183700546, + 0.7516577913014878, + 0.6706888120378312, + 0.5882487760705537, + 0.5190639183458972, + 0.4863138588077732, + 0.4652918617004374, + 0.6603036411218667, + 0.6189718629251246, + 0.5802646281311779, + 0.5410038780854491, + 0.6697462103535458, + 0.6259079034957141, + 0.6246747715872017, + 0.6676260304525075, + 3.1377243999857436, + 2.6752853614525707, + 2.136977079899559, + 1.6970064982433113, + 2.2713553531497497, + 1.9850638882393974, + 1.4820693911808296, + 0.9615863897898643, + 1.0163024380148542, + 0.7191699586662397, + 0.4720839007000451, + 0.42326206126447913 + ], + "peak_layer": 21, + "peak_normalized": 3.1377243999857436, + "population_cv": 0.6918392109604528, + "rest_mean": 1.6060819520875897e-05, + "spike_contrast": 3.2048700555541787, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.147283955011517e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4964432011765894e-05, + 1.7344937077723444e-05, + 1.53827277245e-05, + 1.4339512745209504e-05, + 1.8842309145838954e-05, + 1.7429274521418847e-05, + 1.623125899641309e-05, + 1.4482819096883759e-05, + 1.2702613275905605e-05, + 1.1208639079995919e-05, + 1.0501436008780729e-05, + 1.0047488103737123e-05, + 1.4258562259783503e-05, + 1.3366046005103271e-05, + 1.2530204003269318e-05, + 1.1682409422064666e-05, + 1.446246460545808e-05, + 1.3515822502085939e-05, + 1.3489194316207431e-05, + 1.4416681551665533e-05, + 6.775585643481463e-05, + 5.7769972045207396e-05, + 4.61457711935509e-05, + 3.664506948553026e-05, + 4.9047528591472656e-05, + 4.2865365685429424e-05, + 3.2003728847485036e-05, + 2.076444616250228e-05, + 2.1945981643511914e-05, + 1.5529718439211138e-05, + 1.019415503833443e-05, + 9.13989879336441e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.422064188882359e-05, + "first_to_last_ratio": 0.6376814356469364, + "imbalance_abs_log_ratio": 0.4499164375054873, + "last_quartile_mean": 0.00010070959933727863, + "mean": 8.629182207187114e-05, + "normalized": [ + 0.6940773401510585, + 0.8062528427202595, + 0.7180374505283312, + 0.6647119372719499, + 0.8588689882759897, + 0.7941431696191276, + 0.7473948010342778, + 0.6703237246625169, + 0.591781846497542, + 0.522694550559361, + 0.4893812082436792, + 0.4670959071660659, + 0.6619658672285874, + 0.6164719403371215, + 0.5787012511245688, + 0.5393739243504363, + 0.6707710740427385, + 0.62668399370843, + 0.6317390712162502, + 0.6712190136492653, + 3.138964807406113, + 2.6795820619127526, + 2.136435916928567, + 1.6866743661161174, + 2.264472697470994, + 1.978410075097681, + 1.4778942897678682, + 0.9605324338748193, + 1.022040338716437, + 0.7239267286003249, + 0.47869752189559756, + 0.4306788598251707 + ], + "peak_layer": 21, + "peak_normalized": 3.138964807406113, + "population_cv": 0.6905024564046187, + "rest_mean": 6.421987658271703e-05, + "spike_contrast": 3.1996375366532934, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020548032771330326, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.989319834043272e-05, + 6.957302684895694e-05, + 6.196075992193073e-05, + 5.7359204220119864e-05, + 7.411336991935968e-05, + 6.852806109236553e-05, + 6.449405918829143e-05, + 5.784345557913184e-05, + 5.1065933803329244e-05, + 4.5104265154805034e-05, + 4.222959614708088e-05, + 4.03065569116734e-05, + 5.712224083254114e-05, + 5.319648698787205e-05, + 4.993718539481051e-05, + 4.654355871025473e-05, + 5.788205817225389e-05, + 5.4077703680377454e-05, + 5.4513915529241785e-05, + 5.792071169707924e-05, + 0.0002708669926505536, + 0.00023122601851355284, + 0.00018435694801155478, + 0.00014554620429407805, + 0.00019540547509677708, + 0.0001707206101855263, + 0.00012753019109368324, + 8.288609387818724e-05, + 8.81937230587937e-05, + 6.246895645745099e-05, + 4.1307681385660544e-05, + 3.716406354214996e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05695150885730982, + "first_to_last_ratio": 0.6376814663645546, + "imbalance_abs_log_ratio": 0.4499163893346989, + "last_quartile_mean": 0.08931027771905065, + "mean": 0.07652445149142295, + "normalized": [ + 0.6940773558515339, + 0.8062528453975966, + 0.7180374361894372, + 0.6647119756359035, + 0.8588690718004696, + 0.7941431432641803, + 0.7473947840325739, + 0.6703237318272384, + 0.5917818305157142, + 0.5226945671206367, + 0.4893812028028422, + 0.46709594515141906, + 0.6619658291829611, + 0.6164719318106706, + 0.5787012336125282, + 0.5393739103061749, + 0.6707710619535627, + 0.6266839981127393, + 0.6317390378687008, + 0.6712190249334776, + 3.138964907679676, + 2.679582133407049, + 2.136436026253706, + 1.6866743790728473, + 2.2644726279739604, + 1.9784099533155926, + 1.4778942270811004, + 0.9605324250805215, + 1.0220403417906543, + 0.7239267230206866, + 0.4786975145327867, + 0.43067882342105884 + ], + "peak_layer": 21, + "peak_normalized": 3.138964907679676, + "population_cv": 0.6905024682030625, + "rest_mean": 0.05695082868138949, + "spike_contrast": 3.1996376327558256, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18222201466560364, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05311388894915581, + 0.061698056757450104, + 0.054947420954704285, + 0.05086671933531761, + 0.06572448462247849, + 0.06077136844396591, + 0.057193975895643234, + 0.05129615589976311, + 0.04528577998280525, + 0.039998915046453476, + 0.03744962811470032, + 0.035744260996580124, + 0.05065657198429108, + 0.047175176441669464, + 0.044284794479608536, + 0.04127529263496399, + 0.051330387592315674, + 0.04795664921402931, + 0.0483434833586216, + 0.05136466771364212, + 0.24020756781101227, + 0.20505355298519135, + 0.16348959505558014, + 0.12907183170318604, + 0.1732875257730484, + 0.1513967365026474, + 0.11309504508972168, + 0.07350421696901321, + 0.07821107655763626, + 0.05539809539914131, + 0.03663206472992897, + 0.03295746073126793 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.592033183645981e-06, + "first_to_last_ratio": 0.3776007228014791, + "imbalance_abs_log_ratio": 0.9739179305137912, + "last_quartile_mean": 6.864481520096888e-06, + "mean": 4.875440396290287e-06, + "normalized": [ + 0.58913988046903, + 0.5446521181917551, + 0.5278862817384906, + 0.53898932284961, + 0.5219441801529001, + 0.509889287242028, + 0.5048315077449682, + 0.5158761132703409, + 0.4368296481828547, + 0.4047199700448834, + 0.3956716882892422, + 0.40126275663148436, + 0.5110508169296636, + 0.5085656955747676, + 0.5112234187675316, + 0.511163584085558, + 0.5662253891905324, + 0.5872486754624854, + 0.609842074156892, + 0.6494088419115644, + 3.025689231381945, + 2.8686186674565652, + 2.4420116557657345, + 2.053486551003887, + 2.698068277749679, + 2.429082514259166, + 1.8298867229318267, + 1.2191127574146867, + 1.182073363720965, + 0.8592876866491284, + 0.5581944369380665, + 0.48806688384176716 + ], + "peak_layer": 21, + "peak_normalized": 3.025689231381945, + "population_cv": 0.8235639638153561, + "rest_mean": 3.4149978226330994e-06, + "spike_contrast": 3.736995148125518, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2761830294039101e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.87231637230434e-06, + 2.6554189389571548e-06, + 2.573678102635313e-06, + 2.6278103177901357e-06, + 2.5447077405260643e-06, + 2.485934828655445e-06, + 2.461275926179951e-06, + 2.5151232421194436e-06, + 2.1297369130479638e-06, + 1.9731880911422195e-06, + 1.92907373275375e-06, + 1.9563326532079373e-06, + 2.491597797416034e-06, + 2.4794817363726906e-06, + 2.4924393073888496e-06, + 2.492147586963256e-06, + 2.7605981358647114e-06, + 2.863095915017766e-06, + 2.973248683701968e-06, + 3.166154101563734e-06, + 1.4751567505300045e-05, + 1.3985779332870152e-05, + 1.1905882274731994e-05, + 1.0011651284003165e-05, + 1.3154271073290147e-05, + 1.1842847015941516e-05, + 8.92150364961708e-06, + 5.943711585132405e-06, + 5.763128228863934e-06, + 4.18940589952399e-06, + 2.7214437068323605e-06, + 2.3795410015736707e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.8627718115312746e-05, + "first_to_last_ratio": 0.778294860341054, + "imbalance_abs_log_ratio": 0.2506498287376175, + "last_quartile_mean": 6.247981400520075e-05, + "mean": 6.0048676687074476e-05, + "normalized": [ + 0.768607661628769, + 0.8690595287737466, + 0.7879569251847977, + 0.7385864958726717, + 0.9274786983184008, + 0.8617431831538573, + 0.8054883103443636, + 0.7195191288766225, + 0.6496482220749353, + 0.5799654886485536, + 0.5429742744436054, + 0.5176928240267351, + 0.720980532018643, + 0.6651070950014618, + 0.6225467122437304, + 0.5784265987882409, + 0.7443177261701239, + 0.697888823545442, + 0.701818835999622, + 0.7379727210931422, + 3.1748914800668615, + 2.647277084571918, + 2.0581907003495648, + 1.5579720703470805, + 2.1379505239352077, + 1.7650981274587072, + 1.310550617537982, + 0.8840683850126465, + 0.8342765345805514, + 0.5872947992080164, + 0.42062138312406294, + 0.3840285075999366 + ], + "peak_layer": 21, + "peak_normalized": 3.1748914800668615, + "population_cv": 0.6535432926484233, + "rest_mean": 4.5422861014096134e-05, + "spike_contrast": 3.0607513093024403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013902808132115752, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.615387297235429e-05, + 5.218587466515601e-05, + 4.7315770643763244e-05, + 4.435114169609733e-05, + 5.569386848947033e-05, + 5.174653779249638e-05, + 4.83685071230866e-05, + 4.3206171540077776e-05, + 3.901051604771055e-05, + 3.482616011751816e-05, + 3.2604886655462906e-05, + 3.1086769013199955e-05, + 4.329392686486244e-05, + 3.993880091002211e-05, + 3.738310624612495e-05, + 3.473375181783922e-05, + 4.4695294491248205e-05, + 4.1907300328603014e-05, + 4.214329237584025e-05, + 4.431428533280268e-05, + 0.00019064803200308233, + 0.0001589654857525602, + 0.00012359162792563438, + 9.355416113976389e-05, + 0.00012838109978474677, + 0.00010599180677672848, + 7.869683031458408e-05, + 5.3087136620888487e-05, + 5.009720189264044e-05, + 3.52662755176425e-05, + 2.525775744288694e-05, + 2.3060403691488318e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.645518927442026e-05, + "first_to_last_ratio": 0.9865830246987611, + "imbalance_abs_log_ratio": 0.013507796190281324, + "last_quartile_mean": 3.6950959383830195e-05, + "mean": 4.1335310356771515e-05, + "normalized": [ + 0.8560051014711698, + 0.9467981965967722, + 0.8689980494059457, + 0.8159268191754907, + 1.0103083717536356, + 0.9426554996877177, + 0.8489560918930947, + 0.765857482001633, + 0.7201445334320749, + 0.6498760503731635, + 0.6109444686941845, + 0.5791405787907885, + 0.7989198470081217, + 0.7291371869136885, + 0.6790968935040115, + 0.6275849613644364, + 0.8454614229708973, + 0.7893993011096712, + 0.7856135784658201, + 0.8181223517247423, + 3.2763150867214206, + 2.615409622894913, + 1.9276349807221327, + 1.3402369957405367, + 1.940332562376331, + 1.4667183642001942, + 1.0945246384976166, + 0.8103162680270223, + 0.6598082696260246, + 0.4694457579306494, + 0.3666062775483438, + 0.3437043893777548 + ], + "peak_layer": 21, + "peak_normalized": 3.2763150867214206, + "population_cv": 0.6207682663855968, + "rest_mean": 3.199670040716969e-05, + "spike_contrast": 2.867914594845514, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.176380408462136e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5383236536290497e-05, + 3.913619730155915e-05, + 3.592030407162383e-05, + 3.37265882990323e-05, + 4.176141010248102e-05, + 3.896495763910934e-05, + 3.509186353767291e-05, + 3.1656956707593054e-05, + 2.9767397791147232e-05, + 2.6862828235607594e-05, + 2.5253579224226996e-05, + 2.3938955564517528e-05, + 3.3023599826265126e-05, + 3.0139111913740635e-05, + 2.8070680855307728e-05, + 2.594141915324144e-05, + 3.4947410313179716e-05, + 3.263006510678679e-05, + 3.2473581086378545e-05, + 3.381734131835401e-05, + 0.0001354275009362027, + 0.00010810876847244799, + 7.967939018271863e-05, + 5.539911217056215e-05, + 8.020424866117537e-05, + 6.062725879019126e-05, + 4.5242515625432134e-05, + 3.349467442603782e-05, + 2.7273379600956105e-05, + 1.9404686099733226e-05, + 1.5153784261201508e-05, + 1.4207127605914138e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012982768839719938, + "first_to_last_ratio": 0.6037915882942751, + "imbalance_abs_log_ratio": 0.5045261930845532, + "last_quartile_mean": 0.00021502069739653962, + "mean": 0.0001790727353636612, + "normalized": [ + 0.6739765257680471, + 0.7636067305383979, + 0.68849644503889, + 0.6447045606228033, + 0.8477445590156324, + 0.7818709864142286, + 0.7391626168129651, + 0.6604355769831363, + 0.5800901925673246, + 0.5130947085748045, + 0.4841531977734674, + 0.46136480942786895, + 0.6373637447050293, + 0.5885397553357715, + 0.5617314689244236, + 0.5244758601010534, + 0.6356987553893274, + 0.6101287890369049, + 0.6118132813754859, + 0.6643470301154188, + 3.09815074914609, + 2.693324715468958, + 2.18351992306606, + 1.7462447599227975, + 2.367437166894702, + 2.0445672235354038, + 1.5627333414952298, + 1.026078331181794, + 1.036030398706582, + 0.6947075889876938, + 0.46018158540857673, + 0.4142246216651318 + ], + "peak_layer": 21, + "peak_normalized": 3.09815074914609, + "population_cv": 0.7095514290759102, + "rest_mean": 0.00013205833400004648, + "spike_contrast": 3.278479211520481, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004329505027271807, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012069082004018128, + 0.00013674114597961307, + 0.00012329094170127064, + 0.00011544900917215273, + 0.00015180793707259, + 0.00014001177623867989, + 0.0001323638716712594, + 0.00011826600530184805, + 0.00010387833754066378, + 9.188127296511084e-05, + 8.669863746035844e-05, + 8.261785842478275e-05, + 0.00011413446918595582, + 0.00010539142385823652, + 0.00010059079068014398, + 9.391932690050453e-05, + 0.00011383631499484181, + 0.00010925743117695674, + 0.00010955907782772556, + 0.00011896643991349265, + 0.0005547943292185664, + 0.0004823010240215808, + 0.0003910088853444904, + 0.0003127048257738352, + 0.00042394344927743077, + 0.0003661262453533709, + 0.0002798429341055453, + 0.0001837426534621045, + 0.00018552479741629213, + 0.0001244031882379204, + 8.240597526310012e-05, + 7.417633605655283e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13583384454250336, + 0.0415157787501812, + 0.018140379339456558, + 0.039518751204013824, + 0.022877270355820656, + 0.015532293356955051, + 0.02314678393304348, + 0.05432315170764923, + 0.03743875026702881, + 0.04006272554397583, + 0.045583780854940414, + 0.05995568633079529, + 0.038651175796985626, + 0.04559725895524025, + 0.05632574483752251, + 0.07952121645212173, + 0.042337141931056976, + 0.05044656619429588, + 0.04418695345520973, + 0.06310147047042847, + 0.029010163620114326, + 0.04586305841803551, + 0.06033914536237717, + 0.07656749337911606, + 0.045884523540735245, + 0.051604680716991425, + 0.06502768397331238, + 0.06792232394218445, + 0.060306549072265625, + 0.08025261014699936, + 0.09663844108581543, + 0.12230555713176727 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5986858215910615e-05, + "first_to_last_ratio": 0.8170320074925627, + "imbalance_abs_log_ratio": 0.20207700803204537, + "last_quartile_mean": 1.9566991340980167e-05, + "mean": 1.9039267499465495e-05, + "normalized": [ + 0.7853354674160987, + 0.9097466615865485, + 0.8087937381555731, + 0.7523818308832718, + 0.9858416226009721, + 0.9045115156777315, + 0.8288509697382657, + 0.741963942115696, + 0.6090758738841542, + 0.5628736284922787, + 0.5329013290216021, + 0.5131548852635935, + 0.7265824288035061, + 0.6757039396775449, + 0.6225875966900762, + 0.5898833165121294, + 0.7532717273459274, + 0.6927780667296713, + 0.6897910451069404, + 0.809449516707258, + 3.207633782072864, + 2.5002059817245676, + 1.9824571594317655, + 1.5924827370882133, + 2.273874867822024, + 1.6940739554643818, + 1.0968135124011316, + 0.8719494928225496, + 0.815651324531277, + 0.538892521487639, + 0.48196570767333513, + 0.44851985507141146 + ], + "peak_layer": 21, + "peak_normalized": 3.207633782072864, + "population_cv": 0.6428607369119902, + "rest_mean": 1.4415789741582961e-05, + "spike_contrast": 3.0526282764165082, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4006047392031175e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4952212040952872e-05, + 1.7320910046692006e-05, + 1.539884033263661e-05, + 1.4324798939924221e-05, + 1.8769702364807017e-05, + 1.7221236703335308e-05, + 1.578071533003822e-05, + 1.4126449968898669e-05, + 1.1596358490351122e-05, + 1.0716701581259258e-05, + 1.0146050954062957e-05, + 9.770093129191082e-06, + 1.3833597222401295e-05, + 1.2864908057963476e-05, + 1.18536117952317e-05, + 1.1230946256546304e-05, + 1.434174191672355e-05, + 1.318998693022877e-05, + 1.3133116226526909e-05, + 1.541132587590255e-05, + 6.107099761720747e-05, + 4.760209048981778e-05, + 3.77445321646519e-05, + 3.0319704819703475e-05, + 4.3292911868775263e-05, + 3.225392720196396e-05, + 2.088252585963346e-05, + 1.660127963987179e-05, + 1.5529403754044324e-05, + 1.0260118870064616e-05, + 9.176274033961818e-06, + 8.539489499526098e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.365280069076107e-05, + "first_to_last_ratio": 0.8123684077728273, + "imbalance_abs_log_ratio": 0.2078013375593398, + "last_quartile_mean": 7.83545988269907e-05, + "mean": 7.614439766712167e-05, + "normalized": [ + 0.7854660713099145, + 0.9130496523819581, + 0.8147298517264512, + 0.752236196131495, + 0.9689924576782674, + 0.8892579571020611, + 0.8242433765322099, + 0.7396130336387524, + 0.6125904066287632, + 0.5670744382794471, + 0.5359292329408729, + 0.5145351492524995, + 0.7290151059417195, + 0.673146260009013, + 0.621425875046763, + 0.5883337897938153, + 0.7535116607120764, + 0.6934400787203782, + 0.6978162466203766, + 0.8168476312813977, + 3.2190719364475022, + 2.5076839554965473, + 1.973830744692153, + 1.575947332799018, + 2.2647184639164535, + 1.6880790923847124, + 1.0924366554795215, + 0.8732736568641918, + 0.8206255787733402, + 0.5459821164389519, + 0.48994786526283923, + 0.4571481297165359 + ], + "peak_layer": 21, + "peak_normalized": 3.2190719364475022, + "population_cv": 0.642032487569562, + "rest_mean": 5.76970003883751e-05, + "spike_contrast": 3.0462648281411457, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001757603429723531, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.9808840887853876e-05, + 6.952361582079902e-05, + 6.203711382113397e-05, + 5.727857205783948e-05, + 7.378334703389555e-05, + 6.771201151423156e-05, + 6.276151543715969e-05, + 5.6317388953175396e-05, + 4.664532752940431e-05, + 4.3179541535209864e-05, + 4.0808008634485304e-05, + 3.917896901839413e-05, + 5.551041613216512e-05, + 5.125631651026197e-05, + 4.731809895019978e-05, + 4.479832205106504e-05, + 5.7375691540073603e-05, + 5.280157711240463e-05, + 5.3134797781240195e-05, + 6.219837086973712e-05, + 0.00024511429364793, + 0.00019094608433078974, + 0.0001502961531514302, + 0.00011999956041108817, + 0.00017244562332052737, + 0.00012853776570409536, + 8.318293112097308e-05, + 6.649489660048857e-05, + 6.248604040592909e-05, + 4.157347939326428e-05, + 3.7306785088730976e-05, + 3.480926898191683e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05644794087857008, + "first_to_last_ratio": 0.8123684042388034, + "imbalance_abs_log_ratio": 0.2078013419096122, + "last_quartile_mean": 0.06948564294725657, + "mean": 0.06752561370376498, + "normalized": [ + 0.7854660588804813, + 0.9130496786513765, + 0.814729886022376, + 0.7522361645698925, + 0.968992465622299, + 0.8892579649660906, + 0.8242433717205022, + 0.7396130482968516, + 0.6125904375263521, + 0.5670744516239514, + 0.535929266538159, + 0.5145351229333333, + 0.7290151143815181, + 0.673146253269631, + 0.6214258465577596, + 0.5883338184574164, + 0.7535116614810096, + 0.6934400601144122, + 0.6978162502950979, + 0.8168476412095322, + 3.21907174067581, + 2.5076838612014067, + 1.9738308079311102, + 1.5759473804424433, + 2.264718470873259, + 1.6880792267982687, + 1.0924366234670762, + 0.8732736495232089, + 0.8206255836175469, + 0.5459820799959341, + 0.4899478787351448, + 0.45714813362074885 + ], + "peak_layer": 21, + "peak_normalized": 3.21907174067581, + "population_cv": 0.6420324679566953, + "rest_mean": 0.05116627765474496, + "spike_contrast": 3.0462647570380494, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15586602836847305, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.053039077669382095, + 0.061654239892959595, + 0.05501513555645943, + 0.05079520866274834, + 0.06543181091547012, + 0.0600476898252964, + 0.05565753951668739, + 0.049942824989557266, + 0.041365545243024826, + 0.0382920503616333, + 0.03618895262479782, + 0.034744299948215485, + 0.049227192997932434, + 0.045454613864421844, + 0.0419621616601944, + 0.039727602154016495, + 0.050881337374448776, + 0.04682496562600136, + 0.04712047055363655, + 0.055158138275146484, + 0.21736979484558105, + 0.16933289170265198, + 0.13328413665294647, + 0.10641681402921677, + 0.152926504611969, + 0.11398858577013016, + 0.07376745343208313, + 0.05896833911538124, + 0.055413246154785156, + 0.03686777502298355, + 0.03308403119444847, + 0.03086920827627182 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.57929028180115e-06, + "first_to_last_ratio": 0.4759084167214945, + "imbalance_abs_log_ratio": 0.7425298451079264, + "last_quartile_mean": 5.41971982670475e-06, + "mean": 4.213722270662856e-06, + "normalized": [ + 0.6771451422985035, + 0.6338625145334242, + 0.6120949335724296, + 0.6258725064817227, + 0.6050121598754559, + 0.5852465596314514, + 0.5785033580390577, + 0.5791973952882404, + 0.4696864977471604, + 0.4522091645384124, + 0.4457992216966758, + 0.45590579514105095, + 0.5773211960070957, + 0.5667516162949074, + 0.5644132471304436, + 0.568160519465189, + 0.6503432281671621, + 0.663170344529487, + 0.6837490184222774, + 0.7583004470800568, + 3.021070651777523, + 2.6274098317695387, + 2.3058345630897463, + 2.003282670865861, + 2.819487095674051, + 2.140274896548892, + 1.4825743986898414, + 1.122684410298403, + 0.9855482185618173, + 0.6429507763662341, + 0.5708772581826982, + 0.5252603622351902 + ], + "peak_layer": 21, + "peak_normalized": 3.021070651777523, + "population_cv": 0.7577432386283482, + "rest_mean": 3.000000956658537e-06, + "spike_contrast": 3.5892713108597136, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0767817366286181e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8533015665743733e-06, + 2.670920594027848e-06, + 2.579198053354048e-06, + 2.6372529191576177e-06, + 2.549353212089045e-06, + 2.4660664621478645e-06, + 2.4376524834224256e-06, + 2.4405769636359764e-06, + 1.9791284557868494e-06, + 1.9054838276133523e-06, + 1.8784741087074508e-06, + 1.921060402310104e-06, + 2.4326711809408152e-06, + 2.3881339075160213e-06, + 2.378280669290689e-06, + 2.394070634181844e-06, + 2.7403657441027462e-06, + 2.7944156499870587e-06, + 2.881128466469818e-06, + 3.1952674817148363e-06, + 1.27299526866409e-05, + 1.1071175322285853e-05, + 9.71614645095542e-06, + 8.441276804660447e-06, + 1.1880535566888284e-05, + 9.018523996928707e-06, + 6.247156761673978e-06, + 4.730680302600376e-06, + 4.152826477366034e-06, + 2.7092160053143743e-06, + 2.4055182166193845e-06, + 2.2133012862468604e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.826902750210138e-05, + "first_to_last_ratio": 0.9723604213825404, + "imbalance_abs_log_ratio": 0.028028739360968564, + "last_quartile_mean": 4.964108620697516e-05, + "mean": 5.395586174472555e-05, + "normalized": [ + 0.8546738942457572, + 0.9667007478721396, + 0.8780654782701063, + 0.82151592085777, + 1.0287569123437954, + 0.9492731152891766, + 0.8750422046820956, + 0.7827885676521403, + 0.6672237173045179, + 0.6202571530123847, + 0.5854234638750982, + 0.5620770546607131, + 0.7815896821357725, + 0.715521927114241, + 0.659703864497871, + 0.6215694475292887, + 0.8222045332968153, + 0.7616919988241317, + 0.764120987673305, + 0.8753289635877315, + 3.236716069528407, + 2.479479992593944, + 1.8829977352397056, + 1.4470254883420661, + 2.056662650357765, + 1.5065673712263334, + 1.0009483781315074, + 0.8155586445002443, + 0.6595748803016609, + 0.47597993512775083, + 0.43187920683965, + 0.4130800130861142 + ], + "peak_layer": 21, + "peak_normalized": 3.236716069528407, + "population_cv": 0.610766278714797, + "rest_mean": 4.176007456003895e-05, + "spike_contrast": 2.8690828214345374, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001198131125420332, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.6114666474750265e-05, + 5.2159171900711954e-05, + 4.737677954835817e-05, + 4.432559944689274e-05, + 5.550746573135257e-05, + 5.121884896652773e-05, + 4.721365621662699e-05, + 4.223603173159063e-05, + 3.600063064368442e-05, + 3.3466509194113314e-05, + 3.158702747896314e-05, + 3.032735185115598e-05, + 4.217134483042173e-05, + 3.860660217469558e-05, + 3.5594890505308285e-05, + 3.353731517563574e-05, + 4.4362754124449566e-05, + 4.109774818061851e-05, + 4.122880636714399e-05, + 4.722912854049355e-05, + 0.00017463980475440621, + 0.00013378247967921197, + 0.00010159876546822488, + 7.80755071900785e-05, + 0.00011096900561824441, + 8.128814079100266e-05, + 5.400703230407089e-05, + 4.400416946737096e-05, + 3.558793105185032e-05, + 2.568190757301636e-05, + 2.3302414774661884e-05, + 2.22880880755838e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.626883517426904e-05, + "first_to_last_ratio": 1.1972713068748915, + "imbalance_abs_log_ratio": 0.1800450565949769, + "last_quartile_mean": 3.0292912697404972e-05, + "mean": 3.794855416572318e-05, + "normalized": [ + 0.934241229967689, + 1.0326674049103126, + 0.9483113974161322, + 0.8884949951731808, + 1.1009339220259078, + 1.0148817561286225, + 0.9027362835592571, + 0.8236285600155873, + 0.7333418938739371, + 0.6823440677494514, + 0.6463725772705814, + 0.6166567793546124, + 0.850307320786337, + 0.7701976529963521, + 0.7042915982263429, + 0.6605966097637871, + 0.9142336932467707, + 0.8480352948917127, + 0.839474071242197, + 0.9443144532969522, + 3.2998505290664992, + 2.4485943259044243, + 1.7642227386757172, + 1.2451697951023746, + 1.763789999228952, + 1.2716404306318911, + 0.9094502638769789, + 0.7804639985266583, + 0.5161011792987255, + 0.40746526774101827, + 0.37633786588868356, + 0.36085204416235356 + ], + "peak_layer": 21, + "peak_normalized": 3.2998505290664992, + "population_cv": 0.5857907121700514, + "rest_mean": 3.01878957947742e-05, + "spike_contrast": 2.6453022731935993, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.985610936884768e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.545310391928069e-05, + 3.918823495041579e-05, + 3.5987046430818737e-05, + 3.3717100450303406e-05, + 4.177885057288222e-05, + 3.851329529425129e-05, + 3.425753675401211e-05, + 3.12555130221881e-05, + 2.7829264581669122e-05, + 2.5893970814649947e-05, + 2.452890475979075e-05, + 2.3401233192998916e-05, + 3.2267933420371264e-05, + 2.9227887353044935e-05, + 2.672684786375612e-05, + 2.5068686227314174e-05, + 3.469384682830423e-05, + 3.218171332264319e-05, + 3.185682726325467e-05, + 3.583536818041466e-05, + 0.00012522455654107034, + 9.292061440646648e-05, + 6.694970215903595e-05, + 4.72523934149649e-05, + 6.693328032270074e-05, + 4.825691576115787e-05, + 3.4512322599766776e-05, + 2.961748032248579e-05, + 1.9585293557611294e-05, + 1.546271778352093e-05, + 1.4281477888289373e-05, + 1.3693813343707006e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012846768913732376, + "first_to_last_ratio": 0.7741677488258334, + "imbalance_abs_log_ratio": 0.2559666991257704, + "last_quartile_mean": 0.00016594296175753698, + "mean": 0.0001574651118971815, + "normalized": [ + 0.7618335990450072, + 0.8698560675479529, + 0.7822046486498666, + 0.7351395196601278, + 0.9594102821252986, + 0.8815927774487453, + 0.812858049389013, + 0.7238937569203876, + 0.5978223149190788, + 0.5606129559633132, + 0.5319350341866579, + 0.513003926829316, + 0.6981154037320007, + 0.6408130676433959, + 0.6075107239537738, + 0.5706379352682298, + 0.7181374534396903, + 0.6783796513052409, + 0.6819318446184696, + 0.7908903240994326, + 3.2180960304949577, + 2.5452212715171636, + 2.028532306848687, + 1.6608547003669625, + 2.321647122642583, + 1.7738587692821628, + 1.1712480907534195, + 0.9285924776215382, + 0.7896984661161683, + 0.5235677697382035, + 0.4733567899608167, + 0.4487468679123384 + ], + "peak_layer": 21, + "peak_normalized": 3.2180960304949577, + "population_cv": 0.6626251267593958, + "rest_mean": 0.00011795681536939271, + "spike_contrast": 3.1436073616095452, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037080991314724087, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011996221292065457, + 0.00013697198301088065, + 0.00012316994252614677, + 0.00011575882672332227, + 0.0001510736474301666, + 0.0001388201053487137, + 0.0001279967837035656, + 0.00011398801143513992, + 9.413615771336481e-05, + 8.82769818417728e-05, + 8.376120968023315e-05, + 8.078022074187174e-05, + 0.00010992882016580552, + 0.00010090570140164346, + 9.566174412611872e-05, + 8.985556632978842e-05, + 0.0001130815944634378, + 0.00010682112770155072, + 0.00010738047421909869, + 0.00012453763338271528, + 0.0005067378515377641, + 0.0004007835523225367, + 0.0003194230666849762, + 0.0002615266712382436, + 0.0003655784239526838, + 0.0002793208695948124, + 0.00018443071166984737, + 0.0001462209183955565, + 0.00012434995733201504, + 8.244365744758397e-05, + 7.453717989847064e-05, + 7.066197576932609e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.35791492462158203, + 0.3940892517566681, + 0.4057151973247528, + 0.09380833059549332, + 0.11286329478025436, + 0.13231655955314636, + 0.1621972620487213, + 0.05156813934445381, + 0.08533531427383423, + 0.12003090232610703, + 0.16063299775123596, + 0.046871740370988846, + 0.079384446144104, + 0.11459959298372269, + 0.15625107288360596, + 0.08397205173969269, + 0.11654024571180344, + 0.15294702351093292, + 0.21572710573673248, + 0.03708760067820549, + 0.06553087383508682, + 0.09390194714069366, + 0.12852425873279572, + 0.025487789884209633, + 0.06059965491294861, + 0.08860157430171967, + 0.12100546807050705, + 0.024166326969861984, + 0.06877554953098297, + 0.12682415544986725, + 0.2025238573551178 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6127134472299076e-05, + "first_to_last_ratio": 0.6403082943802734, + "imbalance_abs_log_ratio": 0.4458055086442437, + "last_quartile_mean": 2.5186515017594502e-05, + "mean": 2.159394537670778e-05, + "normalized": [ + 0.6929830574423667, + 0.8032565154251953, + 0.7123500282965459, + 0.6640666207712503, + 0.8725494553308608, + 0.8071350816916083, + 0.7516540798235798, + 0.6706926732818747, + 0.5882490907185235, + 0.5190611932100045, + 0.48631129869562834, + 0.46529523801872363, + 0.6602888008616835, + 0.6189890701429912, + 0.5802627939183058, + 0.5409897412703016, + 0.6697392892795451, + 0.6259125602673482, + 0.6246778698614331, + 0.6676379780381497, + 3.137658015393984, + 2.675256160517513, + 2.137000554072058, + 1.6970284098395438, + 2.2713857488305376, + 1.9850977916906365, + 1.4820653354312663, + 0.9615821107638199, + 1.0163133272453195, + 0.7191715506408339, + 0.4720786856528992, + 0.423259873575668 + ], + "peak_layer": 21, + "peak_normalized": 3.137658015393984, + "population_cv": 0.691835904675886, + "rest_mean": 1.6060833668571258e-05, + "spike_contrast": 3.20486156963133, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.147274860064499e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4964238289394416e-05, + 1.7345477317576297e-05, + 1.5382447600131854e-05, + 1.43398183354293e-05, + 1.8841785276890732e-05, + 1.7429230865673162e-05, + 1.623117714188993e-05, + 1.4482900951406918e-05, + 1.2702618732873816e-05, + 1.1208579053345602e-05, + 1.0501379620109219e-05, + 1.0047559953818563e-05, + 1.4258240298659075e-05, + 1.3366416169446893e-05, + 1.2530163076007739e-05, + 1.1682102922350168e-05, + 1.4462313629337586e-05, + 1.3515921637008432e-05, + 1.3489259799825959e-05, + 1.4416938029171433e-05, + 6.775441579520702e-05, + 5.7769335398916155e-05, + 4.614627323462628e-05, + 3.664553878479637e-05, + 4.9048179789679125e-05, + 4.286609328119084e-05, + 3.200363789801486e-05, + 2.0764351575053297e-05, + 2.1946214474155568e-05, + 1.55297511810204e-05, + 1.0194041351496708e-05, + 9.139850590145215e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.422088154067751e-05, + "first_to_last_ratio": 0.637682021407296, + "imbalance_abs_log_ratio": 0.44991551892756126, + "last_quartile_mean": 0.00010070988264487823, + "mean": 8.629204910448607e-05, + "normalized": [ + 0.6940783808537224, + 0.8062729813904731, + 0.7180303336815479, + 0.6647219507645815, + 0.8588589713696609, + 0.794136864353616, + 0.7473889560340541, + 0.6703283692089702, + 0.5917748510321142, + 0.5226947352403994, + 0.4893764215021349, + 0.4670989784552037, + 0.661952700545094, + 0.6164859593679979, + 0.5786990118719374, + 0.539368710966703, + 0.6707742418503443, + 0.6266861813765591, + 0.6317438172781369, + 0.6712283354793382, + 3.1389231589965445, + 2.679577878793134, + 2.1364533990925567, + 1.686690164787602, + 2.264487313238182, + 1.9784244316759128, + 1.477878596963198, + 0.9605191983702851, + 1.022047261980904, + 0.7239260887312458, + 0.4786954192739982, + 0.4306763354738471 + ], + "peak_layer": 21, + "peak_normalized": 3.1389231589965445, + "population_cv": 0.690500620274875, + "rest_mean": 6.42200389443638e-05, + "spike_contrast": 3.1996384204494492, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002054809039691463, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.98934457229916e-05, + 6.957494770176709e-05, + 6.196030881255865e-05, + 5.736021921620704e-05, + 7.411270053125918e-05, + 6.852769729448482e-05, + 6.449372449424118e-05, + 5.784400855191052e-05, + 5.106546450406313e-05, + 4.51043997600209e-05, + 4.2229294194839895e-05, + 4.030692798551172e-05, + 5.7121254940284416e-05, + 5.319783667800948e-05, + 4.993712354917079e-05, + 4.654323129216209e-05, + 5.788248381577432e-05, + 5.40780347364489e-05, + 5.451446850202046e-05, + 5.7921668485505506e-05, + 0.00027086411137133837, + 0.00023122626589611173, + 0.00018435894162394106, + 0.00014554795052390546, + 0.00019540725043043494, + 0.0001707222982076928, + 0.00012752917245961726, + 8.288516983157024e-05, + 8.819455251796171e-05, + 6.24690655968152e-05, + 4.13076086260844e-05, + 3.716394348884933e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.056951718870550394, + "first_to_last_ratio": 0.6376820191187264, + "imbalance_abs_log_ratio": 0.4499155225164498, + "last_quartile_mean": 0.08931052964180708, + "mean": 0.07652465114369988, + "normalized": [ + 0.6940783685018588, + 0.8062729890693022, + 0.7180303052956253, + 0.6647219248274491, + 0.8588589447083164, + 0.7941368847913186, + 0.7473889396113633, + 0.6703284088385635, + 0.5917748342982165, + 0.5226947125226558, + 0.4893764209845987, + 0.46709901042220126, + 0.6619527107838585, + 0.616485998695461, + 0.5786990422522198, + 0.5393687059717984, + 0.6707742773702546, + 0.6266862088911475, + 0.6317438155472213, + 0.671228324296238, + 3.138923030946758, + 2.6795778685275287, + 2.136453429700619, + 1.6866902298133055, + 2.2644871659665227, + 1.9784244587489672, + 1.4778786878450325, + 0.9605192092561265, + 1.0220472167545944, + 0.7239261000048334, + 0.4786954380387232, + 0.4306763367173204 + ], + "peak_layer": 21, + "peak_normalized": 3.138923030946758, + "population_cv": 0.6905006031140464, + "rest_mean": 0.056950972587973984, + "spike_contrast": 3.199638339154521, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18222251534461975, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05311410501599312, + 0.0616997592151165, + 0.05494701862335205, + 0.05086761340498924, + 0.06572388112545013, + 0.060771048069000244, + 0.05719367787241936, + 0.051296647638082504, + 0.045285362750291824, + 0.03999903053045273, + 0.03744935989379883, + 0.03574458882212639, + 0.05065570026636124, + 0.04717637598514557, + 0.04428474232554436, + 0.04127500206232071, + 0.05133076757192612, + 0.04795694351196289, + 0.048343975096940994, + 0.05136551335453987, + 0.24020498991012573, + 0.20505376160144806, + 0.163491353392601, + 0.1290733814239502, + 0.17328909039497375, + 0.15139824151992798, + 0.11309415102005005, + 0.07350339740514755, + 0.07821180671453476, + 0.05539819225668907, + 0.036632001399993896, + 0.03295735642313957 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.591977136034984e-06, + "first_to_last_ratio": 0.377584641115206, + "imbalance_abs_log_ratio": 0.9739605205507381, + "last_quartile_mean": 6.864625447633443e-06, + "mean": 4.875532418679995e-06, + "normalized": [ + 0.5891470886653372, + 0.5445974877366685, + 0.5278757119769764, + 0.5390247128297342, + 0.5219320902964267, + 0.5098223481982903, + 0.5047854170324074, + 0.5158515929518226, + 0.43682532071303765, + 0.4047462819890774, + 0.3956760190748421, + 0.40124851416781243, + 0.5110306315304836, + 0.5086020328656453, + 0.5111981468235955, + 0.5112038363741968, + 0.5662160078662817, + 0.5872378713533353, + 0.609816433193417, + 0.6494206487580901, + 3.0257058078799917, + 2.8686033250815077, + 2.442085511360172, + 2.0535509509100836, + 2.6979608311079244, + 2.429139638501957, + 1.8298842703872822, + 1.2190711864897739, + 1.1821681083908706, + 0.8593060718331675, + 0.5582216762792698, + 0.48804442738052095 + ], + "peak_layer": 21, + "peak_normalized": 3.0257058078799917, + "population_cv": 0.8235699242655958, + "rest_mean": 3.415056493462611e-06, + "spike_contrast": 3.7370106290435205, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2762102414853871e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.872405730158789e-06, + 2.6552027065918082e-06, + 2.573675146777532e-06, + 2.6280324618710438e-06, + 2.544696826589643e-06, + 2.4856553864083253e-06, + 2.4610976652184036e-06, + 2.515051164664328e-06, + 2.129756012436701e-06, + 1.973353619177942e-06, + 1.929131258293637e-06, + 1.956300138772349e-06, + 2.491546410965384e-06, + 2.4797056994430022e-06, + 2.492363137207576e-06, + 2.49239087679598e-06, + 2.7606045023276238e-06, + 2.8630972792598186e-06, + 2.9731797894783085e-06, + 3.1662714263802627e-06, + 1.4751926755707245e-05, + 1.398596850776812e-05, + 1.1906467079825234e-05, + 1.0012154234573245e-05, + 1.315399549639551e-05, + 1.1843349057016894e-05, + 8.921660082705785e-06, + 5.943621090409579e-06, + 5.7636989367892966e-06, + 4.189574610791169e-06, + 2.7216278795094695e-06, + 2.3794764274498448e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.862782361669815e-05, + "first_to_last_ratio": 0.7782948608369358, + "imbalance_abs_log_ratio": 0.25064982810047876, + "last_quartile_mean": 6.247994951991132e-05, + "mean": 6.004880788168521e-05, + "normalized": [ + 0.7686076787185943, + 0.8690658694356951, + 0.7879558094938062, + 0.7385895471539009, + 0.9274752179545845, + 0.8617380288980451, + 0.805485823509466, + 0.7195218583140428, + 0.6496432282856196, + 0.579963373368429, + 0.542973633406911, + 0.5176967820022496, + 0.7209744736270646, + 0.6651136389238139, + 0.6225458367735092, + 0.578423456947426, + 0.7443181598307745, + 0.6978871776308329, + 0.7018192413444355, + 0.7379772277240845, + 3.17488720924795, + 2.647274451154299, + 2.058197351015695, + 1.557976178868823, + 2.137956515672529, + 1.7650930593990968, + 1.3105561147977765, + 0.8840594863751997, + 0.8342799220515003, + 0.5872955759322932, + 0.4206210094039151, + 0.3840270627376372 + ], + "peak_layer": 21, + "peak_normalized": 3.17488720924795, + "population_cv": 0.6535433957622422, + "rest_mean": 4.5422938354830774e-05, + "spike_contrast": 3.060755388404179, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013902850332669914, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.615397483576089e-05, + 5.218636943027377e-05, + 4.7315807023551315e-05, + 4.4351421820465475e-05, + 5.569378117797896e-05, + 5.17463413416408e-05, + 4.8368463467340916e-05, + 4.320642983657308e-05, + 3.9010301406960934e-05, + 3.482610918581486e-05, + 3.260491939727217e-05, + 3.108707460341975e-05, + 4.329365765443072e-05, + 3.9939281123224646e-05, + 3.738313534995541e-05, + 3.47336390404962e-05, + 4.4695418182527646e-05, + 4.19072930526454e-05, + 4.2143408791162074e-05, + 4.43146527686622e-05, + 0.00019064819207414985, + 0.00015896567492745817, + 0.0001235922973137349, + 9.355461224913597e-05, + 0.00012838174006901681, + 0.00010599173401715234, + 7.869733235565946e-05, + 5.308671825332567e-05, + 5.009751475881785e-05, + 3.526639920892194e-05, + 2.5257790184696205e-05, + 2.3060367311700247e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.645344395408756e-05, + "first_to_last_ratio": 0.9865245275167812, + "imbalance_abs_log_ratio": 0.01356709065898008, + "last_quartile_mean": 3.695138127568498e-05, + "mean": 4.1334421752026174e-05, + "normalized": [ + 0.8559009893021382, + 0.9469134291395908, + 0.8691128415923758, + 0.8158459610499553, + 1.0100395593992384, + 0.9426509450849102, + 0.8490005706596633, + 0.7658551995239993, + 0.720151565776352, + 0.6499584076998435, + 0.6111014164691825, + 0.5791973878203037, + 0.798669989787535, + 0.7292203680582068, + 0.6791147051708346, + 0.6276623067938342, + 0.8453470506184937, + 0.7894265691211152, + 0.7857081832788244, + 0.8180697050541997, + 3.276793902438668, + 2.6146895715088023, + 1.9277214837041228, + 1.3401559674726915, + 1.9403857172500238, + 1.4668533992566692, + 1.0945373428822949, + 0.8102509556533821, + 0.6598620161196013, + 0.46946557551492035, + 0.3666267887349235, + 0.3437101280633033 + ], + "peak_layer": 21, + "peak_normalized": 3.276793902438668, + "population_cv": 0.6207652283001258, + "rest_mean": 3.1996292111418574e-05, + "spike_contrast": 2.867842357851226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.176032181130722e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5378172469791025e-05, + 3.9140119042713195e-05, + 3.5924276744481176e-05, + 3.372252103872597e-05, + 4.174940113443881e-05, + 3.8963931729085743e-05, + 3.509294765535742e-05, + 3.165618181810714e-05, + 2.976704854518175e-05, + 2.6865654945140705e-05, + 2.5259523681597784e-05, + 2.3940789105836302e-05, + 3.301256219856441e-05, + 3.0141902243485674e-05, + 2.8070813641534187e-05, + 2.5944058506865986e-05, + 3.4941931517096236e-05, + 3.263049075030722e-05, + 3.2476793421665207e-05, + 3.381443821126595e-05, + 0.0001354443811578676, + 0.00010807668149936944, + 7.968125282786787e-05, + 5.53945719730109e-05, + 8.020472159842029e-05, + 6.06315370532684e-05, + 4.524206815403886e-05, + 3.349125472595915e-05, + 2.7275014872429892e-05, + 1.940508809639141e-05, + 1.5154306311160326e-05, + 1.4207059393811505e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012982848875253694, + "first_to_last_ratio": 0.6036924070795879, + "imbalance_abs_log_ratio": 0.5046904705661434, + "last_quartile_mean": 0.00021505734912352636, + "mean": 0.00017907720302900998, + "normalized": [ + 0.6739415088564852, + 0.7635979187076084, + 0.6884132846665824, + 0.6448880523181134, + 0.847636297931654, + 0.7820437425942102, + 0.7390348805248977, + 0.6603333703627584, + 0.580097091853396, + 0.5130765445550399, + 0.4840585582545864, + 0.4614791311662735, + 0.637447225260648, + 0.5884103323685723, + 0.5619029319457929, + 0.5245290839622635, + 0.6356966288449003, + 0.6100272686595811, + 0.6118819192326126, + 0.6643605628507145, + 3.097964891544288, + 2.6928767346712488, + 2.1827809090098653, + 1.7461631642015993, + 2.367937988760414, + 2.0449314566347367, + 1.563314534665903, + 1.0260809297257236, + 1.0359670091642164, + 0.6947146354097079, + 0.460142110411202, + 0.41426930088440395 + ], + "peak_layer": 21, + "peak_normalized": 3.097964891544288, + "population_cv": 0.7095182139952283, + "rest_mean": 0.00013206795362445214, + "spike_contrast": 3.278063587209748, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004329271498136222, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012068756041117013, + 0.00013674297952093184, + 0.0001232791255461052, + 0.0001154847486759536, + 0.00015179233741946518, + 0.0001400462060701102, + 0.00013234429934527725, + 0.00011825065303128213, + 0.00010388216469436884, + 9.188031253870577e-05, + 8.668385271448642e-05, + 8.264039206551388e-05, + 0.00011415226617828012, + 0.00010537087655393407, + 0.00010062400542665273, + 9.393120126333088e-05, + 0.00011383877426851541, + 0.00010924197704298422, + 0.00010957410268019885, + 0.00011897183139808476, + 0.0005547748878598213, + 0.0004822328337468207, + 0.00039088630001060665, + 0.0003126980154775083, + 0.0004240437119733542, + 0.00036620060564018786, + 0.0002799539943225682, + 0.00018374770297668874, + 0.00018551807443145663, + 0.0001244075538124889, + 8.240096212830395e-05, + 7.418618770316243e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5986864127626177e-05, + "first_to_last_ratio": 0.8170274391577745, + "imbalance_abs_log_ratio": 0.20208259942567244, + "last_quartile_mean": 1.956710798367567e-05, + "mean": 1.9039307829871177e-05, + "normalized": [ + 0.7853167979867789, + 0.9097665173048055, + 0.808777694116954, + 0.752402497635975, + 0.9858540561935626, + 0.9044938358034963, + 0.8288447236919866, + 0.7419578801196516, + 0.6090850929460484, + 0.5628647453138417, + 0.5328979550345672, + 0.5131403750850531, + 0.7265824660927316, + 0.6757007408890954, + 0.6225826951824261, + 0.5898791530514104, + 0.7532772493405897, + 0.6927950381940383, + 0.6897848547806317, + 0.8094505726944553, + 3.2076147584918284, + 2.5002121502474965, + 1.982484105644761, + 1.5924612114390058, + 2.273889158865966, + 1.6940625327896233, + 1.0968336406315982, + 0.8719471681044353, + 0.815653609383569, + 0.5388945327422838, + 0.4819664542042268, + 0.448525735997107 + ], + "peak_layer": 21, + "peak_normalized": 3.2076147584918284, + "population_cv": 0.6428608089466793, + "rest_mean": 1.4415815443229534e-05, + "spike_contrast": 3.052631111367605, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.400616671773605e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4951888260839041e-05, + 1.7321324776276015e-05, + 1.5398567484226078e-05, + 1.4325222764455248e-05, + 1.8769978851196356e-05, + 1.7220936570083722e-05, + 1.5780629837536253e-05, + 1.4126364476396702e-05, + 1.1596558579185512e-05, + 1.0716555152612273e-05, + 1.0146008207811974e-05, + 9.769837561179884e-06, + 1.3833627235726453e-05, + 1.286487440665951e-05, + 1.1853543583129067e-05, + 1.1230890777369495e-05, + 1.4341877431434114e-05, + 1.3190337995183654e-05, + 1.3133026186551433e-05, + 1.541137862659525e-05, + 6.107076478656381e-05, + 4.760230876854621e-05, + 3.7745125155197456e-05, + 3.03193592117168e-05, + 4.329327566665597e-05, + 3.225377804483287e-05, + 2.0882953322143294e-05, + 1.6601270544924773e-05, + 1.5529480151599273e-05, + 1.0260178896714933e-05, + 9.176307685265783e-06, + 8.539619557268452e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "rest_mean": 5.7697108584559625e-05, + "spike_contrast": 3.04626897714722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017576091195223854, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05644787335768342, + "first_to_last_ratio": 0.8123632868395458, + "imbalance_abs_log_ratio": 0.2078076412872703, + "last_quartile_mean": 0.06948599754832685, + "mean": 0.06752577290171757, + "normalized": [ + 0.7854618348329956, + 0.9130540359301327, + 0.8147131249195937, + 0.7522410113175043, + 0.9690071730110875, + 0.88924395208314, + 0.8242353599678512, + 0.7396083806676593, + 0.6126026750590019, + 0.5670643980854334, + 0.5359283892166958, + 0.5145269586503932, + 0.7290174229589911, + 0.6731434525628229, + 0.6214223402600404, + 0.5883307763560162, + 0.7535191533057549, + 0.6934459281737582, + 0.6978155981634587, + 0.8168473704719704, + 3.2190670201887848, + 2.507709505453073, + 1.9738347607250095, + 1.5759251284181852, + 2.264726151347608, + 1.68807248858185, + 1.0924542395938426, + 0.8732750663142743, + 0.820625855657368, + 0.5459870819951741, + 0.48994655813463567, + 0.45714680759589343 + ], + "peak_layer": 21, + "peak_normalized": 3.2190670201887848, + "population_cv": 0.6420341436220799, + "rest_mean": 0.05116637251167386, + "spike_contrast": 3.0462690113979045, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15586653500795364, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05303891748189926, + 0.06165467947721481, + 0.05501413345336914, + 0.05079565569758415, + 0.06543295830488205, + 0.060046885162591934, + 0.05565712973475456, + 0.04994262754917145, + 0.041366469115018845, + 0.038291461765766144, + 0.03618897870182991, + 0.03474383056163788, + 0.04922746494412422, + 0.04545453190803528, + 0.041962023824453354, + 0.03972749039530754, + 0.05088196322321892, + 0.04682547226548195, + 0.04712053760886192, + 0.05515825003385544, + 0.21736998856067657, + 0.1693350225687027, + 0.13328471779823303, + 0.10641556233167648, + 0.15292738378047943, + 0.11398839950561523, + 0.07376881688833237, + 0.058968573808670044, + 0.055413395166397095, + 0.036868199706077576, + 0.033084020018577576, + 0.030869191512465477 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.579253447265728e-06, + "first_to_last_ratio": 0.47590496457778414, + "imbalance_abs_log_ratio": 0.7425370989326324, + "last_quartile_mean": 5.419681741614113e-06, + "mean": 4.2137082871818166e-06, + "normalized": [ + 0.6771144196047112, + 0.6338426021784607, + 0.6121020910961615, + 0.6258737201152189, + 0.6050362914422536, + 0.5852204423685657, + 0.5784631886764509, + 0.5792281322851218, + 0.4697078599262646, + 0.4522117174548025, + 0.44580914592548354, + 0.4558904724294877, + 0.5773213311917524, + 0.5667240346830847, + 0.5644321177223985, + 0.5681434108598962, + 0.6503548834184826, + 0.6631947770191623, + 0.6837887360559592, + 0.7582966501774209, + 3.020973835679826, + 2.627473806530212, + 2.3058644468737186, + 2.00331262981993, + 2.819406877957528, + 2.1402761714678284, + 1.4825745701937771, + 1.1227288221975413, + 0.9855094000056028, + 0.6429466506282437, + 0.5709200007532207, + 0.5252567632614316 + ], + "peak_layer": 21, + "peak_normalized": 3.020973835679826, + "population_cv": 0.7577358346351384, + "rest_mean": 2.999999306094078e-06, + "spike_contrast": 3.589246425218318, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0767736785055605e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8531626412586775e-06, + 2.670827825568267e-06, + 2.5792196538532153e-06, + 2.6372492811788106e-06, + 2.5494464352959767e-06, + 2.465948227836634e-06, + 2.43747513195558e-06, + 2.440698381178663e-06, + 1.979211901925737e-06, + 1.9054882614000235e-06, + 1.8785096926876577e-06, + 1.920989461723366e-06, + 2.4326636776095256e-06, + 2.3880097614892293e-06, + 2.3783522919984534e-06, + 2.393990598648088e-06, + 2.7404057618696243e-06, + 2.7945093279413413e-06, + 2.8812862638005754e-06, + 3.1952408789948095e-06, + 1.2729502486763522e-05, + 1.1071408152929507e-05, + 9.716240128909703e-06, + 8.441375030088238e-06, + 1.1880158126587048e-05, + 9.018499440571759e-06, + 6.247136752790539e-06, + 4.7308517423516605e-06, + 4.152649125899188e-06, + 2.709189629968023e-06, + 2.405690338491695e-06, + 2.2132787762529915e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.826905797017389e-05, + "first_to_last_ratio": 0.9723593605402814, + "imbalance_abs_log_ratio": 0.028029830358518852, + "last_quartile_mean": 4.964117169947713e-05, + "mean": 5.395593586854375e-05, + "normalized": [ + 0.8546740011831107, + 0.9667023865336807, + 0.8780594173977533, + 0.8215201862718362, + 1.028762848378348, + 0.9492707323927138, + 0.8750383055629682, + 0.7827836490446655, + 0.6672270484578459, + 0.6202534690629704, + 0.5854228619045188, + 0.5620710233383177, + 0.781592991024926, + 0.7155185842665527, + 0.6597031604820425, + 0.621567177701999, + 0.8222052916650392, + 0.7616961441483728, + 0.7641199379358804, + 0.8753280307745216, + 3.236716477578425, + 2.479496544122304, + 1.8830004075567146, + 1.4470156791415152, + 2.05665793704697, + 1.5065644924277526, + 1.0009572516438536, + 0.8155596816984735, + 0.6595753226879756, + 0.4759799891964506, + 0.4318790517932509, + 0.413079917578251 + ], + "peak_layer": 21, + "peak_normalized": 3.236716477578425, + "population_cv": 0.6107674157639074, + "rest_mean": 4.176012171902349e-05, + "spike_contrast": 2.869084843241085, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011981333227595314, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.61147355963476e-05, + 5.2159331971779466e-05, + 4.737651761388406e-05, + 4.4325890485197306e-05, + 5.550786227104254e-05, + 5.121879075886682e-05, + 4.72135106974747e-05, + 4.2235824366798624e-05, + 3.6000859836349264e-05, + 3.3466356399003416e-05, + 3.158703839289956e-05, + 3.0327068088809028e-05, + 4.217158129904419e-05, + 3.860647484543733e-05, + 3.559490141924471e-05, + 3.353723877808079e-05, + 4.4362855987856165e-05, + 4.1098028304986656e-05, + 4.122880636714399e-05, + 4.7229143092408776e-05, + 0.00017464006668888032, + 0.00013378355652093887, + 0.00010159904923057184, + 7.807508518453687e-05, + 0.00011096890375483781, + 8.128809713525698e-05, + 5.400758527684957e-05, + 4.4004285882692784e-05, + 3.558800381142646e-05, + 2.5681945771793835e-05, + 2.330243842152413e-05, + 2.228811354143545e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.626825309765991e-05, + "first_to_last_ratio": 1.1971952602722011, + "imbalance_abs_log_ratio": 0.17998153797771219, + "last_quartile_mean": 3.0294350722215313e-05, + "mean": 3.794959516767449e-05, + "normalized": [ + 0.9342317076920968, + 1.032527492584725, + 0.9483990781630639, + 0.8885231558783968, + 1.1008949985426397, + 1.0148066560537, + 0.9027782413985963, + 0.8234017777861798, + 0.7333936750261243, + 0.6823073758552121, + 0.6464481216407498, + 0.616777331910669, + 0.8501710687237772, + 0.7702178906455867, + 0.7042577074377577, + 0.660638930723233, + 0.914060026408416, + 0.847914539167618, + 0.8394293783449699, + 0.9442746494437388, + 3.300220154812763, + 2.4489522165880904, + 1.7640184700183157, + 1.245126339879113, + 1.7643693302152461, + 1.271277790934411, + 0.9095623054467832, + 0.780451313056054, + 0.5159478762516835, + 0.40747033934370974, + 0.37633772798986104, + 0.36081233203671814 + ], + "peak_layer": 21, + "peak_normalized": 3.300220154812763, + "population_cv": 0.5858778602410362, + "rest_mean": 3.018723526582579e-05, + "spike_contrast": 2.6456990159702416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.986633863765746e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5453715099720284e-05, + 3.9184000343084335e-05, + 3.599136107368395e-05, + 3.371909406268969e-05, + 4.1778519516810775e-05, + 3.85115017706994e-05, + 3.4260068787261844e-05, + 3.124776412732899e-05, + 2.783199306577444e-05, + 2.5893288693623617e-05, + 2.453244451317005e-05, + 2.3406450054608285e-05, + 3.226364788133651e-05, + 2.9229457140900195e-05, + 2.6726294890977442e-05, + 2.5070979972952046e-05, + 3.468820796115324e-05, + 3.217801349819638e-05, + 3.185600508004427e-05, + 3.5834840673487633e-05, + 0.00012524201883934438, + 9.293674520449713e-05, + 6.694378680549562e-05, + 4.725204053102061e-05, + 6.695710180792958e-05, + 4.8244477511616424e-05, + 3.451752127148211e-05, + 2.9617811378557235e-05, + 1.95800130313728e-05, + 1.546333442092873e-05, + 1.4281864423537627e-05, + 1.3692681932298001e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012846494701079791, + "first_to_last_ratio": 0.7741545804954827, + "imbalance_abs_log_ratio": 0.25598370893064326, + "last_quartile_mean": 0.00016594224234722788, + "mean": 0.000157460202672155, + "normalized": [ + 0.7615886502877881, + 0.869859991005712, + 0.7823534284439785, + 0.735201115807512, + 0.9596786286759813, + 0.8813992955944876, + 0.8128285893567467, + 0.7239431731448671, + 0.5977864740547744, + 0.5606030792342883, + 0.5318673348078621, + 0.5130567490014816, + 0.6987134321316841, + 0.6407269063222457, + 0.6075335461477536, + 0.5704889146486948, + 0.7180428901779269, + 0.6784140170824181, + 0.6819219612414157, + 0.7908928021790416, + 3.2176296651190497, + 2.545498211708922, + 2.0288816729118655, + 1.6601468184271124, + 2.321862751456626, + 1.7738386620365922, + 1.1711907122839438, + 0.9284857615000071, + 0.7896970255450443, + 0.5236030848875767, + 0.47341239614817127, + 0.4488522586284289 + ], + "peak_layer": 21, + "peak_normalized": 3.2176296651190497, + "population_cv": 0.6625973254291921, + "rest_mean": 0.0001179550758765631, + "spike_contrast": 3.143466990402101, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037078788736835124, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011991990322712809, + 0.00013696833048015833, + 0.00012318952940404415, + 0.00011576491669984534, + 0.0001511111913714558, + 0.00013878531171940267, + 0.00012798815441783518, + 0.00011399223876651376, + 9.412757935933769e-05, + 8.82726744748652e-05, + 8.374793833354488e-05, + 8.078601968009025e-05, + 0.000110019558633212, + 0.00010088898852700368, + 9.566235530655831e-05, + 8.982930012280121e-05, + 0.0001130631790147163, + 0.00010682320862542838, + 0.00010737557022366673, + 0.00012453414092306048, + 0.0005066486191935837, + 0.00040081466431729496, + 0.00031946811941452324, + 0.0002614070544950664, + 0.0003656009794212878, + 0.00027930899523198605, + 0.00018441592692397535, + 0.00014619955618400127, + 0.00012434585369192064, + 8.24466478661634e-05, + 7.454361184500158e-05, + 7.067636761348695e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0626930519938469, + 0.09030644595623016, + 0.20229491591453552, + 0.20300470292568207, + 0.19049881398677826, + 0.12695962190628052, + 0.12132706493139267, + 0.11585728824138641, + 0.13427142798900604, + 0.10485932976007462, + 0.10419609397649765, + 0.10852954536676407, + 0.1286265105009079, + 0.09096142649650574, + 0.10080574452877045, + 0.10566592961549759, + 0.1287592649459839, + 0.09650235623121262, + 0.1065765842795372, + 0.11446042358875275, + 0.12395420670509338, + 0.07221997529268265, + 0.07677769660949707, + 0.08416415005922318, + 0.09094838052988052, + 0.059439923614263535, + 0.062182098627090454, + 0.06896103918552399, + 0.0776243805885315, + 0.06285123527050018, + 0.07231374084949493, + 0.08016443997621536 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0311521975836513e-05, + "first_to_last_ratio": 1.1418129560476864, + "imbalance_abs_log_ratio": 0.13261731152138012, + "last_quartile_mean": 9.030832870848826e-06, + "mean": 1.3945096057454975e-05, + "normalized": [ + 0.7145337044030123, + 0.932335477800909, + 0.645487324949067, + 0.558162738870377, + 0.34661223918607953, + 0.7511338800577496, + 0.9447212820009926, + 1.0225104899919193, + 0.6315142693979814, + 0.5614582240786927, + 0.4574638139025647, + 0.40615350248375953, + 0.4194348381092698, + 0.7040945119904685, + 0.6483274463519548, + 0.5709286433652233, + 0.8217341836062412, + 1.1893729541204203, + 1.2408864574521172, + 1.5700693567578727, + 2.5502861886942485, + 4.114342551839228, + 2.785136777749787, + 2.232505646034574, + 1.1806687356339707, + 1.2053474712098855, + 0.7898562388784587, + 0.5945521897391881, + 0.46830824879649285, + 0.2867747481020556, + 0.32588927285831853, + 0.32939659158711965 + ], + "peak_layer": 22, + "peak_normalized": 4.114342551839228, + "population_cv": 0.8262244130468728, + "rest_mean": 9.884005235276331e-06, + "spike_contrast": 3.629600110811422, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.587498649721965e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.964241144189145e-06, + 1.3001507795706857e-05, + 9.001382750284392e-06, + 7.783633009239566e-06, + 4.833540970139438e-06, + 1.0474634109414183e-05, + 1.3174229025025852e-05, + 1.425900700269267e-05, + 8.80652714840835e-06, + 7.82958886702545e-06, + 6.3793768276809715e-06, + 5.663849606207805e-06, + 5.849059107276844e-06, + 9.818665603233967e-06, + 9.040988516062498e-06, + 7.961654773680493e-06, + 1.1459162124083377e-05, + 1.658592009334825e-05, + 1.730428084556479e-05, + 2.189476799685508e-05, + 3.556398587534204e-05, + 5.7374902098672464e-05, + 3.883899989887141e-05, + 3.113250568276271e-05, + 1.6464538930449635e-05, + 1.68086862686323e-05, + 1.1014621122740209e-05, + 8.291087397083174e-06, + 6.530603513965616e-06, + 3.999101409135619e-06, + 4.544557214103406e-06, + 4.593467110680649e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.944479362871789e-05, + "first_to_last_ratio": 1.1135282861542686, + "imbalance_abs_log_ratio": 0.1075336103151947, + "last_quartile_mean": 3.54232524841791e-05, + "mean": 5.533300526394669e-05, + "normalized": [ + 0.5280207224829799, + 0.8999098868607942, + 0.6284748654319019, + 0.5778568458898549, + 0.34988006202074273, + 0.7764713506345448, + 0.9190909808587969, + 1.023190827540606, + 0.6226417920568781, + 0.5519145939388094, + 0.4568960180345979, + 0.4213932325876586, + 0.4074494455716704, + 0.7465319443901276, + 0.6474109861274324, + 0.5764712279753077, + 0.7979703549327781, + 1.2369697451324837, + 1.2324489817728508, + 1.5808985864286207, + 2.5860289117498976, + 4.232714993086349, + 2.835646318943954, + 2.2422528664695807, + 1.1646676435019565, + 1.2175941057769888, + 0.783401544111113, + 0.5289102464485725, + 0.46735876466607296, + 0.2966943922188728, + 0.3266155865457417, + 0.3362221758114643 + ], + "peak_layer": 22, + "peak_normalized": 4.232714993086349, + "population_cv": 0.8496980089222349, + "rest_mean": 3.881239232820614e-05, + "spike_contrast": 3.7241794809927478, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014454431511694564, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.921697341662366e-05, + 4.9794718506745994e-05, + 3.477540303720161e-05, + 3.197455589543097e-05, + 1.9359915313543752e-05, + 4.296449333196506e-05, + 5.085606608190574e-05, + 5.661622344632633e-05, + 3.4452641557436436e-05, + 3.053909313166514e-05, + 2.5281429770984687e-05, + 2.3316953956964426e-05, + 2.2545402316609398e-05, + 4.1307856008643284e-05, + 3.582319550332613e-05, + 3.189788549207151e-05, + 4.415409784996882e-05, + 6.844525341875851e-05, + 6.819510599598289e-05, + 8.747586980462074e-05, + 0.0001430927513865754, + 0.00023420884099323303, + 0.00015690483269281685, + 0.00012407058966346085, + 6.444456084864214e-05, + 6.737314106430858e-05, + 4.3347961764084175e-05, + 2.9266193450894207e-05, + 2.5860364985419437e-05, + 1.641699236643035e-05, + 1.807262196962256e-05, + 1.8604183424031362e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03498003748245537, + "first_to_last_ratio": 1.1135283344223057, + "imbalance_abs_log_ratio": 0.10753365366212776, + "last_quartile_mean": 0.03141369321383536, + "mean": 0.04906986153218895, + "normalized": [ + 0.528020735774383, + 0.8999099163028887, + 0.6284749107820657, + 0.5778568640801447, + 0.34988006993508275, + 0.776471348923419, + 0.919091002023713, + 1.0231908018037408, + 0.622641782085556, + 0.5519145889381298, + 0.45689603549337854, + 0.4213932430966296, + 0.4074494429214576, + 0.7465319861987265, + 0.6474110467766695, + 0.5764712070514831, + 0.7979703658517325, + 1.236969773835328, + 1.2324490791670337, + 1.5808986244806464, + 2.5860289675304746, + 4.232714870782151, + 2.835646138092529, + 2.242252864087746, + 1.1646675839359508, + 1.2175940341247224, + 0.7834015320190452, + 0.528910268066999, + 0.467358763362216, + 0.2966943974186558, + 0.32661557776013733, + 0.33622217729716364 + ], + "peak_layer": 22, + "peak_normalized": 4.232714870782151, + "population_cv": 0.8496979833458974, + "rest_mean": 0.034419217595347655, + "spike_contrast": 3.72417933196884, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12818333879113197, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0259099043905735, + 0.0441584549844265, + 0.03083917684853077, + 0.02835535630583763, + 0.017168566584587097, + 0.03810134157538414, + 0.04509966820478439, + 0.05020783096551895, + 0.030552946031093597, + 0.027082372456789017, + 0.022419825196266174, + 0.020677708089351654, + 0.019993487745523453, + 0.036632221192121506, + 0.03176837041974068, + 0.028287362307310104, + 0.039156295359134674, + 0.06069793552160263, + 0.06047610566020012, + 0.0775744765996933, + 0.12689608335494995, + 0.2076987326145172, + 0.13914476335048676, + 0.11002703756093979, + 0.05715007707476616, + 0.05974717065691948, + 0.038441404700279236, + 0.02595355361700058, + 0.022933229804039, + 0.014558752998709679, + 0.01602698117494583, + 0.016498375684022903 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.750052800593039e-06, + "first_to_last_ratio": 0.4308304317433478, + "imbalance_abs_log_ratio": 0.8420406961220296, + "last_quartile_mean": 6.383144267374519e-06, + "mean": 4.5530042669383874e-06, + "normalized": [ + 2.0777537806616246, + 0.8353410859824805, + 0.1910970826520623, + 0.2941663520688202, + 0.18051642612757607, + 0.3343816911052641, + 0.3564217354179324, + 0.562389007550559, + 0.34578362377189115, + 0.35956396375636174, + 0.3051237753595574, + 0.33780495264365074, + 0.3038063276744268, + 0.5221079484446093, + 0.49376666786286805, + 0.5929821493588614, + 0.3686987288124583, + 0.7392110518834059, + 0.5965749796632107, + 0.9039444986896679, + 1.6537371809131796, + 3.0803086468030596, + 2.8630898340929525, + 2.485723197420275, + 1.724168620325731, + 2.41211172564755, + 1.9299601312238395, + 1.6334426636441552, + 1.3339260437132254, + 0.9558102182238872, + 0.6013349904117006, + 0.6249509180931564 + ], + "peak_layer": 22, + "peak_normalized": 3.0803086468030596, + "population_cv": 0.8333981127885269, + "rest_mean": 3.4051366684353625e-06, + "spike_contrast": 3.1574325337710922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0751489298854722e-05, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 1 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.460021828999743e-06, + 3.80331152882718e-06, + 8.700658327143174e-07, + 1.3393406561590382e-06, + 8.218920584113221e-07, + 1.5224412663883413e-06, + 1.622789682187431e-06, + 2.56055955105694e-06, + 1.5743543144708383e-06, + 1.6370962612199946e-06, + 1.389229851156415e-06, + 1.5380273907794617e-06, + 1.383231506224547e-06, + 2.3771597170707537e-06, + 2.248121745651588e-06, + 2.699850256249192e-06, + 1.678686885497882e-06, + 3.3656310733931605e-06, + 2.71620842795528e-06, + 4.1156631596095394e-06, + 7.529472441092366e-06, + 1.4024658412381541e-05, + 1.3035660231253132e-05, + 1.1317508324282244e-05, + 7.850147085264325e-06, + 1.0982354979205411e-05, + 8.787116712483112e-06, + 7.437071417371044e-06, + 6.073370968806557e-06, + 4.35180800195667e-06, + 2.737880777203827e-06, + 2.8454041967052035e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6044932951663213e-05, + "first_to_last_ratio": 1.1246393261284713, + "imbalance_abs_log_ratio": 0.11746238525662386, + "last_quartile_mean": 2.315847609679622e-05, + "mean": 3.690820258839267e-05, + "normalized": [ + 0.45689793146127605, + 0.8738188272728166, + 0.6345310201461108, + 0.5870234156151962, + 0.3959041381169584, + 0.8145991894129532, + 0.9605505534893847, + 0.9220179377165436, + 0.6534489876582346, + 0.574567881594872, + 0.47330901138441, + 0.4295574998000546, + 0.4505301728221963, + 0.8093943395842734, + 0.6650197233282829, + 0.5723463492360998, + 0.7892367771238881, + 1.2091363576952083, + 1.2417470885367035, + 1.5367390457984618, + 2.460873737770247, + 4.227774989574034, + 3.004456821926061, + 2.2368262162922274, + 1.1669129504749511, + 1.3012136399895533, + 0.7651380597325796, + 0.47754123850698527, + 0.4891606669175266, + 0.2974921422063077, + 0.24654864385331124, + 0.27568464496229084 + ], + "peak_layer": 22, + "peak_normalized": 4.227774989574034, + "population_cv": 0.8549059646786937, + "rest_mean": 2.5840054991864168e-05, + "spike_contrast": 3.7413310319999478, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.667619960964657e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.6863281416590326e-05, + 3.2251082302536815e-05, + 2.341939944017213e-05, + 2.166597914765589e-05, + 1.4612110135203693e-05, + 3.006539191119373e-05, + 3.545219442457892e-05, + 3.4030024835374206e-05, + 2.4117627617670223e-05, + 2.120626777468715e-05, + 1.7468984879087657e-05, + 1.5854195225983858e-05, + 1.6628258890705183e-05, + 2.9873290259274654e-05, + 2.4544682673877105e-05, + 2.1124275008332916e-05, + 2.9129310860298574e-05, + 4.462704964680597e-05, + 4.583065310725942e-05, + 5.671827602782287e-05, + 9.082642645807937e-05, + 0.00015603957581333816, + 0.00011088910105172545, + 8.255723514594138e-05, + 4.3068659579148516e-05, + 4.8025456635514274e-05, + 2.823987051669974e-05, + 1.7625188775127754e-05, + 1.805404099286534e-05, + 1.0979900253005326e-05, + 9.099667295231484e-06, + 1.0175024726777337e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.700510824775847e-05, + "first_to_last_ratio": 1.1234226056354075, + "imbalance_abs_log_ratio": 0.1163799234305708, + "last_quartile_mean": 1.5136875617827172e-05, + "mean": 2.416565462226572e-05, + "normalized": [ + 0.4144047676027832, + 0.875113510868426, + 0.648392503192907, + 0.5978987553197436, + 0.4472461776170949, + 0.8581546509904864, + 0.9915956817879598, + 0.7967069167230741, + 0.6805807808463835, + 0.6196150111411964, + 0.5091007553108428, + 0.44255818216184833, + 0.49857092482915655, + 0.8652430602969526, + 0.6897944860164179, + 0.5800177420993363, + 0.7901724317041942, + 1.1871172558602836, + 1.243251110659502, + 1.5081298165112624, + 2.2807123105787017, + 4.096637658310102, + 3.1477300512572848, + 2.2202178168572866, + 1.1479124533540228, + 1.3987972507734237, + 0.7581679674914413, + 0.4356681137302007, + 0.511763265149204, + 0.3118761594955322, + 0.21315449183792956, + 0.2336979396250189 + ], + "peak_layer": 22, + "peak_normalized": 4.096637658310102, + "population_cv": 0.8395518179977153, + "rest_mean": 1.710104003938829e-05, + "spike_contrast": 3.6439054716132255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.231457336980384e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.001436248770915e-05, + 2.114769085892476e-05, + 1.5668829291826114e-05, + 1.4448614820139483e-05, + 1.0807996659423225e-05, + 2.0737868908327073e-05, + 2.396255877101794e-05, + 1.9252944184700027e-05, + 1.644668009248562e-05, + 1.497340235800948e-05, + 1.2302753020776436e-05, + 1.0694708180380985e-05, + 1.2048292774125002e-05, + 2.090916495944839e-05, + 1.6669335309416056e-05, + 1.4016508430358954e-05, + 1.9095034076599404e-05, + 2.868746560125146e-05, + 3.0043976948945783e-05, + 3.644494427135214e-05, + 5.511490599019453e-05, + 9.899793076328933e-05, + 7.606695726281032e-05, + 5.365301694837399e-05, + 2.7740055884351023e-05, + 3.3802851248765364e-05, + 1.8321625248063356e-05, + 1.0528205166338012e-05, + 1.236709431395866e-05, + 7.536691555287689e-06, + 5.1510178309399635e-06, + 5.647463694913313e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.642002219654387e-05, + "first_to_last_ratio": 1.1203608181269622, + "imbalance_abs_log_ratio": 0.11365079246685211, + "last_quartile_mean": 6.821018814662239e-05, + "mean": 0.00011010971388714097, + "normalized": [ + 0.5163109854936148, + 0.8673817094155928, + 0.5924673517780332, + 0.5654643004921971, + 0.34399113467095715, + 0.7609201839652954, + 0.9076117604661024, + 0.9981345295043799, + 0.6327009050241252, + 0.5203303165972624, + 0.4437849422865529, + 0.41423566265400885, + 0.39884159677379066, + 0.7686876572920235, + 0.6339609024838357, + 0.5703063832410542, + 0.781268072457197, + 1.1797447140196398, + 1.2760717444836593, + 1.5599326259342914, + 2.6126342040159805, + 4.419910912775975, + 2.9287458133972417, + 2.3507635425478965, + 1.1736586910474214, + 1.2201215929679614, + 0.7838637940961916, + 0.4869646019277815, + 0.46617791476107356, + 0.26920984270675297, + 0.25672408594104645, + 0.2990775247810641 + ], + "peak_layer": 22, + "peak_normalized": 4.419910912775975, + "population_cv": 0.8912016880930539, + "rest_mean": 7.550380838370916e-05, + "spike_contrast": 3.9333327677514873, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002969816036056727, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.685085488948971e-05, + 9.550715185469016e-05, + 6.523641059175134e-05, + 6.226311234058812e-05, + 3.787676541833207e-05, + 8.378470374736935e-05, + 9.993687126552686e-05, + 0.00010990430746460333, + 6.966651562834159e-05, + 5.729342228733003e-05, + 4.886503302259371e-05, + 4.561137029668316e-05, + 4.3916334107052535e-05, + 8.463997801300138e-05, + 6.980525358812883e-05, + 6.279627268668264e-05, + 8.602520392742008e-05, + 0.00012990135292056948, + 0.0001405078946845606, + 0.00017176373512484133, + 0.0002876764046959579, + 0.0004866751260124147, + 0.0003224833635613322, + 0.0002588419010862708, + 0.00012923122267238796, + 0.0001343472395092249, + 8.631101809442043e-05, + 5.3619532991433516e-05, + 5.13307168148458e-05, + 2.9642618756042793e-05, + 2.8267815650906414e-05, + 3.293134068371728e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059514161199331284, + 0.08157797157764435, + 0.14292451739311218, + 0.10997853428125381, + 0.0991024449467659, + 0.1065012738108635, + 0.12155740708112717, + 0.1095810979604721, + 0.06964563578367233, + 0.08311934769153595, + 0.09363401681184769, + 0.0941167026758194, + 0.08564268052577972, + 0.09493856877088547, + 0.09063401818275452, + 0.09473779052495956, + 0.10505300015211105, + 0.09355910867452621, + 0.09618847072124481, + 0.11416846513748169, + 0.047205954790115356, + 0.053352247923612595, + 0.06512638926506042, + 0.07829347252845764, + 0.05368548259139061, + 0.05310077592730522, + 0.061235878616571426, + 0.07320034503936768, + 0.05304349213838577, + 0.0639304369688034, + 0.07404901832342148, + 0.08157160878181458 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.6955231027823174e-06, + "first_to_last_ratio": 0.14308275980953641, + "imbalance_abs_log_ratio": 1.9443320762007668, + "last_quartile_mean": 3.2816833481774665e-05, + "mean": 1.7864957968072304e-05, + "normalized": [ + 0.5367890577532618, + 0.2021050510263675, + 0.1411976282823933, + 0.2660218188543664, + 0.19890690834720978, + 0.16151559172232732, + 0.2167464101659875, + 0.3793916565295996, + 0.424775345287454, + 0.3093373886676644, + 0.2783613720705816, + 0.2825637413001494, + 0.3733172205433837, + 0.3886765425479794, + 0.44060863402002287, + 0.44323494933287805, + 0.3667341230574315, + 0.45728447370296305, + 0.42556973592368313, + 0.5658685710706646, + 2.6227066493164064, + 3.0085127149589037, + 2.58733070947387, + 2.226933740479959, + 2.8743975419651484, + 2.84666799381699, + 2.39504866979417, + 1.5908588860103314, + 1.8185955715005, + 1.4035945808786974, + 0.9127606447015644, + 0.8535860768970909 + ], + "peak_layer": 22, + "peak_normalized": 3.0085127149589037, + "population_cv": 0.9632431009960319, + "rest_mean": 1.2359982755933069e-05, + "spike_contrast": 3.850476578600324, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.759182411362417e-05, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.589713954483159e-06, + 3.6105982417211635e-06, + 2.5224896944564534e-06, + 4.7524686124233995e-06, + 3.5534635571821127e-06, + 2.8854692573077045e-06, + 3.872165507345926e-06, + 6.777815997338621e-06, + 7.588593689433765e-06, + 5.526299446501071e-06, + 4.972914211975876e-06, + 5.047989361628424e-06, + 6.669296453765128e-06, + 6.9436900957953185e-06, + 7.871454727137461e-06, + 7.918373739812523e-06, + 6.5516896938788705e-06, + 8.1693679021555e-06, + 7.6027854447602294e-06, + 1.0109218237630557e-05, + 4.685454405262135e-05, + 5.3746953199151903e-05, + 4.622255437425338e-05, + 3.97840776713565e-05, + 5.135099127073772e-05, + 5.085560405859724e-05, + 4.278744381736033e-05, + 2.8420627131708898e-05, + 3.248913344577886e-05, + 2.507515819161199e-05, + 1.6306430552504025e-05, + 1.5249279385898262e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.8878105947806034e-05, + "first_to_last_ratio": 0.1437388462831867, + "imbalance_abs_log_ratio": 1.939757193388504, + "last_quartile_mean": 0.00013133614493199275, + "mean": 7.144442986373178e-05, + "normalized": [ + 0.543944422065902, + 0.20498362677674017, + 0.14117605251679072, + 0.2675823635170904, + 0.19429426405696296, + 0.15941273479469137, + 0.21634544427316113, + 0.386139638870723, + 0.42865458135412043, + 0.3129364237141443, + 0.2788876084512782, + 0.2815826978190512, + 0.3736543391719335, + 0.3909795972451435, + 0.4389936626557141, + 0.44232523153890874, + 0.3688366584716598, + 0.45947150441297413, + 0.42734480687548726, + 0.5800824758005463, + 2.6187902433353907, + 2.987231504916195, + 2.580721746390286, + 2.2092452217225382, + 2.879951595578818, + 2.8406783220158585, + 2.397028447242086, + 1.5906421267626043, + 1.8247593500794728, + 1.4018234858691145, + 0.9198949147470885, + 0.8516049069575247 + ], + "peak_layer": 22, + "peak_normalized": 2.987231504916195, + "population_cv": 0.9601858610516129, + "rest_mean": 4.954554700547036e-05, + "spike_contrast": 3.828767846228416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00018969839729834347, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.886179911205545e-05, + 1.4644938346464187e-05, + 1.008624258247437e-05, + 1.911726940306835e-05, + 1.3881242921343073e-05, + 1.1389151950425003e-05, + 1.5456676919711754e-05, + 2.7587526346906088e-05, + 3.062498217332177e-05, + 2.2357564375852235e-05, + 1.9924966181861237e-05, + 2.0117515305173583e-05, + 2.6695521228248253e-05, + 2.7933314413530752e-05, + 3.13636519422289e-05, + 3.160167398164049e-05, + 2.6351324777351692e-05, + 3.282667967141606e-05, + 3.0531406082445756e-05, + 4.144366175751202e-05, + 0.00018709797586780041, + 0.00021342105173971504, + 0.0001843781938077882, + 0.00015783826529514045, + 0.0002057564997812733, + 0.0002029506431426853, + 0.00017125433078035712, + 0.00011364251986378804, + 0.00013036889140494168, + 0.00010015247971750796, + 6.572136771865189e-05, + 6.08424270467367e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.016741293016821146, + "first_to_last_ratio": 0.14373884545887533, + "imbalance_abs_log_ratio": 1.9397571991232891, + "last_quartile_mean": 0.11647020652890205, + "mean": 0.06335763493552804, + "normalized": [ + 0.5439444372437541, + 0.20498362168576487, + 0.14117604516877305, + 0.2675823445899417, + 0.1942942529879486, + 0.15941273039509152, + 0.2163454315588509, + 0.38613966530889043, + 0.4286545697273512, + 0.3129364189494029, + 0.278887605667477, + 0.2815826915705289, + 0.3736543321474427, + 0.3909795777927573, + 0.43899363576159356, + 0.44232520797796643, + 0.3688366746353626, + 0.45947150248290297, + 0.4273447896704886, + 0.580082454376481, + 2.6187903327724635, + 2.987231533325295, + 2.5807218070831297, + 2.2092452282911403, + 2.8799516329199686, + 2.84067822247012, + 2.397028505583422, + 1.5906421518499794, + 1.8247592808395534, + 1.4018234450909468, + 0.9198949415397243, + 0.8516049285354858 + ], + "peak_layer": 22, + "peak_normalized": 2.987231533325295, + "population_cv": 0.9601858709388058, + "rest_mean": 0.04393748607900408, + "spike_contrast": 3.8287679558695995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.16822643876075744, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.03446303308010101, + 0.01298727747052908, + 0.00894458033144474, + 0.01695338450372219, + 0.012310024350881577, + 0.010100013576447964, + 0.013707134872674942, + 0.024464895948767662, + 0.02715853974223137, + 0.01982691138982773, + 0.017669659107923508, + 0.017840413376688957, + 0.023673854768276215, + 0.024771541357040405, + 0.02781359851360321, + 0.02802467904984951, + 0.02336861938238144, + 0.029111027717590332, + 0.02707555517554283, + 0.03675265237689018, + 0.16592036187648773, + 0.18926392495632172, + 0.16350843012332916, + 0.13997255265712738, + 0.18246692419052124, + 0.1799786537885666, + 0.15187005698680878, + 0.10077932476997375, + 0.11561243236064911, + 0.08881621807813644, + 0.05828236788511276, + 0.053955674171447754 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0825735472508313e-06, + "first_to_last_ratio": 0.12187576812955331, + "imbalance_abs_log_ratio": 2.104753047082524, + "last_quartile_mean": 8.882598763193528e-06, + "mean": 4.656443639561303e-06, + "normalized": [ + 0.5462983207096154, + 0.1799290732737773, + 0.13161762512632208, + 0.2712494331106157, + 0.1613911681124062, + 0.13237661267085615, + 0.15721616286614848, + 0.2798363906909718, + 0.34662434040104295, + 0.2682122377700395, + 0.24445710198520895, + 0.24681111820039023, + 0.3364691861798734, + 0.3619231630019147, + 0.42744951901080175, + 0.43112099033231693, + 0.36249737820633765, + 0.4368387762566429, + 0.42578881412518377, + 0.38711372693052853, + 2.644134376023103, + 3.0274176414789977, + 2.680305221816438, + 2.2521785297578925, + 2.965193895729998, + 2.901136489590974, + 2.603841896063781, + 1.6862657755052308, + 1.872923926339135, + 1.3894795988927409, + 0.9615945171774488, + 0.8803069926632656 + ], + "peak_layer": 22, + "peak_normalized": 3.0274176414789977, + "population_cv": 1.0082602587575777, + "rest_mean": 3.1785867888713795e-06, + "spike_contrast": 3.9756254815914156, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2636870633286889e-05, + "top_five_layers": [ + 22, + 25, + 26, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.543807340771309e-06, + 8.3782958881784e-07, + 6.128700533736264e-07, + 1.2630576975425356e-06, + 7.515088782383827e-07, + 6.164042360978783e-07, + 7.32068201614311e-07, + 1.3030423815507675e-06, + 1.6140367051775684e-06, + 1.2489151686168043e-06, + 1.138300717684615e-06, + 1.14926206151722e-06, + 1.5667498018956394e-06, + 1.6852748103701742e-06, + 1.990394594031386e-06, + 2.0074905933142873e-06, + 1.687948611106549e-06, + 2.034115141213988e-06, + 1.9826616153295618e-06, + 1.8025732515525306e-06, + 1.2312262697378173e-05, + 1.409699962096056e-05, + 1.2480690202210099e-05, + 1.0487142390047666e-05, + 1.3807258255837951e-05, + 1.3508978554455098e-05, + 1.2124643035349436e-05, + 7.85200154496124e-06, + 8.721164704184048e-06, + 6.470033440564293e-06, + 4.477610673347954e-06, + 4.099099896848202e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.2557812397062662e-05, + "first_to_last_ratio": 0.15959566659574662, + "imbalance_abs_log_ratio": 1.8351117459907644, + "last_quartile_mean": 7.868517150200205e-05, + "mean": 4.5438396199415365e-05, + "normalized": [ + 0.5876869810338485, + 0.2186593308079107, + 0.115659623677568, + 0.24728992953724943, + 0.2138823134903866, + 0.17309343907534852, + 0.22828222046505, + 0.4264066618299725, + 0.47941738273017553, + 0.34947398487973863, + 0.31642185290041636, + 0.3326608895247323, + 0.42753132054545495, + 0.43329608717407087, + 0.4807098955780001, + 0.491094994316893, + 0.41296023678331367, + 0.5280992844682807, + 0.4827771475203192, + 0.6138123378227869, + 2.7117823469683264, + 3.061642080099734, + 2.646250140140893, + 2.167597408443836, + 2.9446942269972274, + 2.694165709502317, + 2.239316799454017, + 1.5139318638986488, + 1.6066586018929343, + 1.2634426578175135, + 0.8290147862338136, + 0.7622874643892231 + ], + "peak_layer": 22, + "peak_normalized": 3.061642080099734, + "population_cv": 0.9412918554701797, + "rest_mean": 3.107991987808082e-05, + "spike_contrast": 3.9567080229621103, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012297416833462194, + "top_five_layers": [ + 22, + 25, + 21, + 26, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.6703553885454312e-05, + 9.935529305948876e-06, + 5.255387804936618e-06, + 1.1236457794439048e-05, + 9.718469300423749e-06, + 7.86508826422505e-06, + 1.0372777978773229e-05, + 1.9375234842300415e-05, + 2.178395698138047e-05, + 1.587953738635406e-05, + 1.4377701518242247e-05, + 1.511557729827473e-05, + 1.9426337530603632e-05, + 1.968827928067185e-05, + 2.1842686692252755e-05, + 2.231456892332062e-05, + 1.876425085356459e-05, + 2.3995984520297498e-05, + 2.1936619305051863e-05, + 2.789064819808118e-05, + 0.00012321904068812728, + 0.0001391161058563739, + 0.00012024136231048033, + 9.849214984569699e-05, + 0.00013380218297243118, + 0.00012241856893524528, + 0.00010175096394959837, + 6.879063585074618e-05, + 7.300399011000991e-05, + 5.740880806115456e-05, + 3.766910231206566e-05, + 3.463711982476525e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.367274347165221e-06, + "first_to_last_ratio": 0.1920014554991945, + "imbalance_abs_log_ratio": 1.6502523262581177, + "last_quartile_mean": 4.3579223529377487e-05, + "mean": 2.787530080183842e-05, + "normalized": [ + 0.6522901960458961, + 0.23732362933486428, + 0.10383521258256681, + 0.24073339013803968, + 0.24132146388193534, + 0.19483084771733855, + 0.2475207738927628, + 0.483488789493259, + 0.550030749638162, + 0.4005903921146771, + 0.3626794403805512, + 0.398664404925254, + 0.5057551655640742, + 0.49633936232785464, + 0.5444163811542189, + 0.5645330449101175, + 0.481087325236049, + 0.6303807295780811, + 0.5534160889368265, + 0.6675682370643486, + 2.937265214456223, + 3.231070824381384, + 2.7157464748293116, + 2.0522050942602807, + 2.8791759008165254, + 2.406039643209466, + 1.9404796965168836, + 1.3618287054639213, + 1.3825321378732942, + 1.123237294980051, + 0.7355691904186271, + 0.6780441978771551 + ], + "peak_layer": 22, + "peak_normalized": 3.231070824381384, + "population_cv": 0.9137826743997257, + "rest_mean": 1.8774052764288002e-05, + "spike_contrast": 4.1025792976954865, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.702204020461067e-05, + "top_five_layers": [ + 22, + 21, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.8182785424869508e-05, + 6.615467555093346e-06, + 2.8944377845618874e-06, + 6.710515663144179e-06, + 6.726908395648934e-06, + 5.430968485597987e-06, + 6.899716026964597e-06, + 1.3477395441441331e-05, + 1.5332272596424446e-05, + 1.1166577678523026e-05, + 1.0109798495250288e-05, + 1.111289020627737e-05, + 1.409807737218216e-05, + 1.3835609024681617e-05, + 1.5175770386122167e-05, + 1.5736528439447284e-05, + 1.3410453902906738e-05, + 1.7572052456671372e-05, + 1.5426639947691e-05, + 1.86086654139217e-05, + 8.187715138774365e-05, + 9.006707114167511e-05, + 7.570224988739938e-05, + 5.720583430957049e-05, + 8.025789429666474e-05, + 6.706907879561186e-05, + 5.409145524026826e-05, + 3.796138480538502e-05, + 3.8538499211426824e-05, + 3.131057746941224e-05, + 2.0504212443483993e-05, + 1.890068597276695e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.742885974133969e-05, + "first_to_last_ratio": 0.13285814939480156, + "imbalance_abs_log_ratio": 2.018473265828943, + "last_quartile_mean": 0.00028172046586405486, + "mean": 0.00014909372896454443, + "normalized": [ + 0.5351570886371335, + 0.1998816464903299, + 0.1134003859562838, + 0.2373819769757024, + 0.18971959520637977, + 0.1492636791238371, + 0.21004413698536548, + 0.3734913450927867, + 0.42927632587153325, + 0.3024206473119055, + 0.27462175768495817, + 0.27240928064339276, + 0.3512855646324889, + 0.3655094888356237, + 0.41973324501001746, + 0.42402062842094873, + 0.360798486332806, + 0.4559662563090299, + 0.41985007515873884, + 0.5597866310716989, + 2.4844541141001115, + 2.96085466615825, + 2.5443412288142064, + 2.249909733543605, + 2.9984626521607822, + 2.895026684700318, + 2.601768175835175, + 1.7136369023458127, + 1.8313353225818492, + 1.363588302533558, + 0.8854239763588411, + 0.82717999911653 + ], + "peak_layer": 25, + "peak_normalized": 2.9984626521607822, + "population_cv": 0.9800748961570518, + "rest_mean": 0.00010360345199673989, + "spike_contrast": 3.810116525877056, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003947412245906889, + "top_five_layers": [ + 25, + 22, + 26, + 27, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.978856592671946e-05, + 2.980110002681613e-05, + 1.6907286408240907e-05, + 3.53921641362831e-05, + 2.8286001906963065e-05, + 2.2254278519540094e-05, + 3.131626363028772e-05, + 5.568521737586707e-05, + 6.400240818038583e-05, + 4.508902202360332e-05, + 4.0944381908047944e-05, + 4.061451545567252e-05, + 5.2374474762473255e-05, + 5.449517266242765e-05, + 6.257959466893226e-05, + 6.321881664916873e-05, + 5.379279173212126e-05, + 6.79817094351165e-05, + 6.259701331146061e-05, + 8.34606762509793e-05, + 0.0003704165283124894, + 0.00044144486309960485, + 0.0003793453215621412, + 0.0003354474320076406, + 0.0004470519779715687, + 0.0004316303238738328, + 0.00038790731923654675, + 0.00025549251586198807, + 0.00027304061222821474, + 0.00020330246479716152, + 0.00013201116234995425, + 0.00012332735059317201 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 1.8740129107275643, + "forward": { + "activation_sha256": { + "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" + }, + "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", + "loss_nats": 1.2989667654037476, + "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" + }, + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22886337339878082, + "group": 1, + "latest_source_mass": 0.06350618600845337, + "layer": 1, + "max_source_mass": 0.9364938139915466, + "mixer_index": 1, + "normalized_entropy": 0.3301800538435402, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9364938139915466, + "median": 0.9452826976776123, + "p05": 0.8725647330284119, + "p95": 0.9712384939193726 + }, + { + "label": "current_group_partial", + "mean": 0.06350618600845337, + "median": 0.0547172985970974, + "p05": 0.028761589899659157, + "p95": 0.12743529677391052 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.603729784488678, + "group": 1, + "latest_source_mass": 0.4133155345916748, + "layer": 2, + "max_source_mass": 0.5866844654083252, + "mixer_index": 2, + "normalized_entropy": 0.8709979661187783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5866844654083252, + "median": 0.5734395384788513, + "p05": 0.29813364148139954, + "p95": 0.8843642473220825 + }, + { + "label": "current_group_partial", + "mean": 0.4133155345916748, + "median": 0.4265604615211487, + "p05": 0.11563576757907867, + "p95": 0.7018663287162781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3967982828617096, + "group": 1, + "latest_source_mass": 0.14110636711120605, + "layer": 2, + "max_source_mass": 0.858893632888794, + "mixer_index": 3, + "normalized_entropy": 0.5724589149178446, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.858893632888794, + "median": 0.8705973029136658, + "p05": 0.7599387168884277, + "p95": 0.9270997643470764 + }, + { + "label": "current_group_partial", + "mean": 0.14110636711120605, + "median": 0.12940272688865662, + "p05": 0.07290027290582657, + "p95": 0.2400612235069275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6852496862411499, + "group": 1, + "latest_source_mass": 0.518975019454956, + "layer": 3, + "max_source_mass": 0.518975019454956, + "mixer_index": 4, + "normalized_entropy": 0.9886063241108252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48102498054504395, + "median": 0.48102718591690063, + "p05": 0.38786420226097107, + "p95": 0.580143928527832 + }, + { + "label": "current_group_partial", + "mean": 0.518975019454956, + "median": 0.5189728140830994, + "p05": 0.41985607147216797, + "p95": 0.6121357679367065 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5890734195709229, + "group": 1, + "latest_source_mass": 0.3123423159122467, + "layer": 3, + "max_source_mass": 0.6876576542854309, + "mixer_index": 5, + "normalized_entropy": 0.8498533011344741, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6876576542854309, + "median": 0.6909762620925903, + "p05": 0.48803940415382385, + "p95": 0.8900076746940613 + }, + { + "label": "current_group_partial", + "mean": 0.3123423159122467, + "median": 0.3090237081050873, + "p05": 0.10999235510826111, + "p95": 0.5119606256484985 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6873323321342468, + "group": 1, + "latest_source_mass": 0.4831485152244568, + "layer": 4, + "max_source_mass": 0.5168514251708984, + "mixer_index": 6, + "normalized_entropy": 0.9916109470127238, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5168514251708984, + "median": 0.5219584703445435, + "p05": 0.4251725673675537, + "p95": 0.5947866439819336 + }, + { + "label": "current_group_partial", + "mean": 0.4831485152244568, + "median": 0.47804147005081177, + "p05": 0.405213326215744, + "p95": 0.5748274326324463 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.487301766872406, + "group": 1, + "latest_source_mass": 0.2060384750366211, + "layer": 4, + "max_source_mass": 0.7939615249633789, + "mixer_index": 7, + "normalized_entropy": 0.7030278424832499, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7939615249633789, + "median": 0.7953404188156128, + "p05": 0.6590759754180908, + "p95": 0.9329437017440796 + }, + { + "label": "current_group_partial", + "mean": 0.2060384750366211, + "median": 0.20465964078903198, + "p05": 0.06705628335475922, + "p95": 0.3409239649772644 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674697756767273, + "group": 2, + "latest_source_mass": 0.42952626943588257, + "layer": 5, + "max_source_mass": 0.5704737305641174, + "mixer_index": 8, + "normalized_entropy": 0.9733831077870527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5704737305641174, + "median": 0.5696132779121399, + "p05": 0.4695202708244324, + "p95": 0.6750621199607849 + }, + { + "label": "completed_group_1", + "mean": 0.42952626943588257, + "median": 0.4303867816925049, + "p05": 0.3249378800392151, + "p95": 0.5304797887802124 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9691112041473389, + "group": 2, + "latest_source_mass": 0.2681158185005188, + "layer": 5, + "max_source_mass": 0.5506307482719421, + "mixer_index": 9, + "normalized_entropy": 0.8821230329784768, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5506307482719421, + "median": 0.543993353843689, + "p05": 0.41262108087539673, + "p95": 0.7161091566085815 + }, + { + "label": "completed_group_1", + "mean": 0.18125343322753906, + "median": 0.1794438362121582, + "p05": 0.11575750261545181, + "p95": 0.25031501054763794 + }, + { + "label": "current_group_partial", + "mean": 0.2681158185005188, + "median": 0.2664111256599426, + "p05": 0.1564716249704361, + "p95": 0.3817881643772125 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0745112895965576, + "group": 2, + "latest_source_mass": 0.412794291973114, + "layer": 6, + "max_source_mass": 0.412794291973114, + "mixer_index": 10, + "normalized_entropy": 0.9780623252441762, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34262996912002563, + "median": 0.3412924110889435, + "p05": 0.29556700587272644, + "p95": 0.39125555753707886 + }, + { + "label": "completed_group_1", + "mean": 0.24457570910453796, + "median": 0.24382859468460083, + "p05": 0.22120657563209534, + "p95": 0.2734297513961792 + }, + { + "label": "current_group_partial", + "mean": 0.412794291973114, + "median": 0.4151499271392822, + "p05": 0.37735897302627563, + "p95": 0.43749260902404785 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074045658111572, + "group": 2, + "latest_source_mass": 0.3775783181190491, + "layer": 6, + "max_source_mass": 0.4399106502532959, + "mixer_index": 11, + "normalized_entropy": 0.9169791528842925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4399106502532959, + "median": 0.44856905937194824, + "p05": 0.22063353657722473, + "p95": 0.6359403133392334 + }, + { + "label": "completed_group_1", + "mean": 0.18251106142997742, + "median": 0.18562838435173035, + "p05": 0.1196163073182106, + "p95": 0.23045453429222107 + }, + { + "label": "current_group_partial", + "mean": 0.3775783181190491, + "median": 0.3675365447998047, + "p05": 0.23817461729049683, + "p95": 0.5557451844215393 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.060497522354126, + "group": 2, + "latest_source_mass": 0.4025024175643921, + "layer": 7, + "max_source_mass": 0.4025024175643921, + "mixer_index": 12, + "normalized_entropy": 0.9653064445872968, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3786153197288513, + "median": 0.3907395303249359, + "p05": 0.28039106726646423, + "p95": 0.4436405599117279 + }, + { + "label": "completed_group_1", + "mean": 0.2188822478055954, + "median": 0.21226292848587036, + "p05": 0.1885872334241867, + "p95": 0.2672030031681061 + }, + { + "label": "current_group_partial", + "mean": 0.4025024175643921, + "median": 0.39815032482147217, + "p05": 0.3533596992492676, + "p95": 0.460100919008255 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9941019415855408, + "group": 2, + "latest_source_mass": 0.4801652431488037, + "layer": 7, + "max_source_mass": 0.4801652431488037, + "mixer_index": 13, + "normalized_entropy": 0.9048705824970601, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32296955585479736, + "median": 0.2803739905357361, + "p05": 0.1784922033548355, + "p95": 0.6440245509147644 + }, + { + "label": "completed_group_1", + "mean": 0.19686520099639893, + "median": 0.2053825855255127, + "p05": 0.09813719242811203, + "p95": 0.2504696547985077 + }, + { + "label": "current_group_partial", + "mean": 0.4801652431488037, + "median": 0.5057140588760376, + "p05": 0.2529003322124481, + "p95": 0.598870575428009 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0503522157669067, + "group": 2, + "latest_source_mass": 0.4001518487930298, + "layer": 8, + "max_source_mass": 0.40130966901779175, + "mixer_index": 14, + "normalized_entropy": 0.9560717885654542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40130966901779175, + "median": 0.396759569644928, + "p05": 0.3479611873626709, + "p95": 0.4733521044254303 + }, + { + "label": "completed_group_1", + "mean": 0.19853851199150085, + "median": 0.19769755005836487, + "p05": 0.17136667668819427, + "p95": 0.22788912057876587 + }, + { + "label": "current_group_partial", + "mean": 0.4001518487930298, + "median": 0.40291696786880493, + "p05": 0.3506356179714203, + "p95": 0.43941178917884827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0129402875900269, + "group": 2, + "latest_source_mass": 0.42161279916763306, + "layer": 8, + "max_source_mass": 0.42161279916763306, + "mixer_index": 15, + "normalized_entropy": 0.9220179839951123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4093267023563385, + "median": 0.4046899676322937, + "p05": 0.2895161807537079, + "p95": 0.5762131810188293 + }, + { + "label": "completed_group_1", + "mean": 0.16906049847602844, + "median": 0.16888955235481262, + "p05": 0.11233551055192947, + "p95": 0.22477403283119202 + }, + { + "label": "current_group_partial", + "mean": 0.42161279916763306, + "median": 0.4227708578109741, + "p05": 0.3131049573421478, + "p95": 0.519075334072113 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0774019956588745, + "group": 3, + "latest_source_mass": 0.3560434877872467, + "layer": 9, + "max_source_mass": 0.38510820269584656, + "mixer_index": 16, + "normalized_entropy": 0.9806935592947451, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38510820269584656, + "median": 0.3892287313938141, + "p05": 0.2934730052947998, + "p95": 0.4718902111053467 + }, + { + "label": "completed_group_1", + "mean": 0.25884830951690674, + "median": 0.25734007358551025, + "p05": 0.22217319905757904, + "p95": 0.29551801085472107 + }, + { + "label": "completed_group_2", + "mean": 0.3560434877872467, + "median": 0.35412120819091797, + "p05": 0.29444968700408936, + "p95": 0.41908568143844604 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2740377187728882, + "group": 3, + "latest_source_mass": 0.3633861243724823, + "layer": 9, + "max_source_mass": 0.3633861243724823, + "mixer_index": 17, + "normalized_entropy": 0.9190239493895668, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29550763964653015, + "median": 0.30360478162765503, + "p05": 0.16904757916927338, + "p95": 0.45146965980529785 + }, + { + "label": "completed_group_1", + "mean": 0.09541244804859161, + "median": 0.09486871212720871, + "p05": 0.061360105872154236, + "p95": 0.13037073612213135 + }, + { + "label": "completed_group_2", + "mean": 0.24569380283355713, + "median": 0.24333015084266663, + "p05": 0.1698218435049057, + "p95": 0.31589236855506897 + }, + { + "label": "current_group_partial", + "mean": 0.3633861243724823, + "median": 0.36472851037979126, + "p05": 0.2795379161834717, + "p95": 0.4399876892566681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.360302448272705, + "group": 3, + "latest_source_mass": 0.28569501638412476, + "layer": 10, + "max_source_mass": 0.3109527826309204, + "mixer_index": 18, + "normalized_entropy": 0.9812507981160736, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3109527826309204, + "median": 0.32172977924346924, + "p05": 0.23520725965499878, + "p95": 0.37052470445632935 + }, + { + "label": "completed_group_1", + "mean": 0.20984819531440735, + "median": 0.20958667993545532, + "p05": 0.1891367882490158, + "p95": 0.23184512555599213 + }, + { + "label": "completed_group_2", + "mean": 0.19350400567054749, + "median": 0.19153150916099548, + "p05": 0.16607457399368286, + "p95": 0.22877082228660583 + }, + { + "label": "current_group_partial", + "mean": 0.28569501638412476, + "median": 0.28060320019721985, + "p05": 0.23824527859687805, + "p95": 0.3439165949821472 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2722704410552979, + "group": 3, + "latest_source_mass": 0.29912811517715454, + "layer": 10, + "max_source_mass": 0.35036465525627136, + "mixer_index": 19, + "normalized_entropy": 0.9177491279900463, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35036465525627136, + "median": 0.36210548877716064, + "p05": 0.12807878851890564, + "p95": 0.5797606706619263 + }, + { + "label": "completed_group_1", + "mean": 0.12631990015506744, + "median": 0.1264108121395111, + "p05": 0.06977735459804535, + "p95": 0.1755298376083374 + }, + { + "label": "completed_group_2", + "mean": 0.22418735921382904, + "median": 0.22563719749450684, + "p05": 0.1176326721906662, + "p95": 0.32001155614852905 + }, + { + "label": "current_group_partial", + "mean": 0.29912811517715454, + "median": 0.28589946031570435, + "p05": 0.193109393119812, + "p95": 0.44119665026664734 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3558036088943481, + "group": 3, + "latest_source_mass": 0.314197838306427, + "layer": 11, + "max_source_mass": 0.314197838306427, + "mixer_index": 20, + "normalized_entropy": 0.978005571485618, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298976868391037, + "median": 0.2961939573287964, + "p05": 0.24950163066387177, + "p95": 0.35835936665534973 + }, + { + "label": "completed_group_1", + "mean": 0.1941717267036438, + "median": 0.1918032318353653, + "p05": 0.16809208691120148, + "p95": 0.22895419597625732 + }, + { + "label": "completed_group_2", + "mean": 0.1926535815000534, + "median": 0.19306811690330505, + "p05": 0.16983944177627563, + "p95": 0.2155935913324356 + }, + { + "label": "current_group_partial", + "mean": 0.314197838306427, + "median": 0.31541913747787476, + "p05": 0.2673022747039795, + "p95": 0.35941022634506226 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2696614265441895, + "group": 3, + "latest_source_mass": 0.2862822115421295, + "layer": 11, + "max_source_mass": 0.3667730689048767, + "mixer_index": 21, + "normalized_entropy": 0.9158671218416545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3667730689048767, + "median": 0.4067659378051758, + "p05": 0.10593366622924805, + "p95": 0.5672274827957153 + }, + { + "label": "completed_group_1", + "mean": 0.15310388803482056, + "median": 0.15050218999385834, + "p05": 0.08651046454906464, + "p95": 0.22459912300109863 + }, + { + "label": "completed_group_2", + "mean": 0.19384080171585083, + "median": 0.18732509016990662, + "p05": 0.1267009675502777, + "p95": 0.281107634305954 + }, + { + "label": "current_group_partial", + "mean": 0.2862822115421295, + "median": 0.26271265745162964, + "p05": 0.17024393379688263, + "p95": 0.475044310092926 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3606936931610107, + "group": 3, + "latest_source_mass": 0.3028184771537781, + "layer": 12, + "max_source_mass": 0.3028184771537781, + "mixer_index": 22, + "normalized_entropy": 0.9815330216461395, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298227459192276, + "median": 0.2954302132129669, + "p05": 0.25287362933158875, + "p95": 0.35405808687210083 + }, + { + "label": "completed_group_1", + "mean": 0.18611131608486176, + "median": 0.1850002110004425, + "p05": 0.15901757776737213, + "p95": 0.21725836396217346 + }, + { + "label": "completed_group_2", + "mean": 0.21284276247024536, + "median": 0.21402311325073242, + "p05": 0.18819917738437653, + "p95": 0.23418807983398438 + }, + { + "label": "current_group_partial", + "mean": 0.3028184771537781, + "median": 0.3041122555732727, + "p05": 0.25661808252334595, + "p95": 0.34458184242248535 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.269316554069519, + "group": 3, + "latest_source_mass": 0.2653314769268036, + "layer": 12, + "max_source_mass": 0.4081852436065674, + "mixer_index": 23, + "normalized_entropy": 0.9156183489371815, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4081852436065674, + "median": 0.4254342317581177, + "p05": 0.20765496790409088, + "p95": 0.587238073348999 + }, + { + "label": "completed_group_1", + "mean": 0.14420127868652344, + "median": 0.1438048928976059, + "p05": 0.1064920499920845, + "p95": 0.1871720254421234 + }, + { + "label": "completed_group_2", + "mean": 0.18228203058242798, + "median": 0.17507661879062653, + "p05": 0.12183455377817154, + "p95": 0.26505547761917114 + }, + { + "label": "current_group_partial", + "mean": 0.2653314769268036, + "median": 0.24924281239509583, + "p05": 0.15609969198703766, + "p95": 0.4061311185359955 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3636136054992676, + "group": 4, + "latest_source_mass": 0.3136237859725952, + "layer": 13, + "max_source_mass": 0.3136237859725952, + "mixer_index": 24, + "normalized_entropy": 0.9836392931712563, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27427661418914795, + "median": 0.27403637766838074, + "p05": 0.22928935289382935, + "p95": 0.319773405790329 + }, + { + "label": "completed_group_1", + "mean": 0.21341991424560547, + "median": 0.20697325468063354, + "p05": 0.18552586436271667, + "p95": 0.26688331365585327 + }, + { + "label": "completed_group_2", + "mean": 0.19867971539497375, + "median": 0.197545126080513, + "p05": 0.1794913411140442, + "p95": 0.222100168466568 + }, + { + "label": "completed_group_3", + "mean": 0.3136237859725952, + "median": 0.31875866651535034, + "p05": 0.24156072735786438, + "p95": 0.3695686459541321 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.570773959159851, + "group": 4, + "latest_source_mass": 0.2294243425130844, + "layer": 13, + "max_source_mass": 0.2658740282058716, + "mixer_index": 25, + "normalized_entropy": 0.9759767351225285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2658740282058716, + "median": 0.2747163474559784, + "p05": 0.17927880585193634, + "p95": 0.3512483835220337 + }, + { + "label": "completed_group_1", + "mean": 0.13419361412525177, + "median": 0.13374894857406616, + "p05": 0.11072897166013718, + "p95": 0.1586270034313202 + }, + { + "label": "completed_group_2", + "mean": 0.1907704919576645, + "median": 0.19074805080890656, + "p05": 0.14699944853782654, + "p95": 0.23316305875778198 + }, + { + "label": "completed_group_3", + "mean": 0.17973749339580536, + "median": 0.17616114020347595, + "p05": 0.1363743096590042, + "p95": 0.23202598094940186 + }, + { + "label": "current_group_partial", + "mean": 0.2294243425130844, + "median": 0.22995705902576447, + "p05": 0.17975632846355438, + "p95": 0.2789157032966614 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5560038089752197, + "group": 4, + "latest_source_mass": 0.32468360662460327, + "layer": 14, + "max_source_mass": 0.32468360662460327, + "mixer_index": 26, + "normalized_entropy": 0.9667995248241249, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20477107167243958, + "median": 0.206403911113739, + "p05": 0.16619175672531128, + "p95": 0.2431035190820694 + }, + { + "label": "completed_group_1", + "mean": 0.15721669793128967, + "median": 0.15658551454544067, + "p05": 0.13878680765628815, + "p95": 0.17778633534908295 + }, + { + "label": "completed_group_2", + "mean": 0.13811248540878296, + "median": 0.13743364810943604, + "p05": 0.12352626770734787, + "p95": 0.15553249418735504 + }, + { + "label": "completed_group_3", + "mean": 0.1752161681652069, + "median": 0.17309701442718506, + "p05": 0.15223677456378937, + "p95": 0.20474471151828766 + }, + { + "label": "current_group_partial", + "mean": 0.32468360662460327, + "median": 0.32497870922088623, + "p05": 0.2946181893348694, + "p95": 0.35203880071640015 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5659973621368408, + "group": 4, + "latest_source_mass": 0.26703086495399475, + "layer": 14, + "max_source_mass": 0.26703086495399475, + "mixer_index": 27, + "normalized_entropy": 0.9730088685238187, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20531430840492249, + "median": 0.20229148864746094, + "p05": 0.10698921978473663, + "p95": 0.3393132984638214 + }, + { + "label": "completed_group_1", + "mean": 0.15986156463623047, + "median": 0.15952220559120178, + "p05": 0.11969029903411865, + "p95": 0.20180393755435944 + }, + { + "label": "completed_group_2", + "mean": 0.21557100117206573, + "median": 0.21654027700424194, + "p05": 0.1673085242509842, + "p95": 0.2610440254211426 + }, + { + "label": "completed_group_3", + "mean": 0.15222224593162537, + "median": 0.1510905921459198, + "p05": 0.11800260841846466, + "p95": 0.18893754482269287 + }, + { + "label": "current_group_partial", + "mean": 0.26703086495399475, + "median": 0.258539617061615, + "p05": 0.19330690801143646, + "p95": 0.36521556973457336 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5552496910095215, + "group": 4, + "latest_source_mass": 0.31774693727493286, + "layer": 15, + "max_source_mass": 0.31774693727493286, + "mixer_index": 28, + "normalized_entropy": 0.9663309649872576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2162160575389862, + "median": 0.2145705670118332, + "p05": 0.17727568745613098, + "p95": 0.26878347992897034 + }, + { + "label": "completed_group_1", + "mean": 0.17982497811317444, + "median": 0.18024154007434845, + "p05": 0.15453973412513733, + "p95": 0.2030555009841919 + }, + { + "label": "completed_group_2", + "mean": 0.1307673156261444, + "median": 0.13107559084892273, + "p05": 0.11463287472724915, + "p95": 0.14630235731601715 + }, + { + "label": "completed_group_3", + "mean": 0.1554446816444397, + "median": 0.15516987442970276, + "p05": 0.13627474009990692, + "p95": 0.1754152774810791 + }, + { + "label": "current_group_partial", + "mean": 0.31774693727493286, + "median": 0.319039523601532, + "p05": 0.27606821060180664, + "p95": 0.35650834441185 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5068919658660889, + "group": 4, + "latest_source_mass": 0.3298327624797821, + "layer": 15, + "max_source_mass": 0.3298327624797821, + "mixer_index": 29, + "normalized_entropy": 0.9362846209998111, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20646697282791138, + "median": 0.18775638937950134, + "p05": 0.0806753933429718, + "p95": 0.4274584949016571 + }, + { + "label": "completed_group_1", + "mean": 0.1197461187839508, + "median": 0.11989086866378784, + "p05": 0.0809398666024208, + "p95": 0.15743419528007507 + }, + { + "label": "completed_group_2", + "mean": 0.19215428829193115, + "median": 0.19620640575885773, + "p05": 0.1148109883069992, + "p95": 0.2491212636232376 + }, + { + "label": "completed_group_3", + "mean": 0.15179988741874695, + "median": 0.15212899446487427, + "p05": 0.10697861760854721, + "p95": 0.19542087614536285 + }, + { + "label": "current_group_partial", + "mean": 0.3298327624797821, + "median": 0.32837340235710144, + "p05": 0.22746729850769043, + "p95": 0.4339301884174347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5435223579406738, + "group": 4, + "latest_source_mass": 0.3224271535873413, + "layer": 16, + "max_source_mass": 0.3224271535873413, + "mixer_index": 30, + "normalized_entropy": 0.9590443632623663, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22410975396633148, + "median": 0.22760415077209473, + "p05": 0.14593973755836487, + "p95": 0.30093055963516235 + }, + { + "label": "completed_group_1", + "mean": 0.16986176371574402, + "median": 0.16963240504264832, + "p05": 0.13569067418575287, + "p95": 0.20651353895664215 + }, + { + "label": "completed_group_2", + "mean": 0.13080403208732605, + "median": 0.13075244426727295, + "p05": 0.10945598781108856, + "p95": 0.15187400579452515 + }, + { + "label": "completed_group_3", + "mean": 0.15279728174209595, + "median": 0.15036612749099731, + "p05": 0.12810495495796204, + "p95": 0.18468116223812103 + }, + { + "label": "current_group_partial", + "mean": 0.3224271535873413, + "median": 0.3164973258972168, + "p05": 0.2600978910923004, + "p95": 0.40530598163604736 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4949920177459717, + "group": 4, + "latest_source_mass": 0.31847020983695984, + "layer": 16, + "max_source_mass": 0.31847020983695984, + "mixer_index": 31, + "normalized_entropy": 0.9288907675133354, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27174079418182373, + "median": 0.2585451304912567, + "p05": 0.13639721274375916, + "p95": 0.4698348045349121 + }, + { + "label": "completed_group_1", + "mean": 0.12178683280944824, + "median": 0.11714950203895569, + "p05": 0.08594125509262085, + "p95": 0.1723237782716751 + }, + { + "label": "completed_group_2", + "mean": 0.1529826819896698, + "median": 0.15223106741905212, + "p05": 0.10371473431587219, + "p95": 0.203575000166893 + }, + { + "label": "completed_group_3", + "mean": 0.1350194811820984, + "median": 0.13363957405090332, + "p05": 0.08964549005031586, + "p95": 0.18468207120895386 + }, + { + "label": "current_group_partial", + "mean": 0.31847020983695984, + "median": 0.3185216188430786, + "p05": 0.21444711089134216, + "p95": 0.4301546812057495 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.544264793395996, + "group": 5, + "latest_source_mass": 0.2861037850379944, + "layer": 17, + "max_source_mass": 0.2861037850379944, + "mixer_index": 32, + "normalized_entropy": 0.9595056643474137, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24016115069389343, + "median": 0.23868846893310547, + "p05": 0.18435582518577576, + "p95": 0.2983453571796417 + }, + { + "label": "completed_group_1", + "mean": 0.20730282366275787, + "median": 0.1998077780008316, + "p05": 0.16255050897598267, + "p95": 0.28107208013534546 + }, + { + "label": "completed_group_2", + "mean": 0.09692573547363281, + "median": 0.0967404842376709, + "p05": 0.08098706603050232, + "p95": 0.11221462488174438 + }, + { + "label": "completed_group_3", + "mean": 0.1695065200328827, + "median": 0.1696585714817047, + "p05": 0.136396586894989, + "p95": 0.20130634307861328 + }, + { + "label": "completed_group_4", + "mean": 0.2861037850379944, + "median": 0.28883200883865356, + "p05": 0.22401469945907593, + "p95": 0.3409370183944702 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.686682939529419, + "group": 5, + "latest_source_mass": 0.12281867861747742, + "layer": 17, + "max_source_mass": 0.2885604202747345, + "mixer_index": 33, + "normalized_entropy": 0.9413556721740635, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16755425930023193, + "median": 0.1327776312828064, + "p05": 0.0790192037820816, + "p95": 0.3667412996292114 + }, + { + "label": "completed_group_1", + "mean": 0.10962115228176117, + "median": 0.10525335371494293, + "p05": 0.07840172946453094, + "p95": 0.15405339002609253 + }, + { + "label": "completed_group_2", + "mean": 0.1783454418182373, + "median": 0.17896884679794312, + "p05": 0.10802385956048965, + "p95": 0.24048911035060883 + }, + { + "label": "completed_group_3", + "mean": 0.13310004770755768, + "median": 0.13033035397529602, + "p05": 0.0904831737279892, + "p95": 0.17934636771678925 + }, + { + "label": "completed_group_4", + "mean": 0.2885604202747345, + "median": 0.2831825315952301, + "p05": 0.16967125236988068, + "p95": 0.4189499616622925 + }, + { + "label": "current_group_partial", + "mean": 0.12281867861747742, + "median": 0.12062528729438782, + "p05": 0.07461707293987274, + "p95": 0.1782219111919403 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7070834636688232, + "group": 5, + "latest_source_mass": 0.2988377809524536, + "layer": 18, + "max_source_mass": 0.2988377809524536, + "mixer_index": 34, + "normalized_entropy": 0.9527414214834803, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.181177020072937, + "median": 0.18419909477233887, + "p05": 0.13624237477779388, + "p95": 0.22192169725894928 + }, + { + "label": "completed_group_1", + "mean": 0.14557164907455444, + "median": 0.144476979970932, + "p05": 0.1293315887451172, + "p95": 0.16486899554729462 + }, + { + "label": "completed_group_2", + "mean": 0.08318156749010086, + "median": 0.0820261612534523, + "p05": 0.07088497281074524, + "p95": 0.0990293100476265 + }, + { + "label": "completed_group_3", + "mean": 0.11424209177494049, + "median": 0.11267061531543732, + "p05": 0.0979941189289093, + "p95": 0.1353992223739624 + }, + { + "label": "completed_group_4", + "mean": 0.17698991298675537, + "median": 0.17597860097885132, + "p05": 0.14837387204170227, + "p95": 0.21050144731998444 + }, + { + "label": "current_group_partial", + "mean": 0.2988377809524536, + "median": 0.2989804744720459, + "p05": 0.2721370458602905, + "p95": 0.32572072744369507 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7419235706329346, + "group": 5, + "latest_source_mass": 0.26016175746917725, + "layer": 18, + "max_source_mass": 0.26016175746917725, + "mixer_index": 35, + "normalized_entropy": 0.972186055410333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1532188355922699, + "median": 0.1451343297958374, + "p05": 0.11514341086149216, + "p95": 0.22103679180145264 + }, + { + "label": "completed_group_1", + "mean": 0.11420352011919022, + "median": 0.11261284351348877, + "p05": 0.0928458571434021, + "p95": 0.14123988151550293 + }, + { + "label": "completed_group_2", + "mean": 0.16597424447536469, + "median": 0.1655328869819641, + "p05": 0.13871800899505615, + "p95": 0.19611573219299316 + }, + { + "label": "completed_group_3", + "mean": 0.1250365972518921, + "median": 0.12369821965694427, + "p05": 0.10375828295946121, + "p95": 0.14912742376327515 + }, + { + "label": "completed_group_4", + "mean": 0.18140508234500885, + "median": 0.1796584278345108, + "p05": 0.14511024951934814, + "p95": 0.22286373376846313 + }, + { + "label": "current_group_partial", + "mean": 0.26016175746917725, + "median": 0.26266711950302124, + "p05": 0.1889389455318451, + "p95": 0.3234381675720215 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7127439975738525, + "group": 5, + "latest_source_mass": 0.2907913327217102, + "layer": 19, + "max_source_mass": 0.2907913327217102, + "mixer_index": 36, + "normalized_entropy": 0.9559006256078307, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18731768429279327, + "median": 0.19042012095451355, + "p05": 0.13879702985286713, + "p95": 0.23792000114917755 + }, + { + "label": "completed_group_1", + "mean": 0.16715484857559204, + "median": 0.1667211651802063, + "p05": 0.14574480056762695, + "p95": 0.19026139378547668 + }, + { + "label": "completed_group_2", + "mean": 0.08496320247650146, + "median": 0.08452782034873962, + "p05": 0.07410907745361328, + "p95": 0.097386933863163 + }, + { + "label": "completed_group_3", + "mean": 0.11362075805664062, + "median": 0.11248823255300522, + "p05": 0.0977015346288681, + "p95": 0.13219426572322845 + }, + { + "label": "completed_group_4", + "mean": 0.15615218877792358, + "median": 0.15540224313735962, + "p05": 0.1289147436618805, + "p95": 0.18598109483718872 + }, + { + "label": "current_group_partial", + "mean": 0.2907913327217102, + "median": 0.2900415062904358, + "p05": 0.25822532176971436, + "p95": 0.32475030422210693 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076834201812744, + "group": 5, + "latest_source_mass": 0.3738824725151062, + "layer": 19, + "max_source_mass": 0.3738824725151062, + "mixer_index": 37, + "normalized_entropy": 0.8972652009334232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15032559633255005, + "median": 0.12544110417366028, + "p05": 0.06976518034934998, + "p95": 0.35840943455696106 + }, + { + "label": "completed_group_1", + "mean": 0.08624927699565887, + "median": 0.08602266013622284, + "p05": 0.061998721212148666, + "p95": 0.11138717085123062 + }, + { + "label": "completed_group_2", + "mean": 0.15736976265907288, + "median": 0.15802210569381714, + "p05": 0.10121746361255646, + "p95": 0.20687000453472137 + }, + { + "label": "completed_group_3", + "mean": 0.08584291487932205, + "median": 0.0827617198228836, + "p05": 0.060561805963516235, + "p95": 0.11351694166660309 + }, + { + "label": "completed_group_4", + "mean": 0.14632999897003174, + "median": 0.1426895558834076, + "p05": 0.10106660425662994, + "p95": 0.20179018378257751 + }, + { + "label": "current_group_partial", + "mean": 0.3738824725151062, + "median": 0.3799295425415039, + "p05": 0.23631010949611664, + "p95": 0.4884527325630188 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.709290862083435, + "group": 5, + "latest_source_mass": 0.2901708781719208, + "layer": 20, + "max_source_mass": 0.2901708781719208, + "mixer_index": 38, + "normalized_entropy": 0.9539733939957076, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18991519510746002, + "median": 0.1924753338098526, + "p05": 0.14201581478118896, + "p95": 0.23709386587142944 + }, + { + "label": "completed_group_1", + "mean": 0.17091524600982666, + "median": 0.17006167769432068, + "p05": 0.15297070145606995, + "p95": 0.19153237342834473 + }, + { + "label": "completed_group_2", + "mean": 0.07822999358177185, + "median": 0.07778562605381012, + "p05": 0.06707640737295151, + "p95": 0.09089364111423492 + }, + { + "label": "completed_group_3", + "mean": 0.116765096783638, + "median": 0.11516137421131134, + "p05": 0.10162825882434845, + "p95": 0.13653048872947693 + }, + { + "label": "completed_group_4", + "mean": 0.1540035903453827, + "median": 0.15287795662879944, + "p05": 0.12593276798725128, + "p95": 0.18590238690376282 + }, + { + "label": "current_group_partial", + "mean": 0.2901708781719208, + "median": 0.29104921221733093, + "p05": 0.25643545389175415, + "p95": 0.3205650746822357 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6690887212753296, + "group": 5, + "latest_source_mass": 0.3342471718788147, + "layer": 20, + "max_source_mass": 0.3342471718788147, + "mixer_index": 39, + "normalized_entropy": 0.9315361520005944, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19029226899147034, + "median": 0.19644704461097717, + "p05": 0.09456793963909149, + "p95": 0.30468082427978516 + }, + { + "label": "completed_group_1", + "mean": 0.1100241169333458, + "median": 0.10808776319026947, + "p05": 0.09035106748342514, + "p95": 0.1359245330095291 + }, + { + "label": "completed_group_2", + "mean": 0.11471512168645859, + "median": 0.11494918912649155, + "p05": 0.08799737691879272, + "p95": 0.14024731516838074 + }, + { + "label": "completed_group_3", + "mean": 0.09999909996986389, + "median": 0.09682698547840118, + "p05": 0.07921088486909866, + "p95": 0.1264391839504242 + }, + { + "label": "completed_group_4", + "mean": 0.1507222205400467, + "median": 0.1476232260465622, + "p05": 0.11397356539964676, + "p95": 0.19991227984428406 + }, + { + "label": "current_group_partial", + "mean": 0.3342471718788147, + "median": 0.33802270889282227, + "p05": 0.2548266053199768, + "p95": 0.40130624175071716 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7458693981170654, + "group": 6, + "latest_source_mass": 0.2361394464969635, + "layer": 21, + "max_source_mass": 0.2361394464969635, + "mixer_index": 40, + "normalized_entropy": 0.9743882636597644, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20528709888458252, + "median": 0.2140858769416809, + "p05": 0.13168928027153015, + "p95": 0.27098023891448975 + }, + { + "label": "completed_group_1", + "mean": 0.1722964197397232, + "median": 0.17238952219486237, + "p05": 0.15053534507751465, + "p95": 0.19365043938159943 + }, + { + "label": "completed_group_2", + "mean": 0.10435444861650467, + "median": 0.10361101478338242, + "p05": 0.08888213336467743, + "p95": 0.12212536484003067 + }, + { + "label": "completed_group_3", + "mean": 0.12075909972190857, + "median": 0.1180175170302391, + "p05": 0.10358668118715286, + "p95": 0.14458929002285004 + }, + { + "label": "completed_group_4", + "mean": 0.16116347908973694, + "median": 0.15841388702392578, + "p05": 0.1337081342935562, + "p95": 0.19791515171527863 + }, + { + "label": "completed_group_5", + "mean": 0.2361394464969635, + "median": 0.23709020018577576, + "p05": 0.20446214079856873, + "p95": 0.26509594917297363 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4612125158309937, + "group": 6, + "latest_source_mass": 0.5543455481529236, + "layer": 21, + "max_source_mass": 0.5543455481529236, + "mixer_index": 41, + "normalized_entropy": 0.7509146897354808, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08934466540813446, + "median": 0.07911523431539536, + "p05": 0.04488275945186615, + "p95": 0.1737232506275177 + }, + { + "label": "completed_group_1", + "mean": 0.054379627108573914, + "median": 0.05344422906637192, + "p05": 0.04221750423312187, + "p95": 0.06924191862344742 + }, + { + "label": "completed_group_2", + "mean": 0.07567165046930313, + "median": 0.07438549399375916, + "p05": 0.05811355262994766, + "p95": 0.09632256627082825 + }, + { + "label": "completed_group_3", + "mean": 0.05758609622716904, + "median": 0.05612001568078995, + "p05": 0.044792283326387405, + "p95": 0.07347827404737473 + }, + { + "label": "completed_group_4", + "mean": 0.07884950935840607, + "median": 0.07757881283760071, + "p05": 0.05758974328637123, + "p95": 0.10452556610107422 + }, + { + "label": "completed_group_5", + "mean": 0.08982288837432861, + "median": 0.08912864327430725, + "p05": 0.0663854256272316, + "p95": 0.11604885756969452 + }, + { + "label": "current_group_partial", + "mean": 0.5543455481529236, + "median": 0.5596089363098145, + "p05": 0.45455610752105713, + "p95": 0.6367484331130981 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7241814136505127, + "group": 6, + "latest_source_mass": 0.39484071731567383, + "layer": 22, + "max_source_mass": 0.39484071731567383, + "mixer_index": 42, + "normalized_entropy": 0.886053970419732, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13360010087490082, + "median": 0.14072942733764648, + "p05": 0.07699814438819885, + "p95": 0.17506597936153412 + }, + { + "label": "completed_group_1", + "mean": 0.10609637200832367, + "median": 0.10586297512054443, + "p05": 0.08648421615362167, + "p95": 0.12602579593658447 + }, + { + "label": "completed_group_2", + "mean": 0.06363911926746368, + "median": 0.06324908882379532, + "p05": 0.054980404675006866, + "p95": 0.07316673547029495 + }, + { + "label": "completed_group_3", + "mean": 0.07426095008850098, + "median": 0.07362888753414154, + "p05": 0.06318790465593338, + "p95": 0.08665015548467636 + }, + { + "label": "completed_group_4", + "mean": 0.09383076429367065, + "median": 0.09294568747282028, + "p05": 0.07638302445411682, + "p95": 0.11445100605487823 + }, + { + "label": "completed_group_5", + "mean": 0.13373197615146637, + "median": 0.13327741622924805, + "p05": 0.11399447172880173, + "p95": 0.15641780197620392 + }, + { + "label": "current_group_partial", + "mean": 0.39484071731567383, + "median": 0.3879151940345764, + "p05": 0.3302172124385834, + "p95": 0.4852263331413269 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3979300260543823, + "group": 6, + "latest_source_mass": 0.5830056071281433, + "layer": 22, + "max_source_mass": 0.5830056071281433, + "mixer_index": 43, + "normalized_entropy": 0.7183939231382495, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08959861099720001, + "median": 0.07544538378715515, + "p05": 0.04165133833885193, + "p95": 0.20099139213562012 + }, + { + "label": "completed_group_1", + "mean": 0.05414731800556183, + "median": 0.05336771905422211, + "p05": 0.04069708660244942, + "p95": 0.07046990841627121 + }, + { + "label": "completed_group_2", + "mean": 0.07668881118297577, + "median": 0.07607124000787735, + "p05": 0.0555989071726799, + "p95": 0.09980960935354233 + }, + { + "label": "completed_group_3", + "mean": 0.05603286623954773, + "median": 0.054573021829128265, + "p05": 0.04162490367889404, + "p95": 0.0730118378996849 + }, + { + "label": "completed_group_4", + "mean": 0.06979228556156158, + "median": 0.06884932518005371, + "p05": 0.04576495662331581, + "p95": 0.09806846082210541 + }, + { + "label": "completed_group_5", + "mean": 0.07073447108268738, + "median": 0.07023533433675766, + "p05": 0.04956750199198723, + "p95": 0.09379896521568298 + }, + { + "label": "current_group_partial", + "mean": 0.5830056071281433, + "median": 0.5862616896629333, + "p05": 0.4658474922180176, + "p95": 0.6849814057350159 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7459560632705688, + "group": 6, + "latest_source_mass": 0.37691548466682434, + "layer": 23, + "max_source_mass": 0.37691548466682434, + "mixer_index": 44, + "normalized_entropy": 0.897243926765161, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14318783581256866, + "median": 0.1493408977985382, + "p05": 0.08292607218027115, + "p95": 0.19464610517024994 + }, + { + "label": "completed_group_1", + "mean": 0.1094653457403183, + "median": 0.10858799517154694, + "p05": 0.09432064741849899, + "p95": 0.1279931515455246 + }, + { + "label": "completed_group_2", + "mean": 0.06349228322505951, + "median": 0.06325754523277283, + "p05": 0.05439673364162445, + "p95": 0.0737837627530098 + }, + { + "label": "completed_group_3", + "mean": 0.07859592139720917, + "median": 0.0775163471698761, + "p05": 0.06603655964136124, + "p95": 0.09357087314128876 + }, + { + "label": "completed_group_4", + "mean": 0.09567482024431229, + "median": 0.09468592703342438, + "p05": 0.07671141624450684, + "p95": 0.11774884164333344 + }, + { + "label": "completed_group_5", + "mean": 0.13266831636428833, + "median": 0.13199329376220703, + "p05": 0.11244359612464905, + "p95": 0.15504513680934906 + }, + { + "label": "current_group_partial", + "mean": 0.37691548466682434, + "median": 0.37180638313293457, + "p05": 0.31207239627838135, + "p95": 0.4595233201980591 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4055393934249878, + "group": 6, + "latest_source_mass": 0.5772827863693237, + "layer": 23, + "max_source_mass": 0.5772827863693237, + "mixer_index": 45, + "normalized_entropy": 0.7223043644164862, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10154955089092255, + "median": 0.08766824007034302, + "p05": 0.04901552200317383, + "p95": 0.20287738740444183 + }, + { + "label": "completed_group_1", + "mean": 0.05619276314973831, + "median": 0.055477917194366455, + "p05": 0.041608311235904694, + "p95": 0.07371112704277039 + }, + { + "label": "completed_group_2", + "mean": 0.07791627198457718, + "median": 0.07736755907535553, + "p05": 0.055920060724020004, + "p95": 0.10158214718103409 + }, + { + "label": "completed_group_3", + "mean": 0.053915880620479584, + "median": 0.05215863138437271, + "p05": 0.03798118978738785, + "p95": 0.07156119495630264 + }, + { + "label": "completed_group_4", + "mean": 0.06398355215787888, + "median": 0.06229078769683838, + "p05": 0.040616974234580994, + "p95": 0.09270931780338287 + }, + { + "label": "completed_group_5", + "mean": 0.06915920227766037, + "median": 0.0684593990445137, + "p05": 0.046986356377601624, + "p95": 0.09408729523420334 + }, + { + "label": "current_group_partial", + "mean": 0.5772827863693237, + "median": 0.5824689269065857, + "p05": 0.4518115520477295, + "p95": 0.6926202178001404 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7853620052337646, + "group": 6, + "latest_source_mass": 0.3407441973686218, + "layer": 24, + "max_source_mass": 0.3407441973686218, + "mixer_index": 46, + "normalized_entropy": 0.9174945750195659, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15284113585948944, + "median": 0.1593906283378601, + "p05": 0.08678612112998962, + "p95": 0.211342453956604 + }, + { + "label": "completed_group_1", + "mean": 0.12206703424453735, + "median": 0.12153112143278122, + "p05": 0.1060115247964859, + "p95": 0.1399766057729721 + }, + { + "label": "completed_group_2", + "mean": 0.06508444249629974, + "median": 0.06482555717229843, + "p05": 0.055350616574287415, + "p95": 0.07557765394449234 + }, + { + "label": "completed_group_3", + "mean": 0.08347313106060028, + "median": 0.08190013468265533, + "p05": 0.06966532766819, + "p95": 0.10050469636917114 + }, + { + "label": "completed_group_4", + "mean": 0.09811970591545105, + "median": 0.09664031863212585, + "p05": 0.07688035070896149, + "p95": 0.12369741499423981 + }, + { + "label": "completed_group_5", + "mean": 0.13767032325267792, + "median": 0.1373325139284134, + "p05": 0.11642646044492722, + "p95": 0.1606694608926773 + }, + { + "label": "current_group_partial", + "mean": 0.3407441973686218, + "median": 0.33805108070373535, + "p05": 0.271301805973053, + "p95": 0.41781002283096313 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.509842872619629, + "group": 6, + "latest_source_mass": 0.5227814316749573, + "layer": 24, + "max_source_mass": 0.5227814316749573, + "mixer_index": 47, + "normalized_entropy": 0.7759057494780099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11788781732320786, + "median": 0.10097194463014603, + "p05": 0.06265684217214584, + "p95": 0.2408325970172882 + }, + { + "label": "completed_group_1", + "mean": 0.06822571158409119, + "median": 0.0673636943101883, + "p05": 0.05209728330373764, + "p95": 0.08648915588855743 + }, + { + "label": "completed_group_2", + "mean": 0.08556114137172699, + "median": 0.08463940769433975, + "p05": 0.06234566867351532, + "p95": 0.1125786080956459 + }, + { + "label": "completed_group_3", + "mean": 0.06067090481519699, + "median": 0.058974385261535645, + "p05": 0.043267179280519485, + "p95": 0.07997901737689972 + }, + { + "label": "completed_group_4", + "mean": 0.07228855788707733, + "median": 0.06972275674343109, + "p05": 0.04689513146877289, + "p95": 0.106512151658535 + }, + { + "label": "completed_group_5", + "mean": 0.07258446514606476, + "median": 0.07082539796829224, + "p05": 0.050949547439813614, + "p95": 0.09970614314079285 + }, + { + "label": "current_group_partial", + "mean": 0.5227814316749573, + "median": 0.5290273427963257, + "p05": 0.38773614168167114, + "p95": 0.6392589807510376 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.839367389678955, + "group": 7, + "latest_source_mass": 0.29583215713500977, + "layer": 25, + "max_source_mass": 0.29583215713500977, + "mixer_index": 48, + "normalized_entropy": 0.9452478525649903, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15906387567520142, + "median": 0.15846745669841766, + "p05": 0.09519976377487183, + "p95": 0.24517810344696045 + }, + { + "label": "completed_group_1", + "mean": 0.13139380514621735, + "median": 0.1308426856994629, + "p05": 0.11478274315595627, + "p95": 0.15000829100608826 + }, + { + "label": "completed_group_2", + "mean": 0.07717745751142502, + "median": 0.07715500891208649, + "p05": 0.06616044044494629, + "p95": 0.08791667222976685 + }, + { + "label": "completed_group_3", + "mean": 0.09607303142547607, + "median": 0.09436552226543427, + "p05": 0.08152313530445099, + "p95": 0.11384639143943787 + }, + { + "label": "completed_group_4", + "mean": 0.1107850894331932, + "median": 0.1089654266834259, + "p05": 0.08814431726932526, + "p95": 0.139452263712883 + }, + { + "label": "completed_group_5", + "mean": 0.12967455387115479, + "median": 0.1299237608909607, + "p05": 0.10832332074642181, + "p95": 0.15138982236385345 + }, + { + "label": "completed_group_6", + "mean": 0.29583215713500977, + "median": 0.29647016525268555, + "p05": 0.2276897132396698, + "p95": 0.3604946434497833 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5841834545135498, + "group": 7, + "latest_source_mass": 0.40767741203308105, + "layer": 25, + "max_source_mass": 0.40767741203308105, + "mixer_index": 49, + "normalized_entropy": 0.7618312045616817, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07896879315376282, + "median": 0.06809917092323303, + "p05": 0.025932887569069862, + "p95": 0.19368338584899902 + }, + { + "label": "completed_group_1", + "mean": 0.048752427101135254, + "median": 0.04767637699842453, + "p05": 0.034443892538547516, + "p95": 0.0667153000831604 + }, + { + "label": "completed_group_2", + "mean": 0.055707164108753204, + "median": 0.0549345538020134, + "p05": 0.038696639239788055, + "p95": 0.0750277042388916 + }, + { + "label": "completed_group_3", + "mean": 0.04139826446771622, + "median": 0.03878612071275711, + "p05": 0.026673054322600365, + "p95": 0.05746251344680786 + }, + { + "label": "completed_group_4", + "mean": 0.04449018836021423, + "median": 0.04196460545063019, + "p05": 0.026162121444940567, + "p95": 0.07164917886257172 + }, + { + "label": "completed_group_5", + "mean": 0.04285799711942673, + "median": 0.04109475016593933, + "p05": 0.029393533244729042, + "p95": 0.06277710199356079 + }, + { + "label": "completed_group_6", + "mean": 0.2801477909088135, + "median": 0.2763082981109619, + "p05": 0.1455708146095276, + "p95": 0.4254714846611023 + }, + { + "label": "current_group_partial", + "mean": 0.40767741203308105, + "median": 0.4120897948741913, + "p05": 0.21399420499801636, + "p95": 0.5853031277656555 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8125314712524414, + "group": 7, + "latest_source_mass": 0.3700707256793976, + "layer": 26, + "max_source_mass": 0.3700707256793976, + "mixer_index": 50, + "normalized_entropy": 0.8716433883436914, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10889513790607452, + "median": 0.11333495378494263, + "p05": 0.04850132763385773, + "p95": 0.1620434820652008 + }, + { + "label": "completed_group_1", + "mean": 0.08656036853790283, + "median": 0.08640051633119583, + "p05": 0.0650845319032669, + "p95": 0.1084962785243988 + }, + { + "label": "completed_group_2", + "mean": 0.048123255372047424, + "median": 0.04800213500857353, + "p05": 0.03576180338859558, + "p95": 0.06107594817876816 + }, + { + "label": "completed_group_3", + "mean": 0.06561170518398285, + "median": 0.06379580497741699, + "p05": 0.050968848168849945, + "p95": 0.08332888036966324 + }, + { + "label": "completed_group_4", + "mean": 0.06917259097099304, + "median": 0.06724178791046143, + "p05": 0.050207968801259995, + "p95": 0.09373342245817184 + }, + { + "label": "completed_group_5", + "mean": 0.08286689221858978, + "median": 0.08183546364307404, + "p05": 0.06535022705793381, + "p95": 0.10340933501720428 + }, + { + "label": "completed_group_6", + "mean": 0.16869932413101196, + "median": 0.16509854793548584, + "p05": 0.11059704422950745, + "p95": 0.23654375970363617 + }, + { + "label": "current_group_partial", + "mean": 0.3700707256793976, + "median": 0.3694133758544922, + "p05": 0.23441238701343536, + "p95": 0.5025027990341187 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.50089693069458, + "group": 7, + "latest_source_mass": 0.49096518754959106, + "layer": 26, + "max_source_mass": 0.49096518754959106, + "mixer_index": 51, + "normalized_entropy": 0.7217788529328457, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07974998652935028, + "median": 0.07402504235506058, + "p05": 0.04579866677522659, + "p95": 0.1359037309885025 + }, + { + "label": "completed_group_1", + "mean": 0.04166398197412491, + "median": 0.04038450866937637, + "p05": 0.027992289513349533, + "p95": 0.05969971418380737 + }, + { + "label": "completed_group_2", + "mean": 0.04607272148132324, + "median": 0.044438350945711136, + "p05": 0.030038591474294662, + "p95": 0.06808309257030487 + }, + { + "label": "completed_group_3", + "mean": 0.03586764633655548, + "median": 0.03317485377192497, + "p05": 0.02203454077243805, + "p95": 0.05572832003235817 + }, + { + "label": "completed_group_4", + "mean": 0.04223340004682541, + "median": 0.03930673003196716, + "p05": 0.02465122565627098, + "p95": 0.06968094408512115 + }, + { + "label": "completed_group_5", + "mean": 0.04095568507909775, + "median": 0.03919431194663048, + "p05": 0.026239857077598572, + "p95": 0.06058088690042496 + }, + { + "label": "completed_group_6", + "mean": 0.22249138355255127, + "median": 0.2125401496887207, + "p05": 0.10647299885749817, + "p95": 0.3711881935596466 + }, + { + "label": "current_group_partial", + "mean": 0.49096518754959106, + "median": 0.4988659620285034, + "p05": 0.2878011167049408, + "p95": 0.6676807403564453 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8059929609298706, + "group": 7, + "latest_source_mass": 0.37139976024627686, + "layer": 27, + "max_source_mass": 0.37139976024627686, + "mixer_index": 52, + "normalized_entropy": 0.8684990295379666, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10711923241615295, + "median": 0.10972198843955994, + "p05": 0.04896187782287598, + "p95": 0.16201043128967285 + }, + { + "label": "completed_group_1", + "mean": 0.07651707530021667, + "median": 0.07666032761335373, + "p05": 0.05141128972172737, + "p95": 0.10152198374271393 + }, + { + "label": "completed_group_2", + "mean": 0.058284737169742584, + "median": 0.05812416970729828, + "p05": 0.039273619651794434, + "p95": 0.07745087146759033 + }, + { + "label": "completed_group_3", + "mean": 0.06609587371349335, + "median": 0.06383530795574188, + "p05": 0.04481930285692215, + "p95": 0.09336265176534653 + }, + { + "label": "completed_group_4", + "mean": 0.07139429450035095, + "median": 0.0694514587521553, + "p05": 0.0485839769244194, + "p95": 0.10034126043319702 + }, + { + "label": "completed_group_5", + "mean": 0.07493151724338531, + "median": 0.07319007813930511, + "p05": 0.051876723766326904, + "p95": 0.10402432829141617 + }, + { + "label": "completed_group_6", + "mean": 0.17425751686096191, + "median": 0.16937679052352905, + "p05": 0.10449051856994629, + "p95": 0.2624811828136444 + }, + { + "label": "current_group_partial", + "mean": 0.37139976024627686, + "median": 0.3732905089855194, + "p05": 0.23830299079418182, + "p95": 0.5014542937278748 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5622732639312744, + "group": 7, + "latest_source_mass": 0.4768889844417572, + "layer": 27, + "max_source_mass": 0.4768889844417572, + "mixer_index": 53, + "normalized_entropy": 0.7512946301290215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08775827288627625, + "median": 0.08504067361354828, + "p05": 0.0428270623087883, + "p95": 0.144285649061203 + }, + { + "label": "completed_group_1", + "mean": 0.047765426337718964, + "median": 0.04646281152963638, + "p05": 0.031814731657505035, + "p95": 0.06807165592908859 + }, + { + "label": "completed_group_2", + "mean": 0.053928617388010025, + "median": 0.052262187004089355, + "p05": 0.035437777638435364, + "p95": 0.0784272700548172 + }, + { + "label": "completed_group_3", + "mean": 0.04293380305171013, + "median": 0.03996016085147858, + "p05": 0.02556057646870613, + "p95": 0.06623870134353638 + }, + { + "label": "completed_group_4", + "mean": 0.04174989461898804, + "median": 0.03903091698884964, + "p05": 0.025013400241732597, + "p95": 0.06869364529848099 + }, + { + "label": "completed_group_5", + "mean": 0.04734005779027939, + "median": 0.04609068110585213, + "p05": 0.029884062707424164, + "p95": 0.06911227852106094 + }, + { + "label": "completed_group_6", + "mean": 0.20163494348526, + "median": 0.19222964346408844, + "p05": 0.09963390231132507, + "p95": 0.3307305574417114 + }, + { + "label": "current_group_partial", + "mean": 0.4768889844417572, + "median": 0.48383551836013794, + "p05": 0.276212215423584, + "p95": 0.6487575173377991 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8757455348968506, + "group": 7, + "latest_source_mass": 0.3109493851661682, + "layer": 28, + "max_source_mass": 0.3109493851661682, + "mixer_index": 54, + "normalized_entropy": 0.9020429270551009, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11563669890165329, + "median": 0.12087491154670715, + "p05": 0.05733206868171692, + "p95": 0.1688953936100006 + }, + { + "label": "completed_group_1", + "mean": 0.08108697831630707, + "median": 0.08185897022485733, + "p05": 0.05175650864839554, + "p95": 0.10753051191568375 + }, + { + "label": "completed_group_2", + "mean": 0.07280858606100082, + "median": 0.07357072830200195, + "p05": 0.04870978742837906, + "p95": 0.0948689877986908 + }, + { + "label": "completed_group_3", + "mean": 0.07553227245807648, + "median": 0.07362619787454605, + "p05": 0.056317005306482315, + "p95": 0.09756095707416534 + }, + { + "label": "completed_group_4", + "mean": 0.07606014609336853, + "median": 0.07456053793430328, + "p05": 0.05528895929455757, + "p95": 0.1022825762629509 + }, + { + "label": "completed_group_5", + "mean": 0.06935927271842957, + "median": 0.06843917071819305, + "p05": 0.04978285729885101, + "p95": 0.09167985618114471 + }, + { + "label": "completed_group_6", + "mean": 0.19856664538383484, + "median": 0.19508783519268036, + "p05": 0.1281973123550415, + "p95": 0.27796900272369385 + }, + { + "label": "current_group_partial", + "mean": 0.3109493851661682, + "median": 0.3105033040046692, + "p05": 0.18172834813594818, + "p95": 0.43916669487953186 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6924852132797241, + "group": 7, + "latest_source_mass": 0.3315581679344177, + "layer": 28, + "max_source_mass": 0.3315581679344177, + "mixer_index": 55, + "normalized_entropy": 0.8139133413255193, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08638577908277512, + "median": 0.058608733117580414, + "p05": 0.026544082909822464, + "p95": 0.2740764319896698 + }, + { + "label": "completed_group_1", + "mean": 0.059912845492362976, + "median": 0.05901291221380234, + "p05": 0.03629481792449951, + "p95": 0.08707985281944275 + }, + { + "label": "completed_group_2", + "mean": 0.06609142571687698, + "median": 0.0647939145565033, + "p05": 0.038691695779561996, + "p95": 0.09703366458415985 + }, + { + "label": "completed_group_3", + "mean": 0.056745611131191254, + "median": 0.05418796092271805, + "p05": 0.032174739986658096, + "p95": 0.08370660990476608 + }, + { + "label": "completed_group_4", + "mean": 0.06277773529291153, + "median": 0.0600711889564991, + "p05": 0.03501531109213829, + "p95": 0.10094858705997467 + }, + { + "label": "completed_group_5", + "mean": 0.05352145433425903, + "median": 0.05179818347096443, + "p05": 0.032271772623062134, + "p95": 0.08084158599376678 + }, + { + "label": "completed_group_6", + "mean": 0.2830069661140442, + "median": 0.2759752869606018, + "p05": 0.12981083989143372, + "p95": 0.46204638481140137 + }, + { + "label": "current_group_partial", + "mean": 0.3315581679344177, + "median": 0.3204721510410309, + "p05": 0.13446375727653503, + "p95": 0.5667697787284851 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8980522155761719, + "group": 8, + "latest_source_mass": 0.20123262703418732, + "layer": 29, + "max_source_mass": 0.26602888107299805, + "mixer_index": 56, + "normalized_entropy": 0.9127701729200177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11603894829750061, + "median": 0.09991352260112762, + "p05": 0.030035395175218582, + "p95": 0.2941743731498718 + }, + { + "label": "completed_group_1", + "mean": 0.08804173022508621, + "median": 0.08880250155925751, + "p05": 0.05664174258708954, + "p95": 0.11779069900512695 + }, + { + "label": "completed_group_2", + "mean": 0.07975024729967117, + "median": 0.07984748482704163, + "p05": 0.05428915470838547, + "p95": 0.10499341040849686 + }, + { + "label": "completed_group_3", + "mean": 0.09027000516653061, + "median": 0.08898856490850449, + "p05": 0.06189651042222977, + "p95": 0.11996553838253021 + }, + { + "label": "completed_group_4", + "mean": 0.08430293202400208, + "median": 0.0819794088602066, + "p05": 0.05464346706867218, + "p95": 0.12159879505634308 + }, + { + "label": "completed_group_5", + "mean": 0.07433462142944336, + "median": 0.07287181913852692, + "p05": 0.052520088851451874, + "p95": 0.1010315865278244 + }, + { + "label": "completed_group_6", + "mean": 0.26602888107299805, + "median": 0.2639757990837097, + "p05": 0.15474572777748108, + "p95": 0.3846370279788971 + }, + { + "label": "completed_group_7", + "mean": 0.20123262703418732, + "median": 0.18140269815921783, + "p05": 0.08419936150312424, + "p95": 0.3782624900341034 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7512003183364868, + "group": 8, + "latest_source_mass": 0.33482813835144043, + "layer": 29, + "max_source_mass": 0.33482813835144043, + "mixer_index": 57, + "normalized_entropy": 0.7970056117156376, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07744461297988892, + "median": 0.06970229744911194, + "p05": 0.014401955530047417, + "p95": 0.17144526541233063 + }, + { + "label": "completed_group_1", + "mean": 0.046086348593235016, + "median": 0.04394831880927086, + "p05": 0.028066720813512802, + "p95": 0.0718703418970108 + }, + { + "label": "completed_group_2", + "mean": 0.05025433003902435, + "median": 0.04861236736178398, + "p05": 0.029644077643752098, + "p95": 0.07761462777853012 + }, + { + "label": "completed_group_3", + "mean": 0.03748488053679466, + "median": 0.03434944897890091, + "p05": 0.020774798467755318, + "p95": 0.05938909202814102 + }, + { + "label": "completed_group_4", + "mean": 0.04005061462521553, + "median": 0.037498839199543, + "p05": 0.019541850313544273, + "p95": 0.06922487914562225 + }, + { + "label": "completed_group_5", + "mean": 0.041623204946517944, + "median": 0.039552345871925354, + "p05": 0.022081367671489716, + "p95": 0.06821337342262268 + }, + { + "label": "completed_group_6", + "mean": 0.198423832654953, + "median": 0.18318486213684082, + "p05": 0.07548617571592331, + "p95": 0.3721975088119507 + }, + { + "label": "completed_group_7", + "mean": 0.17380405962467194, + "median": 0.1514919549226761, + "p05": 0.05777408182621002, + "p95": 0.36326974630355835 + }, + { + "label": "current_group_partial", + "mean": 0.33482813835144043, + "median": 0.33602219820022583, + "p05": 0.0795319601893425, + "p95": 0.5979493856430054 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0160694122314453, + "group": 8, + "latest_source_mass": 0.21711516380310059, + "layer": 30, + "max_source_mass": 0.21711516380310059, + "mixer_index": 58, + "normalized_entropy": 0.9175527313077866, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09385271370410919, + "median": 0.09211117774248123, + "p05": 0.032377734780311584, + "p95": 0.16496515274047852 + }, + { + "label": "completed_group_1", + "mean": 0.06208505108952522, + "median": 0.06140740588307381, + "p05": 0.0407925546169281, + "p95": 0.0863042101264 + }, + { + "label": "completed_group_2", + "mean": 0.06560768187046051, + "median": 0.06516212224960327, + "p05": 0.041897669434547424, + "p95": 0.09085705131292343 + }, + { + "label": "completed_group_3", + "mean": 0.07029052078723907, + "median": 0.06780634075403214, + "p05": 0.04846132919192314, + "p95": 0.09917552024126053 + }, + { + "label": "completed_group_4", + "mean": 0.06935097277164459, + "median": 0.06708167493343353, + "p05": 0.04668639227747917, + "p95": 0.09815650433301926 + }, + { + "label": "completed_group_5", + "mean": 0.056832700967788696, + "median": 0.05595892667770386, + "p05": 0.038822539150714874, + "p95": 0.0771557092666626 + }, + { + "label": "completed_group_6", + "mean": 0.19425815343856812, + "median": 0.19145764410495758, + "p05": 0.12249615788459778, + "p95": 0.27496030926704407 + }, + { + "label": "completed_group_7", + "mean": 0.17060703039169312, + "median": 0.16317546367645264, + "p05": 0.09239593893289566, + "p95": 0.2726203501224518 + }, + { + "label": "current_group_partial", + "mean": 0.21711516380310059, + "median": 0.21198657155036926, + "p05": 0.11019522696733475, + "p95": 0.3391069769859314 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7837426662445068, + "group": 8, + "latest_source_mass": 0.29026859998703003, + "layer": 30, + "max_source_mass": 0.29026859998703003, + "mixer_index": 59, + "normalized_entropy": 0.8118162725118464, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08836327493190765, + "median": 0.07904411852359772, + "p05": 0.02631576545536518, + "p95": 0.17534467577934265 + }, + { + "label": "completed_group_1", + "mean": 0.04953558370471001, + "median": 0.04658038914203644, + "p05": 0.025181813165545464, + "p95": 0.08318150043487549 + }, + { + "label": "completed_group_2", + "mean": 0.04813014343380928, + "median": 0.045740917325019836, + "p05": 0.026714935898780823, + "p95": 0.07843323051929474 + }, + { + "label": "completed_group_3", + "mean": 0.0373263955116272, + "median": 0.03401809185743332, + "p05": 0.01990434154868126, + "p95": 0.06290259957313538 + }, + { + "label": "completed_group_4", + "mean": 0.037015400826931, + "median": 0.03405017405748367, + "p05": 0.018646355718374252, + "p95": 0.06544196605682373 + }, + { + "label": "completed_group_5", + "mean": 0.03390160948038101, + "median": 0.03198609501123428, + "p05": 0.018189728260040283, + "p95": 0.05570102855563164 + }, + { + "label": "completed_group_6", + "mean": 0.1840541511774063, + "median": 0.17327937483787537, + "p05": 0.07966600358486176, + "p95": 0.32982927560806274 + }, + { + "label": "completed_group_7", + "mean": 0.23140482604503632, + "median": 0.22406929731369019, + "p05": 0.08831030130386353, + "p95": 0.40350067615509033 + }, + { + "label": "current_group_partial", + "mean": 0.29026859998703003, + "median": 0.2745037376880646, + "p05": 0.10280418395996094, + "p95": 0.5399085879325867 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9973840713500977, + "group": 8, + "latest_source_mass": 0.19101998209953308, + "layer": 31, + "max_source_mass": 0.20831164717674255, + "mixer_index": 60, + "normalized_entropy": 0.9090486661912383, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10261696577072144, + "median": 0.10389091819524765, + "p05": 0.01371697522699833, + "p95": 0.21163305640220642 + }, + { + "label": "completed_group_1", + "mean": 0.06039813905954361, + "median": 0.06085003539919853, + "p05": 0.026107093319296837, + "p95": 0.09348798543214798 + }, + { + "label": "completed_group_2", + "mean": 0.06671924144029617, + "median": 0.06656201928853989, + "p05": 0.034684643149375916, + "p95": 0.09819839894771576 + }, + { + "label": "completed_group_3", + "mean": 0.08377155661582947, + "median": 0.0804532840847969, + "p05": 0.05448959022760391, + "p95": 0.12039553374052048 + }, + { + "label": "completed_group_4", + "mean": 0.07002593576908112, + "median": 0.06737451255321503, + "p05": 0.044841229915618896, + "p95": 0.10655929893255234 + }, + { + "label": "completed_group_5", + "mean": 0.05768897384405136, + "median": 0.056077782064676285, + "p05": 0.039495617151260376, + "p95": 0.08242568373680115 + }, + { + "label": "completed_group_6", + "mean": 0.20831164717674255, + "median": 0.2021285891532898, + "p05": 0.11652229726314545, + "p95": 0.31706833839416504 + }, + { + "label": "completed_group_7", + "mean": 0.1594475358724594, + "median": 0.1546279788017273, + "p05": 0.06593266129493713, + "p95": 0.2735573351383209 + }, + { + "label": "current_group_partial", + "mean": 0.19101998209953308, + "median": 0.1713225245475769, + "p05": 0.06408411264419556, + "p95": 0.3850628137588501 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.969987154006958, + "group": 8, + "latest_source_mass": 0.16864418983459473, + "layer": 31, + "max_source_mass": 0.24503237009048462, + "mixer_index": 61, + "normalized_entropy": 0.8965797917640489, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08120635151863098, + "median": 0.061164144426584244, + "p05": 0.0259944386780262, + "p95": 0.23622018098831177 + }, + { + "label": "completed_group_1", + "mean": 0.07894632965326309, + "median": 0.07658770680427551, + "p05": 0.0496862456202507, + "p95": 0.11596276611089706 + }, + { + "label": "completed_group_2", + "mean": 0.08615730702877045, + "median": 0.08467075228691101, + "p05": 0.050409119576215744, + "p95": 0.12811316549777985 + }, + { + "label": "completed_group_3", + "mean": 0.07609779387712479, + "median": 0.0722871869802475, + "p05": 0.04099677503108978, + "p95": 0.11950768530368805 + }, + { + "label": "completed_group_4", + "mean": 0.058728985488414764, + "median": 0.056080836802721024, + "p05": 0.03323862701654434, + "p95": 0.09344407916069031 + }, + { + "label": "completed_group_5", + "mean": 0.05076971277594566, + "median": 0.04946017265319824, + "p05": 0.03047419898211956, + "p95": 0.07563400268554688 + }, + { + "label": "completed_group_6", + "mean": 0.24503237009048462, + "median": 0.24110376834869385, + "p05": 0.10613486170768738, + "p95": 0.3991503119468689 + }, + { + "label": "completed_group_7", + "mean": 0.1544169783592224, + "median": 0.1400495171546936, + "p05": 0.06199025735259056, + "p95": 0.2995939254760742 + }, + { + "label": "current_group_partial", + "mean": 0.16864418983459473, + "median": 0.15222924947738647, + "p05": 0.0543956533074379, + "p95": 0.3329208791255951 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.071648359298706, + "group": 8, + "latest_source_mass": 0.11520501971244812, + "layer": 32, + "max_source_mass": 0.20955611765384674, + "mixer_index": 62, + "normalized_entropy": 0.9428478002054053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10569609701633453, + "median": 0.08689188957214355, + "p05": 0.03173588588833809, + "p95": 0.24571751058101654 + }, + { + "label": "completed_group_1", + "mean": 0.09922641515731812, + "median": 0.09795533865690231, + "p05": 0.06407676637172699, + "p95": 0.14120742678642273 + }, + { + "label": "completed_group_2", + "mean": 0.08180510997772217, + "median": 0.080217644572258, + "p05": 0.054607540369033813, + "p95": 0.11442367732524872 + }, + { + "label": "completed_group_3", + "mean": 0.10646799206733704, + "median": 0.10510502755641937, + "p05": 0.07415887713432312, + "p95": 0.14216329157352448 + }, + { + "label": "completed_group_4", + "mean": 0.0826798751950264, + "median": 0.08022409677505493, + "p05": 0.05491747707128525, + "p95": 0.11860857903957367 + }, + { + "label": "completed_group_5", + "mean": 0.06615597754716873, + "median": 0.06485182791948318, + "p05": 0.04477996751666069, + "p95": 0.09238749742507935 + }, + { + "label": "completed_group_6", + "mean": 0.20955611765384674, + "median": 0.206287682056427, + "p05": 0.129981130361557, + "p95": 0.2996031939983368 + }, + { + "label": "completed_group_7", + "mean": 0.13320739567279816, + "median": 0.12465103715658188, + "p05": 0.06303662061691284, + "p95": 0.23065781593322754 + }, + { + "label": "current_group_partial", + "mean": 0.11520501971244812, + "median": 0.10386130958795547, + "p05": 0.03912060335278511, + "p95": 0.22735844552516937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781701564788818, + "group": 8, + "latest_source_mass": 0.16112206876277924, + "layer": 32, + "max_source_mass": 0.22251944243907928, + "mixer_index": 63, + "normalized_entropy": 0.9003040366848136, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09446848928928375, + "median": 0.074150450527668, + "p05": 0.042592622339725494, + "p95": 0.19443637132644653 + }, + { + "label": "completed_group_1", + "mean": 0.07056106626987457, + "median": 0.06868506222963333, + "p05": 0.04818308353424072, + "p95": 0.0982733890414238 + }, + { + "label": "completed_group_2", + "mean": 0.08033448457717896, + "median": 0.08047192543745041, + "p05": 0.048296794295310974, + "p95": 0.11253741383552551 + }, + { + "label": "completed_group_3", + "mean": 0.0658978745341301, + "median": 0.06256154924631119, + "p05": 0.04223886504769325, + "p95": 0.0967884361743927 + }, + { + "label": "completed_group_4", + "mean": 0.058234501630067825, + "median": 0.0563921295106411, + "p05": 0.03529864549636841, + "p95": 0.08846980333328247 + }, + { + "label": "completed_group_5", + "mean": 0.04316519945859909, + "median": 0.042268507182598114, + "p05": 0.029443470761179924, + "p95": 0.05996086448431015 + }, + { + "label": "completed_group_6", + "mean": 0.2036968618631363, + "median": 0.19645483791828156, + "p05": 0.09398645162582397, + "p95": 0.3395216464996338 + }, + { + "label": "completed_group_7", + "mean": 0.22251944243907928, + "median": 0.2130982130765915, + "p05": 0.11867167800664902, + "p95": 0.35465875267982483 + }, + { + "label": "current_group_partial", + "mean": 0.16112206876277924, + "median": 0.14023703336715698, + "p05": 0.06008770316839218, + "p95": 0.3247341811656952 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.6036605834960938, + "group": null, + "latest_source_mass": 0.31639498472213745, + "layer": null, + "max_source_mass": 0.3412485122680664, + "mixer_index": null, + "normalized_entropy": 0.7298573846467136, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09759747982025146, + "median": 0.09915056824684143, + "p05": 0.025461740791797638, + "p95": 0.16398483514785767 + }, + { + "label": "completed_group_1", + "mean": 0.04215940088033676, + "median": 0.04050712287425995, + "p05": 0.025720715522766113, + "p95": 0.06398092210292816 + }, + { + "label": "completed_group_2", + "mean": 0.041145481169223785, + "median": 0.03881259262561798, + "p05": 0.023510465398430824, + "p95": 0.0662468671798706 + }, + { + "label": "completed_group_3", + "mean": 0.029554028064012527, + "median": 0.027420252561569214, + "p05": 0.01627083122730255, + "p95": 0.04924973472952843 + }, + { + "label": "completed_group_4", + "mean": 0.0289996936917305, + "median": 0.027226522564888, + "p05": 0.01560397818684578, + "p95": 0.048351090401411057 + }, + { + "label": "completed_group_5", + "mean": 0.03647579252719879, + "median": 0.0347352996468544, + "p05": 0.020744025707244873, + "p95": 0.058257561177015305 + }, + { + "label": "completed_group_6", + "mean": 0.06642461568117142, + "median": 0.057809896767139435, + "p05": 0.022837381809949875, + "p95": 0.14078694581985474 + }, + { + "label": "completed_group_7", + "mean": 0.3412485122680664, + "median": 0.3336990475654602, + "p05": 0.0819513276219368, + "p95": 0.6423859596252441 + }, + { + "label": "completed_group_8", + "mean": 0.31639498472213745, + "median": 0.30713334679603577, + "p05": 0.05841828137636185, + "p95": 0.6156357526779175 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.24537226557731628, + 0.07698935270309448, + 0.0523640513420105, + 0.09380833059549332, + 0.03647654876112938, + 0.039120711386203766, + 0.06371714919805527, + 0.05156813934445381, + 0.03728010877966881, + 0.03983496502041817, + 0.040284499526023865, + 0.046871740370988846, + 0.04314620420336723, + 0.04033593460917473, + 0.04471700266003609, + 0.08397205173969269, + 0.0528092160820961, + 0.04443390294909477, + 0.07560808956623077, + 0.03708760067820549, + 0.03599191829562187, + 0.033304885029792786, + 0.04061548039317131, + 0.025487789884209633, + 0.021565141156315804, + 0.02342291921377182, + 0.017244853079319, + 0.024166326969861984, + 0.016819356009364128, + 0.021590016782283783, + 0.026201345026493073 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.459922923975682e-06, + "first_to_last_ratio": 2.328407895763024, + "imbalance_abs_log_ratio": 0.8451847274778195, + "last_quartile_mean": 3.6333509001451603e-06, + "mean": 5.48445950698806e-06, + "normalized": [ + 1.7981492239015215, + 1.9357513407040896, + 1.6672073870519302, + 1.5017840953582675, + 1.4824821685817133, + 1.4014170610824435, + 1.3280238040790173, + 1.2253951633993214, + 1.156695262052324, + 1.0641431906294738, + 1.0106344255365511, + 0.9662588157958233, + 0.9475559543612537, + 0.9090700068173221, + 0.8837794995113782, + 0.8574858790556309, + 0.8607721566700557, + 0.837358195636748, + 0.8360590737297285, + 0.824566471544106, + 0.8217232949937042, + 0.8060327579708028, + 0.7938760845984737, + 0.7839296928623085, + 0.7744755711517991, + 0.7571739836659549, + 0.7307992717355416, + 0.68650330244616, + 0.6478013597166409, + 0.6078235943688355, + 0.5594006660114121, + 0.5358712449796666 + ], + "peak_layer": 2, + "peak_normalized": 1.9357513407040896, + "population_cv": 0.35649081874965816, + "rest_mean": 5.691642602161866e-06, + "spike_contrast": 0.7670317864637676, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.365670793049503e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.8618766060099e-06, + 1.0616549843689427e-05, + 9.14373140403768e-06, + 8.236474059231114e-06, + 8.130613423418254e-06, + 7.686015123908874e-06, + 7.283492777787615e-06, + 6.720630153722595e-06, + 6.343848326650914e-06, + 5.836250238644425e-06, + 5.542783583223354e-06, + 5.299407348502427e-06, + 5.196832262299722e-06, + 4.985757641406963e-06, + 4.847052878176328e-06, + 4.7028465814946685e-06, + 4.720870037999703e-06, + 4.59245711681433e-06, + 4.585332135320641e-06, + 4.522301424003672e-06, + 4.506708137341775e-06, + 4.420654022396775e-06, + 4.353981239546556e-06, + 4.2994306568289176e-06, + 4.247579909133492e-06, + 4.152690053160768e-06, + 4.008039013569942e-06, + 3.7650995636795415e-06, + 3.552840325937723e-06, + 3.333583890707814e-06, + 3.068010300921742e-06, + 2.93896414405026e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.357288642291678e-05, + "first_to_last_ratio": 2.3058080996835404, + "imbalance_abs_log_ratio": 0.8354312005538636, + "last_quartile_mean": 1.456013899314712e-05, + "mean": 2.1838724165945678e-05, + "normalized": [ + 1.8092622254038118, + 1.9431844817635626, + 1.6767107875669782, + 1.4950940749576347, + 1.4674045069810522, + 1.3807616632021802, + 1.3135886403618173, + 1.2124726583027057, + 1.1458806799401573, + 1.058561922179424, + 1.0064538267556071, + 0.9652676350426349, + 0.9479497443118112, + 0.9127224235896364, + 0.8880734273823512, + 0.8621668896719061, + 0.8678176638273486, + 0.8415578850574983, + 0.8383782990632195, + 0.8250966517885899, + 0.8217974585881554, + 0.8072470254003167, + 0.7943189523527976, + 0.7845346497712516, + 0.7759095209279144, + 0.7590478200773931, + 0.7335564093275885, + 0.6907997066984136, + 0.653253330518448, + 0.6137700422385202, + 0.5655418513252468, + 0.5418171456240263 + ], + "peak_layer": 2, + "peak_normalized": 1.9431844817635626, + "population_cv": 0.3545327058404385, + "rest_mean": 2.266066288332998e-05, + "spike_contrast": 0.7678616985591674, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.740025509207044e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.951197868445888e-05, + 4.2436669900780544e-05, + 3.661722439574078e-05, + 3.2650947105139494e-05, + 3.204624226782471e-05, + 3.01540731015848e-05, + 2.8687099984381348e-05, + 2.6478855943423696e-05, + 2.502457209629938e-05, + 2.3117641831049696e-05, + 2.197966750827618e-05, + 2.1080213628010824e-05, + 2.0702012989204377e-05, + 1.99326932488475e-05, + 1.9394390619709156e-05, + 1.8828624888556078e-05, + 1.895203058666084e-05, + 1.8378550521447323e-05, + 1.8309112419956364e-05, + 1.8019058188656345e-05, + 1.7947008018381894e-05, + 1.762924512149766e-05, + 1.7346912500215694e-05, + 1.7133235814981163e-05, + 1.694487400527578e-05, + 1.6576635971432552e-05, + 1.6019936083466746e-05, + 1.5086184248502832e-05, + 1.4266219295677729e-05, + 1.3403954653767869e-05, + 1.2350712495390326e-05, + 1.1832595191663131e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.029772771056741476, + "first_to_last_ratio": 2.3058080657449813, + "imbalance_abs_log_ratio": 0.8354311858351369, + "last_quartile_mean": 0.012912076897919178, + "mean": 0.01936679895152338, + "normalized": [ + 1.8092622688469415, + 1.9431844502992837, + 1.6767106129751548, + 1.4950940451422423, + 1.4674045233760984, + 1.3807616273635772, + 1.3135886581915137, + 1.2124727235166237, + 1.1458806648362485, + 1.0585619314023544, + 1.0064538755889925, + 0.965267610260004, + 0.9479497452013365, + 0.91272242891937, + 0.8880734542201515, + 0.8621668660127682, + 0.8678176632029413, + 0.8415579121933264, + 0.838378389023671, + 0.8250966015781978, + 0.8217974339297908, + 0.8072470687715084, + 0.7943189250144355, + 0.7845346707619233, + 0.7759095445302433, + 0.7590478000025082, + 0.7335564570017248, + 0.69079971362327, + 0.6532533237931528, + 0.6137700206530842, + 0.56554185062535, + 0.5418171391422113 + ], + "peak_layer": 2, + "peak_normalized": 1.9431844502992837, + "population_cv": 0.35453269447871943, + "rest_mean": 0.020095702398706367, + "spike_contrast": 0.7678617064775293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01543072033673525, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.035039618611335754, + 0.0376332625746727, + 0.03247251734137535, + 0.028955185785889626, + 0.028418928384780884, + 0.02674093283712864, + 0.02544000744819641, + 0.023481715470552444, + 0.02219204045832157, + 0.02050095610320568, + 0.019491789862513542, + 0.018694143742322922, + 0.018358752131462097, + 0.01767651177942753, + 0.017199140042066574, + 0.016697412356734276, + 0.01680685020983219, + 0.016298282891511917, + 0.01623670570552349, + 0.015979479998350143, + 0.015915585681796074, + 0.01563379168510437, + 0.015383414924144745, + 0.015193925239145756, + 0.015026884153485298, + 0.014700326137244701, + 0.014206640422344208, + 0.013378579169511795, + 0.012651425786316395, + 0.011886760592460632, + 0.01095273531973362, + 0.010493263602256775 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2918531382410947e-06, + "first_to_last_ratio": 1.348602552566167, + "imbalance_abs_log_ratio": 0.2990689100691611, + "last_quartile_mean": 9.57919837674126e-07, + "mean": 1.0475390865849477e-06, + "normalized": [ + 2.4104888609370185, + 1.341971980631633, + 1.2193546382838214, + 1.0858339683324785, + 0.9788660798673142, + 0.9472203180914999, + 0.9376150883665126, + 0.9444624163032238, + 0.9288836138476254, + 0.8900745462090585, + 0.8740769359854389, + 0.8711050178927995, + 0.8545471342557541, + 0.8634403690695975, + 0.8786271700589888, + 0.8875789012156576, + 0.8758221673193468, + 0.9032444171738859, + 0.9303783636244995, + 0.9552629683285443, + 0.9757615411235937, + 1.0157402642998257, + 1.045251506463968, + 1.0688091449644375, + 1.0624071055149489, + 1.0529656438443655, + 1.0284038000374691, + 0.9523781978486857, + 0.8838469105126692, + 0.8403571044364565, + 0.7713914356307573, + 0.7238323895281237 + ], + "peak_layer": 1, + "peak_normalized": 2.4104888609370185, + "population_cv": 0.279812062407651, + "rest_mean": 1.0410222465132527e-06, + "spike_contrast": 1.040064250882766, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0827300229721004e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 24 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.5250812996091554e-06, + 1.4057681028134539e-06, + 1.2773216440109536e-06, + 1.1374535233699135e-06, + 1.0254004791931948e-06, + 9.922503068082733e-07, + 9.821884532357217e-07, + 9.893612968880916e-07, + 9.730418923936668e-07, + 9.323878771283489e-07, + 9.156297551271564e-07, + 9.125165547629877e-07, + 8.951715244620573e-07, + 9.044875355357362e-07, + 9.203963031723106e-07, + 9.297735914515215e-07, + 9.174579531645577e-07, + 9.461838317292859e-07, + 9.746077012096066e-07, + 1.0006752972913091e-06, + 1.0221483535133302e-06, + 1.0640276286721928e-06, + 1.0949418083328055e-06, + 1.1196193554496858e-06, + 1.1129129688924877e-06, + 1.103022668758058e-06, + 1.0772931773317396e-06, + 9.976533874578308e-07, + 9.258641853193694e-07, + 8.803069135865371e-07, + 8.08062679880095e-07, + 7.582427201668906e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.57942979260406e-05, + "first_to_last_ratio": 2.5994532681374096, + "imbalance_abs_log_ratio": 0.9553011414295731, + "last_quartile_mean": 9.922970434672607e-06, + "mean": 1.6124295910913133e-05, + "normalized": [ + 1.8750767487966784, + 2.0012252521436076, + 1.745806554118046, + 1.5722176381119681, + 1.5471673963385955, + 1.4484453745095827, + 1.357829461828116, + 1.2499614439371438, + 1.1713452937825866, + 1.079951327288068, + 1.0230197185281116, + 0.9769120487640461, + 0.96163457802513, + 0.918624119720732, + 0.8885326600597652, + 0.8580725357821459, + 0.8699071437806652, + 0.8325299891167686, + 0.8310410601256768, + 0.8095951692808093, + 0.8030390759939747, + 0.7755468823771603, + 0.7515517005264444, + 0.7277277649695101, + 0.7158336494479539, + 0.6950266607011315, + 0.6717468100420292, + 0.6353600749013578, + 0.6043990157307626, + 0.5679033620811016, + 0.527236654105826, + 0.5057328350845046 + ], + "peak_layer": 2, + "peak_normalized": 2.0012252521436076, + "population_cv": 0.3945192515369112, + "rest_mean": 1.6856638096753698e-05, + "spike_contrast": 0.7219498951999067, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2169648107374087e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.0234292353270575e-05, + 3.226834814995527e-05, + 2.8149901481810957e-05, + 2.5350902433274314e-05, + 2.4946984922280535e-05, + 2.3355161829385906e-05, + 2.189404403907247e-05, + 2.0154748199274763e-05, + 1.8887118130805902e-05, + 1.741345477057621e-05, + 1.6495472664246336e-05, + 1.575201895320788e-05, + 1.550568049424328e-05, + 1.4812167137279175e-05, + 1.4326963537314441e-05, + 1.3835815479978919e-05, + 1.4026640201336704e-05, + 1.3423959899228066e-05, + 1.3399951967585366e-05, + 1.305415207752958e-05, + 1.2948439689353108e-05, + 1.2505147424235474e-05, + 1.2118242011638358e-05, + 1.1734097824955825e-05, + 1.154231358668767e-05, + 1.1206815543118864e-05, + 1.0831444342329632e-05, + 1.0244733857689425e-05, + 9.745508577907458e-06, + 9.157041858998127e-06, + 8.501319825882092e-06, + 8.154585884767585e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.933050248226209e-05, + "first_to_last_ratio": 2.5350541410379415, + "imbalance_abs_log_ratio": 0.930214994226497, + "last_quartile_mean": 7.6252819098954205e-06, + "mean": 1.202020879986776e-05, + "normalized": [ + 1.8583274986098608, + 2.0026775386155595, + 1.77731729208076, + 1.615557162502337, + 1.5930029936690748, + 1.4693486681854961, + 1.3292468480215616, + 1.2198575705059131, + 1.1444407348951777, + 1.053479376420273, + 1.0059382961834429, + 0.9516573870963836, + 0.9421936613647472, + 0.8964225087904136, + 0.8652818603489327, + 0.841617098226079, + 0.8649772378810286, + 0.8234169272301419, + 0.8325562065776818, + 0.8056805753420667, + 0.8017546832979361, + 0.7661992033221119, + 0.7431607861543773, + 0.720913207047878, + 0.7139696155781592, + 0.6926020821557612, + 0.6726446950161074, + 0.6435870710508805, + 0.6274403778164145, + 0.5970363543218487, + 0.569837411659171, + 0.5578570700324221 + ], + "peak_layer": 2, + "peak_normalized": 2.0026775386155595, + "population_cv": 0.3945939876253568, + "rest_mean": 1.2578481834713818e-05, + "spike_contrast": 0.7159476421745723, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.005534411699046e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.2337484551826492e-05, + 2.4072602172964253e-05, + 2.136372495442629e-05, + 1.941933442139998e-05, + 1.91482286027167e-05, + 1.7661877791397274e-05, + 1.5977824659785256e-05, + 1.4662942703580484e-05, + 1.3756416592514142e-05, + 1.2663042070926167e-05, + 1.20915883599082e-05, + 1.143912049883511e-05, + 1.1325364539516158e-05, + 1.0775185728562064e-05, + 1.0400868632132187e-05, + 1.0116413250216283e-05, + 1.039720700646285e-05, + 9.897643394651823e-06, + 1.0007499440689571e-05, + 9.68444874160923e-06, + 9.63725869951304e-06, + 9.209874406224117e-06, + 8.932947821449488e-06, + 8.66552727529779e-06, + 8.582063856010791e-06, + 8.325221642735414e-06, + 8.08532968221698e-06, + 7.736050974926911e-06, + 7.5419643508212175e-06, + 7.176501640060451e-06, + 6.849564670119435e-06, + 6.705558462272165e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.031241830190993e-05, + "first_to_last_ratio": 2.261102148101721, + "imbalance_abs_log_ratio": 0.8158523705600311, + "last_quartile_mean": 3.109652448074485e-05, + "mean": 4.619880877498872e-05, + "normalized": [ + 1.7997064857450336, + 1.904359030499371, + 1.6374224610397496, + 1.477941957137803, + 1.4602990373876177, + 1.3729930118356377, + 1.3118711870701552, + 1.2110312951649602, + 1.137824927128826, + 1.0604309170276205, + 1.0061171703075658, + 0.9633053240172016, + 0.949330006688662, + 0.9216248238136175, + 0.906779203920096, + 0.8687912686710114, + 0.8800028320072268, + 0.8475838252231725, + 0.8463171144866274, + 0.8286039531613457, + 0.8201877997739442, + 0.8102510580287974, + 0.7986912791711968, + 0.7937156242787844, + 0.7857004463402506, + 0.7744091131435279, + 0.7453649305566711, + 0.6981461341444608, + 0.6620680295240863, + 0.614979715499695, + 0.5681574453782887, + 0.5359925918269958 + ], + "peak_layer": 2, + "peak_normalized": 1.904359030499371, + "population_cv": 0.3447198298829091, + "rest_mean": 4.7895252632938706e-05, + "spike_contrast": 0.7733128004546086, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.70380119420588e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.314429578604177e-05, + 8.797911868896335e-05, + 7.564696716144681e-05, + 6.827915785834193e-05, + 6.746407598257065e-05, + 6.343064160319045e-05, + 6.060688610887155e-05, + 5.594820322585292e-05, + 5.256615622784011e-05, + 4.899064515484497e-05, + 4.648141475627199e-05, + 4.4503558456199244e-05, + 4.3857915443368256e-05, + 4.2577968997647986e-05, + 4.189211904304102e-05, + 4.013712168671191e-05, + 4.06550825573504e-05, + 3.915736306225881e-05, + 3.909884253516793e-05, + 3.828051558230072e-05, + 3.789169932133518e-05, + 3.74326336896047e-05, + 3.689858567668125e-05, + 3.666871634777635e-05, + 3.629842467489652e-05, + 3.577677853172645e-05, + 3.443497189437039e-05, + 3.225351974833757e-05, + 3.058675429201685e-05, + 2.8411330276867375e-05, + 2.624819717311766e-05, + 2.476221925462596e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13583384454250336, + 0.0415157787501812, + 0.018140379339456558, + 0.039518751204013824, + 0.022877270355820656, + 0.015532293356955051, + 0.02314678393304348, + 0.05432315170764923, + 0.03743875026702881, + 0.04006272554397583, + 0.045583780854940414, + 0.05995568633079529, + 0.038651175796985626, + 0.04559725895524025, + 0.05632574483752251, + 0.07952121645212173, + 0.042337141931056976, + 0.05044656619429588, + 0.04418695345520973, + 0.06310147047042847, + 0.029010163620114326, + 0.04586305841803551, + 0.06033914536237717, + 0.07656749337911606, + 0.045884523540735245, + 0.051604680716991425, + 0.06502768397331238, + 0.06792232394218445, + 0.060306549072265625, + 0.08025261014699936, + 0.09663844108581543, + 0.12230555713176727 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.247537380157155e-06, + "first_to_last_ratio": 2.3850511827649155, + "imbalance_abs_log_ratio": 0.8692205843888265, + "last_quartile_mean": 3.458012741930361e-06, + "mean": 5.339330805043119e-06, + "normalized": [ + 1.6764909218312523, + 1.9617954794601216, + 1.6992151272822646, + 1.4983517248278486, + 1.5139649706461762, + 1.4256520166778042, + 1.3385091192829222, + 1.2434304822395104, + 1.131080096564001, + 1.0546107928438107, + 1.0037428856625374, + 0.962940037127743, + 0.9549245789646706, + 0.9150321056456306, + 0.8876946170122294, + 0.8650767880832936, + 0.8770248559584294, + 0.8519848957954361, + 0.8533470944296186, + 0.848236507394223, + 0.8353817265855179, + 0.8173190953401269, + 0.8073880080861434, + 0.7956133450140085, + 0.7777311024585761, + 0.7505722139587175, + 0.705927759165694, + 0.665757931848141, + 0.624138138877991, + 0.5752620873586926, + 0.5511417007015933, + 0.530661792875274 + ], + "peak_layer": 2, + "peak_normalized": 1.9617954794601216, + "population_cv": 0.3568485733648799, + "rest_mean": 5.530472267141535e-06, + "spike_contrast": 0.7788063481128092, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.307166909711668e-06, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.95133962330874e-06, + 1.0474675036675762e-05, + 9.07267167349346e-06, + 8.000195521162823e-06, + 8.08355980552733e-06, + 7.612027729919646e-06, + 7.146742973418441e-06, + 6.639086677751038e-06, + 6.039210802555317e-06, + 5.630915893561905e-06, + 5.359315309760859e-06, + 5.141455403645523e-06, + 5.098658220958896e-06, + 4.885659109277185e-06, + 4.7396952140843496e-06, + 4.6189311433408875e-06, + 4.682725830207346e-06, + 4.549029199552024e-06, + 4.556302428682102e-06, + 4.52901531389216e-06, + 4.460379386728164e-06, + 4.363937023299513e-06, + 4.310911663196748e-06, + 4.2480428419366945e-06, + 4.1525636333972216e-06, + 4.007553343399195e-06, + 3.7691818306484492e-06, + 3.5547018342185766e-06, + 3.3324799915135372e-06, + 3.071514584007673e-06, + 2.942727860499872e-06, + 2.833378857758362e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.269888588874892e-05, + "first_to_last_ratio": 2.35686858292009, + "imbalance_abs_log_ratio": 0.8573338663198418, + "last_quartile_mean": 1.3873868965674774e-05, + "mean": 2.1262430664137355e-05, + "normalized": [ + 1.6815888767055769, + 1.9686225336479115, + 1.7075392779747713, + 1.4904980805963362, + 1.4972655559630919, + 1.4034192556792997, + 1.3238794766841318, + 1.2301588485530683, + 1.121552358208752, + 1.0499507133536148, + 1.0011277236394078, + 0.963324273966619, + 0.9560838812000779, + 0.9203885412533225, + 0.8930508699850467, + 0.8703824880945366, + 0.8841528709344532, + 0.8558102501916335, + 0.8550558750774316, + 0.8483637687825304, + 0.8350347355926667, + 0.8178886567183555, + 0.8078366810386547, + 0.7969743808996977, + 0.7797462600958652, + 0.7532932557652127, + 0.7097594040214169, + 0.6713244282550651, + 0.6299481338236045, + 0.5812721180769863, + 0.5573951790036309, + 0.5373112462172303 + ], + "peak_layer": 2, + "peak_normalized": 1.9686225336479115, + "population_cv": 0.35389080715057486, + "rest_mean": 2.202041212984585e-05, + "spike_contrast": 0.7797006998811187, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7169330749311485e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5754666896536946e-05, + 4.1857700125547126e-05, + 3.630643550422974e-05, + 3.169161209370941e-05, + 3.183550506946631e-05, + 2.9840104616596363e-05, + 2.8148895580670796e-05, + 2.615616722323466e-05, + 2.3846929252613336e-05, + 2.232450424344279e-05, + 2.1286408809828572e-05, + 2.0482615582295693e-05, + 2.0328667233115993e-05, + 1.9569697542465292e-05, + 1.89884322026046e-05, + 1.850644730438944e-05, + 1.8799239114741795e-05, + 1.819660610635765e-05, + 1.818056625779718e-05, + 1.8038275811704807e-05, + 1.7754868167685345e-05, + 1.7390300854458474e-05, + 1.717657141853124e-05, + 1.6945612514973618e-05, + 1.6579300790908746e-05, + 1.601684562047012e-05, + 1.5091210116224829e-05, + 1.4273989108914975e-05, + 1.339422851742711e-05, + 1.2359258107608184e-05, + 1.1851576346089132e-05, + 1.1424543117755093e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.02899769856594503, + "first_to_last_ratio": 2.35686858681256, + "imbalance_abs_log_ratio": 0.8573338679713849, + "last_quartile_mean": 0.012303485535085201, + "mean": 0.01885573595063761, + "normalized": [ + 1.6815888871597915, + 1.9686224147635019, + 1.7075392481901774, + 1.4904981407970117, + 1.4972654379699921, + 1.4034192484903574, + 1.3238795515861768, + 1.230158916579243, + 1.1215523220126975, + 1.0499507103306343, + 1.001127702787151, + 0.9633242487753337, + 0.9560838749935686, + 0.9203885704718483, + 0.8930508868493195, + 0.8703824268783725, + 0.8841529174326813, + 0.8558103041085099, + 0.8550558906482283, + 0.8483637681044575, + 0.8350347432767428, + 0.8178887044245051, + 0.8078366908478906, + 0.796974401455123, + 0.7797462730931964, + 0.7532932002519721, + 0.7097594341790385, + 0.671324454011456, + 0.6299481306480355, + 0.5812721137067387, + 0.5573951795880433, + 0.5373112055882039 + ], + "peak_layer": 2, + "peak_normalized": 1.9686224147635019, + "population_cv": 0.35389079505518783, + "rest_mean": 0.019527921418624895, + "spike_contrast": 0.7797007217053022, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01522593442350626, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03170759603381157, + 0.037119824439287186, + 0.03219690918922424, + 0.02810443937778473, + 0.028232041746377945, + 0.026462502777576447, + 0.02496272325515747, + 0.023195551708340645, + 0.021147694438695908, + 0.019797593355178833, + 0.018876999616622925, + 0.01816418766975403, + 0.018027665093541145, + 0.017354603856801987, + 0.016839131712913513, + 0.01641170121729374, + 0.016671353951096535, + 0.016136933118104935, + 0.016122708097100258, + 0.015996523201465607, + 0.015745194628834724, + 0.015421893447637558, + 0.015232355333864689, + 0.015027538873255253, + 0.014702689833939075, + 0.014203897677361965, + 0.013383036479353905, + 0.012658316642045975, + 0.011878135614097118, + 0.010960313491523266, + 0.010510096326470375, + 0.01013139821588993 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2314776824950968e-06, + "first_to_last_ratio": 1.358248310832468, + "imbalance_abs_log_ratio": 0.30619586281350625, + "last_quartile_mean": 9.066660879852861e-07, + "mean": 1.0095481766114744e-06, + "normalized": [ + 2.2510647132267514, + 1.3455279053230764, + 1.2283491143156429, + 1.0526455105583856, + 1.0056690176197902, + 0.9692221599384264, + 0.9572558258596283, + 0.9489099547933215, + 0.9218216280115407, + 0.8905546788106925, + 0.8763660110671972, + 0.8694242377636043, + 0.8685998082234971, + 0.8764693322122661, + 0.8891057616302812, + 0.8990022944596364, + 0.9003146700723137, + 0.9275413402074646, + 0.9581989480285121, + 0.9876176030015524, + 1.000156116631407, + 1.0336678655116023, + 1.068099651182219, + 1.0896881962272793, + 1.081885225695443, + 1.0551976273002857, + 0.9988041017347361, + 0.9238330401452145, + 0.8601167762256441, + 0.7958830056879117, + 0.7583180274077601, + 0.7106898511269164 + ], + "peak_layer": 1, + "peak_normalized": 2.2510647132267514, + "population_cv": 0.2561818803841662, + "rest_mean": 9.993219375953938e-07, + "spike_contrast": 1.0654923375948289, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0647698672983098e-06, + "top_five_layers": [ + 1, + 2, + 3, + 24, + 25 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.2725582766724983e-06, + 1.3583752433987684e-06, + 1.2400776085996768e-06, + 1.0626963558024727e-06, + 1.015271323012712e-06, + 9.784764642972732e-07, + 9.663958735472988e-07, + 9.579703146300744e-07, + 9.306233437200717e-07, + 8.990578521661519e-07, + 8.847337085171603e-07, + 8.777256539360678e-07, + 8.768933525971079e-07, + 8.848380161907698e-07, + 8.975951004686067e-07, + 9.075861271412577e-07, + 9.089110335480655e-07, + 9.363976687382092e-07, + 9.673480008132174e-07, + 9.970475502996123e-07, + 1.0097057838720502e-06, + 1.0435375088491128e-06, + 1.078298055290361e-06, + 1.1000927315762965e-06, + 1.092215256903728e-06, + 1.0652728406057577e-06, + 1.0083408596983645e-06, + 9.326539611720364e-07, + 8.683293231115385e-07, + 8.03482237188291e-07, + 7.655585818611144e-07, + 7.174756433414586e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.5231804784198175e-05, + "first_to_last_ratio": 2.6518592370690897, + "imbalance_abs_log_ratio": 0.975260992886681, + "last_quartile_mean": 9.514760222373297e-06, + "mean": 1.575830160049918e-05, + "normalized": [ + 1.7680072391258728, + 2.020274701367778, + 1.7747981337484644, + 1.5702283057720683, + 1.5745659549387958, + 1.468575316323847, + 1.366589194311488, + 1.2663643116143115, + 1.1465258163389551, + 1.0704343447928568, + 1.0164163108220898, + 0.9740484512911409, + 0.9677541966052103, + 0.9245201462352831, + 0.8913652028193111, + 0.8644158057124367, + 0.8839251267896815, + 0.845522595189969, + 0.8460497665068152, + 0.8329662909412521, + 0.8145821291394524, + 0.7836795820242509, + 0.7613757450733065, + 0.7366672589275867, + 0.716422356677449, + 0.689594392508022, + 0.652884419046894, + 0.6205919149770609, + 0.5818976326649927, + 0.5425153265661802, + 0.5218602418605108, + 0.5045817892866661 + ], + "peak_layer": 2, + "peak_normalized": 2.020274701367778, + "population_cv": 0.39415073781885135, + "rest_mean": 1.645124252212958e-05, + "spike_contrast": 0.7304263253994938, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2016420623695012e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.786079130601138e-05, + 3.1836098060011864e-05, + 2.7967804271611385e-05, + 2.47441312239971e-05, + 2.4812485207803547e-05, + 2.314225275767967e-05, + 2.1535124687943608e-05, + 1.9955750758526847e-05, + 1.8067299606627785e-05, + 1.6868227248778567e-05, + 1.6016994777601212e-05, + 1.5349349268944934e-05, + 1.5250162505253684e-05, + 1.4568867300113197e-05, + 1.4046401702216826e-05, + 1.362172497465508e-05, + 1.3929158740211278e-05, + 1.3324000065040309e-05, + 1.3332307389646303e-05, + 1.3126134035701398e-05, + 1.2836430869356263e-05, + 1.2349459211691283e-05, + 1.1997988622169942e-05, + 1.1608624845393933e-05, + 1.128959956986364e-05, + 1.0866836419154424e-05, + 1.0288349585607648e-05, + 9.77947456703987e-06, + 9.169718396151438e-06, + 8.549120138923172e-06, + 8.223631084547378e-06, + 7.95135201769881e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.902213432458666e-05, + "first_to_last_ratio": 2.570514437449803, + "imbalance_abs_log_ratio": 0.9441060490896753, + "last_quartile_mean": 7.4001274015245144e-06, + "mean": 1.1811577877551827e-05, + "normalized": [ + 1.7836661639182594, + 2.007884486098661, + 1.801766001704916, + 1.6166376372836335, + 1.6162792780207194, + 1.484195322536222, + 1.334217273990145, + 1.2390748916935237, + 1.1152682343795186, + 1.047183670438877, + 0.9989511624599826, + 0.9468241347561922, + 0.942586809823197, + 0.9015408911709663, + 0.8679632288380377, + 0.8434512704540663, + 0.8735354304753469, + 0.8326703084602718, + 0.8456967523674954, + 0.8281893546686905, + 0.8104136880244638, + 0.7723543788975116, + 0.750047939286133, + 0.727483933767052, + 0.7080168021618385, + 0.6848377954753971, + 0.6576536196741141, + 0.6404010158732426, + 0.6084114841371344, + 0.5799389397954393, + 0.5702129585195616, + 0.5626451408493897 + ], + "peak_layer": 2, + "peak_normalized": 2.007884486098661, + "population_cv": 0.39402258867818274, + "rest_mean": 1.2350397237578701e-05, + "spike_contrast": 0.720782753960385, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.901953333406708e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1067911802674644e-05, + 2.3716283976682462e-05, + 2.1281699446262792e-05, + 1.909504135255702e-05, + 1.9090808564214967e-05, + 1.753068863763474e-05, + 1.57592112373095e-05, + 1.4635429579357151e-05, + 1.3173077604733407e-05, + 1.2368891475489363e-05, + 1.1799189451267011e-05, + 1.1183487004018389e-05, + 1.1133437510579824e-05, + 1.0648620445863344e-05, + 1.025201527227182e-05, + 9.962490366888233e-06, + 1.0317831765860319e-05, + 9.835150194703601e-06, + 9.989013051381335e-06, + 9.782223060028628e-06, + 9.572264389134943e-06, + 9.12272389541613e-06, + 8.859249646775424e-06, + 8.592733138357289e-06, + 8.36279559734976e-06, + 8.089014954748563e-06, + 7.767926945234649e-06, + 7.564146471850108e-06, + 7.186299626482651e-06, + 6.849993951618671e-06, + 6.735114766343031e-06, + 6.645726898568682e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.828012237747316e-05, + "first_to_last_ratio": 2.312494882445189, + "imbalance_abs_log_ratio": 0.8383269774053195, + "last_quartile_mean": 2.9526604748753016e-05, + "mean": 4.489596511803029e-05, + "normalized": [ + 1.6577187409379064, + 1.9152258678535772, + 1.6744752052558183, + 1.4840054420576252, + 1.4912917773323804, + 1.3961062438002583, + 1.3297764454700043, + 1.2182169964281564, + 1.1079961846053805, + 1.047881406618081, + 0.999963947398217, + 0.9691026873070004, + 0.9556277478959141, + 0.9353258396988691, + 0.9030474230504885, + 0.8874013307551517, + 0.8935396144488046, + 0.8678070689766392, + 0.8641197392760416, + 0.8577613745480667, + 0.8343135878548937, + 0.8204103169462279, + 0.8193971014627366, + 0.8081501178832716, + 0.7960663239792694, + 0.7673062057216204, + 0.7200712732146042, + 0.6825993169002439, + 0.6333797250658595, + 0.5852510577399719, + 0.5516412156589899, + 0.5250226738579302 + ], + "peak_layer": 2, + "peak_normalized": 1.9152258678535772, + "population_cv": 0.3428609763636811, + "rest_mean": 4.642851807282703e-05, + "spike_contrast": 0.7887432268392901, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6620179162127894e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.442488276865333e-05, + 8.598591375630349e-05, + 7.517718040617183e-05, + 6.662585656158626e-05, + 6.695298361591995e-05, + 6.267953722272068e-05, + 5.970159691059962e-05, + 5.469302777783014e-05, + 4.9744558054953814e-05, + 4.704564707935788e-05, + 4.489434650167823e-05, + 4.350880044512451e-05, + 4.2903830035356805e-05, + 4.199235627311282e-05, + 4.054318560520187e-05, + 3.984073919127695e-05, + 4.0116323361871764e-05, + 3.89610358979553e-05, + 3.879548967233859e-05, + 3.851002475130372e-05, + 3.745731373783201e-05, + 3.683311297209002e-05, + 3.6787623685086146e-05, + 3.628267950261943e-05, + 3.574016591301188e-05, + 3.4448952646926045e-05, + 3.232829476473853e-05, + 3.064595512114465e-05, + 2.8436194043024443e-05, + 2.627541107358411e-05, + 2.4766464775893837e-05, + 2.357139965170063e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.35791492462158203, + 0.3940892517566681, + 0.4057151973247528, + 0.09380833059549332, + 0.11286329478025436, + 0.13231655955314636, + 0.1621972620487213, + 0.05156813934445381, + 0.08533531427383423, + 0.12003090232610703, + 0.16063299775123596, + 0.046871740370988846, + 0.079384446144104, + 0.11459959298372269, + 0.15625107288360596, + 0.08397205173969269, + 0.11654024571180344, + 0.15294702351093292, + 0.21572710573673248, + 0.03708760067820549, + 0.06553087383508682, + 0.09390194714069366, + 0.12852425873279572, + 0.025487789884209633, + 0.06059965491294861, + 0.08860157430171967, + 0.12100546807050705, + 0.024166326969861984, + 0.06877554953098297, + 0.12682415544986725, + 0.2025238573551178 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.4598292460214e-06, + "first_to_last_ratio": 2.328358854359362, + "imbalance_abs_log_ratio": 0.8451636650513356, + "last_quartile_mean": 3.633387194668103e-06, + "mean": 5.48445886039417e-06, + "normalized": [ + 1.7981292044822659, + 1.9357505739332324, + 1.6671812164387254, + 1.501771337574118, + 1.4824621119466825, + 1.4013991506965175, + 1.3280020709214548, + 1.225379388067507, + 1.156718780628782, + 1.0641415748591472, + 1.0106456553801202, + 0.9662668896138349, + 0.9475625334929744, + 0.9090604128642483, + 0.8837694050828951, + 0.8574965104342549, + 0.860785690483099, + 0.8373501686260958, + 0.8360756725074443, + 0.8245861368451315, + 0.8217235577025693, + 0.8060327700829376, + 0.7938754319523651, + 0.7839311948498927, + 0.7744962255348613, + 0.7571785503773947, + 0.7308065715532428, + 0.6865019323584283, + 0.6478069914366877, + 0.6078375958541098, + 0.5594073652125656, + 0.5358673282064139 + ], + "peak_layer": 2, + "peak_normalized": 1.9357505739332324, + "population_cv": 0.3564831577662627, + "rest_mean": 5.691637507307287e-06, + "spike_contrast": 0.7670365797994658, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.365694167063339e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.861765647656284e-06, + 1.0616544386721216e-05, + 9.143586794380099e-06, + 8.236403118644375e-06, + 8.130502465064637e-06, + 7.68591598898638e-06, + 7.283372724486981e-06, + 6.720542842231225e-06, + 6.343976565403864e-06, + 5.836240688950056e-06, + 5.542844519368373e-06, + 5.2994510042481124e-06, + 5.196867732593091e-06, + 4.985704435966909e-06, + 4.846996944252169e-06, + 4.702904334408231e-06, + 4.720943707070546e-06, + 4.592412551573943e-06, + 4.5854226300434675e-06, + 4.522408744378481e-06, + 4.506709046836477e-06, + 4.420653567649424e-06, + 4.353977146820398e-06, + 4.299438387533883e-06, + 4.247692686476512e-06, + 4.152714609517716e-06, + 4.008078576589469e-06, + 3.765091605600901e-06, + 3.552870794010232e-06, + 3.333660288262763e-06, + 3.068046680709813e-06, + 2.9389423161774175e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.3572764550626744e-05, + "first_to_last_ratio": 2.3057895392595262, + "imbalance_abs_log_ratio": 0.8354231510987864, + "last_quartile_mean": 1.456020333989727e-05, + "mean": 2.183873047556517e-05, + "normalized": [ + 1.80924687671537, + 1.9431940819531333, + 1.676696643261768, + 1.4950864798932781, + 1.467401917430866, + 1.3807559335924544, + 1.31358592866812, + 1.212462979303528, + 1.1458997558869217, + 1.0585578682053984, + 1.0064671958441547, + 0.9652690219967264, + 0.9479597986271983, + 0.9127134975291503, + 0.888075836142126, + 0.862166224116184, + 0.867826991667279, + 0.841547063844033, + 0.83837247628253, + 0.8251061585552277, + 0.8217963049445071, + 0.8072479582581175, + 0.7943165572696724, + 0.7845285926712973, + 0.7759202912841717, + 0.7590490167365564, + 0.7335626108658329, + 0.6907982993808188, + 0.6532520589865599, + 0.6137777359935852, + 0.565539938799635, + 0.5418179052937989 + ], + "peak_layer": 2, + "peak_normalized": 1.9431940819531333, + "population_cv": 0.3545305270645445, + "rest_mean": 2.2660667733968392e-05, + "spike_contrast": 0.76786216030628, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7400269280187784e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.951165490434505e-05, + 4.2436891817487776e-05, + 3.6616926081478596e-05, + 3.265079067205079e-05, + 3.204619497410022e-05, + 3.0153956686262973e-05, + 2.868704905267805e-05, + 2.64786522166105e-05, + 2.5024995920830406e-05, + 2.3117559976526536e-05, + 2.197996582253836e-05, + 2.1080250007798895e-05, + 2.0702238543890417e-05, + 1.993250407394953e-05, + 1.939444882737007e-05, + 1.882861579360906e-05, + 1.8952239770442247e-05, + 1.8378319509793073e-05, + 1.8308990547666326e-05, + 1.801927101041656e-05, + 1.7946988009498455e-05, + 1.762927058734931e-05, + 1.7346865206491202e-05, + 1.7133108485722914e-05, + 1.6945114111877047e-05, + 1.6576666894252412e-05, + 1.602007614565082e-05, + 1.508615787315648e-05, + 1.4266195648815483e-05, + 1.3404126548266504e-05, + 1.2350674296612851e-05, + 1.183261520054657e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.029772662557661533, + "first_to_last_ratio": 2.3057895177959966, + "imbalance_abs_log_ratio": 0.8354231417902483, + "last_quartile_mean": 0.01291213370859623, + "mean": 0.019366804277524352, + "normalized": [ + 1.809246767643357, + 1.9431941106945074, + 1.6766966870594806, + 1.495086420690814, + 1.4674019077542524, + 1.3807559578984756, + 1.3135858925156472, + 1.2124629647121858, + 1.1458997775072552, + 1.0585579855568523, + 1.0064672559708643, + 0.9652689798174655, + 0.9479597754699817, + 0.9127135219659304, + 0.888075806778596, + 0.8621662442022726, + 0.8678270422679951, + 0.841547005089253, + 0.8383724840087661, + 0.8251060885690371, + 0.8217963423351232, + 0.8072480008996415, + 0.7943165425842438, + 0.7845285881999872, + 0.7759202953515476, + 0.7590489858287814, + 0.7335626029644629, + 0.6907983214336281, + 0.653252038106227, + 0.6137777383932114, + 0.5655399639078199, + 0.5418179038223359 + ], + "peak_layer": 2, + "peak_normalized": 1.9431941106945074, + "population_cv": 0.35453052191257556, + "rest_mean": 0.020095706399944093, + "spike_contrast": 0.7678621746036595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.015430732816457748, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.035039328038692474, + 0.03763346001505852, + 0.03247225657105446, + 0.028955046087503433, + 0.02841888554394245, + 0.02674083039164543, + 0.02543996088206768, + 0.02348153293132782, + 0.022192416712641716, + 0.02050088532269001, + 0.019492054358124733, + 0.01869417540729046, + 0.018358951434493065, + 0.017676344141364098, + 0.017199190333485603, + 0.01669740490615368, + 0.016807036474347115, + 0.016298076137900352, + 0.016236595809459686, + 0.015979668125510216, + 0.01591556891798973, + 0.01563381403684616, + 0.015383373014628887, + 0.015193811617791653, + 0.01502709649503231, + 0.014700353145599365, + 0.014206763356924057, + 0.01337855588644743, + 0.012651404365897179, + 0.01188691332936287, + 0.010952701792120934, + 0.010493281297385693 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.291835857841761e-06, + "first_to_last_ratio": 1.3485833026747331, + "imbalance_abs_log_ratio": 0.299054636012817, + "last_quartile_mean": 9.579206974308363e-07, + "mean": 1.0475393885656104e-06, + "normalized": [ + 2.4104197937214296, + 1.3419993768147092, + 1.2193873876623726, + 1.0857681046985204, + 0.9788139215354608, + 0.9471793472155275, + 0.9376610507921448, + 0.9444495548464527, + 0.9288857336777386, + 0.8900930106166185, + 0.8741329012581984, + 0.8711139916120108, + 0.8545363607420399, + 0.8634605233315061, + 0.8786425447326217, + 0.887541854523117, + 0.8758151318715187, + 0.9032250016851439, + 0.9304164056283383, + 0.9552919953757107, + 0.9757865467437721, + 1.0156631340115454, + 1.0452801819870623, + 1.0688491005570937, + 1.0624669234868362, + 1.0529715263673087, + 1.0284586355464043, + 0.9523438456637114, + 0.8838175160851754, + 0.8403596296329269, + 0.7713843760240694, + 0.7237845915529142 + ], + "peak_layer": 1, + "peak_normalized": 2.4104197937214296, + "population_cv": 0.27980440532718537, + "rest_mean": 1.0410195853991254e-06, + "spike_contrast": 1.0400825698677958, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0827463256646297e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 24 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.5250096769013908e-06, + 1.4057972066439106e-06, + 1.2773563184964587e-06, + 1.1373848565199296e-06, + 1.0253461368847638e-06, + 9.922076742441277e-07, + 9.82236883828591e-07, + 9.893481092149159e-07, + 9.730443935040967e-07, + 9.324074881078559e-07, + 9.156886449090962e-07, + 9.12526218144194e-07, + 8.951604968387983e-07, + 9.045089086612279e-07, + 9.204126740769425e-07, + 9.297350516135339e-07, + 9.174508477372001e-07, + 9.46163766002428e-07, + 9.746478326633223e-07, + 1.000705992737494e-06, + 1.0221748425465194e-06, + 1.063947138391086e-06, + 1.0949721627184772e-06, + 1.1196615332664805e-06, + 1.1129759514005855e-06, + 1.1030291489078081e-06, + 1.0773509302453022e-06, + 9.976176897907862e-07, + 9.258336604034412e-07, + 8.80309812600899e-07, + 8.080555176093185e-07, + 7.581928684885497e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.5794245857468923e-05, + "first_to_last_ratio": 2.5994382822780953, + "imbalance_abs_log_ratio": 0.9552953764086378, + "last_quartile_mean": 9.923007610268542e-06, + "mean": 1.6124308444886992e-05, + "normalized": [ + 1.875069425095799, + 2.001233172595225, + 1.7457889523480976, + 1.5722148366283213, + 1.547165404000032, + 1.4484413155137394, + 1.3578243451672811, + 1.2499566367465564, + 1.1713539721385289, + 1.0799457497690719, + 1.023025015060715, + 0.976913996826235, + 0.9616437578274982, + 0.9186228415910193, + 0.888533041071715, + 0.8580748018430988, + 0.8699143078938142, + 0.8325216144516643, + 0.8310392860253453, + 0.8096019290349791, + 0.8030372108501562, + 0.7755478024583314, + 0.7515516803716358, + 0.7277252251002508, + 0.7158404256809536, + 0.6950255563813353, + 0.6717498413974154, + 0.6353591861780038, + 0.6043991099627453, + 0.567907771476934, + 0.5272362442668266, + 0.5057355442466751 + ], + "peak_layer": 2, + "peak_normalized": 2.001233172595225, + "population_cv": 0.3945173779757133, + "rest_mean": 1.685664924648504e-05, + "spike_contrast": 0.7219506046728209, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2169668116257526e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.023419776582159e-05, + 3.226850094506517e-05, + 2.8149639547336847e-05, + 2.5350876967422664e-05, + 2.494697218935471e-05, + 2.3355114535661414e-05, + 2.189397855545394e-05, + 2.0154686353635043e-05, + 1.8887272744905204e-05, + 1.741337837302126e-05, + 1.6495570889674127e-05, + 1.5752062608953565e-05, + 1.550584056531079e-05, + 1.4812158042332157e-05, + 1.4326980817713775e-05, + 1.3835862773703411e-05, + 1.402676662110025e-05, + 1.3423835298453923e-05, + 1.339993377769133e-05, + 1.3054271221335512e-05, + 1.294841968046967e-05, + 1.2505171980592422e-05, + 1.2118251106585376e-05, + 1.1734065992641263e-05, + 1.15424318209989e-05, + 1.1206806448171847e-05, + 1.0831501640495844e-05, + 1.0244727491226513e-05, + 9.745517672854476e-06, + 9.15712007554248e-06, + 8.501319825882092e-06, + 8.154635906976182e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9329483848196105e-05, + "first_to_last_ratio": 2.5350525368598045, + "imbalance_abs_log_ratio": 0.9302143614279234, + "last_quartile_mean": 7.624884915458097e-06, + "mean": 1.201984051135696e-05, + "normalized": [ + 1.8583357088751307, + 2.0027277021690506, + 1.7771668453163478, + 1.615570343516219, + 1.592938001514604, + 1.4694493793664982, + 1.32898385240296, + 1.2198799651106356, + 1.144267264787803, + 1.0533998962206053, + 1.0058282278662807, + 0.9519161926020662, + 0.9421831838426558, + 0.8966453451440455, + 0.8650037408265971, + 0.8416374374551236, + 0.8651561323934935, + 0.8233100437191515, + 0.8327440956360556, + 0.8058278405503257, + 0.8017291581294494, + 0.7663098470747268, + 0.7431513228704318, + 0.7209725235957172, + 0.7140172181236788, + 0.6925010270670932, + 0.6728195124367922, + 0.6435648715599169, + 0.6274519981756614, + 0.5969246531502274, + 0.5697514359189264, + 0.5578352325817294 + ], + "peak_layer": 2, + "peak_normalized": 2.0027277021690506, + "population_cv": 0.3945781723821696, + "rest_mean": 1.2578015162876841e-05, + "spike_contrast": 0.7159871630405812, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.005697393149603e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.233689883723855e-05, + 2.407246756774839e-05, + 2.1361262042773888e-05, + 1.941889786394313e-05, + 1.9146860722685233e-05, + 1.766254717949778e-05, + 1.597417394805234e-05, + 1.4662762623629533e-05, + 1.3753910025116056e-05, + 1.2661698747251648e-05, + 1.20898948807735e-05, + 1.144188081525499e-05, + 1.1324891602271236e-05, + 1.0777534043882042e-05, + 1.039720700646285e-05, + 1.0116347766597755e-05, + 1.039903872879222e-05, + 9.896055416902527e-06, + 1.0009451216319576e-05, + 9.685922123026103e-06, + 9.636656614020467e-06, + 9.210922144120559e-06, + 8.932560376706533e-06, + 8.665974746691063e-06, + 8.582373084209394e-06, + 8.32375189929735e-06, + 8.087183232419193e-06, + 7.735547114862129e-06, + 7.541872946603689e-06, + 7.174939128162805e-06, + 6.848321390862111e-06, + 6.705090527248103e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.030938422758481e-05, + "first_to_last_ratio": 2.2606879822899777, + "imbalance_abs_log_ratio": 0.8156691839012263, + "last_quartile_mean": 3.110087936875061e-05, + "mean": 4.619929336513451e-05, + "normalized": [ + 1.8000040071532064, + 1.904408036353138, + 1.637904058859854, + 1.4777606171460849, + 1.4597770726472121, + 1.372671031064488, + 1.3115103960008867, + 1.2109361461416157, + 1.137534863820107, + 1.0602352149716379, + 1.0062919047911225, + 0.9633358523801769, + 0.9491225557491253, + 0.9216727196154594, + 0.9066025950003278, + 0.8688487743640587, + 0.8802774784785187, + 0.8475087887298653, + 0.846208545767534, + 0.8286626678374982, + 0.8203744064059802, + 0.810217596933452, + 0.7988918917164441, + 0.7937267489942904, + 0.7857342550389422, + 0.7744580018919649, + 0.7454551502724039, + 0.6984329250267984, + 0.662100300186566, + 0.6150609478307189, + 0.5682149152709306, + 0.5360595335595907 + ], + "peak_layer": 2, + "peak_normalized": 1.904408036353138, + "population_cv": 0.3446893198253748, + "rest_mean": 4.789507282246767e-05, + "spike_contrast": 0.7734007302345921, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.704208429553546e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.315891318488866e-05, + 8.798230555839837e-05, + 7.567001011921093e-05, + 6.82714962749742e-05, + 6.744066922692582e-05, + 6.341643165796995e-05, + 6.0590853536268696e-05, + 5.5944394262041897e-05, + 5.255330688669346e-05, + 4.8982117732521147e-05, + 4.6489974920405075e-05, + 4.45054356532637e-05, + 4.384879139252007e-05, + 4.258062836015597e-05, + 4.188439925201237e-05, + 4.014019941678271e-05, + 4.066819747094996e-05, + 3.915430716006085e-05, + 3.9094236853998154e-05, + 3.828362969215959e-05, + 3.790071787079796e-05, + 3.743148045032285e-05, + 3.690824087243527e-05, + 3.6669614928541705e-05, + 3.630036735557951e-05, + 3.5779412428382784e-05, + 3.443950117798522e-05, + 3.2267107599182054e-05, + 3.058856600546278e-05, + 2.841538116626907e-05, + 2.6251127565046772e-05, + 2.4765571652096696e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.247530786320567e-06, + "first_to_last_ratio": 2.3850533729557593, + "imbalance_abs_log_ratio": 0.86922150268771, + "last_quartile_mean": 3.45800680179309e-06, + "mean": 5.339328467357518e-06, + "normalized": [ + 1.676519761737885, + 1.9618007671893674, + 1.6991821441606432, + 1.498358513038152, + 1.5139533691039981, + 1.4256686527077282, + 1.3385028917629673, + 1.243419273267211, + 1.1310636430692016, + 1.054615257538969, + 1.0037504793539562, + 0.9629245320498924, + 0.9549124771530781, + 0.9150161537455905, + 0.8877036077745802, + 0.8650752931079952, + 0.8770263471429458, + 0.8519858650005171, + 0.8533548777823373, + 0.8482425851215958, + 0.8353975931646603, + 0.8173266925801457, + 0.8073981560598078, + 0.7956149708934007, + 0.7777226705206242, + 0.7505720315608271, + 0.7059216805334544, + 0.665764185190204, + 0.6241330890536816, + 0.5752674068009126, + 0.5511430066219813, + 0.5306620252116886 + ], + "peak_layer": 2, + "peak_normalized": 1.9618007671893674, + "population_cv": 0.3568479486512281, + "rest_mean": 5.530464544857817e-06, + "spike_contrast": 0.7788122704557785, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.3071936488559e-06, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.951489689934533e-06, + 1.0474698683538008e-05, + 9.072491593542509e-06, + 8.000228262972087e-06, + 8.083494321908802e-06, + 7.612113222421613e-06, + 7.14670659363037e-06, + 6.639023922616616e-06, + 6.039120307832491e-06, + 5.630937266687397e-06, + 5.359353508538334e-06, + 5.141370365890907e-06, + 5.098591373098316e-06, + 4.885571797785815e-06, + 4.739741143566789e-06, + 4.618921138899168e-06, + 4.6827317419229075e-06, + 4.54903238278348e-06, + 4.556341991701629e-06, + 4.529045781964669e-06, + 4.460462150746025e-06, + 4.363975676824339e-06, + 4.3109639591421e-06, + 4.248049663146958e-06, + 4.15251679442008e-06, + 4.00755061491509e-06, + 3.7691477245971328e-06, + 3.5547336665331386e-06, + 3.332451569804107e-06, + 3.0715416414750507e-06, + 2.942733544841758e-06, + 2.833378857758362e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.269887292844942e-05, + "first_to_last_ratio": 2.3568639793204684, + "imbalance_abs_log_ratio": 0.8573319130483905, + "last_quartile_mean": 1.3873890566173941e-05, + "mean": 2.12624314031018e-05, + "normalized": [ + 1.6816001107908207, + 1.968609461712434, + 1.7075260440141944, + 1.4905139409933708, + 1.4972595154646264, + 1.4034185225085787, + 1.3238781474315933, + 1.2301608589864579, + 1.121553773570522, + 1.0499501635664128, + 1.001131281922829, + 0.9633193641679035, + 0.9560899219831485, + 0.9203740514079343, + 0.8930538331784411, + 0.8703809179546808, + 0.8841528402061924, + 0.8558056863273277, + 0.8550540488218812, + 0.8483642525948266, + 0.8350365031100208, + 0.8178976965352276, + 0.8078428125234317, + 0.7969682791900239, + 0.7797478584358322, + 0.7532962238158083, + 0.7097589943815636, + 0.6713244476982704, + 0.629948240254275, + 0.5812766747709892, + 0.5573943041371163, + 0.5373112275432649 + ], + "peak_layer": 2, + "peak_normalized": 1.968609461712434, + "population_cv": 0.35388972104860866, + "rest_mean": 2.2020403102268808e-05, + "spike_contrast": 0.7797034481094922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.716938422759995e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5754907003138214e-05, + 4.185742363915779e-05, + 3.630615537986159e-05, + 3.169195042573847e-05, + 3.183537774020806e-05, + 2.9840090064681135e-05, + 2.8148868295829743e-05, + 2.6156210878980346e-05, + 2.3846960175433196e-05, + 2.2324493329506367e-05, + 2.128648520738352e-05, + 2.048251189989969e-05, + 2.0328796381363645e-05, + 1.9569390133256093e-05, + 1.8988495867233723e-05, + 1.8506414562580176e-05, + 1.8799239114741795e-05, + 1.819650969991926e-05, + 1.8180528059019707e-05, + 1.8038286725641228e-05, + 1.775490636646282e-05, + 1.739049366733525e-05, + 1.7176702385768294e-05, + 1.6945483366725966e-05, + 1.6579335351707414e-05, + 1.6016909285099246e-05, + 1.5091201930772513e-05, + 1.4273990018409677e-05, + 1.3394231245911215e-05, + 1.2359355423541274e-05, + 1.1851558156195097e-05, + 1.1424543117755093e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.02899768785573542, + "first_to_last_ratio": 2.3568640590123793, + "imbalance_abs_log_ratio": 0.8573319468610798, + "last_quartile_mean": 0.012303504627197981, + "mean": 0.018855736823752522, + "normalized": [ + 1.6816000706700753, + 1.9686094816861437, + 1.707526129634271, + 1.4905139760038595, + 1.4972595403829805, + 1.403418590801014, + 1.323878107307901, + 1.2301608352967786, + 1.12155375183925, + 1.0499501677926542, + 1.0011313114380156, + 0.963319363752492, + 0.9560898565462423, + 0.9203740066049206, + 0.8930539078006676, + 0.8703809048152793, + 0.8841528764919796, + 0.8558057204161084, + 0.8550539741588212, + 0.848364321524986, + 0.8350365815065012, + 0.8178976558963745, + 0.8078427780491525, + 0.7969682399430816, + 0.7797478175310808, + 0.7532962276748001, + 0.7097590061776181, + 0.6713244229257764, + 0.6299482002622888, + 0.5812766802470385, + 0.5573943141138397, + 0.5373111807080079 + ], + "peak_layer": 2, + "peak_normalized": 1.9686094816861437, + "population_cv": 0.3538897326937565, + "rest_mean": 0.019527913761083728, + "spike_contrast": 0.7797034311216169, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01522598136216402, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03170780837535858, + 0.037119582295417786, + 0.032196663320064545, + 0.028104739263653755, + 0.02823193185031414, + 0.02646249160170555, + 0.02496269717812538, + 0.02319558896124363, + 0.021147722378373146, + 0.019797584041953087, + 0.018877068534493446, + 0.018164096400141716, + 0.01802777871489525, + 0.01735433004796505, + 0.01683918945491314, + 0.0164116732776165, + 0.016671353951096535, + 0.01613684743642807, + 0.016122672706842422, + 0.015996534377336502, + 0.01574523001909256, + 0.015422062948346138, + 0.015232470817863941, + 0.015027423389256, + 0.014702719636261463, + 0.014203955419361591, + 0.013383029028773308, + 0.012658316642045975, + 0.011878137476742268, + 0.010960400104522705, + 0.010510080493986607, + 0.01013139821588993 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.231474328733384e-06, + "first_to_last_ratio": 1.3582476774197694, + "imbalance_abs_log_ratio": 0.3061953964681102, + "last_quartile_mean": 9.066640416222072e-07, + "mean": 1.0095540616816834e-06, + "normalized": [ + 2.2510689330098104, + 1.3453781719433024, + 1.2282097485564385, + 1.0527316026565288, + 1.0056637182516928, + 0.969318760716823, + 0.9573356047524155, + 0.9488541987585868, + 0.9217951398143368, + 0.8906270200688119, + 0.8763270628110001, + 0.8694286288822738, + 0.8685312885606712, + 0.8765222738855435, + 0.8891328417344724, + 0.8990263889886259, + 0.900299568341834, + 0.9275418452948997, + 0.9582330013794911, + 0.9876130845276129, + 1.0001282145999768, + 1.0337196092881593, + 1.0681495050787642, + 1.0896942312291085, + 1.0818926575150707, + 1.0552217685009913, + 0.9988175358018173, + 0.923772869549939, + 0.860068575983845, + 0.7959012825140968, + 0.7583091587544984, + 0.710685708248561 + ], + "peak_layer": 1, + "peak_normalized": 2.2510689330098104, + "population_cv": 0.25617417026860584, + "rest_mean": 9.993245039704899e-07, + "spike_contrast": 1.065513423409031, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0647936733221285e-06, + "top_five_layers": [ + 1, + 2, + 3, + 24, + 25 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.2725757844455075e-06, + 1.3582319979832391e-06, + 1.2399441402521916e-06, + 1.0627894653225667e-06, + 1.0152718914469006e-06, + 9.785796919459244e-07, + 9.664820481702918e-07, + 9.579196103004506e-07, + 9.306020274379989e-07, + 8.991361255539232e-07, + 8.846995456224249e-07, + 8.777352036304364e-07, + 8.768292900640517e-07, + 8.848966217556153e-07, + 8.976276717476139e-07, + 9.076157425624842e-07, + 9.089010859497648e-07, + 9.364036372971896e-07, + 9.673880185800954e-07, + 9.970488008548273e-07, + 1.0096835012518568e-06, + 1.043595830196864e-06, + 1.0783546713355463e-06, + 1.1001052371284459e-06, + 1.09222912669793e-06, + 1.0653034223651048e-06, + 1.008360300147615e-06, + 9.325986525254848e-07, + 8.682857242092723e-07, + 8.035053724597674e-07, + 7.655540912310244e-07, + 7.174756433414586e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.523178955016192e-05, + "first_to_last_ratio": 2.651853802016533, + "imbalance_abs_log_ratio": 0.9752589433592965, + "last_quartile_mean": 9.514773978480662e-06, + "mean": 1.5758304414248414e-05, + "normalized": [ + 1.768014772712438, + 2.0202664913584756, + 1.7747876589600569, + 1.5702354129512541, + 1.5745618645822212, + 1.4685789787379961, + 1.3665832942020837, + 1.266364662649249, + 1.14652665049396, + 1.0704311524660057, + 1.0164222471522504, + 0.9740486236599476, + 0.9677569672847843, + 0.9245129976091487, + 0.891368391145493, + 0.8644143239142155, + 0.883928662735923, + 0.8455178269951987, + 0.8460475954048016, + 0.8329649879043416, + 0.8145835420025702, + 0.783683597592109, + 0.7613793029015486, + 0.7366657999397114, + 0.716423036769208, + 0.6895960585495816, + 0.6528852259142993, + 0.6205906498609965, + 0.5818984522075571, + 0.54251886575813, + 0.5218602063941854, + 0.5045816991902581 + ], + "peak_layer": 2, + "peak_normalized": 2.0202664913584756, + "population_cv": 0.3941497713845743, + "rest_mean": 1.645124066945519e-05, + "spike_contrast": 0.7304281104125244, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2016448636131827e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.786091499729082e-05, + 3.183597436873242e-05, + 2.7967644200543873e-05, + 2.4744247639318928e-05, + 2.481242518115323e-05, + 2.314231460331939e-05, + 2.1535035557462834e-05, + 1.9955759853473864e-05, + 1.8067315977532417e-05, + 1.6868179955054075e-05, + 1.60170911840396e-05, + 1.5349354725913145e-05, + 1.5250208889483474e-05, + 1.4568757251254283e-05, + 1.404645445290953e-05, + 1.362170405627694e-05, + 1.3929216947872192e-05, + 1.3323927305464167e-05, + 1.3332275557331741e-05, + 1.3126115845807362e-05, + 1.2836455425713211e-05, + 1.234952469530981e-05, + 1.1998046829830855e-05, + 1.1608603927015793e-05, + 1.1289612302789465e-05, + 1.086686461349018e-05, + 1.0288364137522876e-05, + 9.779456377145834e-06, + 9.169732948066667e-06, + 8.549177437089384e-06, + 8.22363199404208e-06, + 7.95135201769881e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9022438777938078e-05, + "first_to_last_ratio": 2.5706811468129205, + "imbalance_abs_log_ratio": 0.9441709014612467, + "last_quartile_mean": 7.3997659342239785e-06, + "mean": 1.1811503938474743e-05, + "normalized": [ + 1.7838575112897632, + 2.0081895041574866, + 1.8013630165639003, + 1.6167453942621992, + 1.616253821452643, + 1.484346910873369, + 1.334088872746376, + 1.2391628829890091, + 1.1149528907165038, + 1.0471548053789776, + 0.9987803140821989, + 0.9467453609714046, + 0.9424805202441363, + 0.9017388056265639, + 0.868090708413055, + 0.8437486142496254, + 0.8736503938091038, + 0.8327134052809684, + 0.8458575878838563, + 0.8280086592496597, + 0.8103372943527495, + 0.7723135523278634, + 0.7500419314219764, + 0.7274729336065492, + 0.7078485216009954, + 0.6848456245402912, + 0.657709866043606, + 0.6402184904077136, + 0.6083853394516672, + 0.5800023997525418, + 0.5702454032899359, + 0.5626486629633098 + ], + "peak_layer": 2, + "peak_normalized": 2.0081895041574866, + "population_cv": 0.39403850931287454, + "rest_mean": 1.2350452262008158e-05, + "spike_contrast": 0.7207171691012223, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.901182991394307e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1070040020276792e-05, + 2.3719738237559795e-05, + 2.1276806364767253e-05, + 1.9096194591838866e-05, + 1.9090388377662748e-05, + 1.7532369383843616e-05, + 1.5757595974719152e-05, + 1.4636377272836398e-05, + 1.3169270459911786e-05, + 1.2368473107926548e-05, + 1.1797097613452934e-05, + 1.1182486559846438e-05, + 1.1132112376799341e-05, + 1.065089145413367e-05, + 1.025345682137413e-05, + 9.965940080292057e-06, + 1.031912506732624e-05, + 9.835597666096874e-06, + 9.990850230678916e-06, + 9.780027539818548e-06, + 9.571302143740468e-06, + 9.122184565057978e-06, + 8.859123227011878e-06, + 8.592549420427531e-06, + 8.360755600733683e-06, + 8.089056791504845e-06, + 7.76854267314775e-06, + 7.5619432209350634e-06, + 7.185945833043661e-06, + 6.850700629001949e-06, + 6.735455826856196e-06, + 6.645726898568682e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.828027426308836e-05, + "first_to_last_ratio": 2.3127230006738477, + "imbalance_abs_log_ratio": 0.838425618478933, + "last_quartile_mean": 2.9523758030336467e-05, + "mean": 4.4895063126659807e-05, + "normalized": [ + 1.6579137880786858, + 1.9150708402023993, + 1.6745616807855777, + 1.4835198878876907, + 1.4915155698146907, + 1.3960120954471311, + 1.3299713054901032, + 1.2185230612071285, + 1.1078646449578662, + 1.0477053876101448, + 0.999765654013293, + 0.9692189920089408, + 0.955488933285625, + 0.9355193384387992, + 0.9031699366923724, + 0.8872109049801036, + 0.8933262176117003, + 0.8675434009211797, + 0.864331741509242, + 0.8580824004003949, + 0.8343570910048359, + 0.8204903297173597, + 0.8195590181762232, + 0.8083415477202021, + 0.7958344380770943, + 0.7671025897223802, + 0.7201771453938138, + 0.6826439856557369, + 0.6331928257241357, + 0.58535097991478, + 0.5516010454074354, + 0.5250332221429341 + ], + "peak_layer": 2, + "peak_normalized": 1.9150708402023993, + "population_cv": 0.34285892534032564, + "rest_mean": 4.642717794922646e-05, + "spike_contrast": 0.788797525553888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.662164308479987e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.443214417435229e-05, + 8.597722626291215e-05, + 7.517955236835405e-05, + 6.660271901637316e-05, + 6.696168566122651e-05, + 6.267405115067959e-05, + 5.9709145716624334e-05, + 5.470566975418478e-05, + 4.9737653171177953e-05, + 4.703679951489903e-05, + 4.4884542148793116e-05, + 4.351314782979898e-05, + 4.2896735976682976e-05, + 4.200019975542091e-05, + 4.0547871321905404e-05, + 3.983138958574273e-05, + 4.010593693237752e-05, + 3.89484157494735e-05, + 3.8804228097433224e-05, + 3.8523663533851504e-05, + 3.745851427083835e-05, + 3.683596514747478e-05, + 3.6794153857044876e-05, + 3.629054481280036e-05, + 3.5729037335840985e-05, + 3.4439119190210477e-05, + 3.233239840483293e-05, + 3.064734482904896e-05, + 2.8427231882233173e-05, + 2.6279369194526225e-05, + 2.4764163754298352e-05, + 2.357139965170063e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0626930519938469, + 0.09030644595623016, + 0.20229491591453552, + 0.20300470292568207, + 0.19049881398677826, + 0.12695962190628052, + 0.12132706493139267, + 0.11585728824138641, + 0.13427142798900604, + 0.10485932976007462, + 0.10419609397649765, + 0.10852954536676407, + 0.1286265105009079, + 0.09096142649650574, + 0.10080574452877045, + 0.10566592961549759, + 0.1287592649459839, + 0.09650235623121262, + 0.1065765842795372, + 0.11446042358875275, + 0.12395420670509338, + 0.07221997529268265, + 0.07677769660949707, + 0.08416415005922318, + 0.09094838052988052, + 0.059439923614263535, + 0.062182098627090454, + 0.06896103918552399, + 0.0776243805885315, + 0.06285123527050018, + 0.07231374084949493, + 0.08016443997621536 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 6.4383448545868305e-06, + "first_to_last_ratio": 5.042948145238695, + "imbalance_abs_log_ratio": 1.6179908605191557, + "last_quartile_mean": 1.276702569441568e-06, + "mean": 3.687052599232743e-06, + "normalized": [ + 3.304965820363612, + 2.227169292436081, + 1.6521910965438824, + 1.3947865539668498, + 0.5800793227551304, + 1.2714767915022098, + 1.750193241091026, + 1.7887674026051423, + 1.1778703519658833, + 1.1179830646270124, + 0.8417156564271726, + 0.7655624105040819, + 0.5268426976300161, + 1.047779560839775, + 0.8380697743291855, + 0.7915788420213868, + 0.7099070375965003, + 1.3022194692601015, + 1.0739447359104763, + 1.3540969285822078, + 0.6285621976492703, + 1.167508624430284, + 0.9740363058232884, + 0.9425613189042933, + 0.35040916273624656, + 0.44668469186889276, + 0.3295723924580137, + 0.4329866871315858, + 0.2917160480921282, + 0.22501238083060152, + 0.3167851028104899, + 0.37696503630717276 + ], + "peak_layer": 1, + "peak_normalized": 3.304965820363612, + "population_cv": 0.641995540013263, + "rest_mean": 3.814996382010981e-06, + "spike_contrast": 0.7853627820875961, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.9961561722302575e-06, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.218558281834703e-05, + 8.211690328607801e-06, + 6.091715476941317e-06, + 5.142651389178354e-06, + 2.1387829747254727e-06, + 4.688001808972331e-06, + 6.4530545387242455e-06, + 6.5952795011980925e-06, + 4.342869942774996e-06, + 4.122062364331214e-06, + 3.1034498988447012e-06, + 2.8226688755239593e-06, + 1.942496737683541e-06, + 3.863218353217235e-06, + 3.090007339778822e-06, + 2.918592826972599e-06, + 2.6174645881837932e-06, + 4.80135167890694e-06, + 3.959690729971044e-06, + 4.992626600142103e-06, + 2.3175418846221874e-06, + 4.3046657083323225e-06, + 3.5913230931328144e-06, + 3.475273160802317e-06, + 1.2919770142616471e-06, + 1.646949954192678e-06, + 1.215150746247673e-06, + 1.5964446902216878e-06, + 1.0755724133559852e-06, + 8.296324836010172e-07, + 1.1680033367156284e-06, + 1.3898899169362267e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.5929461344276206e-05, + "first_to_last_ratio": 5.201390647867945, + "imbalance_abs_log_ratio": 1.6489260221160402, + "last_quartile_mean": 4.985101696775018e-06, + "mean": 1.4714017034123117e-05, + "normalized": [ + 3.407292787305506, + 2.2558164668132807, + 1.6137698282608655, + 1.433949322998548, + 0.5790123126574623, + 1.305711460245382, + 1.7258967842855322, + 1.7763796775082434, + 1.1315333512208903, + 1.0856488588758444, + 0.8714057435432361, + 0.7593394863607054, + 0.5216263168993628, + 1.0262173986737884, + 0.8250528440437026, + 0.7676154211606976, + 0.723733557926003, + 1.3830217582174593, + 1.0774149558606307, + 1.3018941088272185, + 0.646511650324857, + 1.1377981887421273, + 0.9823485776546648, + 0.9506130966721329, + 0.33641925298333475, + 0.4509311419254927, + 0.32185944273605893, + 0.4000452689662174, + 0.290655292962286, + 0.22712751082346105, + 0.30970536495859485, + 0.3736527695664126 + ], + "peak_layer": 1, + "peak_normalized": 3.407292787305506, + "population_cv": 0.6554716761859383, + "rest_mean": 1.522972111497775e-05, + "spike_contrast": 0.7832851900207317, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1929214997508097e-05, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.0134964112658054e-05, + 3.3192121918546036e-05, + 2.374503674218431e-05, + 2.1099154764669947e-05, + 8.519597031408921e-06, + 1.921226066770032e-05, + 2.539487468311563e-05, + 2.6137680833926424e-05, + 1.6649401004542597e-05, + 1.59742558025755e-05, + 1.2821878954127897e-05, + 1.1172934136993717e-05, + 7.675218512304127e-06, + 1.5099780284799635e-05, + 1.2139841601310764e-05, + 1.1294706382614095e-05, + 1.0649027899489738e-05, + 2.03498057089746e-05, + 1.5853102013352327e-05, + 1.9156092093908228e-05, + 9.512783435638994e-06, + 1.674158193054609e-05, + 1.4454293705057353e-05, + 1.3987337297294289e-05, + 4.9500786190037616e-06, + 6.635008503508288e-06, + 4.735845323011745e-06, + 5.886272901989287e-06, + 4.276706931705121e-06, + 3.3419580631743884e-06, + 4.557010015560081e-06, + 5.497933216247475e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.022994505590759218, + "first_to_last_ratio": 5.20139065184949, + "imbalance_abs_log_ratio": 1.6489260228815172, + "last_quartile_mean": 0.004420838027726859, + "mean": 0.01304853729379829, + "normalized": [ + 3.4072926237702874, + 2.255816491845768, + 1.613769818466143, + 1.4339493120942854, + 0.5790123351120128, + 1.3057114456299215, + 1.725896924613148, + 1.7763798383286113, + 1.13153331687909, + 1.0856488688373112, + 0.8714057478512427, + 0.7593395234826519, + 0.5216262984837251, + 1.0262174043007157, + 0.8250528027479058, + 0.7676154484189387, + 0.7237335326830917, + 1.3830217401378855, + 1.0774149116451128, + 1.3018940938870247, + 0.6465115988335997, + 1.1377982160374271, + 0.9823485499546105, + 0.9506130843152019, + 0.3364192535702373, + 0.45093116963769325, + 0.3218594440675733, + 0.4000452442436667, + 0.2906553022018551, + 0.22712752175948964, + 0.30970535698369017, + 0.3736527791800823 + ], + "peak_layer": 1, + "peak_normalized": 3.4072926237702874, + "population_cv": 0.6554716682951498, + "rest_mean": 0.013505868855173941, + "spike_contrast": 0.7832851759342461, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.010578946862369775, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.04446018487215042, + 0.029435105621814728, + 0.021057335659861565, + 0.018710941076278687, + 0.007555264048278332, + 0.01703762449324131, + 0.022520430386066437, + 0.023179158568382263, + 0.014764854684472084, + 0.014166129752993584, + 0.01137057039886713, + 0.009908270090818405, + 0.0068064602091908455, + 0.013390636071562767, + 0.010765732266008854, + 0.010016258805990219, + 0.009443663991987705, + 0.018046410754323006, + 0.014058688655495644, + 0.016987813636660576, + 0.008436030708253384, + 0.014846602454781532, + 0.012818211689591408, + 0.01240411028265953, + 0.0043897791765630245, + 0.005883992183953524, + 0.004199794959276915, + 0.005220005288720131, + 0.0037926265504211187, + 0.0029636819381266832, + 0.004041201900690794, + 0.004875622224062681 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.464145257619066e-06, + "first_to_last_ratio": 2.649593113901558, + "imbalance_abs_log_ratio": 0.9744060862854225, + "last_quartile_mean": 9.300089303110326e-07, + "mean": 1.2621179932281734e-06, + "normalized": [ + 9.98073298735891, + 2.6081595638248576, + 0.448046131778418, + 0.5712169168045845, + 0.31851900459136234, + 0.5125315286968626, + 0.5097468218730781, + 0.6701588608628778, + 0.5743479667542774, + 0.6374764102214231, + 0.546790403537672, + 0.5390341640801587, + 0.35740547847929505, + 0.6625310250792219, + 0.6002964531361203, + 0.6405992632336756, + 0.4466467073820306, + 0.8660901213820976, + 0.747225340722879, + 0.8852501473817866, + 0.3909436983971323, + 0.8018997241352653, + 0.8266134004412924, + 0.9628283052981523, + 0.5300152809900267, + 0.7412196878662661, + 0.7889073080638102, + 1.074944484309473, + 0.6781360573263348, + 0.7166665274120905, + 0.6792494446365528, + 0.6857707839420146 + ], + "peak_layer": 1, + "peak_normalized": 9.98073298735891, + "population_cv": 1.6572947531819389, + "rest_mean": 1.3316606755240543e-06, + "spike_contrast": 0.6657758429950736, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.865875088304165e-07, + "top_five_layers": [ + 1, + 2, + 28, + 24, + 20 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.259686268895166e-05, + 3.2918051147134975e-06, + 5.654870847138227e-07, + 7.209431487353868e-07, + 4.020085668798856e-07, + 6.468752644650522e-07, + 6.433606358768884e-07, + 8.45819556616334e-07, + 7.248949032145902e-07, + 8.045704475989623e-07, + 6.901140068293898e-07, + 6.803247174502758e-07, + 4.51087885267043e-07, + 8.361923278243921e-07, + 7.576449547741504e-07, + 8.08511856575933e-07, + 5.637208460029797e-07, + 1.0931079259535181e-06, + 9.430865475223982e-07, + 1.1172901395184454e-06, + 4.934170760861889e-07, + 1.012092070595827e-06, + 1.0432836461404804e-06, + 1.2152029285061872e-06, + 6.68941822823399e-07, + 9.355067049909849e-07, + 9.956941084965365e-07, + 1.356706775368366e-06, + 8.558877198083792e-07, + 9.045177193911513e-07, + 8.572929459660372e-07, + 8.655236456434068e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.6010561125767708e-05, + "first_to_last_ratio": 4.917014092262474, + "imbalance_abs_log_ratio": 1.5927014544542644, + "last_quartile_mean": 3.2561552245624625e-06, + "mean": 9.725274459526645e-06, + "normalized": [ + 2.0495551017102773, + 2.324610334145404, + 1.6789860752319785, + 1.469067342376247, + 0.683331640000088, + 1.4508358782825064, + 1.8574881471610558, + 1.6563953320037643, + 1.2394147680445438, + 1.1757596976660154, + 0.9385611021097731, + 0.8217156066495895, + 0.5863388014820525, + 1.1122247987743321, + 0.8990502135337656, + 0.7981592831866082, + 0.7526244364476551, + 1.397025330921264, + 1.1617275022678442, + 1.3251897151639205, + 0.6397940246687674, + 1.2096957514764108, + 1.0918700868694384, + 1.0020693481814167, + 0.3582271576029711, + 0.48201484763579705, + 0.332690365953382, + 0.356368076645877, + 0.30908080698932155, + 0.24694487721404904, + 0.26695462213676374, + 0.32622892746712 + ], + "peak_layer": 2, + "peak_normalized": 2.324610334145404, + "population_cv": 0.5441079654169187, + "rest_mean": 9.97681458843376e-06, + "spike_contrast": 0.83864019815785, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.366957763428217e-06, + "top_five_layers": [ + 2, + 1, + 7, + 3, + 8 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.9932485884055495e-05, + 2.2607473511015996e-05, + 1.6328600395354442e-05, + 1.42870831041364e-05, + 6.645587745879311e-06, + 1.4109777112025768e-05, + 1.8064582036458887e-05, + 1.6108899217215367e-05, + 1.2053648788423743e-05, + 1.143458575825207e-05, + 9.127764315053355e-06, + 7.991409802343696e-06, + 5.7023057706828695e-06, + 1.0816691428772174e-05, + 8.743510079511907e-06, + 7.762318091408815e-06, + 7.319479209400015e-06, + 1.3586454770120326e-05, + 1.1298118806735147e-05, + 1.2887833690911066e-05, + 6.222172487468924e-06, + 1.176462319563143e-05, + 1.0618736268952489e-05, + 9.745399438543245e-06, + 3.4838574265450006e-06, + 4.6877266868250445e-06, + 3.2355051189369988e-06, + 3.465777353994781e-06, + 3.0058956781431334e-06, + 2.4016067072807346e-06, + 2.5962069685192546e-06, + 3.1726658562547527e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.92899242646672e-06, + "first_to_last_ratio": 4.574804953054558, + "imbalance_abs_log_ratio": 1.520564064606545, + "last_quartile_mean": 2.1703640982195793e-06, + "mean": 6.422408389994416e-06, + "normalized": [ + 1.2694651099378356, + 2.2660081023317744, + 1.6776221325435599, + 1.4243775268645795, + 0.778686696628982, + 1.544586331363982, + 1.9675189416357637, + 1.4396717003002808, + 1.3121814531183145, + 1.2749800755885756, + 1.028924014649809, + 0.867611835816526, + 0.6467747339714943, + 1.1792708375805625, + 0.9308967888836265, + 0.8222106571344964, + 0.7916222394567357, + 1.4570716548377307, + 1.2330890513227566, + 1.308271526002021, + 0.6046831152920485, + 1.2497078701132496, + 1.207744340136328, + 1.0135338985983424, + 0.3691120245165777, + 0.5161808316774584, + 0.34931346918180795, + 0.3250152051676304, + 0.32914583570336414, + 0.26959120942422543, + 0.2525212479205583, + 0.29260954229900277 + ], + "peak_layer": 2, + "peak_normalized": 2.2660081023317744, + "population_cv": 0.5120965787375209, + "rest_mean": 6.5544765961762836e-06, + "spike_contrast": 0.8710443912398678, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.7092400766123316e-06, + "top_five_layers": [ + 2, + 7, + 3, + 6, + 18 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153023372869939e-06, + 1.4553229448210914e-05, + 1.0774374459288083e-05, + 9.147934179054573e-06, + 5.001043973607011e-06, + 9.919964213622734e-06, + 1.2636210158234462e-05, + 9.24615960684605e-06, + 8.427365173702128e-06, + 8.188442734535784e-06, + 6.608170224353671e-06, + 5.572157533606514e-06, + 4.153851477894932e-06, + 7.5737589213531464e-06, + 5.978599347145064e-06, + 5.280572622723412e-06, + 5.084121312393108e-06, + 9.357909220852889e-06, + 7.919401468825527e-06, + 8.402254024986178e-06, + 3.883521912939614e-06, + 8.026134310057387e-06, + 7.756627383059822e-06, + 6.509328613901744e-06, + 2.3705881631030934e-06, + 3.3151241041196045e-06, + 2.2434337552112993e-06, + 2.087380380544346e-06, + 2.1139089767530095e-06, + 1.731424845274887e-06, + 1.6217945812968537e-06, + 1.8792579794535413e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 4.949699314238387e-05, + "first_to_last_ratio": 5.262607641337885, + "imbalance_abs_log_ratio": 1.6606266532103107, + "last_quartile_mean": 9.405412015439651e-06, + "mean": 2.8804784136582384e-05, + "normalized": [ + 2.499900864670067, + 2.416668640808701, + 1.6691856383541526, + 1.5215442251975408, + 0.6043399354947968, + 1.3939313584258168, + 1.817946933469592, + 1.823363590450555, + 1.1883363380697656, + 1.0810267152424702, + 0.8648382834598256, + 0.8031803638827597, + 0.533176912017834, + 1.0261239559732709, + 0.8554824003261805, + 0.7869370867194827, + 0.7081002132573654, + 1.3561635382832782, + 1.1101608226630673, + 1.398379059233539, + 0.6739882703135641, + 1.211767597856563, + 0.9998718364198766, + 1.043404895274106, + 0.34963454859286996, + 0.4704717001014101, + 0.31939467298460794, + 0.36230101093546613, + 0.28800141494502707, + 0.22353921217823322, + 0.2657184225315432, + 0.3331195418666723 + ], + "peak_layer": 1, + "peak_normalized": 2.499900864670067, + "population_cv": 0.5921886777524846, + "rest_mean": 2.957433365803876e-05, + "spike_contrast": 0.8334665120685795, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4649216720717958e-05, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.200910476967692e-05, + 6.961161852814257e-05, + 4.8080531996674836e-05, + 4.382775296107866e-05, + 1.7407881387043744e-05, + 4.01518918806687e-05, + 5.2365568990353495e-05, + 5.2521594625432044e-05, + 3.4229771699756384e-05, + 3.113874117843807e-05, + 2.4911480068112724e-05, + 2.3135437004384585e-05, + 1.5358045857283287e-05, + 2.9557279049186036e-05, + 2.4641985874040984e-05, + 2.2667552912025712e-05, + 2.0396673789946362e-05, + 3.906399797415361e-05, + 3.197794285370037e-05, + 4.028000694233924e-05, + 1.941408663697075e-05, + 3.490470407996327e-05, + 2.880109241232276e-05, + 3.0055052775423974e-05, + 1.0071147698909044e-05, + 1.3551835763792042e-05, + 9.200094609695952e-06, + 1.0436002412461676e-05, + 8.295818588521797e-06, + 6.438998752855696e-06, + 7.65396180213429e-06, + 9.595436495146714e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059514161199331284, + 0.08157797157764435, + 0.14292451739311218, + 0.10997853428125381, + 0.0991024449467659, + 0.1065012738108635, + 0.12155740708112717, + 0.1095810979604721, + 0.06964563578367233, + 0.08311934769153595, + 0.09363401681184769, + 0.0941167026758194, + 0.08564268052577972, + 0.09493856877088547, + 0.09063401818275452, + 0.09473779052495956, + 0.10505300015211105, + 0.09355910867452621, + 0.09618847072124481, + 0.11416846513748169, + 0.047205954790115356, + 0.053352247923612595, + 0.06512638926506042, + 0.07829347252845764, + 0.05368548259139061, + 0.05310077592730522, + 0.061235878616571426, + 0.07320034503936768, + 0.05304349213838577, + 0.0639304369688034, + 0.07404901832342148, + 0.08157160878181458 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.6960831434053034e-06, + "first_to_last_ratio": 0.38455700861487224, + "imbalance_abs_log_ratio": 0.9556632341481542, + "last_quartile_mean": 7.0108802674440085e-06, + "mean": 4.081287496404684e-06, + "normalized": [ + 1.6016557623725205, + 0.5624254649150265, + 0.3792516557450063, + 0.6472346627621453, + 0.3825774509297101, + 0.309883335456179, + 0.41924156145633357, + 0.9824999654437843, + 0.9295888075476517, + 0.6942195909151134, + 0.6392271159962603, + 0.6622046687372564, + 0.5739498407816973, + 0.6102779288338825, + 0.725372269896936, + 0.7725414351920691, + 0.5151773567621832, + 0.6691928665493493, + 0.6340666932552725, + 0.5067672404198594, + 0.8579891398085714, + 1.216818523393502, + 1.4001261788695554, + 1.56522304857102, + 1.4389835581991173, + 1.8374458775865694, + 2.199543673756739, + 1.6451073151800026, + 2.0248581159734593, + 1.9770863776726855, + 1.3258318688286215, + 1.2936306481919198 + ], + "peak_layer": 27, + "peak_normalized": 2.199543673756739, + "population_cv": 0.5384151707957241, + "rest_mean": 3.857702406801562e-06, + "spike_contrast": 1.3709318196595643, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.288646980261546e-06, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.5368176365154795e-06, + 2.295420017617289e-06, + 1.5478350405828678e-06, + 2.6415507363708457e-06, + 1.5614085668858024e-06, + 1.2647229823414818e-06, + 1.7110453427449102e-06, + 4.009864824183751e-06, + 3.793919177041971e-06, + 2.833309736161027e-06, + 2.6088696358783636e-06, + 2.7026476345781703e-06, + 2.3424543087458005e-06, + 2.4907196802814724e-06, + 2.960452775369049e-06, + 3.1529636999039212e-06, + 2.1025869045843137e-06, + 2.7311684789310675e-06, + 2.587808467069408e-06, + 2.068262801913079e-06, + 3.501700348351733e-06, + 4.966186224919511e-06, + 5.7143174672091845e-06, + 6.388125257217325e-06, + 5.872905603609979e-06, + 7.499144885514397e-06, + 8.976970093499403e-06, + 6.714155915688025e-06, + 8.264028110716026e-06, + 8.06905791250756e-06, + 5.411101028585108e-06, + 5.279678589431569e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.0839275830676343e-05, + "first_to_last_ratio": 0.38713025500084114, + "imbalance_abs_log_ratio": 0.9489940663518645, + "last_quartile_mean": 2.7999040867143776e-05, + "mean": 1.6318590368769037e-05, + "normalized": [ + 1.6422297380818347, + 0.577857691983563, + 0.3782290994995378, + 0.6502872006236952, + 0.3822780102727073, + 0.31178153988993046, + 0.41970720095668634, + 0.9514590030920431, + 0.920750817583778, + 0.6902360262942715, + 0.6326124977684876, + 0.6666642010329482, + 0.573439016269761, + 0.6073224038931471, + 0.7243939579070396, + 0.7687525976401163, + 0.5168423803555005, + 0.6726938023393403, + 0.6400255756173949, + 0.5037402338382693, + 0.8679547154047897, + 1.2120933210153013, + 1.403510673963058, + 1.5589326681275335, + 1.4338206424993105, + 1.8383886875461704, + 2.207829035366123, + 1.646625228594747, + 2.022240419628589, + 1.965711109970758, + 1.3282014630518373, + 1.283389039891731 + ], + "peak_layer": 27, + "peak_normalized": 2.207829035366123, + "population_cv": 0.5389668648883295, + "rest_mean": 1.542631884553379e-05, + "spike_contrast": 1.3701814934519443, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1136856594239363e-05, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.6798874387168325e-05, + 9.429822966922075e-06, + 6.1721657402813435e-06, + 1.061177044903161e-05, + 6.238238256628392e-06, + 5.087835234007798e-06, + 6.849029887234792e-06, + 1.5526469724136405e-05, + 1.5025355423858855e-05, + 1.1263678970863111e-05, + 1.0323344213247765e-05, + 1.0879020010179374e-05, + 9.357716407976113e-06, + 9.910645530908369e-06, + 1.18210882646963e-05, + 1.254495873581618e-05, + 8.434139090240933e-06, + 1.097741460398538e-05, + 1.044431519403588e-05, + 8.220330528274644e-06, + 1.4163797459332272e-05, + 1.9779654394369572e-05, + 2.2903315766598098e-05, + 2.5439583623665385e-05, + 2.339793172723148e-05, + 2.9999911930644885e-05, + 3.602865763241425e-05, + 2.687060259631835e-05, + 3.300011303508654e-05, + 3.2077634386951104e-05, + 2.1674375602742657e-05, + 2.094310002576094e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.009612377849407494, + "first_to_last_ratio": 0.38713024225946485, + "imbalance_abs_log_ratio": 0.9489940992642404, + "last_quartile_mean": 0.024829829344525933, + "mean": 0.014471489033894613, + "normalized": [ + 1.6422296302631012, + 0.5778576663317349, + 0.37822910199946935, + 0.6502872213280481, + 0.38227800767762105, + 0.3117815365380894, + 0.4197072091098399, + 0.9514589510980404, + 0.9207507998752875, + 0.6902360121427672, + 0.6326125119311165, + 0.6666641983580384, + 0.573439005071423, + 0.607322401384804, + 0.7243939498038978, + 0.7687525851502272, + 0.5168424285086333, + 0.6726938110990978, + 0.6400255788101445, + 0.5037402548123486, + 0.8679547482284273, + 1.2120933573235502, + 1.4035107155843847, + 1.5589326526958214, + 1.4338205694791435, + 1.8383887153176988, + 2.2078290833379297, + 1.6466252521885574, + 2.022240548084217, + 1.9657110301441079, + 1.3282014367981623, + 1.28338902952427 + ], + "peak_layer": 27, + "peak_normalized": 2.2078290833379297, + "population_cv": 0.5389668665215603, + "rest_mean": 0.01368021371739882, + "spike_contrast": 1.3701814993673933, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.018744375742971896, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.023765508085489273, + 0.008362460881471634, + 0.005473538301885128, + 0.009410624392330647, + 0.0055321319960057735, + 0.004511943086981773, + 0.006073788274079561, + 0.013769027777016163, + 0.013324635103344917, + 0.009988742880523205, + 0.009154845029115677, + 0.009647623635828495, + 0.008298516273498535, + 0.008788859471678734, + 0.010483059100806713, + 0.011124994605779648, + 0.007479479536414146, + 0.009734881110489368, + 0.00926212314516306, + 0.007289871573448181, + 0.012560597620904446, + 0.017540795728564262, + 0.020310889929533005, + 0.022560076788067818, + 0.020749518647789955, + 0.026604222133755684, + 0.03195057436823845, + 0.02382911927998066, + 0.02926483191549778, + 0.02844676561653614, + 0.019221052527427673, + 0.018572550266981125 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.596527093449822e-07, + "first_to_last_ratio": 0.3487620963761673, + "imbalance_abs_log_ratio": 1.053365261498487, + "last_quartile_mean": 1.8914116992618801e-06, + "mean": 1.0618384331806396e-06, + "normalized": [ + 1.9623284636711533, + 0.5051262881892306, + 0.3546331093387114, + 0.6524291934355505, + 0.33242707700757573, + 0.26547608492472724, + 0.31949906663212685, + 0.5779720956168373, + 0.775638331440552, + 0.5998773794374831, + 0.5826887294747466, + 0.6213115726756598, + 0.5572960370183113, + 0.6050525236179888, + 0.7278412668111575, + 0.7634285200712856, + 0.5211325543229903, + 0.6528362049921996, + 0.6569013419877625, + 0.48939266573007467, + 0.8730686968435768, + 1.2548058690120942, + 1.4927712810480218, + 1.6059753116097122, + 1.4417731559107498, + 1.8108704859544356, + 2.3560676855554643, + 1.7166277442710838, + 2.056050650260877, + 2.0539911278871554, + 1.42644194116846, + 1.3882675440822447 + ], + "peak_layer": 27, + "peak_normalized": 2.3560676855554643, + "population_cv": 0.600205151665452, + "rest_mean": 9.962249070785792e-07, + "spike_contrast": 1.4215178360523206, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.416151474131766e-06, + "top_five_layers": [ + 27, + 29, + 30, + 1, + 26 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.0836757812503492e-06, + 5.363625064092048e-07, + 3.765630651741958e-07, + 6.927743925189134e-07, + 3.52983846596544e-07, + 2.818927100634028e-07, + 3.3925638831533433e-07, + 6.137129844319134e-07, + 8.236025905716815e-07, + 6.369728566824051e-07, + 6.187212875374826e-07, + 6.597325068469218e-07, + 5.917583507653035e-07, + 6.424680236705171e-07, + 7.728498303549713e-07, + 8.106377435979084e-07, + 5.533585749617487e-07, + 6.932065730325121e-07, + 6.975230917305453e-07, + 5.19655941388919e-07, + 9.270578971154464e-07, + 1.332401097897673e-06, + 1.5850819181650877e-06, + 1.7052863086064463e-06, + 1.5309301488741767e-06, + 1.9228518794989213e-06, + 2.50176321969775e-06, + 1.8227813143312233e-06, + 2.183193601013045e-06, + 2.181006721002632e-06, + 1.5146508758334676e-06, + 1.4741158338438254e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 7.227550952393358e-06, + "first_to_last_ratio": 0.3980164784812622, + "imbalance_abs_log_ratio": 0.9212618713355372, + "last_quartile_mean": 1.8158923921873793e-05, + "mean": 1.0911241304256691e-05, + "normalized": [ + 1.6660127430322156, + 0.5876271935840073, + 0.29483709953854387, + 0.5894374739528777, + 0.41635452404712836, + 0.33657065533584085, + 0.43430341472813805, + 0.9740163595711268, + 0.9875415328473797, + 0.7356703350916547, + 0.6756170470367248, + 0.7197599061272355, + 0.6198568191216928, + 0.6512295663928718, + 0.7665848858812202, + 0.8116271754128533, + 0.5305907755231244, + 0.6901913960659463, + 0.6428575406488269, + 0.5176364098703187, + 0.8723657576961961, + 1.2199632610317463, + 1.415147460346549, + 1.5302808917079092, + 1.4588334149411595, + 1.8140206258329823, + 2.1102391081606, + 1.5869337150428715, + 1.998597539089171, + 1.88124956086161, + 1.2686189428642605, + 1.195426868615217 + ], + "peak_layer": 27, + "peak_normalized": 2.1102391081606, + "population_cv": 0.5159943803714161, + "rest_mean": 1.0306438963636066e-05, + "spike_contrast": 1.375564731293613, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4177173943608068e-05, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.81782670551911e-05, + 6.411742106138263e-06, + 3.2170387385122012e-06, + 6.431494512071367e-06, + 4.542944679997163e-06, + 3.672403636301169e-06, + 4.738789357361384e-06, + 1.0627727533574216e-05, + 1.0775303962873295e-05, + 8.027076546568424e-06, + 7.371820629487047e-06, + 7.85347401688341e-06, + 6.763407327525783e-06, + 7.105722943379078e-06, + 8.364392670046072e-06, + 8.855859960021917e-06, + 5.789403985545505e-06, + 7.530844868597342e-06, + 7.014373750280356e-06, + 5.648055775964167e-06, + 9.51859328779392e-06, + 1.3311313523445278e-05, + 1.5441015420947224e-05, + 1.66972640727181e-05, + 1.591768341313582e-05, + 1.979321677936241e-05, + 2.3025328118819743e-05, + 1.7315416698693298e-05, + 2.180718001909554e-05, + 2.0526767912087962e-05, + 1.384220740874298e-05, + 1.3043591025052592e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.632263923554092e-06, + "first_to_last_ratio": 0.3802213248794799, + "imbalance_abs_log_ratio": 0.9670017619171267, + "last_quartile_mean": 1.218307238559646e-05, + "mean": 7.265074810192118e-06, + "normalized": [ + 1.6008181275970341, + 0.5739917816715392, + 0.235460636588888, + 0.49956890329522624, + 0.4370066400923714, + 0.3438728691875936, + 0.4329980193133239, + 0.9771405297973998, + 0.9929176905310494, + 0.7133552305353121, + 0.6552622145802756, + 0.7166704391631107, + 0.6237781231575423, + 0.6649176564292614, + 0.7618691062673217, + 0.8379753775300484, + 0.5325892299976221, + 0.6974715354559842, + 0.6405465790924313, + 0.531438133277865, + 0.87605151631772, + 1.2349821890311432, + 1.4485483745005174, + 1.55527350554067, + 1.5475592171159964, + 1.891436311373116, + 2.0687037039687666, + 1.6168445987295306, + 1.977149264052817, + 1.9131038449622708, + 1.2696779592298815, + 1.1310206916163696 + ], + "peak_layer": 27, + "peak_normalized": 2.0687037039687666, + "population_cv": 0.5243879132228086, + "rest_mean": 6.817757480348519e-06, + "spike_contrast": 1.4199080004313516, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.680588391347555e-06, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1630063454504125e-05, + 4.170093234279193e-06, + 1.7106391396737308e-06, + 3.6294054552854504e-06, + 3.1748859328217804e-06, + 2.4982621198432753e-06, + 3.1457630029763095e-06, + 7.098999049048871e-06, + 7.21362130207126e-06, + 5.182579116080888e-06, + 4.760529009217862e-06, + 5.206664354773238e-06, + 4.531794729700778e-06, + 4.8306765165762044e-06, + 5.535036052606301e-06, + 6.0879538068547845e-06, + 3.869300599035341e-06, + 5.067182883067289e-06, + 4.653618816519156e-06, + 3.860937795252539e-06, + 6.364579803630477e-06, + 8.972237992566079e-06, + 1.0523812306928448e-05, + 1.1299178368062712e-05, + 1.1243133485550061e-05, + 1.3741426300839521e-05, + 1.502928716945462e-05, + 1.1746496966225095e-05, + 1.4364137314260006e-05, + 1.3898842553317081e-05, + 9.224305358657148e-06, + 8.216949936468154e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.248317332487204e-05, + "first_to_last_ratio": 0.36732053671673337, + "imbalance_abs_log_ratio": 1.001520415068512, + "last_quartile_mean": 6.120859325164929e-05, + "mean": 3.520552513691655e-05, + "normalized": [ + 1.6113007958889953, + 0.5636281288577828, + 0.3099745506097867, + 0.6128259302643165, + 0.3776724202532394, + 0.30228081296349946, + 0.41660627301599373, + 0.9147212761628488, + 0.9143135144743936, + 0.6818048236562128, + 0.6378236024718279, + 0.669448797145322, + 0.5777616284561341, + 0.6047361595032282, + 0.735292441295308, + 0.776547208280745, + 0.5187662829625476, + 0.6862775953042356, + 0.6442540642333291, + 0.503974019865995, + 0.8786895478652295, + 1.1802818108868638, + 1.3947424485633424, + 1.5774151382314772, + 1.403240136016062, + 1.8387857304360302, + 2.291845133717591, + 1.6909973322209617, + 2.0609348356508104, + 1.9921347125886224, + 1.3276799111046451, + 1.3032429370526224 + ], + "peak_layer": 27, + "peak_normalized": 2.291845133717591, + "population_cv": 0.5527267740898714, + "rest_mean": 3.3335239331117247e-05, + "spike_contrast": 1.3590743428664134, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.530506848823279e-05, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.672669067280367e-05, + 1.9842824258375913e-05, + 1.0912816833297256e-05, + 2.1574858692474663e-05, + 1.329615588474553e-05, + 1.064195475919405e-05, + 1.4666842616861686e-05, + 3.220324288122356e-05, + 3.2188887416850775e-05, + 2.4003296857699752e-05, + 2.2454914869740605e-05, + 2.356829645577818e-05, + 2.0340401533758268e-05, + 2.1290054064593278e-05, + 2.5886356525006704e-05, + 2.733875226113014e-05, + 1.826343941502273e-05, + 2.416076313238591e-05, + 2.2681302652927116e-05, + 1.7742670024745166e-05, + 3.093472696491517e-05, + 4.155244096182287e-05, + 4.910264033242129e-05, + 5.553372830036096e-05, + 4.940180588164367e-05, + 6.473541725426912e-05, + 8.068561146501452e-05, + 5.953244908596389e-05, + 7.255629316205159e-05, + 7.013414870016277e-05, + 4.6741668484173715e-05, + 4.588135197991505e-05 + ] + } + } + } + } + } + }, + "step": 8000 + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "cross_entropy_nats": 3.5801993310451508, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "cross_entropy_nats": 2.155670017004013, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "cross_entropy_nats": 1.358029618859291, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "cross_entropy_nats": 1.249566838145256, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "cross_entropy_nats": 1.1869123727083206, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "parent_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "parent_path": "experiments/k3/attnres_gradient/manifest.json", + "path": "experiments/k3/attnres_spike/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "aggregation_groups": 8, + "blocks_per_group": 4, + "d_ff": 768, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + } + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", + "round05_equivalence": { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true + }, + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 224.20719212223366, + "measured_steps": 7980, + "median_ms": 227.38306851533707, + "p95_ms": 230.37451171875, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + }, + { + "bits_per_byte": 6.611855126224988, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.582988739013672, + "step": 30, + "unclipped_grad_norm": 3.075686454772949 + }, + { + "bits_per_byte": 6.3915955694037825, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.43031644821167, + "step": 40, + "unclipped_grad_norm": 3.2318925857543945 + }, + { + "bits_per_byte": 6.093941755902502, + "learning_rate": 3.75e-05, + "loss_nats": 4.223998546600342, + "step": 50, + "unclipped_grad_norm": 2.8704335689544678 + }, + { + "bits_per_byte": 5.8858000966610895, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.079725742340088, + "step": 60, + "unclipped_grad_norm": 2.669306755065918 + }, + { + "bits_per_byte": 5.715495996470702, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9616799354553223, + "step": 70, + "unclipped_grad_norm": 3.1737496852874756 + }, + { + "bits_per_byte": 5.565897919513593, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8579864501953125, + "step": 80, + "unclipped_grad_norm": 2.5544967651367188 + }, + { + "bits_per_byte": 5.354669171689194, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.711573839187622, + "step": 90, + "unclipped_grad_norm": 2.425950050354004 + }, + { + "bits_per_byte": 5.180876231443403, + "learning_rate": 7.5e-05, + "loss_nats": 3.5911097526550293, + "step": 100, + "unclipped_grad_norm": 2.2783501148223877 + }, + { + "bits_per_byte": 4.992272489270609, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4603796005249023, + "step": 110, + "unclipped_grad_norm": 2.1105899810791016 + }, + { + "bits_per_byte": 4.775783480086919, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3103208541870117, + "step": 120, + "unclipped_grad_norm": 1.8969556093215942 + }, + { + "bits_per_byte": 4.582375918510225, + "learning_rate": 9.75e-05, + "loss_nats": 3.1762609481811523, + "step": 130, + "unclipped_grad_norm": 1.7015109062194824 + }, + { + "bits_per_byte": 4.427903165297061, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0691885948181152, + "step": 140, + "unclipped_grad_norm": 1.446854829788208 + }, + { + "bits_per_byte": 4.271120341117507, + "learning_rate": 0.0001125, + "loss_nats": 2.960515022277832, + "step": 150, + "unclipped_grad_norm": 1.2307006120681763 + }, + { + "bits_per_byte": 4.166711049639903, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.888144016265869, + "step": 160, + "unclipped_grad_norm": 1.0598198175430298 + }, + { + "bits_per_byte": 4.026318516052877, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7908313274383545, + "step": 170, + "unclipped_grad_norm": 0.9654788374900818 + }, + { + "bits_per_byte": 3.94755837315965, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.736238956451416, + "step": 180, + "unclipped_grad_norm": 0.8419675230979919 + }, + { + "bits_per_byte": 3.838975750676209, + "learning_rate": 0.0001425, + "loss_nats": 2.660975217819214, + "step": 190, + "unclipped_grad_norm": 0.817121684551239 + }, + { + "bits_per_byte": 3.774983758012572, + "learning_rate": 0.00015, + "loss_nats": 2.616619348526001, + "step": 200, + "unclipped_grad_norm": 0.955824613571167 + }, + { + "bits_per_byte": 3.7344016998190783, + "learning_rate": 0.0001575, + "loss_nats": 2.5884900093078613, + "step": 210, + "unclipped_grad_norm": 0.8557147979736328 + }, + { + "bits_per_byte": 3.63166201591605, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5172762870788574, + "step": 220, + "unclipped_grad_norm": 0.6891153454780579 + }, + { + "bits_per_byte": 3.6208855830129867, + "learning_rate": 0.0001725, + "loss_nats": 2.5098066329956055, + "step": 230, + "unclipped_grad_norm": 1.036860466003418 + }, + { + "bits_per_byte": 3.4987607026353063, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4251561164855957, + "step": 240, + "unclipped_grad_norm": 0.6406477689743042 + }, + { + "bits_per_byte": 3.5387748741156506, + "learning_rate": 0.0001875, + "loss_nats": 2.4528918266296387, + "step": 250, + "unclipped_grad_norm": 1.106186866760254 + }, + { + "bits_per_byte": 3.5260464381252943, + "learning_rate": 0.000195, + "loss_nats": 2.4440691471099854, + "step": 260, + "unclipped_grad_norm": 0.8556521534919739 + }, + { + "bits_per_byte": 3.5219934949750615, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4412598609924316, + "step": 270, + "unclipped_grad_norm": 0.9917709827423096 + }, + { + "bits_per_byte": 3.474740573719901, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4085066318511963, + "step": 280, + "unclipped_grad_norm": 1.2901222705841064 + }, + { + "bits_per_byte": 3.363479149466621, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.331386089324951, + "step": 290, + "unclipped_grad_norm": 0.7895627617835999 + }, + { + "bits_per_byte": 3.4037392561376993, + "learning_rate": 0.000225, + "loss_nats": 2.3592922687530518, + "step": 300, + "unclipped_grad_norm": 1.5281883478164673 + }, + { + "bits_per_byte": 3.3541442751430894, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.324915647506714, + "step": 310, + "unclipped_grad_norm": 0.8648361563682556 + }, + { + "bits_per_byte": 3.3434108379027414, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3174757957458496, + "step": 320, + "unclipped_grad_norm": 1.4613573551177979 + }, + { + "bits_per_byte": 3.3325662998699364, + "learning_rate": 0.0002475, + "loss_nats": 2.3099589347839355, + "step": 330, + "unclipped_grad_norm": 1.1215400695800781 + }, + { + "bits_per_byte": 3.4180712583647006, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3692264556884766, + "step": 340, + "unclipped_grad_norm": 1.1166281700134277 + }, + { + "bits_per_byte": 3.3577222022116393, + "learning_rate": 0.0002625, + "loss_nats": 2.3273956775665283, + "step": 350, + "unclipped_grad_norm": 1.0810235738754272 + }, + { + "bits_per_byte": 3.2923828974611427, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2821059226989746, + "step": 360, + "unclipped_grad_norm": 1.128420114517212 + }, + { + "bits_per_byte": 3.291989401155969, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.2818331718444824, + "step": 370, + "unclipped_grad_norm": 1.0846341848373413 + }, + { + "bits_per_byte": 3.3389695579268603, + "learning_rate": 0.000285, + "loss_nats": 2.3143973350524902, + "step": 380, + "unclipped_grad_norm": 1.0868512392044067 + }, + { + "bits_per_byte": 3.256548592327368, + "learning_rate": 0.0002925, + "loss_nats": 2.257267475128174, + "step": 390, + "unclipped_grad_norm": 1.1476690769195557 + }, + { + "bits_per_byte": 3.2058776878689605, + "learning_rate": 0.0003, + "loss_nats": 2.2221450805664062, + "step": 400, + "unclipped_grad_norm": 1.1263198852539062 + }, + { + "bits_per_byte": 3.195324144481415, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.214829921722412, + "step": 410, + "unclipped_grad_norm": 1.1377208232879639 + }, + { + "bits_per_byte": 3.227808915506986, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.237346649169922, + "step": 420, + "unclipped_grad_norm": 1.2929860353469849 + }, + { + "bits_per_byte": 3.1876103786249166, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2094831466674805, + "step": 430, + "unclipped_grad_norm": 1.071694016456604 + }, + { + "bits_per_byte": 3.2374495749837515, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2440290451049805, + "step": 440, + "unclipped_grad_norm": 0.9089367985725403 + }, + { + "bits_per_byte": 3.155426921203657, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.1871752738952637, + "step": 450, + "unclipped_grad_norm": 1.1129212379455566 + }, + { + "bits_per_byte": 3.188086770567806, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.209813356399536, + "step": 460, + "unclipped_grad_norm": 1.3905333280563354 + }, + { + "bits_per_byte": 3.128408790718412, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.168447732925415, + "step": 470, + "unclipped_grad_norm": 1.059533953666687 + }, + { + "bits_per_byte": 3.136691131218663, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1741886138916016, + "step": 480, + "unclipped_grad_norm": 1.0161854028701782 + }, + { + "bits_per_byte": 3.0887341129908705, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1409473419189453, + "step": 490, + "unclipped_grad_norm": 1.385905385017395 + }, + { + "bits_per_byte": 3.033835874974255, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1028947830200195, + "step": 500, + "unclipped_grad_norm": 1.1716443300247192 + }, + { + "bits_per_byte": 3.0607071322756636, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.121520519256592, + "step": 510, + "unclipped_grad_norm": 0.947066605091095 + }, + { + "bits_per_byte": 2.997100036784657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0774314403533936, + "step": 520, + "unclipped_grad_norm": 1.352820634841919 + }, + { + "bits_per_byte": 3.0643050093317137, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.124014377593994, + "step": 530, + "unclipped_grad_norm": 1.2646857500076294 + }, + { + "bits_per_byte": 3.008989197438803, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.085672378540039, + "step": 540, + "unclipped_grad_norm": 1.0451104640960693 + }, + { + "bits_per_byte": 2.978105584892765, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.064265489578247, + "step": 550, + "unclipped_grad_norm": 1.2619587182998657 + }, + { + "bits_per_byte": 2.968300166036473, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.057468891143799, + "step": 560, + "unclipped_grad_norm": 0.9861598610877991 + }, + { + "bits_per_byte": 2.9221661639078262, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.025491237640381, + "step": 570, + "unclipped_grad_norm": 1.1261875629425049 + }, + { + "bits_per_byte": 2.96999178739036, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0586414337158203, + "step": 580, + "unclipped_grad_norm": 0.8640331625938416 + }, + { + "bits_per_byte": 2.883698804389342, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9988276958465576, + "step": 590, + "unclipped_grad_norm": 1.0034456253051758 + }, + { + "bits_per_byte": 2.9404706214044567, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.0381789207458496, + "step": 600, + "unclipped_grad_norm": 0.8714175224304199 + }, + { + "bits_per_byte": 2.8380133330141457, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.967160940170288, + "step": 610, + "unclipped_grad_norm": 0.9427422881126404 + }, + { + "bits_per_byte": 2.894644124255553, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.0064144134521484, + "step": 620, + "unclipped_grad_norm": 1.0238139629364014 + }, + { + "bits_per_byte": 2.81883675149499, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9538687467575073, + "step": 630, + "unclipped_grad_norm": 0.9875481128692627 + }, + { + "bits_per_byte": 2.7787291934352267, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9260683059692383, + "step": 640, + "unclipped_grad_norm": 1.0125378370285034 + }, + { + "bits_per_byte": 2.728423580127252, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8911991119384766, + "step": 650, + "unclipped_grad_norm": 1.5507988929748535 + }, + { + "bits_per_byte": 2.649404087396524, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8364269733428955, + "step": 660, + "unclipped_grad_norm": 0.9040125012397766 + }, + { + "bits_per_byte": 2.7024973674923345, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8732284307479858, + "step": 670, + "unclipped_grad_norm": 1.205861210823059 + }, + { + "bits_per_byte": 2.641403626461037, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8308814764022827, + "step": 680, + "unclipped_grad_norm": 1.3370649814605713 + }, + { + "bits_per_byte": 2.7325929555321187, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8940891027450562, + "step": 690, + "unclipped_grad_norm": 1.6124995946884155 + }, + { + "bits_per_byte": 2.6130706886099455, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8112425804138184, + "step": 700, + "unclipped_grad_norm": 1.08383309841156 + }, + { + "bits_per_byte": 2.628106443844277, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.821664571762085, + "step": 710, + "unclipped_grad_norm": 1.0767779350280762 + }, + { + "bits_per_byte": 2.6725746220166666, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.852487564086914, + "step": 720, + "unclipped_grad_norm": 1.296352505683899 + }, + { + "bits_per_byte": 2.608218370098502, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8078792095184326, + "step": 730, + "unclipped_grad_norm": 1.0864766836166382 + }, + { + "bits_per_byte": 2.555699340073923, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7714757919311523, + "step": 740, + "unclipped_grad_norm": 1.0339913368225098 + }, + { + "bits_per_byte": 2.515936435246489, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.743914246559143, + "step": 750, + "unclipped_grad_norm": 1.1017143726348877 + }, + { + "bits_per_byte": 2.4657125136976736, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.709101676940918, + "step": 760, + "unclipped_grad_norm": 1.7173939943313599 + }, + { + "bits_per_byte": 2.601046349592221, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.802907943725586, + "step": 770, + "unclipped_grad_norm": 0.9487742781639099 + }, + { + "bits_per_byte": 2.4794518637098015, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7186250686645508, + "step": 780, + "unclipped_grad_norm": 0.9247332811355591 + }, + { + "bits_per_byte": 2.466098098800908, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7093689441680908, + "step": 790, + "unclipped_grad_norm": 1.9121034145355225 + }, + { + "bits_per_byte": 2.4025570446478333, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.66532564163208, + "step": 800, + "unclipped_grad_norm": 1.1767630577087402 + }, + { + "bits_per_byte": 2.455517898102479, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7020353078842163, + "step": 810, + "unclipped_grad_norm": 1.1299240589141846 + }, + { + "bits_per_byte": 2.400107839716896, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6636279821395874, + "step": 820, + "unclipped_grad_norm": 1.0006046295166016 + }, + { + "bits_per_byte": 2.4500607166079513, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6982526779174805, + "step": 830, + "unclipped_grad_norm": 1.0554603338241577 + }, + { + "bits_per_byte": 2.5153210813217015, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7434877157211304, + "step": 840, + "unclipped_grad_norm": 1.0736713409423828 + }, + { + "bits_per_byte": 2.399556119372928, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6632455587387085, + "step": 850, + "unclipped_grad_norm": 0.8279580473899841 + }, + { + "bits_per_byte": 2.414819407654351, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6738252639770508, + "step": 860, + "unclipped_grad_norm": 1.2342168092727661 + }, + { + "bits_per_byte": 2.3222158652452247, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6096373796463013, + "step": 870, + "unclipped_grad_norm": 1.0648512840270996 + }, + { + "bits_per_byte": 2.3629336177502487, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6378607749938965, + "step": 880, + "unclipped_grad_norm": 0.9258757829666138 + }, + { + "bits_per_byte": 2.2961121665146167, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5915436744689941, + "step": 890, + "unclipped_grad_norm": 1.0717874765396118 + }, + { + "bits_per_byte": 2.3158979105831285, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6052581071853638, + "step": 900, + "unclipped_grad_norm": 0.9598665833473206 + }, + { + "bits_per_byte": 2.3301258633063577, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6151201725006104, + "step": 910, + "unclipped_grad_norm": 1.0995051860809326 + }, + { + "bits_per_byte": 2.3123376977276173, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.602790355682373, + "step": 920, + "unclipped_grad_norm": 1.1837356090545654 + }, + { + "bits_per_byte": 2.3340250539667196, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6178228855133057, + "step": 930, + "unclipped_grad_norm": 1.1502224206924438 + }, + { + "bits_per_byte": 2.330418921743428, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6153233051300049, + "step": 940, + "unclipped_grad_norm": 1.0088714361190796 + }, + { + "bits_per_byte": 2.2995397807463123, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5939195156097412, + "step": 950, + "unclipped_grad_norm": 0.9483612179756165 + }, + { + "bits_per_byte": 2.3536873144919115, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6314517259597778, + "step": 960, + "unclipped_grad_norm": 0.9582105278968811 + }, + { + "bits_per_byte": 2.2526236015215417, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5613996982574463, + "step": 970, + "unclipped_grad_norm": 1.2002545595169067 + }, + { + "bits_per_byte": 2.3230659754884395, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6102266311645508, + "step": 980, + "unclipped_grad_norm": 0.9191399812698364 + }, + { + "bits_per_byte": 2.285470911975131, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.584167718887329, + "step": 990, + "unclipped_grad_norm": 1.053741216659546 + }, + { + "bits_per_byte": 2.248061417742104, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5582374334335327, + "step": 1000, + "unclipped_grad_norm": 0.9024814963340759 + }, + { + "bits_per_byte": 2.2402749032242903, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.552840232849121, + "step": 1010, + "unclipped_grad_norm": 1.048907995223999 + }, + { + "bits_per_byte": 2.2041726491204714, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5278160572052002, + "step": 1020, + "unclipped_grad_norm": 1.0486118793487549 + }, + { + "bits_per_byte": 2.216881995036582, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5366255044937134, + "step": 1030, + "unclipped_grad_norm": 0.9243784546852112 + }, + { + "bits_per_byte": 2.2610579746851553, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5672459602355957, + "step": 1040, + "unclipped_grad_norm": 1.102870225906372 + }, + { + "bits_per_byte": 2.2528270569975124, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5615407228469849, + "step": 1050, + "unclipped_grad_norm": 1.0764544010162354 + }, + { + "bits_per_byte": 2.14889054582713, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.489497423171997, + "step": 1060, + "unclipped_grad_norm": 0.971049427986145 + }, + { + "bits_per_byte": 2.2350196293618754, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5491975545883179, + "step": 1070, + "unclipped_grad_norm": 1.0203380584716797 + }, + { + "bits_per_byte": 2.217042282867186, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5367366075515747, + "step": 1080, + "unclipped_grad_norm": 1.023221492767334 + }, + { + "bits_per_byte": 2.186512266651322, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5155748128890991, + "step": 1090, + "unclipped_grad_norm": 1.111006498336792 + }, + { + "bits_per_byte": 2.2323391077655277, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5473395586013794, + "step": 1100, + "unclipped_grad_norm": 0.9547863602638245 + }, + { + "bits_per_byte": 2.116467344590568, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4670233726501465, + "step": 1110, + "unclipped_grad_norm": 1.1085364818572998 + }, + { + "bits_per_byte": 2.143238164023873, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.485579490661621, + "step": 1120, + "unclipped_grad_norm": 0.901618242263794 + }, + { + "bits_per_byte": 2.077939275196329, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4403177499771118, + "step": 1130, + "unclipped_grad_norm": 0.8683149814605713 + }, + { + "bits_per_byte": 2.224320932617002, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5417817831039429, + "step": 1140, + "unclipped_grad_norm": 1.030465006828308 + }, + { + "bits_per_byte": 2.0642131678483175, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4308035373687744, + "step": 1150, + "unclipped_grad_norm": 0.827274739742279 + }, + { + "bits_per_byte": 2.103388407957753, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4579577445983887, + "step": 1160, + "unclipped_grad_norm": 0.9158841967582703 + }, + { + "bits_per_byte": 2.1566975262803965, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4949088096618652, + "step": 1170, + "unclipped_grad_norm": 0.9397322535514832 + }, + { + "bits_per_byte": 2.1322636071070153, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4779725074768066, + "step": 1180, + "unclipped_grad_norm": 0.8510220050811768 + }, + { + "bits_per_byte": 2.068760561119781, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4339555501937866, + "step": 1190, + "unclipped_grad_norm": 0.9270830154418945 + }, + { + "bits_per_byte": 2.0310528489701576, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4078185558319092, + "step": 1200, + "unclipped_grad_norm": 0.8815417885780334 + }, + { + "bits_per_byte": 2.1008361854189492, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.456188678741455, + "step": 1210, + "unclipped_grad_norm": 1.2037686109542847 + }, + { + "bits_per_byte": 2.1123913557651197, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.464198112487793, + "step": 1220, + "unclipped_grad_norm": 0.8964646458625793 + }, + { + "bits_per_byte": 2.0711254945517954, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4355947971343994, + "step": 1230, + "unclipped_grad_norm": 0.8661414980888367 + }, + { + "bits_per_byte": 2.0159510523978943, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.397350788116455, + "step": 1240, + "unclipped_grad_norm": 0.9392534494400024 + }, + { + "bits_per_byte": 2.104896867788471, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4590033292770386, + "step": 1250, + "unclipped_grad_norm": 0.8430818915367126 + }, + { + "bits_per_byte": 2.08180441329182, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.442996859550476, + "step": 1260, + "unclipped_grad_norm": 0.8917548656463623 + }, + { + "bits_per_byte": 2.087352401626003, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4468424320220947, + "step": 1270, + "unclipped_grad_norm": 0.9866762161254883 + }, + { + "bits_per_byte": 1.9607164163161914, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.359065055847168, + "step": 1280, + "unclipped_grad_norm": 0.8596939444541931 + }, + { + "bits_per_byte": 2.088993460080536, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4479799270629883, + "step": 1290, + "unclipped_grad_norm": 1.007126808166504 + }, + { + "bits_per_byte": 2.034349412421896, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4101035594940186, + "step": 1300, + "unclipped_grad_norm": 0.8474438786506653 + }, + { + "bits_per_byte": 2.0247273270714237, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4034340381622314, + "step": 1310, + "unclipped_grad_norm": 0.8226730823516846 + }, + { + "bits_per_byte": 2.020511000402871, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4005115032196045, + "step": 1320, + "unclipped_grad_norm": 0.9414357542991638 + }, + { + "bits_per_byte": 2.085571349293668, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4456079006195068, + "step": 1330, + "unclipped_grad_norm": 0.9917200207710266 + }, + { + "bits_per_byte": 2.0422444479924042, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4155759811401367, + "step": 1340, + "unclipped_grad_norm": 0.8823419213294983 + }, + { + "bits_per_byte": 2.0447034559344406, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4172804355621338, + "step": 1350, + "unclipped_grad_norm": 0.8861486315727234 + }, + { + "bits_per_byte": 2.0988803987133373, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4548330307006836, + "step": 1360, + "unclipped_grad_norm": 0.9090827107429504 + }, + { + "bits_per_byte": 2.082760980795919, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4436599016189575, + "step": 1370, + "unclipped_grad_norm": 0.8366561532020569 + }, + { + "bits_per_byte": 2.0317923743688677, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4083311557769775, + "step": 1380, + "unclipped_grad_norm": 0.8379060626029968 + }, + { + "bits_per_byte": 2.0432778917414387, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4162923097610474, + "step": 1390, + "unclipped_grad_norm": 0.8665340542793274 + }, + { + "bits_per_byte": 1.9746810636009497, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3687446117401123, + "step": 1400, + "unclipped_grad_norm": 0.8112426996231079 + }, + { + "bits_per_byte": 2.0034977886489185, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.388718843460083, + "step": 1410, + "unclipped_grad_norm": 0.8745980262756348 + }, + { + "bits_per_byte": 2.019574210903621, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.3998621702194214, + "step": 1420, + "unclipped_grad_norm": 0.8596643209457397 + }, + { + "bits_per_byte": 2.0775151659793014, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4400237798690796, + "step": 1430, + "unclipped_grad_norm": 0.913244903087616 + }, + { + "bits_per_byte": 2.027370700415185, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.405266284942627, + "step": 1440, + "unclipped_grad_norm": 1.0067864656448364 + }, + { + "bits_per_byte": 1.9631205617926026, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3607314825057983, + "step": 1450, + "unclipped_grad_norm": 0.8125852942466736 + }, + { + "bits_per_byte": 1.9959419028659118, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.383481502532959, + "step": 1460, + "unclipped_grad_norm": 0.9857028126716614 + }, + { + "bits_per_byte": 1.9364156117145783, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3422210216522217, + "step": 1470, + "unclipped_grad_norm": 0.8698804974555969 + }, + { + "bits_per_byte": 1.996789089404062, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3840687274932861, + "step": 1480, + "unclipped_grad_norm": 0.8397032022476196 + }, + { + "bits_per_byte": 2.017818268038312, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.398645043373108, + "step": 1490, + "unclipped_grad_norm": 0.8181726932525635 + }, + { + "bits_per_byte": 1.996446843928845, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.38383150100708, + "step": 1500, + "unclipped_grad_norm": 0.8242432475090027 + }, + { + "bits_per_byte": 2.0162594172908914, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3975645303726196, + "step": 1510, + "unclipped_grad_norm": 0.9299814701080322 + }, + { + "bits_per_byte": 1.980743108078562, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3729465007781982, + "step": 1520, + "unclipped_grad_norm": 0.8446369171142578 + }, + { + "bits_per_byte": 2.047233664693934, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4190342426300049, + "step": 1530, + "unclipped_grad_norm": 0.9717140197753906 + }, + { + "bits_per_byte": 1.993557363411699, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3818286657333374, + "step": 1540, + "unclipped_grad_norm": 0.8417568802833557 + }, + { + "bits_per_byte": 2.049996737962696, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4209494590759277, + "step": 1550, + "unclipped_grad_norm": 0.8546327352523804 + }, + { + "bits_per_byte": 1.966114779744124, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3628069162368774, + "step": 1560, + "unclipped_grad_norm": 0.9258830547332764 + }, + { + "bits_per_byte": 2.0427641795633114, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4159362316131592, + "step": 1570, + "unclipped_grad_norm": 0.8567822575569153 + }, + { + "bits_per_byte": 1.9830621221427962, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.374553918838501, + "step": 1580, + "unclipped_grad_norm": 0.8522606492042542 + }, + { + "bits_per_byte": 1.9876789964052082, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3777540922164917, + "step": 1590, + "unclipped_grad_norm": 0.8731296062469482 + }, + { + "bits_per_byte": 2.0122188568915207, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3947638273239136, + "step": 1600, + "unclipped_grad_norm": 0.8627029657363892 + }, + { + "bits_per_byte": 1.9498011934138815, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3514991998672485, + "step": 1610, + "unclipped_grad_norm": 0.7740172147750854 + }, + { + "bits_per_byte": 1.9095543294069197, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3236021995544434, + "step": 1620, + "unclipped_grad_norm": 0.7910023331642151 + }, + { + "bits_per_byte": 1.9407452749500496, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3452221155166626, + "step": 1630, + "unclipped_grad_norm": 0.8484107851982117 + }, + { + "bits_per_byte": 1.9911346438089947, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3801493644714355, + "step": 1640, + "unclipped_grad_norm": 0.8722788691520691 + }, + { + "bits_per_byte": 2.0368601871418424, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4118438959121704, + "step": 1650, + "unclipped_grad_norm": 0.8482462763786316 + }, + { + "bits_per_byte": 1.9665676100638458, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3631207942962646, + "step": 1660, + "unclipped_grad_norm": 0.8444085121154785 + }, + { + "bits_per_byte": 1.9945896032821775, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3825441598892212, + "step": 1670, + "unclipped_grad_norm": 0.8560684323310852 + }, + { + "bits_per_byte": 2.0061820938635853, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3905794620513916, + "step": 1680, + "unclipped_grad_norm": 0.8526912927627563 + }, + { + "bits_per_byte": 2.0514603103215387, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4219639301300049, + "step": 1690, + "unclipped_grad_norm": 0.8917502760887146 + }, + { + "bits_per_byte": 1.9422809080396035, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3462865352630615, + "step": 1700, + "unclipped_grad_norm": 0.7474871873855591 + }, + { + "bits_per_byte": 1.878015978909047, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3017414808273315, + "step": 1710, + "unclipped_grad_norm": 0.8072315454483032 + }, + { + "bits_per_byte": 1.8905051870320533, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3103983402252197, + "step": 1720, + "unclipped_grad_norm": 0.845556914806366 + }, + { + "bits_per_byte": 1.8762399140735866, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3005104064941406, + "step": 1730, + "unclipped_grad_norm": 0.9344940185546875 + }, + { + "bits_per_byte": 1.9632232354351677, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3608026504516602, + "step": 1740, + "unclipped_grad_norm": 0.7860697507858276 + }, + { + "bits_per_byte": 1.989613629244764, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3790950775146484, + "step": 1750, + "unclipped_grad_norm": 0.8348115086555481 + }, + { + "bits_per_byte": 1.9869124697303127, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3772227764129639, + "step": 1760, + "unclipped_grad_norm": 0.8862095475196838 + }, + { + "bits_per_byte": 1.9928130224987648, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3813127279281616, + "step": 1770, + "unclipped_grad_norm": 0.816246509552002 + }, + { + "bits_per_byte": 1.8514405317671223, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.2833207845687866, + "step": 1780, + "unclipped_grad_norm": 0.8118871450424194 + }, + { + "bits_per_byte": 1.9588221993995885, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3577520847320557, + "step": 1790, + "unclipped_grad_norm": 0.839783251285553 + }, + { + "bits_per_byte": 1.8827697513615458, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3050365447998047, + "step": 1800, + "unclipped_grad_norm": 0.8949951529502869 + }, + { + "bits_per_byte": 1.92937567388416, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.33734130859375, + "step": 1810, + "unclipped_grad_norm": 0.7295374274253845 + }, + { + "bits_per_byte": 1.8974312723476003, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3151991367340088, + "step": 1820, + "unclipped_grad_norm": 0.7755441069602966 + }, + { + "bits_per_byte": 1.8886371114783813, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3091034889221191, + "step": 1830, + "unclipped_grad_norm": 0.7666956186294556 + }, + { + "bits_per_byte": 1.8876932706904463, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3084492683410645, + "step": 1840, + "unclipped_grad_norm": 0.8784642219543457 + }, + { + "bits_per_byte": 1.8964055678178544, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.314488172531128, + "step": 1850, + "unclipped_grad_norm": 0.8603965044021606 + }, + { + "bits_per_byte": 1.8927124123732264, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3119282722473145, + "step": 1860, + "unclipped_grad_norm": 0.8056252002716064 + }, + { + "bits_per_byte": 1.8818489562488263, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3043982982635498, + "step": 1870, + "unclipped_grad_norm": 0.8151036500930786 + }, + { + "bits_per_byte": 1.8811881989042114, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3039402961730957, + "step": 1880, + "unclipped_grad_norm": 0.79389888048172 + }, + { + "bits_per_byte": 1.8949089747900463, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.313450813293457, + "step": 1890, + "unclipped_grad_norm": 0.8192400932312012 + }, + { + "bits_per_byte": 1.9233574849825177, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3331698179244995, + "step": 1900, + "unclipped_grad_norm": 0.7950363159179688 + }, + { + "bits_per_byte": 1.8611783614416253, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2900705337524414, + "step": 1910, + "unclipped_grad_norm": 0.8037525415420532 + }, + { + "bits_per_byte": 1.855479888287937, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2861206531524658, + "step": 1920, + "unclipped_grad_norm": 0.7507341504096985 + }, + { + "bits_per_byte": 1.9602173226633874, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3587191104888916, + "step": 1930, + "unclipped_grad_norm": 0.8329893350601196 + }, + { + "bits_per_byte": 1.8066889261861108, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2523013353347778, + "step": 1940, + "unclipped_grad_norm": 0.7595474123954773 + }, + { + "bits_per_byte": 1.9033319970987033, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.319289207458496, + "step": 1950, + "unclipped_grad_norm": 0.7690061330795288 + }, + { + "bits_per_byte": 1.816397862775269, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.259031057357788, + "step": 1960, + "unclipped_grad_norm": 0.7758741974830627 + }, + { + "bits_per_byte": 1.909067618504978, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3232648372650146, + "step": 1970, + "unclipped_grad_norm": 0.7503845691680908 + }, + { + "bits_per_byte": 1.9405849871194454, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3451110124588013, + "step": 1980, + "unclipped_grad_norm": 0.8655099868774414 + }, + { + "bits_per_byte": 1.9436471382180576, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.347233533859253, + "step": 1990, + "unclipped_grad_norm": 0.8436980247497559 + }, + { + "bits_per_byte": 1.9015801818170146, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3180749416351318, + "step": 2000, + "unclipped_grad_norm": 0.8021923303604126 + }, + { + "bits_per_byte": 1.8746163978494417, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.2993850708007812, + "step": 2010, + "unclipped_grad_norm": 0.7083144187927246 + }, + { + "bits_per_byte": 1.9072580170525995, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3220105171203613, + "step": 2020, + "unclipped_grad_norm": 0.7678324580192566 + }, + { + "bits_per_byte": 1.983610230851096, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3749338388442993, + "step": 2030, + "unclipped_grad_norm": 0.8317387104034424 + }, + { + "bits_per_byte": 1.909348466173837, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.323459506034851, + "step": 2040, + "unclipped_grad_norm": 0.8302199840545654 + }, + { + "bits_per_byte": 1.961244747019642, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.359431266784668, + "step": 2050, + "unclipped_grad_norm": 0.7990598082542419 + }, + { + "bits_per_byte": 1.9220292629699043, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3322491645812988, + "step": 2060, + "unclipped_grad_norm": 0.8404503464698792 + }, + { + "bits_per_byte": 1.8663830724046866, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.2936781644821167, + "step": 2070, + "unclipped_grad_norm": 0.7720745801925659 + }, + { + "bits_per_byte": 1.8672120287818452, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.2942527532577515, + "step": 2080, + "unclipped_grad_norm": 0.8028935790061951 + }, + { + "bits_per_byte": 1.857879218250124, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2877837419509888, + "step": 2090, + "unclipped_grad_norm": 0.7713416814804077 + }, + { + "bits_per_byte": 1.8603779541845098, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.289515733718872, + "step": 2100, + "unclipped_grad_norm": 0.7443708181381226 + }, + { + "bits_per_byte": 1.897931741861611, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3155460357666016, + "step": 2110, + "unclipped_grad_norm": 0.7534405589103699 + }, + { + "bits_per_byte": 1.8710701155886507, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2969269752502441, + "step": 2120, + "unclipped_grad_norm": 0.7505779266357422 + }, + { + "bits_per_byte": 1.8349618420920515, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.271898627281189, + "step": 2130, + "unclipped_grad_norm": 0.7835680246353149 + }, + { + "bits_per_byte": 1.9616896661374241, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3597396612167358, + "step": 2140, + "unclipped_grad_norm": 0.7786996960639954 + }, + { + "bits_per_byte": 1.8168395142226847, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2593371868133545, + "step": 2150, + "unclipped_grad_norm": 0.7194197773933411 + }, + { + "bits_per_byte": 1.8739360344826286, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.2989134788513184, + "step": 2160, + "unclipped_grad_norm": 0.8012002110481262 + }, + { + "bits_per_byte": 1.9634492206384013, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3609592914581299, + "step": 2170, + "unclipped_grad_norm": 0.8270084261894226 + }, + { + "bits_per_byte": 1.8175344961148212, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2598189115524292, + "step": 2180, + "unclipped_grad_norm": 0.7824540734291077 + }, + { + "bits_per_byte": 1.8458584908680682, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.279451608657837, + "step": 2190, + "unclipped_grad_norm": 0.7973959445953369 + }, + { + "bits_per_byte": 1.8557901449900935, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.2863357067108154, + "step": 2200, + "unclipped_grad_norm": 0.7451788783073425 + }, + { + "bits_per_byte": 1.8380872828061812, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2740650177001953, + "step": 2210, + "unclipped_grad_norm": 0.7804975509643555 + }, + { + "bits_per_byte": 1.8298283319464475, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2683403491973877, + "step": 2220, + "unclipped_grad_norm": 0.7694668173789978 + }, + { + "bits_per_byte": 1.9031531351418058, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3191652297973633, + "step": 2230, + "unclipped_grad_norm": 0.8418469429016113 + }, + { + "bits_per_byte": 1.8400618156207371, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2754336595535278, + "step": 2240, + "unclipped_grad_norm": 0.7987614870071411 + }, + { + "bits_per_byte": 1.852736936989326, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2842193841934204, + "step": 2250, + "unclipped_grad_norm": 0.7795421481132507 + }, + { + "bits_per_byte": 1.8339105121473271, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.271169900894165, + "step": 2260, + "unclipped_grad_norm": 0.7681040167808533 + }, + { + "bits_per_byte": 1.825916414569922, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2656288146972656, + "step": 2270, + "unclipped_grad_norm": 0.7933830618858337 + }, + { + "bits_per_byte": 1.8451832869807807, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2789835929870605, + "step": 2280, + "unclipped_grad_norm": 0.7935850024223328 + }, + { + "bits_per_byte": 1.8711039961708706, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2969504594802856, + "step": 2290, + "unclipped_grad_norm": 0.7818295955657959 + }, + { + "bits_per_byte": 1.7944778140095496, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2438372373580933, + "step": 2300, + "unclipped_grad_norm": 0.7245032787322998 + }, + { + "bits_per_byte": 1.8221434591752936, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2630136013031006, + "step": 2310, + "unclipped_grad_norm": 0.7396021485328674 + }, + { + "bits_per_byte": 1.7928696042413317, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.242722511291504, + "step": 2320, + "unclipped_grad_norm": 0.7618010640144348 + }, + { + "bits_per_byte": 1.8702191454321815, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2963371276855469, + "step": 2330, + "unclipped_grad_norm": 0.7263991236686707 + }, + { + "bits_per_byte": 1.7994940319872652, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.247314214706421, + "step": 2340, + "unclipped_grad_norm": 0.7788637280464172 + }, + { + "bits_per_byte": 1.849551646312696, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2820115089416504, + "step": 2350, + "unclipped_grad_norm": 0.7732409238815308 + }, + { + "bits_per_byte": 1.8613248906601605, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2901721000671387, + "step": 2360, + "unclipped_grad_norm": 0.8177725672721863 + }, + { + "bits_per_byte": 1.8822589628884834, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3046824932098389, + "step": 2370, + "unclipped_grad_norm": 0.7931574583053589 + }, + { + "bits_per_byte": 1.8382422391746087, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2741724252700806, + "step": 2380, + "unclipped_grad_norm": 0.8604201674461365 + }, + { + "bits_per_byte": 1.8797817247854554, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3029654026031494, + "step": 2390, + "unclipped_grad_norm": 0.7636454105377197 + }, + { + "bits_per_byte": 1.817860919186159, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2600451707839966, + "step": 2400, + "unclipped_grad_norm": 0.779283881187439 + }, + { + "bits_per_byte": 1.8354538844561699, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2722396850585938, + "step": 2410, + "unclipped_grad_norm": 0.7781111001968384 + }, + { + "bits_per_byte": 1.8069649583407457, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2524926662445068, + "step": 2420, + "unclipped_grad_norm": 0.7548633813858032 + }, + { + "bits_per_byte": 1.8169932667125561, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.259443759918213, + "step": 2430, + "unclipped_grad_norm": 0.7562755942344666 + }, + { + "bits_per_byte": 1.8760644917697067, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3003888130187988, + "step": 2440, + "unclipped_grad_norm": 0.7478408813476562 + }, + { + "bits_per_byte": 1.838183937055966, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2741320133209229, + "step": 2450, + "unclipped_grad_norm": 0.760445237159729 + }, + { + "bits_per_byte": 1.8195946762895068, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.261246919631958, + "step": 2460, + "unclipped_grad_norm": 0.7331485748291016 + }, + { + "bits_per_byte": 1.8163858239897086, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2590227127075195, + "step": 2470, + "unclipped_grad_norm": 0.7682369351387024 + }, + { + "bits_per_byte": 1.8266683227194942, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2661499977111816, + "step": 2480, + "unclipped_grad_norm": 0.7132375240325928 + }, + { + "bits_per_byte": 1.8042156436840528, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.250586986541748, + "step": 2490, + "unclipped_grad_norm": 0.7770085334777832 + }, + { + "bits_per_byte": 1.8313916542427904, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2694239616394043, + "step": 2500, + "unclipped_grad_norm": 0.7582318782806396 + }, + { + "bits_per_byte": 1.7788244690759694, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2329871654510498, + "step": 2510, + "unclipped_grad_norm": 0.8006963133811951 + }, + { + "bits_per_byte": 1.8448090527325032, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.278724193572998, + "step": 2520, + "unclipped_grad_norm": 0.7802237272262573 + }, + { + "bits_per_byte": 1.855150369528884, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2858922481536865, + "step": 2530, + "unclipped_grad_norm": 0.7603861689567566 + }, + { + "bits_per_byte": 1.7974587892969547, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.245903491973877, + "step": 2540, + "unclipped_grad_norm": 0.7829954624176025 + }, + { + "bits_per_byte": 1.8048358131230642, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2510168552398682, + "step": 2550, + "unclipped_grad_norm": 0.8207260966300964 + }, + { + "bits_per_byte": 1.8410618947355044, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2761268615722656, + "step": 2560, + "unclipped_grad_norm": 0.8064310550689697 + }, + { + "bits_per_byte": 1.8072459779922554, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2526874542236328, + "step": 2570, + "unclipped_grad_norm": 0.7300699353218079 + }, + { + "bits_per_byte": 1.8352736466380657, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2721147537231445, + "step": 2580, + "unclipped_grad_norm": 0.7476873993873596 + }, + { + "bits_per_byte": 1.793617556789934, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2432409524917603, + "step": 2590, + "unclipped_grad_norm": 0.7210070490837097 + }, + { + "bits_per_byte": 1.797604974550188, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2460048198699951, + "step": 2600, + "unclipped_grad_norm": 0.7853913903236389 + }, + { + "bits_per_byte": 1.7385629865436842, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2050800323486328, + "step": 2610, + "unclipped_grad_norm": 0.7092626094818115 + }, + { + "bits_per_byte": 1.8079149905041119, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2531511783599854, + "step": 2620, + "unclipped_grad_norm": 0.7522015571594238 + }, + { + "bits_per_byte": 1.8198622812942493, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.261432409286499, + "step": 2630, + "unclipped_grad_norm": 0.7675716876983643 + }, + { + "bits_per_byte": 1.680096624469618, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.164554238319397, + "step": 2640, + "unclipped_grad_norm": 0.6825466156005859 + }, + { + "bits_per_byte": 1.7638260340768748, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2225910425186157, + "step": 2650, + "unclipped_grad_norm": 0.7630164623260498 + }, + { + "bits_per_byte": 1.8201589513669878, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2616380453109741, + "step": 2660, + "unclipped_grad_norm": 0.7772946953773499 + }, + { + "bits_per_byte": 1.811244230659514, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2554588317871094, + "step": 2670, + "unclipped_grad_norm": 0.7582975625991821 + }, + { + "bits_per_byte": 1.74257860945868, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.207863450050354, + "step": 2680, + "unclipped_grad_norm": 0.7818799614906311 + }, + { + "bits_per_byte": 1.9030688636428832, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3191068172454834, + "step": 2690, + "unclipped_grad_norm": 0.7075229287147522 + }, + { + "bits_per_byte": 1.8200497423836899, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2615623474121094, + "step": 2700, + "unclipped_grad_norm": 0.8272333741188049 + }, + { + "bits_per_byte": 1.7354167359288, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2028992176055908, + "step": 2710, + "unclipped_grad_norm": 0.7310962677001953 + }, + { + "bits_per_byte": 1.728331394678554, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.1979880332946777, + "step": 2720, + "unclipped_grad_norm": 0.7193791270256042 + }, + { + "bits_per_byte": 1.76859700479446, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.225898027420044, + "step": 2730, + "unclipped_grad_norm": 0.773657500743866 + }, + { + "bits_per_byte": 1.7790022991369616, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2331104278564453, + "step": 2740, + "unclipped_grad_norm": 0.7759174704551697 + }, + { + "bits_per_byte": 1.7909771071512373, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.241410732269287, + "step": 2750, + "unclipped_grad_norm": 0.770866870880127 + }, + { + "bits_per_byte": 1.81886375002334, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2607402801513672, + "step": 2760, + "unclipped_grad_norm": 0.7793042659759521 + }, + { + "bits_per_byte": 1.704546881994831, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.181501865386963, + "step": 2770, + "unclipped_grad_norm": 0.7179723978042603 + }, + { + "bits_per_byte": 1.7222445846992174, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.1937689781188965, + "step": 2780, + "unclipped_grad_norm": 0.711125910282135 + }, + { + "bits_per_byte": 1.8073385046584196, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2527515888214111, + "step": 2790, + "unclipped_grad_norm": 0.7564381957054138 + }, + { + "bits_per_byte": 1.8564130661515186, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2867674827575684, + "step": 2800, + "unclipped_grad_norm": 0.7982874512672424 + }, + { + "bits_per_byte": 1.7924368958917607, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2424225807189941, + "step": 2810, + "unclipped_grad_norm": 0.827074408531189 + }, + { + "bits_per_byte": 1.781587886310033, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2349026203155518, + "step": 2820, + "unclipped_grad_norm": 0.6945767998695374 + }, + { + "bits_per_byte": 1.862601517877515, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2910569906234741, + "step": 2830, + "unclipped_grad_norm": 0.7473636269569397 + }, + { + "bits_per_byte": 1.7520161574247766, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2144050598144531, + "step": 2840, + "unclipped_grad_norm": 0.7269325852394104 + }, + { + "bits_per_byte": 1.7726036846116109, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2286752462387085, + "step": 2850, + "unclipped_grad_norm": 0.7537488341331482 + }, + { + "bits_per_byte": 1.7375121725469125, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2043516635894775, + "step": 2860, + "unclipped_grad_norm": 0.7839256525039673 + }, + { + "bits_per_byte": 1.836773851301542, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.273154616355896, + "step": 2870, + "unclipped_grad_norm": 0.7615643739700317 + }, + { + "bits_per_byte": 1.8235200083127996, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.263967752456665, + "step": 2880, + "unclipped_grad_norm": 0.7500733137130737 + }, + { + "bits_per_byte": 1.7346435019305209, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2023632526397705, + "step": 2890, + "unclipped_grad_norm": 0.7335168123245239 + }, + { + "bits_per_byte": 1.710140445731493, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1853790283203125, + "step": 2900, + "unclipped_grad_norm": 0.7237004041671753 + }, + { + "bits_per_byte": 1.7640040361205178, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2227144241333008, + "step": 2910, + "unclipped_grad_norm": 0.7653093338012695 + }, + { + "bits_per_byte": 1.7372325287566095, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.204157829284668, + "step": 2920, + "unclipped_grad_norm": 0.7592934966087341 + }, + { + "bits_per_byte": 1.7605243111456106, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2203024625778198, + "step": 2930, + "unclipped_grad_norm": 0.7871323227882385 + }, + { + "bits_per_byte": 1.7773966691085064, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2319974899291992, + "step": 2940, + "unclipped_grad_norm": 0.7994735240936279 + }, + { + "bits_per_byte": 1.761262116717812, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208138704299927, + "step": 2950, + "unclipped_grad_norm": 0.7672474384307861 + }, + { + "bits_per_byte": 1.8009139207527887, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2482984066009521, + "step": 2960, + "unclipped_grad_norm": 0.7721540927886963 + }, + { + "bits_per_byte": 1.7577087831683356, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.218350887298584, + "step": 2970, + "unclipped_grad_norm": 0.7659915685653687 + }, + { + "bits_per_byte": 1.7511909846659368, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2138330936431885, + "step": 2980, + "unclipped_grad_norm": 0.7457980513572693 + }, + { + "bits_per_byte": 1.7677185174138528, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2252891063690186, + "step": 2990, + "unclipped_grad_norm": 0.7551211714744568 + }, + { + "bits_per_byte": 1.7804380103063644, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2341055870056152, + "step": 3000, + "unclipped_grad_norm": 0.7369211316108704 + }, + { + "bits_per_byte": 1.77081558099059, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.227435827255249, + "step": 3010, + "unclipped_grad_norm": 0.7526711225509644 + }, + { + "bits_per_byte": 1.726040929734363, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1964004039764404, + "step": 3020, + "unclipped_grad_norm": 0.8113403916358948 + }, + { + "bits_per_byte": 1.8045116258261877, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.250792145729065, + "step": 3030, + "unclipped_grad_norm": 0.7493385672569275 + }, + { + "bits_per_byte": 1.7229166928987893, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.194234848022461, + "step": 3040, + "unclipped_grad_norm": 0.7606903314590454 + }, + { + "bits_per_byte": 1.7403560776615798, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2063229084014893, + "step": 3050, + "unclipped_grad_norm": 0.7690193057060242 + }, + { + "bits_per_byte": 1.759874388708, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2198519706726074, + "step": 3060, + "unclipped_grad_norm": 0.7480750679969788 + }, + { + "bits_per_byte": 1.6873952242069339, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.169613242149353, + "step": 3070, + "unclipped_grad_norm": 0.7001752853393555 + }, + { + "bits_per_byte": 1.737272084766308, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2041852474212646, + "step": 3080, + "unclipped_grad_norm": 0.8322204351425171 + }, + { + "bits_per_byte": 1.8015143121869506, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.248714566230774, + "step": 3090, + "unclipped_grad_norm": 0.8183287382125854 + }, + { + "bits_per_byte": 1.7001176408045102, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.178431749343872, + "step": 3100, + "unclipped_grad_norm": 0.7750749588012695 + }, + { + "bits_per_byte": 1.7770231227908324, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.231738567352295, + "step": 3110, + "unclipped_grad_norm": 0.7082787752151489 + }, + { + "bits_per_byte": 1.749435729731231, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2126164436340332, + "step": 3120, + "unclipped_grad_norm": 0.736611545085907 + }, + { + "bits_per_byte": 1.7966217497352055, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2453233003616333, + "step": 3130, + "unclipped_grad_norm": 0.7366204857826233 + }, + { + "bits_per_byte": 1.7636881039908827, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.222495436668396, + "step": 3140, + "unclipped_grad_norm": 0.7583194375038147 + }, + { + "bits_per_byte": 1.7359698321339747, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2032825946807861, + "step": 3150, + "unclipped_grad_norm": 0.7600930333137512 + }, + { + "bits_per_byte": 1.7415977924008093, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.207183599472046, + "step": 3160, + "unclipped_grad_norm": 0.7639689445495605 + }, + { + "bits_per_byte": 1.654884311818432, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.1470783948898315, + "step": 3170, + "unclipped_grad_norm": 0.7011478543281555 + }, + { + "bits_per_byte": 1.7584833930278216, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2188878059387207, + "step": 3180, + "unclipped_grad_norm": 0.7645173072814941 + }, + { + "bits_per_byte": 1.7305350083840592, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.199515461921692, + "step": 3190, + "unclipped_grad_norm": 0.7665988206863403 + }, + { + "bits_per_byte": 1.7003714871971838, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.178607702255249, + "step": 3200, + "unclipped_grad_norm": 0.7379894852638245 + }, + { + "bits_per_byte": 1.6959066455981344, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1755129098892212, + "step": 3210, + "unclipped_grad_norm": 0.8013631701469421 + }, + { + "bits_per_byte": 1.7098007799960389, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1851435899734497, + "step": 3220, + "unclipped_grad_norm": 0.7296364903450012 + }, + { + "bits_per_byte": 1.7987765203678656, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.246816873550415, + "step": 3230, + "unclipped_grad_norm": 0.7610434889793396 + }, + { + "bits_per_byte": 1.6862874839527267, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1688454151153564, + "step": 3240, + "unclipped_grad_norm": 0.723595917224884 + }, + { + "bits_per_byte": 1.8345631863073515, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2716223001480103, + "step": 3250, + "unclipped_grad_norm": 0.8353017568588257 + }, + { + "bits_per_byte": 1.6872093109613513, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1694843769073486, + "step": 3260, + "unclipped_grad_norm": 0.7602720260620117 + }, + { + "bits_per_byte": 1.7198954736710823, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1921406984329224, + "step": 3270, + "unclipped_grad_norm": 0.7516167163848877 + }, + { + "bits_per_byte": 1.7692164863028679, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2263274192810059, + "step": 3280, + "unclipped_grad_norm": 0.7487898468971252 + }, + { + "bits_per_byte": 1.7624494849393688, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2216368913650513, + "step": 3290, + "unclipped_grad_norm": 0.7708044052124023 + }, + { + "bits_per_byte": 1.7002387165907176, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1785156726837158, + "step": 3300, + "unclipped_grad_norm": 0.7703046202659607 + }, + { + "bits_per_byte": 1.7332599015043295, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2014042139053345, + "step": 3310, + "unclipped_grad_norm": 0.8132672905921936 + }, + { + "bits_per_byte": 1.7262779218272521, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1965646743774414, + "step": 3320, + "unclipped_grad_norm": 0.7545798420906067 + }, + { + "bits_per_byte": 1.7272033604715449, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1972061395645142, + "step": 3330, + "unclipped_grad_norm": 0.7775130867958069 + }, + { + "bits_per_byte": 1.6975052243379043, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1766209602355957, + "step": 3340, + "unclipped_grad_norm": 0.7105540633201599 + }, + { + "bits_per_byte": 1.7083408192728646, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1841316223144531, + "step": 3350, + "unclipped_grad_norm": 0.7163253426551819 + }, + { + "bits_per_byte": 1.732425613664994, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2008259296417236, + "step": 3360, + "unclipped_grad_norm": 0.7626740336418152 + }, + { + "bits_per_byte": 1.6756070173688444, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1614422798156738, + "step": 3370, + "unclipped_grad_norm": 0.8061453700065613 + }, + { + "bits_per_byte": 1.7092863798873084, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1847870349884033, + "step": 3380, + "unclipped_grad_norm": 0.7406735420227051 + }, + { + "bits_per_byte": 1.7539413312185348, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2157394886016846, + "step": 3390, + "unclipped_grad_norm": 0.7695116400718689 + }, + { + "bits_per_byte": 1.8092177590843976, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2540541887283325, + "step": 3400, + "unclipped_grad_norm": 0.7858016490936279 + }, + { + "bits_per_byte": 1.8255297975707825, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2653608322143555, + "step": 3410, + "unclipped_grad_norm": 0.7603964805603027 + }, + { + "bits_per_byte": 1.6857877023693193, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1684989929199219, + "step": 3420, + "unclipped_grad_norm": 0.7168062329292297 + }, + { + "bits_per_byte": 1.6176447364444573, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1212658882141113, + "step": 3430, + "unclipped_grad_norm": 0.7431134581565857 + }, + { + "bits_per_byte": 1.711189711884407, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1861063241958618, + "step": 3440, + "unclipped_grad_norm": 0.7753933668136597 + }, + { + "bits_per_byte": 1.77535626693867, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2305831909179688, + "step": 3450, + "unclipped_grad_norm": 0.8042635321617126 + }, + { + "bits_per_byte": 1.7083845028661837, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.184161901473999, + "step": 3460, + "unclipped_grad_norm": 0.7639697194099426 + }, + { + "bits_per_byte": 1.6937178224006033, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1739957332611084, + "step": 3470, + "unclipped_grad_norm": 0.748127281665802 + }, + { + "bits_per_byte": 1.6501064617948125, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1437666416168213, + "step": 3480, + "unclipped_grad_norm": 0.7506517767906189 + }, + { + "bits_per_byte": 1.8358463488654388, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2725117206573486, + "step": 3490, + "unclipped_grad_norm": 0.836082398891449 + }, + { + "bits_per_byte": 1.714104301868579, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.188126564025879, + "step": 3500, + "unclipped_grad_norm": 0.7544166445732117 + }, + { + "bits_per_byte": 1.744907598516664, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2094777822494507, + "step": 3510, + "unclipped_grad_norm": 0.7650617361068726 + }, + { + "bits_per_byte": 1.738600994709525, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2051063776016235, + "step": 3520, + "unclipped_grad_norm": 0.7794367671012878 + }, + { + "bits_per_byte": 1.7051804660806096, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.181941032409668, + "step": 3530, + "unclipped_grad_norm": 0.7445787787437439 + }, + { + "bits_per_byte": 1.7158172490711725, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1893138885498047, + "step": 3540, + "unclipped_grad_norm": 0.7629107236862183 + }, + { + "bits_per_byte": 1.7173026632266746, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.1903434991836548, + "step": 3550, + "unclipped_grad_norm": 0.7763495445251465 + }, + { + "bits_per_byte": 1.6560372835098163, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.14787757396698, + "step": 3560, + "unclipped_grad_norm": 0.7450886964797974 + }, + { + "bits_per_byte": 1.6838996768281473, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1671903133392334, + "step": 3570, + "unclipped_grad_norm": 0.7487124800682068 + }, + { + "bits_per_byte": 1.6946102403759304, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1746143102645874, + "step": 3580, + "unclipped_grad_norm": 0.757170557975769 + }, + { + "bits_per_byte": 1.7307594457434352, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1996710300445557, + "step": 3590, + "unclipped_grad_norm": 0.7700981497764587 + }, + { + "bits_per_byte": 1.7491381997452384, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2124102115631104, + "step": 3600, + "unclipped_grad_norm": 0.7877120971679688 + }, + { + "bits_per_byte": 1.688025540622346, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1700501441955566, + "step": 3610, + "unclipped_grad_norm": 0.780867874622345 + }, + { + "bits_per_byte": 1.6436277033541595, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1392759084701538, + "step": 3620, + "unclipped_grad_norm": 0.7423334717750549 + }, + { + "bits_per_byte": 1.6950637586262558, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1749286651611328, + "step": 3630, + "unclipped_grad_norm": 0.7513464689254761 + }, + { + "bits_per_byte": 1.6758522646289749, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1616122722625732, + "step": 3640, + "unclipped_grad_norm": 0.7453336715698242 + }, + { + "bits_per_byte": 1.736063734661346, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2033476829528809, + "step": 3650, + "unclipped_grad_norm": 0.7682862877845764 + }, + { + "bits_per_byte": 1.6360068081291268, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1339935064315796, + "step": 3660, + "unclipped_grad_norm": 0.7531437277793884 + }, + { + "bits_per_byte": 1.7183529612754938, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1910715103149414, + "step": 3670, + "unclipped_grad_norm": 0.7578480243682861 + }, + { + "bits_per_byte": 1.6583936178092877, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1495108604431152, + "step": 3680, + "unclipped_grad_norm": 0.7224898934364319 + }, + { + "bits_per_byte": 1.6372521345040243, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1348567008972168, + "step": 3690, + "unclipped_grad_norm": 0.7246464490890503 + }, + { + "bits_per_byte": 1.6659130432703113, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1547229290008545, + "step": 3700, + "unclipped_grad_norm": 0.7878638505935669 + }, + { + "bits_per_byte": 1.694639821391879, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1746348142623901, + "step": 3710, + "unclipped_grad_norm": 0.7455682754516602 + }, + { + "bits_per_byte": 1.6891824679147003, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1708520650863647, + "step": 3720, + "unclipped_grad_norm": 0.7515487670898438 + }, + { + "bits_per_byte": 1.6091924770851538, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1154072284698486, + "step": 3730, + "unclipped_grad_norm": 0.7353127598762512 + }, + { + "bits_per_byte": 1.6633680440028435, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.152958869934082, + "step": 3740, + "unclipped_grad_norm": 0.7253232002258301 + }, + { + "bits_per_byte": 1.6922951819126664, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1730096340179443, + "step": 3750, + "unclipped_grad_norm": 0.7686370611190796 + }, + { + "bits_per_byte": 1.7713081393026613, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2277772426605225, + "step": 3760, + "unclipped_grad_norm": 0.7601476907730103 + }, + { + "bits_per_byte": 1.7305630415561497, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1995348930358887, + "step": 3770, + "unclipped_grad_norm": 0.7877833247184753 + }, + { + "bits_per_byte": 1.5644979697442287, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.0844273567199707, + "step": 3780, + "unclipped_grad_norm": 0.7386898994445801 + }, + { + "bits_per_byte": 1.6644766441703052, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1537272930145264, + "step": 3790, + "unclipped_grad_norm": 0.7420179843902588 + }, + { + "bits_per_byte": 1.6653939996300076, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1543631553649902, + "step": 3800, + "unclipped_grad_norm": 0.7838358283042908 + }, + { + "bits_per_byte": 1.6952983429620325, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.17509126663208, + "step": 3810, + "unclipped_grad_norm": 0.8048396110534668 + }, + { + "bits_per_byte": 1.6744323758634516, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.160628080368042, + "step": 3820, + "unclipped_grad_norm": 0.7709245681762695 + }, + { + "bits_per_byte": 1.720191799778519, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1923460960388184, + "step": 3830, + "unclipped_grad_norm": 0.7672807574272156 + }, + { + "bits_per_byte": 1.6467005173771254, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1414058208465576, + "step": 3840, + "unclipped_grad_norm": 0.7319741249084473 + }, + { + "bits_per_byte": 1.5964396195583748, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.106567621231079, + "step": 3850, + "unclipped_grad_norm": 0.7756319642066956 + }, + { + "bits_per_byte": 1.6360811046342996, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1340450048446655, + "step": 3860, + "unclipped_grad_norm": 0.766754150390625 + }, + { + "bits_per_byte": 1.7433616464680584, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2084062099456787, + "step": 3870, + "unclipped_grad_norm": 0.7916337847709656 + }, + { + "bits_per_byte": 1.6358671582166262, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1338967084884644, + "step": 3880, + "unclipped_grad_norm": 0.7497678399085999 + }, + { + "bits_per_byte": 1.600699113872294, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1095200777053833, + "step": 3890, + "unclipped_grad_norm": 0.7555440664291382 + }, + { + "bits_per_byte": 1.6778517349279058, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1629981994628906, + "step": 3900, + "unclipped_grad_norm": 0.826478123664856 + }, + { + "bits_per_byte": 1.6805711246033486, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1648831367492676, + "step": 3910, + "unclipped_grad_norm": 0.7849931120872498 + }, + { + "bits_per_byte": 1.6375147520118918, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1350387334823608, + "step": 3920, + "unclipped_grad_norm": 0.7588120102882385 + }, + { + "bits_per_byte": 1.673005091843941, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.15963876247406, + "step": 3930, + "unclipped_grad_norm": 0.742434024810791 + }, + { + "bits_per_byte": 1.6996766772876981, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1781260967254639, + "step": 3940, + "unclipped_grad_norm": 0.7588658928871155 + }, + { + "bits_per_byte": 1.7339400928884918, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2018756866455078, + "step": 3950, + "unclipped_grad_norm": 0.7937244176864624 + }, + { + "bits_per_byte": 1.713467450112434, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1876851320266724, + "step": 3960, + "unclipped_grad_norm": 0.7884036302566528 + }, + { + "bits_per_byte": 1.6516962694193882, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1448686122894287, + "step": 3970, + "unclipped_grad_norm": 0.8004661798477173 + }, + { + "bits_per_byte": 1.6740500584305835, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1603630781173706, + "step": 3980, + "unclipped_grad_norm": 0.7692103385925293 + }, + { + "bits_per_byte": 1.691941929547794, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.172764778137207, + "step": 3990, + "unclipped_grad_norm": 0.7891059517860413 + }, + { + "bits_per_byte": 1.5813677479673616, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.0961205959320068, + "step": 4000, + "unclipped_grad_norm": 0.7436221837997437 + }, + { + "bits_per_byte": 1.6793110077204767, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.164009690284729, + "step": 4010, + "unclipped_grad_norm": 0.8044730424880981 + }, + { + "bits_per_byte": 1.6584589712166156, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1495561599731445, + "step": 4020, + "unclipped_grad_norm": 0.8183668851852417 + }, + { + "bits_per_byte": 1.6850787898824628, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1680076122283936, + "step": 4030, + "unclipped_grad_norm": 0.793128252029419 + }, + { + "bits_per_byte": 1.617821878574846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1213886737823486, + "step": 4040, + "unclipped_grad_norm": 0.7582541108131409 + }, + { + "bits_per_byte": 1.6450140555027646, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1402368545532227, + "step": 4050, + "unclipped_grad_norm": 0.7559582591056824 + }, + { + "bits_per_byte": 1.6581645369183384, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1493520736694336, + "step": 4060, + "unclipped_grad_norm": 0.743245005607605 + }, + { + "bits_per_byte": 1.680131880913045, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1645786762237549, + "step": 4070, + "unclipped_grad_norm": 0.7624887824058533 + }, + { + "bits_per_byte": 1.7564567494700543, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2174830436706543, + "step": 4080, + "unclipped_grad_norm": 0.8132302761077881 + }, + { + "bits_per_byte": 1.6669108866106175, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1554145812988281, + "step": 4090, + "unclipped_grad_norm": 0.8276432752609253 + }, + { + "bits_per_byte": 1.638365894151012, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1356287002563477, + "step": 4100, + "unclipped_grad_norm": 0.7751885652542114 + }, + { + "bits_per_byte": 1.6303674970073352, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1300846338272095, + "step": 4110, + "unclipped_grad_norm": 0.7898787260055542 + }, + { + "bits_per_byte": 1.6896222275529564, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.171156883239746, + "step": 4120, + "unclipped_grad_norm": 0.8148247599601746 + }, + { + "bits_per_byte": 1.6213521664491068, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1238356828689575, + "step": 4130, + "unclipped_grad_norm": 0.8133904933929443 + }, + { + "bits_per_byte": 1.6881359534842002, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1701266765594482, + "step": 4140, + "unclipped_grad_norm": 0.8217440247535706 + }, + { + "bits_per_byte": 1.6129334437067215, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1180002689361572, + "step": 4150, + "unclipped_grad_norm": 0.7947918176651001 + }, + { + "bits_per_byte": 1.651558683298698, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.144773244857788, + "step": 4160, + "unclipped_grad_norm": 0.7860805988311768 + }, + { + "bits_per_byte": 1.6828089628563754, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1664342880249023, + "step": 4170, + "unclipped_grad_norm": 0.8207855224609375 + }, + { + "bits_per_byte": 1.7311515661874024, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1999428272247314, + "step": 4180, + "unclipped_grad_norm": 0.8373809456825256 + }, + { + "bits_per_byte": 1.6433404923272186, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.139076828956604, + "step": 4190, + "unclipped_grad_norm": 0.7621071338653564 + }, + { + "bits_per_byte": 1.682679287937625, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.166344404220581, + "step": 4200, + "unclipped_grad_norm": 0.7672834396362305 + }, + { + "bits_per_byte": 1.661693964879345, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1517984867095947, + "step": 4210, + "unclipped_grad_norm": 0.7330533862113953 + }, + { + "bits_per_byte": 1.6972419188994332, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1764384508132935, + "step": 4220, + "unclipped_grad_norm": 0.7866858243942261 + }, + { + "bits_per_byte": 1.6670219874030747, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.155491590499878, + "step": 4230, + "unclipped_grad_norm": 0.7902593612670898 + }, + { + "bits_per_byte": 1.6114956687455084, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1170036792755127, + "step": 4240, + "unclipped_grad_norm": 0.740251362323761 + }, + { + "bits_per_byte": 1.6335233786506678, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1322721242904663, + "step": 4250, + "unclipped_grad_norm": 0.7777352333068848 + }, + { + "bits_per_byte": 1.6115895712728796, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1170687675476074, + "step": 4260, + "unclipped_grad_norm": 0.774424135684967 + }, + { + "bits_per_byte": 1.619611874005026, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1226294040679932, + "step": 4270, + "unclipped_grad_norm": 0.7803477048873901 + }, + { + "bits_per_byte": 1.6299772683725275, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1298141479492188, + "step": 4280, + "unclipped_grad_norm": 0.7580373287200928 + }, + { + "bits_per_byte": 1.6795242662075467, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.164157509803772, + "step": 4290, + "unclipped_grad_norm": 0.7638103365898132 + }, + { + "bits_per_byte": 1.600082040120998, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.109092354774475, + "step": 4300, + "unclipped_grad_norm": 0.7825598120689392 + }, + { + "bits_per_byte": 1.6771565810531184, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1625163555145264, + "step": 4310, + "unclipped_grad_norm": 0.7940689921379089 + }, + { + "bits_per_byte": 1.6577068910843924, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.149034857749939, + "step": 4320, + "unclipped_grad_norm": 0.7976187467575073 + }, + { + "bits_per_byte": 1.6195226150092283, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1225675344467163, + "step": 4330, + "unclipped_grad_norm": 0.7892199158668518 + }, + { + "bits_per_byte": 1.655279183984813, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1473520994186401, + "step": 4340, + "unclipped_grad_norm": 0.8144169449806213 + }, + { + "bits_per_byte": 1.6394932404274176, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.136410117149353, + "step": 4350, + "unclipped_grad_norm": 0.8119622468948364 + }, + { + "bits_per_byte": 1.6287904160989233, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1289914846420288, + "step": 4360, + "unclipped_grad_norm": 0.7554351091384888 + }, + { + "bits_per_byte": 1.6321829458698431, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1313430070877075, + "step": 4370, + "unclipped_grad_norm": 0.808085024356842 + }, + { + "bits_per_byte": 1.6820371047193032, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1658992767333984, + "step": 4380, + "unclipped_grad_norm": 0.7708722949028015 + }, + { + "bits_per_byte": 1.6522423143358775, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1452471017837524, + "step": 4390, + "unclipped_grad_norm": 0.8123121857643127 + }, + { + "bits_per_byte": 1.6025224739367414, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1107839345932007, + "step": 4400, + "unclipped_grad_norm": 0.7717524766921997 + }, + { + "bits_per_byte": 1.6853694405624209, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1682090759277344, + "step": 4410, + "unclipped_grad_norm": 0.81525057554245 + }, + { + "bits_per_byte": 1.6142167782474597, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1188898086547852, + "step": 4420, + "unclipped_grad_norm": 0.7956154346466064 + }, + { + "bits_per_byte": 1.6755413199962148, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1613967418670654, + "step": 4430, + "unclipped_grad_norm": 0.7907513380050659 + }, + { + "bits_per_byte": 1.6011154838700328, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1098086833953857, + "step": 4440, + "unclipped_grad_norm": 0.7607184648513794 + }, + { + "bits_per_byte": 1.5990663105850023, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1083883047103882, + "step": 4450, + "unclipped_grad_norm": 0.7442869544029236 + }, + { + "bits_per_byte": 1.5859970049806358, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0993293523788452, + "step": 4460, + "unclipped_grad_norm": 0.7537500262260437 + }, + { + "bits_per_byte": 1.6451918855637568, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1403601169586182, + "step": 4470, + "unclipped_grad_norm": 0.7931647896766663 + }, + { + "bits_per_byte": 1.6505943765753102, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1441048383712769, + "step": 4480, + "unclipped_grad_norm": 0.9106901288032532 + }, + { + "bits_per_byte": 1.6301040195862135, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1299020051956177, + "step": 4490, + "unclipped_grad_norm": 0.7690432667732239 + }, + { + "bits_per_byte": 1.6520481459230536, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1451125144958496, + "step": 4500, + "unclipped_grad_norm": 0.7990603446960449 + }, + { + "bits_per_byte": 1.5968599451570833, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1068589687347412, + "step": 4510, + "unclipped_grad_norm": 0.8130239248275757 + }, + { + "bits_per_byte": 1.6430763269754933, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.138893723487854, + "step": 4520, + "unclipped_grad_norm": 0.7745712399482727 + }, + { + "bits_per_byte": 1.660709880151108, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1511163711547852, + "step": 4530, + "unclipped_grad_norm": 0.7695059776306152 + }, + { + "bits_per_byte": 1.707295852686222, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1834073066711426, + "step": 4540, + "unclipped_grad_norm": 0.8436031937599182 + }, + { + "bits_per_byte": 1.5623325361872151, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.0829263925552368, + "step": 4550, + "unclipped_grad_norm": 0.7549967169761658 + }, + { + "bits_per_byte": 1.6257084869954619, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1268552541732788, + "step": 4560, + "unclipped_grad_norm": 0.8449703454971313 + }, + { + "bits_per_byte": 1.6175644205465043, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1212102174758911, + "step": 4570, + "unclipped_grad_norm": 0.7528937458992004 + }, + { + "bits_per_byte": 1.570274179056107, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0884311199188232, + "step": 4580, + "unclipped_grad_norm": 0.790579617023468 + }, + { + "bits_per_byte": 1.609637052237634, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1157153844833374, + "step": 4590, + "unclipped_grad_norm": 0.7935193181037903 + }, + { + "bits_per_byte": 1.6088013885370918, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1151361465454102, + "step": 4600, + "unclipped_grad_norm": 0.7915687561035156 + }, + { + "bits_per_byte": 1.6615916352020816, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.151727557182312, + "step": 4610, + "unclipped_grad_norm": 0.8493216633796692 + }, + { + "bits_per_byte": 1.6675298521710726, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1558436155319214, + "step": 4620, + "unclipped_grad_norm": 0.8179065585136414 + }, + { + "bits_per_byte": 1.6286588493710132, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1289002895355225, + "step": 4630, + "unclipped_grad_norm": 0.8027536273002625 + }, + { + "bits_per_byte": 1.7031861552612044, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.180558681488037, + "step": 4640, + "unclipped_grad_norm": 0.8179929256439209 + }, + { + "bits_per_byte": 1.6059254946493637, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.113142728805542, + "step": 4650, + "unclipped_grad_norm": 0.8228299021720886 + }, + { + "bits_per_byte": 1.5739184914278899, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0909571647644043, + "step": 4660, + "unclipped_grad_norm": 0.820624589920044 + }, + { + "bits_per_byte": 1.7237126286069824, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.194786548614502, + "step": 4670, + "unclipped_grad_norm": 0.8582949638366699 + }, + { + "bits_per_byte": 1.7009104808249877, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1789813041687012, + "step": 4680, + "unclipped_grad_norm": 0.8158925175666809 + }, + { + "bits_per_byte": 1.6679322915740917, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1561225652694702, + "step": 4690, + "unclipped_grad_norm": 0.849236249923706 + }, + { + "bits_per_byte": 1.589747430648001, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.101928949356079, + "step": 4700, + "unclipped_grad_norm": 0.7403190732002258 + }, + { + "bits_per_byte": 1.6742342518496576, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1604907512664795, + "step": 4710, + "unclipped_grad_norm": 0.8582251071929932 + }, + { + "bits_per_byte": 1.607032031025015, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1139097213745117, + "step": 4720, + "unclipped_grad_norm": 0.8400667905807495 + }, + { + "bits_per_byte": 1.5176990826873464, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0519888401031494, + "step": 4730, + "unclipped_grad_norm": 0.7827285528182983 + }, + { + "bits_per_byte": 1.5868581221135059, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.099926233291626, + "step": 4740, + "unclipped_grad_norm": 0.7815871238708496 + }, + { + "bits_per_byte": 1.6377919880450826, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1352308988571167, + "step": 4750, + "unclipped_grad_norm": 0.7846289873123169 + }, + { + "bits_per_byte": 1.6125860387519786, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1177594661712646, + "step": 4760, + "unclipped_grad_norm": 0.8307337164878845 + }, + { + "bits_per_byte": 1.5936545325103024, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1046371459960938, + "step": 4770, + "unclipped_grad_norm": 0.8213374614715576 + }, + { + "bits_per_byte": 1.6121588338472355, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1174633502960205, + "step": 4780, + "unclipped_grad_norm": 0.8224621415138245 + }, + { + "bits_per_byte": 1.5705015401205475, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.0885887145996094, + "step": 4790, + "unclipped_grad_norm": 0.8122685551643372 + }, + { + "bits_per_byte": 1.5722774329733569, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0898196697235107, + "step": 4800, + "unclipped_grad_norm": 0.8455460667610168 + }, + { + "bits_per_byte": 1.6480244398234671, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1423234939575195, + "step": 4810, + "unclipped_grad_norm": 0.8817458748817444 + }, + { + "bits_per_byte": 1.574972745077679, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.0916879177093506, + "step": 4820, + "unclipped_grad_norm": 0.7954387664794922 + }, + { + "bits_per_byte": 1.608869665649484, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1151834726333618, + "step": 4830, + "unclipped_grad_norm": 0.8687117695808411 + }, + { + "bits_per_byte": 1.5985519104762718, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1080317497253418, + "step": 4840, + "unclipped_grad_norm": 0.8505465388298035 + }, + { + "bits_per_byte": 1.6228387844831649, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1248661279678345, + "step": 4850, + "unclipped_grad_norm": 0.7912812232971191 + }, + { + "bits_per_byte": 1.5432041098103009, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0696675777435303, + "step": 4860, + "unclipped_grad_norm": 0.7933070063591003 + }, + { + "bits_per_byte": 1.635918581029234, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.13393235206604, + "step": 4870, + "unclipped_grad_norm": 0.8805916905403137 + }, + { + "bits_per_byte": 1.4766798445953597, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0235564708709717, + "step": 4880, + "unclipped_grad_norm": 0.8066675066947937 + }, + { + "bits_per_byte": 1.5344280071194227, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0635844469070435, + "step": 4890, + "unclipped_grad_norm": 0.8088182806968689 + }, + { + "bits_per_byte": 1.6101516243290155, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1160720586776733, + "step": 4900, + "unclipped_grad_norm": 0.8653903603553772 + }, + { + "bits_per_byte": 1.6779198400576476, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1630454063415527, + "step": 4910, + "unclipped_grad_norm": 0.8391038179397583 + }, + { + "bits_per_byte": 1.6284904783558185, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1287835836410522, + "step": 4920, + "unclipped_grad_norm": 0.8723531365394592 + }, + { + "bits_per_byte": 1.5831391692712484, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0973484516143799, + "step": 4930, + "unclipped_grad_norm": 0.7553638815879822 + }, + { + "bits_per_byte": 1.5755332365368409, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0920764207839966, + "step": 4940, + "unclipped_grad_norm": 0.8158645033836365 + }, + { + "bits_per_byte": 1.530142715407874, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0606141090393066, + "step": 4950, + "unclipped_grad_norm": 0.7773033380508423 + }, + { + "bits_per_byte": 1.5220387209165676, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.0549968481063843, + "step": 4960, + "unclipped_grad_norm": 0.8325244784355164 + }, + { + "bits_per_byte": 1.6031579498316795, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1112244129180908, + "step": 4970, + "unclipped_grad_norm": 0.8591333627700806 + }, + { + "bits_per_byte": 1.6532000857185325, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1459109783172607, + "step": 4980, + "unclipped_grad_norm": 0.8351151347160339 + }, + { + "bits_per_byte": 1.585272098107249, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.0988268852233887, + "step": 4990, + "unclipped_grad_norm": 0.8779996037483215 + }, + { + "bits_per_byte": 1.5394700224947675, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.0670793056488037, + "step": 5000, + "unclipped_grad_norm": 0.7928086519241333 + }, + { + "bits_per_byte": 1.5552827953456976, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0780398845672607, + "step": 5010, + "unclipped_grad_norm": 0.8201308846473694 + }, + { + "bits_per_byte": 1.5898205232746179, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1019796133041382, + "step": 5020, + "unclipped_grad_norm": 0.7774789929389954 + }, + { + "bits_per_byte": 1.5697566832596608, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.088072419166565, + "step": 5030, + "unclipped_grad_norm": 0.7855319380760193 + }, + { + "bits_per_byte": 1.539816567536256, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0673195123672485, + "step": 5040, + "unclipped_grad_norm": 0.7920348048210144 + }, + { + "bits_per_byte": 1.5092143186070297, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0461076498031616, + "step": 5050, + "unclipped_grad_norm": 0.8045132160186768 + }, + { + "bits_per_byte": 1.5339026001210367, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0632202625274658, + "step": 5060, + "unclipped_grad_norm": 0.8044583797454834 + }, + { + "bits_per_byte": 1.6138325690054323, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1186234951019287, + "step": 5070, + "unclipped_grad_norm": 0.8723475337028503 + }, + { + "bits_per_byte": 1.574335549356232, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.091246247291565, + "step": 5080, + "unclipped_grad_norm": 0.8741123080253601 + }, + { + "bits_per_byte": 1.5594206979254568, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.0809080600738525, + "step": 5090, + "unclipped_grad_norm": 0.8465037941932678 + }, + { + "bits_per_byte": 1.5640659493252613, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0841279029846191, + "step": 5100, + "unclipped_grad_norm": 0.783299446105957 + }, + { + "bits_per_byte": 1.488738924108558, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0319151878356934, + "step": 5110, + "unclipped_grad_norm": 0.8348572254180908 + }, + { + "bits_per_byte": 1.5847128105266433, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0984392166137695, + "step": 5120, + "unclipped_grad_norm": 0.8110998868942261 + }, + { + "bits_per_byte": 1.590178763136365, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1022279262542725, + "step": 5130, + "unclipped_grad_norm": 0.8208310604095459 + }, + { + "bits_per_byte": 1.5494388248693793, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.0739891529083252, + "step": 5140, + "unclipped_grad_norm": 0.8678956627845764 + }, + { + "bits_per_byte": 1.6273239200350162, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1279749870300293, + "step": 5150, + "unclipped_grad_norm": 0.848126232624054 + }, + { + "bits_per_byte": 1.5085432423033631, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0456424951553345, + "step": 5160, + "unclipped_grad_norm": 0.7922812104225159 + }, + { + "bits_per_byte": 1.6700107019097687, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1575632095336914, + "step": 5170, + "unclipped_grad_norm": 0.8759046196937561 + }, + { + "bits_per_byte": 1.5869253673299932, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0999728441238403, + "step": 5180, + "unclipped_grad_norm": 0.8647724390029907 + }, + { + "bits_per_byte": 1.5237559676854329, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0561871528625488, + "step": 5190, + "unclipped_grad_norm": 0.813337504863739 + }, + { + "bits_per_byte": 1.556985939537192, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0792204141616821, + "step": 5200, + "unclipped_grad_norm": 0.8388134241104126 + }, + { + "bits_per_byte": 1.5602121620847276, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.0814566612243652, + "step": 5210, + "unclipped_grad_norm": 0.8162683248519897 + }, + { + "bits_per_byte": 1.6432956048553435, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1390457153320312, + "step": 5220, + "unclipped_grad_norm": 0.8667425513267517 + }, + { + "bits_per_byte": 1.5501931407760636, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.074512004852295, + "step": 5230, + "unclipped_grad_norm": 0.8668702840805054 + }, + { + "bits_per_byte": 1.5510165937083948, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.075082778930664, + "step": 5240, + "unclipped_grad_norm": 0.8049055337905884 + }, + { + "bits_per_byte": 1.5393878147876552, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0670223236083984, + "step": 5250, + "unclipped_grad_norm": 0.8209428191184998 + }, + { + "bits_per_byte": 1.540701762240247, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0679330825805664, + "step": 5260, + "unclipped_grad_norm": 0.8524174690246582 + }, + { + "bits_per_byte": 1.5376201771020872, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0657970905303955, + "step": 5270, + "unclipped_grad_norm": 0.8064137697219849 + }, + { + "bits_per_byte": 1.5550475230793173, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0778768062591553, + "step": 5280, + "unclipped_grad_norm": 0.8169121146202087 + }, + { + "bits_per_byte": 1.5423493560355128, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.069075107574463, + "step": 5290, + "unclipped_grad_norm": 0.8263237476348877 + }, + { + "bits_per_byte": 1.5542627662434303, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.077332854270935, + "step": 5300, + "unclipped_grad_norm": 0.8492859601974487 + }, + { + "bits_per_byte": 1.5509104804128124, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0750092267990112, + "step": 5310, + "unclipped_grad_norm": 0.8885292410850525 + }, + { + "bits_per_byte": 1.5517654061702515, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0756018161773682, + "step": 5320, + "unclipped_grad_norm": 0.8326203227043152 + }, + { + "bits_per_byte": 1.5477100552629064, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0727908611297607, + "step": 5330, + "unclipped_grad_norm": 0.8806997537612915 + }, + { + "bits_per_byte": 1.503639328996661, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0422433614730835, + "step": 5340, + "unclipped_grad_norm": 0.8234722018241882 + }, + { + "bits_per_byte": 1.5869049013945407, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0999586582183838, + "step": 5350, + "unclipped_grad_norm": 0.8706242442131042 + }, + { + "bits_per_byte": 1.5072631754329913, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.044755220413208, + "step": 5360, + "unclipped_grad_norm": 0.8708282709121704 + }, + { + "bits_per_byte": 1.471025742965594, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0196373462677002, + "step": 5370, + "unclipped_grad_norm": 0.8303158283233643 + }, + { + "bits_per_byte": 1.5968417149960918, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1068463325500488, + "step": 5380, + "unclipped_grad_norm": 0.883955180644989 + }, + { + "bits_per_byte": 1.531507741707772, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0615602731704712, + "step": 5390, + "unclipped_grad_norm": 0.8507125377655029 + }, + { + "bits_per_byte": 1.5483677169198058, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073246717453003, + "step": 5400, + "unclipped_grad_norm": 0.8522453904151917 + }, + { + "bits_per_byte": 1.571479089508052, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.089266300201416, + "step": 5410, + "unclipped_grad_norm": 0.8313266038894653 + }, + { + "bits_per_byte": 1.5408727129952047, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0680515766143799, + "step": 5420, + "unclipped_grad_norm": 0.8902517557144165 + }, + { + "bits_per_byte": 1.5778336764747818, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0936709642410278, + "step": 5430, + "unclipped_grad_norm": 0.8956695795059204 + }, + { + "bits_per_byte": 1.513927503153925, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0493745803833008, + "step": 5440, + "unclipped_grad_norm": 0.897573709487915 + }, + { + "bits_per_byte": 1.5078418970531444, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0451563596725464, + "step": 5450, + "unclipped_grad_norm": 0.8440007567405701 + }, + { + "bits_per_byte": 1.5030907043404085, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0418630838394165, + "step": 5460, + "unclipped_grad_norm": 0.8436890244483948 + }, + { + "bits_per_byte": 1.5464133060754008, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0718920230865479, + "step": 5470, + "unclipped_grad_norm": 0.8681005239486694 + }, + { + "bits_per_byte": 1.553998600891705, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.077149748802185, + "step": 5480, + "unclipped_grad_norm": 0.8523774147033691 + }, + { + "bits_per_byte": 1.543775436176467, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.070063591003418, + "step": 5490, + "unclipped_grad_norm": 0.8906207084655762 + }, + { + "bits_per_byte": 1.5444891641775478, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0705583095550537, + "step": 5500, + "unclipped_grad_norm": 0.8658390045166016 + }, + { + "bits_per_byte": 1.6370650173798855, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1347270011901855, + "step": 5510, + "unclipped_grad_norm": 0.9161565899848938 + }, + { + "bits_per_byte": 1.5520985365649729, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.075832724571228, + "step": 5520, + "unclipped_grad_norm": 0.845907986164093 + }, + { + "bits_per_byte": 1.5215033389244317, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0546257495880127, + "step": 5530, + "unclipped_grad_norm": 0.8585748672485352 + }, + { + "bits_per_byte": 1.5943249208833656, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1051018238067627, + "step": 5540, + "unclipped_grad_norm": 0.9360165596008301 + }, + { + "bits_per_byte": 1.5176287417831436, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0519400835037231, + "step": 5550, + "unclipped_grad_norm": 0.8308430910110474 + }, + { + "bits_per_byte": 1.5164790377621256, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0511431694030762, + "step": 5560, + "unclipped_grad_norm": 0.894058346748352 + }, + { + "bits_per_byte": 1.4905304673825959, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0331569910049438, + "step": 5570, + "unclipped_grad_norm": 0.8399901390075684 + }, + { + "bits_per_byte": 1.5655759569998369, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.085174560546875, + "step": 5580, + "unclipped_grad_norm": 0.8579773902893066 + }, + { + "bits_per_byte": 1.482571282283316, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0276401042938232, + "step": 5590, + "unclipped_grad_norm": 0.8161733746528625 + }, + { + "bits_per_byte": 1.5496305855250914, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0741220712661743, + "step": 5600, + "unclipped_grad_norm": 0.8440633416175842 + }, + { + "bits_per_byte": 1.5466313800766949, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0720431804656982, + "step": 5610, + "unclipped_grad_norm": 0.8384022116661072 + }, + { + "bits_per_byte": 1.55366271877457, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0769169330596924, + "step": 5620, + "unclipped_grad_norm": 0.8643615245819092 + }, + { + "bits_per_byte": 1.5359743031333302, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0646562576293945, + "step": 5630, + "unclipped_grad_norm": 0.8559057712554932 + }, + { + "bits_per_byte": 1.5154779267514533, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.050449252128601, + "step": 5640, + "unclipped_grad_norm": 0.8564271330833435 + }, + { + "bits_per_byte": 1.5399061704973558, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0673816204071045, + "step": 5650, + "unclipped_grad_norm": 0.899230420589447 + }, + { + "bits_per_byte": 1.6017691899259623, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1102617979049683, + "step": 5660, + "unclipped_grad_norm": 0.8796955943107605 + }, + { + "bits_per_byte": 1.5203328250026595, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.05381441116333, + "step": 5670, + "unclipped_grad_norm": 0.8509314060211182 + }, + { + "bits_per_byte": 1.5301421994599214, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.060613751411438, + "step": 5680, + "unclipped_grad_norm": 0.8624072074890137 + }, + { + "bits_per_byte": 1.6028342784827556, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1110000610351562, + "step": 5690, + "unclipped_grad_norm": 0.8914096355438232 + }, + { + "bits_per_byte": 1.526354281574668, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.057988166809082, + "step": 5700, + "unclipped_grad_norm": 0.8579604625701904 + }, + { + "bits_per_byte": 1.5347007716036911, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.063773512840271, + "step": 5710, + "unclipped_grad_norm": 0.8810272216796875 + }, + { + "bits_per_byte": 1.5491378552303696, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0737805366516113, + "step": 5720, + "unclipped_grad_norm": 0.8394830226898193 + }, + { + "bits_per_byte": 1.5456280332915613, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.071347713470459, + "step": 5730, + "unclipped_grad_norm": 0.865943968296051 + }, + { + "bits_per_byte": 1.5433466834278662, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0697664022445679, + "step": 5740, + "unclipped_grad_norm": 0.886944591999054 + }, + { + "bits_per_byte": 1.5488365416260579, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0735716819763184, + "step": 5750, + "unclipped_grad_norm": 0.9145897626876831 + }, + { + "bits_per_byte": 1.5636783004302166, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0838592052459717, + "step": 5760, + "unclipped_grad_norm": 0.9009037017822266 + }, + { + "bits_per_byte": 1.511110599315443, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0474220514297485, + "step": 5770, + "unclipped_grad_norm": 0.844070315361023 + }, + { + "bits_per_byte": 1.494012772097266, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.035570740699768, + "step": 5780, + "unclipped_grad_norm": 0.8943845629692078 + }, + { + "bits_per_byte": 1.5135893852623288, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.049140214920044, + "step": 5790, + "unclipped_grad_norm": 0.8841454386711121 + }, + { + "bits_per_byte": 1.4905120652389536, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033144235610962, + "step": 5800, + "unclipped_grad_norm": 0.8580894470214844 + }, + { + "bits_per_byte": 1.5735937881830608, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0907320976257324, + "step": 5810, + "unclipped_grad_norm": 0.933175265789032 + }, + { + "bits_per_byte": 1.5042643139498963, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.042676568031311, + "step": 5820, + "unclipped_grad_norm": 0.828717052936554 + }, + { + "bits_per_byte": 1.5156406223391694, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0505620241165161, + "step": 5830, + "unclipped_grad_norm": 0.9511673450469971 + }, + { + "bits_per_byte": 1.5092205099824607, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0461119413375854, + "step": 5840, + "unclipped_grad_norm": 0.8395604491233826 + }, + { + "bits_per_byte": 1.477936693807865, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0244276523590088, + "step": 5850, + "unclipped_grad_norm": 0.9143134355545044 + }, + { + "bits_per_byte": 1.5336375748560571, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.063036561012268, + "step": 5860, + "unclipped_grad_norm": 0.9014923572540283 + }, + { + "bits_per_byte": 1.5402322496033916, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0676076412200928, + "step": 5870, + "unclipped_grad_norm": 0.8728002905845642 + }, + { + "bits_per_byte": 1.555285031120159, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.078041434288025, + "step": 5880, + "unclipped_grad_norm": 0.8936865925788879 + }, + { + "bits_per_byte": 1.564054942435606, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.084120273590088, + "step": 5890, + "unclipped_grad_norm": 0.8752698302268982 + }, + { + "bits_per_byte": 1.5458715607251832, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.071516513824463, + "step": 5900, + "unclipped_grad_norm": 0.8957662582397461 + }, + { + "bits_per_byte": 1.5568008862048637, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0790921449661255, + "step": 5910, + "unclipped_grad_norm": 0.8808028101921082 + }, + { + "bits_per_byte": 1.4896764015384112, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0325649976730347, + "step": 5920, + "unclipped_grad_norm": 0.8660398721694946 + }, + { + "bits_per_byte": 1.5908711652887386, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.102707862854004, + "step": 5930, + "unclipped_grad_norm": 0.9604227542877197 + }, + { + "bits_per_byte": 1.4574508083676907, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.010227918624878, + "step": 5940, + "unclipped_grad_norm": 0.8380697965621948 + }, + { + "bits_per_byte": 1.5131755950043528, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0488533973693848, + "step": 5950, + "unclipped_grad_norm": 0.8392192125320435 + }, + { + "bits_per_byte": 1.5452194025131114, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0710644721984863, + "step": 5960, + "unclipped_grad_norm": 0.932554304599762 + }, + { + "bits_per_byte": 1.5213482105733536, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.054518222808838, + "step": 5970, + "unclipped_grad_norm": 0.927780270576477 + }, + { + "bits_per_byte": 1.5601101763727658, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0813859701156616, + "step": 5980, + "unclipped_grad_norm": 0.8920019268989563 + }, + { + "bits_per_byte": 1.5382338112003657, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0662224292755127, + "step": 5990, + "unclipped_grad_norm": 0.9537093639373779 + }, + { + "bits_per_byte": 1.4675097296513546, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.017200231552124, + "step": 6000, + "unclipped_grad_norm": 0.8773758411407471 + }, + { + "bits_per_byte": 1.506333093257125, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0441105365753174, + "step": 6010, + "unclipped_grad_norm": 0.8785988092422485 + }, + { + "bits_per_byte": 1.4794742187065786, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0254933834075928, + "step": 6020, + "unclipped_grad_norm": 0.8875186443328857 + }, + { + "bits_per_byte": 1.5245675538148544, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.056749701499939, + "step": 6030, + "unclipped_grad_norm": 0.88789963722229 + }, + { + "bits_per_byte": 1.5628784091210537, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.083304762840271, + "step": 6040, + "unclipped_grad_norm": 0.9149079322814941 + }, + { + "bits_per_byte": 1.4881958028971334, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.031538724899292, + "step": 6050, + "unclipped_grad_norm": 0.9299342632293701 + }, + { + "bits_per_byte": 1.4785307218839452, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0248394012451172, + "step": 6060, + "unclipped_grad_norm": 0.8783867359161377 + }, + { + "bits_per_byte": 1.5509557118499895, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.075040578842163, + "step": 6070, + "unclipped_grad_norm": 0.9337034821510315 + }, + { + "bits_per_byte": 1.4914187577743023, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0337727069854736, + "step": 6080, + "unclipped_grad_norm": 0.9420101046562195 + }, + { + "bits_per_byte": 1.4968499698885498, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0375373363494873, + "step": 6090, + "unclipped_grad_norm": 0.8664546012878418 + }, + { + "bits_per_byte": 1.4906561867003767, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0332441329956055, + "step": 6100, + "unclipped_grad_norm": 0.8687640428543091 + }, + { + "bits_per_byte": 1.5538661742505406, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.077057957649231, + "step": 6110, + "unclipped_grad_norm": 0.8986173868179321 + }, + { + "bits_per_byte": 1.5476213122150613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0727293491363525, + "step": 6120, + "unclipped_grad_norm": 0.9189077019691467 + }, + { + "bits_per_byte": 1.4789527673091625, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0251319408416748, + "step": 6130, + "unclipped_grad_norm": 0.8851786851882935 + }, + { + "bits_per_byte": 1.4768556108645414, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0236783027648926, + "step": 6140, + "unclipped_grad_norm": 0.9043483138084412 + }, + { + "bits_per_byte": 1.552988202817886, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0764493942260742, + "step": 6150, + "unclipped_grad_norm": 0.9005390405654907 + }, + { + "bits_per_byte": 1.3994871515820966, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 0.970050573348999, + "step": 6160, + "unclipped_grad_norm": 0.8554620742797852 + }, + { + "bits_per_byte": 1.5302097886417105, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0606606006622314, + "step": 6170, + "unclipped_grad_norm": 0.8954014778137207 + }, + { + "bits_per_byte": 1.4938230752333643, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0354392528533936, + "step": 6180, + "unclipped_grad_norm": 0.892278790473938 + }, + { + "bits_per_byte": 1.516749394489282, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.05133056640625, + "step": 6190, + "unclipped_grad_norm": 0.9185703992843628 + }, + { + "bits_per_byte": 1.5086576107661869, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0457217693328857, + "step": 6200, + "unclipped_grad_norm": 0.8927735090255737 + }, + { + "bits_per_byte": 1.5088584865023946, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.045861005783081, + "step": 6210, + "unclipped_grad_norm": 0.8906724452972412 + }, + { + "bits_per_byte": 1.5118191678369979, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0479131937026978, + "step": 6220, + "unclipped_grad_norm": 0.9077256917953491 + }, + { + "bits_per_byte": 1.4471287536282067, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0030732154846191, + "step": 6230, + "unclipped_grad_norm": 0.8839235305786133 + }, + { + "bits_per_byte": 1.5076786855174757, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0450432300567627, + "step": 6240, + "unclipped_grad_norm": 0.9515005946159363 + }, + { + "bits_per_byte": 1.5408861276419719, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0680608749389648, + "step": 6250, + "unclipped_grad_norm": 0.901561975479126 + }, + { + "bits_per_byte": 1.4200397662908057, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 0.9842965602874756, + "step": 6260, + "unclipped_grad_norm": 0.889621376991272 + }, + { + "bits_per_byte": 1.5216213190229235, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0547075271606445, + "step": 6270, + "unclipped_grad_norm": 0.9411186575889587 + }, + { + "bits_per_byte": 1.5224643779774532, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0552918910980225, + "step": 6280, + "unclipped_grad_norm": 0.9406285881996155 + }, + { + "bits_per_byte": 1.452789390598705, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0069968700408936, + "step": 6290, + "unclipped_grad_norm": 0.8839805126190186 + }, + { + "bits_per_byte": 1.5074908804627336, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0449130535125732, + "step": 6300, + "unclipped_grad_norm": 0.9184122681617737 + }, + { + "bits_per_byte": 1.478450577968643, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0247838497161865, + "step": 6310, + "unclipped_grad_norm": 0.8948619365692139 + }, + { + "bits_per_byte": 1.4214500240278807, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 0.985274076461792, + "step": 6320, + "unclipped_grad_norm": 0.8749227523803711 + }, + { + "bits_per_byte": 1.5185149683830395, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0525543689727783, + "step": 6330, + "unclipped_grad_norm": 0.9816636443138123 + }, + { + "bits_per_byte": 1.4759477144506365, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.023048996925354, + "step": 6340, + "unclipped_grad_norm": 0.8950188159942627 + }, + { + "bits_per_byte": 1.587707716408768, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1005151271820068, + "step": 6350, + "unclipped_grad_norm": 1.01009202003479 + }, + { + "bits_per_byte": 1.4638088349874379, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0146349668502808, + "step": 6360, + "unclipped_grad_norm": 0.9816907644271851 + }, + { + "bits_per_byte": 1.4382974445064014, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 0.9969518184661865, + "step": 6370, + "unclipped_grad_norm": 0.8970034122467041 + }, + { + "bits_per_byte": 1.4818757844432269, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0271580219268799, + "step": 6380, + "unclipped_grad_norm": 0.9058489799499512 + }, + { + "bits_per_byte": 1.527050123380059, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0584704875946045, + "step": 6390, + "unclipped_grad_norm": 0.929339587688446 + }, + { + "bits_per_byte": 1.5156593684481134, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0505750179290771, + "step": 6400, + "unclipped_grad_norm": 0.9009133577346802 + }, + { + "bits_per_byte": 1.4969940913499729, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0376372337341309, + "step": 6410, + "unclipped_grad_norm": 0.957128643989563 + }, + { + "bits_per_byte": 1.4783625228514012, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0247228145599365, + "step": 6420, + "unclipped_grad_norm": 0.9414587616920471 + }, + { + "bits_per_byte": 1.523227980947284, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.055821180343628, + "step": 6430, + "unclipped_grad_norm": 0.9234302639961243 + }, + { + "bits_per_byte": 1.4623229048839832, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.013604998588562, + "step": 6440, + "unclipped_grad_norm": 0.9177728295326233 + }, + { + "bits_per_byte": 1.5306256426914968, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0609488487243652, + "step": 6450, + "unclipped_grad_norm": 0.9453822374343872 + }, + { + "bits_per_byte": 1.5847898587542297, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0984926223754883, + "step": 6460, + "unclipped_grad_norm": 0.9647909998893738 + }, + { + "bits_per_byte": 1.5690988496201106, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0876164436340332, + "step": 6470, + "unclipped_grad_norm": 0.9592546820640564 + }, + { + "bits_per_byte": 1.4544314809491432, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0081350803375244, + "step": 6480, + "unclipped_grad_norm": 0.9334172010421753 + }, + { + "bits_per_byte": 1.5573938823850386, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0795031785964966, + "step": 6490, + "unclipped_grad_norm": 0.967822790145874 + }, + { + "bits_per_byte": 1.5280063469188563, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0591332912445068, + "step": 6500, + "unclipped_grad_norm": 0.9625112414360046 + }, + { + "bits_per_byte": 1.4687796495453256, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.018080472946167, + "step": 6510, + "unclipped_grad_norm": 0.9510544538497925 + }, + { + "bits_per_byte": 1.4284500618832987, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9901261329650879, + "step": 6520, + "unclipped_grad_norm": 0.8985839486122131 + }, + { + "bits_per_byte": 1.4546447394362132, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0082828998565674, + "step": 6530, + "unclipped_grad_norm": 0.907886803150177 + }, + { + "bits_per_byte": 1.4970680438898438, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0376884937286377, + "step": 6540, + "unclipped_grad_norm": 0.9381445050239563 + }, + { + "bits_per_byte": 1.4280130539674563, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 0.9898232221603394, + "step": 6550, + "unclipped_grad_norm": 0.9031082391738892 + }, + { + "bits_per_byte": 1.5210671909218436, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.054323434829712, + "step": 6560, + "unclipped_grad_norm": 0.9433532357215881 + }, + { + "bits_per_byte": 1.479243074023819, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0253331661224365, + "step": 6570, + "unclipped_grad_norm": 0.9311387538909912 + }, + { + "bits_per_byte": 1.5051876888023787, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0433166027069092, + "step": 6580, + "unclipped_grad_norm": 0.9569925665855408 + }, + { + "bits_per_byte": 1.494823842278735, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0361329317092896, + "step": 6590, + "unclipped_grad_norm": 0.8958868384361267 + }, + { + "bits_per_byte": 1.4302109062541577, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9913466572761536, + "step": 6600, + "unclipped_grad_norm": 0.9136191010475159 + }, + { + "bits_per_byte": 1.436879103584736, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 0.9959686994552612, + "step": 6610, + "unclipped_grad_norm": 0.8710504770278931 + }, + { + "bits_per_byte": 1.4278971376607748, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 0.9897428750991821, + "step": 6620, + "unclipped_grad_norm": 0.9145034551620483 + }, + { + "bits_per_byte": 1.4582168190946336, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.010758876800537, + "step": 6630, + "unclipped_grad_norm": 0.8999127149581909 + }, + { + "bits_per_byte": 1.5240701799885592, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.056404948234558, + "step": 6640, + "unclipped_grad_norm": 0.9801992774009705 + }, + { + "bits_per_byte": 1.5169175935218258, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0514471530914307, + "step": 6650, + "unclipped_grad_norm": 0.8882671594619751 + }, + { + "bits_per_byte": 1.5152221885496202, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.050271987915039, + "step": 6660, + "unclipped_grad_norm": 0.9029646515846252 + }, + { + "bits_per_byte": 1.4896082964086694, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0325177907943726, + "step": 6670, + "unclipped_grad_norm": 0.9797500371932983 + }, + { + "bits_per_byte": 1.4764526555135697, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.023398995399475, + "step": 6680, + "unclipped_grad_norm": 0.9356769919395447 + }, + { + "bits_per_byte": 1.4958203097578342, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0368236303329468, + "step": 6690, + "unclipped_grad_norm": 0.939789891242981 + }, + { + "bits_per_byte": 1.5480882451121538, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.073053002357483, + "step": 6700, + "unclipped_grad_norm": 1.0080333948135376 + }, + { + "bits_per_byte": 1.525887864625528, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0576648712158203, + "step": 6710, + "unclipped_grad_norm": 0.9197862148284912 + }, + { + "bits_per_byte": 1.534906462854123, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0639160871505737, + "step": 6720, + "unclipped_grad_norm": 0.9543269872665405 + }, + { + "bits_per_byte": 1.4820493149379474, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0272783041000366, + "step": 6730, + "unclipped_grad_norm": 0.9817740321159363 + }, + { + "bits_per_byte": 1.4384499931177168, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 0.9970575571060181, + "step": 6740, + "unclipped_grad_norm": 0.9710240364074707 + }, + { + "bits_per_byte": 1.511069151496585, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0473933219909668, + "step": 6750, + "unclipped_grad_norm": 0.9612263441085815 + }, + { + "bits_per_byte": 1.4780942299160553, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0245368480682373, + "step": 6760, + "unclipped_grad_norm": 0.9238441586494446 + }, + { + "bits_per_byte": 1.3767874194686909, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9543163180351257, + "step": 6770, + "unclipped_grad_norm": 0.9215719699859619 + }, + { + "bits_per_byte": 1.4785716537548506, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0248677730560303, + "step": 6780, + "unclipped_grad_norm": 0.9403936266899109 + }, + { + "bits_per_byte": 1.4459957319243224, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0022878646850586, + "step": 6790, + "unclipped_grad_norm": 0.9651554822921753 + }, + { + "bits_per_byte": 1.5175859181030786, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.051910400390625, + "step": 6800, + "unclipped_grad_norm": 0.9926899075508118 + }, + { + "bits_per_byte": 1.5110634760691066, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0473893880844116, + "step": 6810, + "unclipped_grad_norm": 0.9793192148208618 + }, + { + "bits_per_byte": 1.5052492585913877, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0433592796325684, + "step": 6820, + "unclipped_grad_norm": 1.0081095695495605 + }, + { + "bits_per_byte": 1.466343515295854, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0163918733596802, + "step": 6830, + "unclipped_grad_norm": 0.960361659526825 + }, + { + "bits_per_byte": 1.4376153613130795, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 0.9964790344238281, + "step": 6840, + "unclipped_grad_norm": 0.9702624678611755 + }, + { + "bits_per_byte": 1.4922177891702109, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0343265533447266, + "step": 6850, + "unclipped_grad_norm": 0.9893983006477356 + }, + { + "bits_per_byte": 1.4659078832412185, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.016089916229248, + "step": 6860, + "unclipped_grad_norm": 0.9693014621734619 + }, + { + "bits_per_byte": 1.413224953750367, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9795728921890259, + "step": 6870, + "unclipped_grad_norm": 0.9252057075500488 + }, + { + "bits_per_byte": 1.4200107012228098, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 0.9842764139175415, + "step": 6880, + "unclipped_grad_norm": 0.8884434700012207 + }, + { + "bits_per_byte": 1.4366857090938405, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9958346486091614, + "step": 6890, + "unclipped_grad_norm": 0.9679457545280457 + }, + { + "bits_per_byte": 1.5416700245646047, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0686042308807373, + "step": 6900, + "unclipped_grad_norm": 0.9985603094100952 + }, + { + "bits_per_byte": 1.4893800754309745, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0323596000671387, + "step": 6910, + "unclipped_grad_norm": 0.9936915636062622 + }, + { + "bits_per_byte": 1.4499423897963222, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.00502347946167, + "step": 6920, + "unclipped_grad_norm": 0.923648476600647 + }, + { + "bits_per_byte": 1.4406846496917027, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9986065030097961, + "step": 6930, + "unclipped_grad_norm": 0.9748247861862183 + }, + { + "bits_per_byte": 1.408974058568991, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 0.9766263961791992, + "step": 6940, + "unclipped_grad_norm": 0.9357670545578003 + }, + { + "bits_per_byte": 1.434796565665438, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9945251941680908, + "step": 6950, + "unclipped_grad_norm": 0.9624595642089844 + }, + { + "bits_per_byte": 1.49675830313564, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0374737977981567, + "step": 6960, + "unclipped_grad_norm": 0.9387544989585876 + }, + { + "bits_per_byte": 1.4929531869853003, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0348362922668457, + "step": 6970, + "unclipped_grad_norm": 0.9343914985656738 + }, + { + "bits_per_byte": 1.4660633555575984, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.016197681427002, + "step": 6980, + "unclipped_grad_norm": 0.9326976537704468 + }, + { + "bits_per_byte": 1.492015881538098, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.034186601638794, + "step": 6990, + "unclipped_grad_norm": 1.0069916248321533 + }, + { + "bits_per_byte": 1.52302985693349, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0556838512420654, + "step": 7000, + "unclipped_grad_norm": 0.9719612002372742 + }, + { + "bits_per_byte": 1.512785538352196, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0485830307006836, + "step": 7010, + "unclipped_grad_norm": 0.9873834848403931 + }, + { + "bits_per_byte": 1.4473323810868282, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0032143592834473, + "step": 7020, + "unclipped_grad_norm": 0.939332127571106 + }, + { + "bits_per_byte": 1.5166639191118032, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0512713193893433, + "step": 7030, + "unclipped_grad_norm": 0.9837853312492371 + }, + { + "bits_per_byte": 1.5072451172546506, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0447427034378052, + "step": 7040, + "unclipped_grad_norm": 0.9734936952590942 + }, + { + "bits_per_byte": 1.4796651194490362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0256257057189941, + "step": 7050, + "unclipped_grad_norm": 1.005554437637329 + }, + { + "bits_per_byte": 1.4927048440374544, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0346641540527344, + "step": 7060, + "unclipped_grad_norm": 0.9903765320777893 + }, + { + "bits_per_byte": 1.4561590466770602, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.009332537651062, + "step": 7070, + "unclipped_grad_norm": 1.009922981262207 + }, + { + "bits_per_byte": 1.4353391709289103, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9949012994766235, + "step": 7080, + "unclipped_grad_norm": 0.9552373290061951 + }, + { + "bits_per_byte": 1.4780041110070032, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0244743824005127, + "step": 7090, + "unclipped_grad_norm": 1.0046122074127197 + }, + { + "bits_per_byte": 1.4728850474040718, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0209261178970337, + "step": 7100, + "unclipped_grad_norm": 0.9868813157081604 + }, + { + "bits_per_byte": 1.4862149067244954, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.030165672302246, + "step": 7110, + "unclipped_grad_norm": 0.9593554735183716 + }, + { + "bits_per_byte": 1.5328834309320238, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.062513828277588, + "step": 7120, + "unclipped_grad_norm": 1.0129024982452393 + }, + { + "bits_per_byte": 1.4811555211014134, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0266587734222412, + "step": 7130, + "unclipped_grad_norm": 1.0546875 + }, + { + "bits_per_byte": 1.4724865636020226, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0206499099731445, + "step": 7140, + "unclipped_grad_norm": 0.9776664972305298 + }, + { + "bits_per_byte": 1.4357171887955067, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 0.9951633214950562, + "step": 7150, + "unclipped_grad_norm": 0.9291411638259888 + }, + { + "bits_per_byte": 1.5232358921492237, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0558266639709473, + "step": 7160, + "unclipped_grad_norm": 1.0189123153686523 + }, + { + "bits_per_byte": 1.4618840051589814, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0133007764816284, + "step": 7170, + "unclipped_grad_norm": 0.9592419266700745 + }, + { + "bits_per_byte": 1.4762450724539784, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0232551097869873, + "step": 7180, + "unclipped_grad_norm": 0.9680334329605103 + }, + { + "bits_per_byte": 1.4441101141402626, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0009808540344238, + "step": 7190, + "unclipped_grad_norm": 1.0148857831954956 + }, + { + "bits_per_byte": 1.472608843266786, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0207346677780151, + "step": 7200, + "unclipped_grad_norm": 0.9935786128044128 + }, + { + "bits_per_byte": 1.5299951542934338, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.060511827468872, + "step": 7210, + "unclipped_grad_norm": 0.9725626111030579 + }, + { + "bits_per_byte": 1.5238232129019202, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0562337636947632, + "step": 7220, + "unclipped_grad_norm": 1.0141370296478271 + }, + { + "bits_per_byte": 1.4390708504873315, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9974879026412964, + "step": 7230, + "unclipped_grad_norm": 1.0297694206237793 + }, + { + "bits_per_byte": 1.4632025961431465, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0142147541046143, + "step": 7240, + "unclipped_grad_norm": 0.9345177412033081 + }, + { + "bits_per_byte": 1.5031187375124992, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0418825149536133, + "step": 7250, + "unclipped_grad_norm": 1.0408201217651367 + }, + { + "bits_per_byte": 1.3820238612308362, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9579459428787231, + "step": 7260, + "unclipped_grad_norm": 0.9544695615768433 + }, + { + "bits_per_byte": 1.4274823155068936, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 0.9894553422927856, + "step": 7270, + "unclipped_grad_norm": 0.9940797686576843 + }, + { + "bits_per_byte": 1.4990891840027836, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0390894412994385, + "step": 7280, + "unclipped_grad_norm": 1.013985514640808 + }, + { + "bits_per_byte": 1.4370805812602216, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.99610835313797, + "step": 7290, + "unclipped_grad_norm": 0.9820091724395752 + }, + { + "bits_per_byte": 1.4451605841717328, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.001708984375, + "step": 7300, + "unclipped_grad_norm": 0.9996907114982605 + }, + { + "bits_per_byte": 1.4502488628801598, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0052359104156494, + "step": 7310, + "unclipped_grad_norm": 0.9723894000053406 + }, + { + "bits_per_byte": 1.4141343980081293, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9802032709121704, + "step": 7320, + "unclipped_grad_norm": 0.9815637469291687 + }, + { + "bits_per_byte": 1.429772436485783, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 0.9910427331924438, + "step": 7330, + "unclipped_grad_norm": 1.0089058876037598 + }, + { + "bits_per_byte": 1.4033304478809279, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9727145433425903, + "step": 7340, + "unclipped_grad_norm": 0.9830491542816162 + }, + { + "bits_per_byte": 1.4800763299672492, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.02591073513031, + "step": 7350, + "unclipped_grad_norm": 0.9842677712440491 + }, + { + "bits_per_byte": 1.4409101189469837, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 0.9987627863883972, + "step": 7360, + "unclipped_grad_norm": 1.0266367197036743 + }, + { + "bits_per_byte": 1.4700960047550296, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0189929008483887, + "step": 7370, + "unclipped_grad_norm": 0.9935671091079712 + }, + { + "bits_per_byte": 1.4111584102175994, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9781404733657837, + "step": 7380, + "unclipped_grad_norm": 0.9791932702064514 + }, + { + "bits_per_byte": 1.4805810990475317, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0262606143951416, + "step": 7390, + "unclipped_grad_norm": 0.9806758761405945 + }, + { + "bits_per_byte": 1.422500752033327, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9860023856163025, + "step": 7400, + "unclipped_grad_norm": 1.014562726020813 + }, + { + "bits_per_byte": 1.419007526420327, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9835810661315918, + "step": 7410, + "unclipped_grad_norm": 0.9609370231628418 + }, + { + "bits_per_byte": 1.483305992167802, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0281493663787842, + "step": 7420, + "unclipped_grad_norm": 0.9924190640449524 + }, + { + "bits_per_byte": 1.4892452410326982, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0322661399841309, + "step": 7430, + "unclipped_grad_norm": 1.0244429111480713 + }, + { + "bits_per_byte": 1.50888548777858, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0458797216415405, + "step": 7440, + "unclipped_grad_norm": 1.0394790172576904 + }, + { + "bits_per_byte": 1.4553271665948369, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0087559223175049, + "step": 7450, + "unclipped_grad_norm": 0.99248868227005 + }, + { + "bits_per_byte": 1.4425173828106221, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.999876856803894, + "step": 7460, + "unclipped_grad_norm": 1.0004469156265259 + }, + { + "bits_per_byte": 1.437790267669007, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9966002702713013, + "step": 7470, + "unclipped_grad_norm": 1.0109968185424805 + }, + { + "bits_per_byte": 1.4682612938356252, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.017721176147461, + "step": 7480, + "unclipped_grad_norm": 1.0299010276794434 + }, + { + "bits_per_byte": 1.5222762289574092, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.055161476135254, + "step": 7490, + "unclipped_grad_norm": 1.0327476263046265 + }, + { + "bits_per_byte": 1.4747042798848986, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0221871137619019, + "step": 7500, + "unclipped_grad_norm": 0.9886637926101685 + }, + { + "bits_per_byte": 1.4352613487793948, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 0.9948473572731018, + "step": 7510, + "unclipped_grad_norm": 0.9710773825645447 + }, + { + "bits_per_byte": 1.4570311706995855, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.009937047958374, + "step": 7520, + "unclipped_grad_norm": 1.0144232511520386 + }, + { + "bits_per_byte": 1.502727820947088, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0416115522384644, + "step": 7530, + "unclipped_grad_norm": 1.0307852029800415 + }, + { + "bits_per_byte": 1.4572943041554056, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0101194381713867, + "step": 7540, + "unclipped_grad_norm": 0.9732069969177246 + }, + { + "bits_per_byte": 1.4747278415080667, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0222034454345703, + "step": 7550, + "unclipped_grad_norm": 1.0250486135482788 + }, + { + "bits_per_byte": 1.4513507557242378, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0059996843338013, + "step": 7560, + "unclipped_grad_norm": 1.0031380653381348 + }, + { + "bits_per_byte": 1.4127096077370564, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.979215681552887, + "step": 7570, + "unclipped_grad_norm": 0.9720339775085449 + }, + { + "bits_per_byte": 1.587541409185384, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.1003998517990112, + "step": 7580, + "unclipped_grad_norm": 1.080464482307434 + }, + { + "bits_per_byte": 1.4421111597892842, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9995952844619751, + "step": 7590, + "unclipped_grad_norm": 0.9815735220909119 + }, + { + "bits_per_byte": 1.4267507013101233, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9889482259750366, + "step": 7600, + "unclipped_grad_norm": 1.029758334159851 + }, + { + "bits_per_byte": 1.4539179407536669, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0077791213989258, + "step": 7610, + "unclipped_grad_norm": 1.031221628189087 + }, + { + "bits_per_byte": 1.4028547438686412, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 0.9723848104476929, + "step": 7620, + "unclipped_grad_norm": 0.9562389850616455 + }, + { + "bits_per_byte": 1.3938100042771153, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9661154747009277, + "step": 7630, + "unclipped_grad_norm": 1.023301124572754 + }, + { + "bits_per_byte": 1.4184527103886437, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.983196496963501, + "step": 7640, + "unclipped_grad_norm": 0.9981763958930969 + }, + { + "bits_per_byte": 1.5608753271864544, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.081916332244873, + "step": 7650, + "unclipped_grad_norm": 1.0295648574829102 + }, + { + "bits_per_byte": 1.4665378556913289, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0165265798568726, + "step": 7660, + "unclipped_grad_norm": 1.0587892532348633 + }, + { + "bits_per_byte": 1.4931012640476933, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.034938931465149, + "step": 7670, + "unclipped_grad_norm": 1.0342050790786743 + }, + { + "bits_per_byte": 1.4465245785757257, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0026544332504272, + "step": 7680, + "unclipped_grad_norm": 1.0567405223846436 + }, + { + "bits_per_byte": 1.4541416901824393, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0079342126846313, + "step": 7690, + "unclipped_grad_norm": 1.0190099477767944 + }, + { + "bits_per_byte": 1.3949941048283057, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 0.9669362306594849, + "step": 7700, + "unclipped_grad_norm": 1.002891182899475 + }, + { + "bits_per_byte": 1.4762163513512843, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0232352018356323, + "step": 7710, + "unclipped_grad_norm": 0.9984713792800903 + }, + { + "bits_per_byte": 1.480895139368007, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0264782905578613, + "step": 7720, + "unclipped_grad_norm": 1.0342158079147339 + }, + { + "bits_per_byte": 1.486845911070511, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.030603051185608, + "step": 7730, + "unclipped_grad_norm": 1.027107834815979 + }, + { + "bits_per_byte": 1.4387832095037634, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9972885251045227, + "step": 7740, + "unclipped_grad_norm": 1.0455139875411987 + }, + { + "bits_per_byte": 1.4381068877292456, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9968197345733643, + "step": 7750, + "unclipped_grad_norm": 1.0159157514572144 + }, + { + "bits_per_byte": 1.4538223183997872, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0077128410339355, + "step": 7760, + "unclipped_grad_norm": 0.9867706894874573 + }, + { + "bits_per_byte": 1.3986834766546148, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9694935083389282, + "step": 7770, + "unclipped_grad_norm": 0.9943742156028748 + }, + { + "bits_per_byte": 1.4760859885019304, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0231448411941528, + "step": 7780, + "unclipped_grad_norm": 1.0422483682632446 + }, + { + "bits_per_byte": 1.4162891686407895, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9816968441009521, + "step": 7790, + "unclipped_grad_norm": 0.9937946796417236 + }, + { + "bits_per_byte": 1.45477768202533, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0083750486373901, + "step": 7800, + "unclipped_grad_norm": 1.0428839921951294 + }, + { + "bits_per_byte": 1.4292601001688625, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9906876087188721, + "step": 7810, + "unclipped_grad_norm": 1.028254508972168 + }, + { + "bits_per_byte": 1.4070698666586379, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.975306510925293, + "step": 7820, + "unclipped_grad_norm": 1.0253064632415771 + }, + { + "bits_per_byte": 1.4530960356651936, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0072094202041626, + "step": 7830, + "unclipped_grad_norm": 1.0885143280029297 + }, + { + "bits_per_byte": 1.4330722676078875, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9933300018310547, + "step": 7840, + "unclipped_grad_norm": 1.0074925422668457 + }, + { + "bits_per_byte": 1.3858685333908742, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9606108665466309, + "step": 7850, + "unclipped_grad_norm": 0.9880329966545105 + }, + { + "bits_per_byte": 1.4517502714221922, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0062766075134277, + "step": 7860, + "unclipped_grad_norm": 1.0082415342330933 + }, + { + "bits_per_byte": 1.4675547891058807, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0172314643859863, + "step": 7870, + "unclipped_grad_norm": 1.023923397064209 + }, + { + "bits_per_byte": 1.3867994754799946, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9612561464309692, + "step": 7880, + "unclipped_grad_norm": 1.0151392221450806 + }, + { + "bits_per_byte": 1.4568015738606837, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0097779035568237, + "step": 7890, + "unclipped_grad_norm": 1.0505098104476929 + }, + { + "bits_per_byte": 1.446776189193938, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.00282883644104, + "step": 7900, + "unclipped_grad_norm": 1.0279992818832397 + }, + { + "bits_per_byte": 1.461830518554563, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0132637023925781, + "step": 7910, + "unclipped_grad_norm": 1.0125948190689087 + }, + { + "bits_per_byte": 1.4593548282953928, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0115476846694946, + "step": 7920, + "unclipped_grad_norm": 1.0411789417266846 + }, + { + "bits_per_byte": 1.4317973602170415, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9924463033676147, + "step": 7930, + "unclipped_grad_norm": 1.0330348014831543 + }, + { + "bits_per_byte": 1.431196452834927, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9920297861099243, + "step": 7940, + "unclipped_grad_norm": 1.0142537355422974 + }, + { + "bits_per_byte": 1.49434246283897, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.035799264907837, + "step": 7950, + "unclipped_grad_norm": 1.0716009140014648 + }, + { + "bits_per_byte": 1.4186909923514142, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9833616614341736, + "step": 7960, + "unclipped_grad_norm": 1.0115852355957031 + }, + { + "bits_per_byte": 1.4420433126335188, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9995482563972473, + "step": 7970, + "unclipped_grad_norm": 1.0335451364517212 + }, + { + "bits_per_byte": 1.4872165336831205, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0308599472045898, + "step": 7980, + "unclipped_grad_norm": 1.0748283863067627 + }, + { + "bits_per_byte": 1.5234063269562288, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.055944800376892, + "step": 7990, + "unclipped_grad_norm": 1.076164722442627 + }, + { + "bits_per_byte": 1.4309937712808851, + "learning_rate": 3e-05, + "loss_nats": 0.9918892979621887, + "step": 8000, + "unclipped_grad_norm": 1.0322442054748535 + } + ] +} diff --git a/experiments/k3/attnres_spike/results/raw/formal-seed-2026073002.json b/experiments/k3/attnres_spike/results/raw/formal-seed-2026073002.json new file mode 100644 index 0000000..42c847c --- /dev/null +++ b/experiments/k3/attnres_spike/results/raw/formal-seed-2026073002.json @@ -0,0 +1,86460 @@ +{ + "architecture": "block", + "artifacts": { + "protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", + "runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", + "scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" + }, + "batch_size": 32, + "canonical_sha256_without_self": "5352c74eca853b375c0e85933dafd7c5916c39fc59052e742ca14ffd6d68bc78", + "depth": 32, + "diagnostics": [ + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "initialization_negative_control": { + "comparisons": { + "detached_learned": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + }, + "uniform_value_backward": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + } + }, + "passed": true + }, + "loss_scale_gate": { + "checks": { + "attention_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "mlp_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_attention_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_mlp_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_attention_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_mlp_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 8.038491115106986, + "forward": { + "activation_sha256": { + "attention_branch_output": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", + "mlp_branch_output": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", + "post_attention_state": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", + "post_mlp_state": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", + "pre_attention_input": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", + "pre_mlp_input": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd" + }, + "logits_sha256": "5e5a50bae6f74c41645e35a857f0cba662c8b3f5ba9a8a4635e860fdaa3b0c88", + "loss_nats": 5.571857452392578, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.571857452392578, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0023513585329055786, + 0.0023208872880786657, + 0.002778682392090559, + 0.002497476991266012, + 0.002628475893288851, + 0.002920303260907531, + 0.002932395786046982, + 0.0032228652853518724, + 0.003561305580660701, + 0.003354643704369664, + 0.0034162327647209167, + 0.0036349669098854065, + 0.0035210708156228065, + 0.0038089773152023554, + 0.003915763925760984, + 0.0037244565319269896, + 0.004162859171628952, + 0.004167158622294664, + 0.004440344404429197, + 0.0041700867004692554, + 0.004737259820103645, + 0.004552462603896856, + 0.004706811159849167, + 0.005186605267226696, + 0.005580419674515724, + 0.004905648995190859, + 0.005227713380008936, + 0.005349088925868273, + 0.005235649179667234, + 0.004925078712403774, + 0.005810525268316269, + 0.005564783234149218 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024914435198297724, + "first_to_last_ratio": 5.005386356076312, + "imbalance_abs_log_ratio": 1.61051460380912, + "last_quartile_mean": 4.977524895366514e-05, + "mean": 0.00012233095264946314, + "normalized": [ + 2.946204230178919, + 2.5883907308658176, + 2.285503889899197, + 2.0157125547168477, + 1.8168856694000293, + 1.687577213316311, + 1.555459627812154, + 1.3974018168018878, + 1.2756749767931916, + 1.1684180974434406, + 1.0925035606597346, + 1.0000212864934148, + 0.922022236502042, + 0.863306019148705, + 0.7954975128821132, + 0.747136294956448, + 0.6955650087701328, + 0.6567702950630189, + 0.6067855157189815, + 0.5734103460233209, + 0.549704279492861, + 0.527795032775898, + 0.5009080420334019, + 0.4762252632696638, + 0.45754883501456406, + 0.43797117166433, + 0.4282553488048722, + 0.41570642703193256, + 0.40058854659018955, + 0.39033614610916745, + 0.3703926654711839, + 0.3543213582962283 + ], + "peak_layer": 1, + "peak_normalized": 2.946204230178919, + "population_cv": 0.6883820030702144, + "rest_mean": 0.00013360270127628205, + "spike_contrast": 0.46004691130861425, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.146351006464101e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036041197017766535, + 0.0003166403039358556, + 0.00027958786813542247, + 0.0002465840370859951, + 0.00022226135479286313, + 0.0002064429281745106, + 0.00019028085807804018, + 0.0001709454954834655, + 0.00015605453518219292, + 0.00014293369895312935, + 0.00013364700134843588, + 0.00012233355664648116, + 0.0001127918585552834, + 0.00010560904775047675, + 9.731396858114749e-05, + 9.139789472101256e-05, + 8.508913015248254e-05, + 8.03433358669281e-05, + 7.42286501917988e-05, + 7.014583388809115e-05, + 6.724584818584844e-05, + 6.456566916313022e-05, + 6.127655797172338e-05, + 5.825709013151936e-05, + 5.597238487098366e-05, + 5.3577430662699044e-05, + 5.2388884796528146e-05, + 5.085376324132085e-05, + 4.900437852484174e-05, + 4.775019260705449e-05, + 4.5310487621463835e-05, + 4.334446930442937e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002986813742609229, + "first_to_last_ratio": 2.3520308544062645, + "imbalance_abs_log_ratio": 0.8552791483203458, + "last_quartile_mean": 0.0001269887143280357, + "mean": 0.00018719729814620223, + "normalized": [ + 2.1746021240914017, + 1.94336263487061, + 1.7490368812517914, + 1.5797937988337565, + 1.4558020735282082, + 1.372777068369229, + 1.290232936323773, + 1.1987380034727069, + 1.1282687150208124, + 1.0677045109732362, + 1.0238712087163733, + 0.9743741602296072, + 0.9332108464677616, + 0.9012687760530631, + 0.8680188022864366, + 0.8423083374767277, + 0.8168968442148058, + 0.7977443195284458, + 0.774889242886524, + 0.759928616461841, + 0.7477907292059667, + 0.7367409087088999, + 0.7235787757136496, + 0.7121132246295925, + 0.7033660136533255, + 0.6946945949961153, + 0.6884818791691059, + 0.6826464924874314, + 0.674695217464828, + 0.6690870522688999, + 0.6604539573176332, + 0.6535212533274407 + ], + "peak_layer": 1, + "peak_normalized": 2.1746021240914017, + "population_cv": 0.3960989175249597, + "rest_mean": 0.00019674027215741161, + "spike_contrast": 0.68956516628749, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013566523848567157, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004070796421729028, + 0.0003637922345660627, + 0.0003274149785283953, + 0.0002957331307698041, + 0.0002725222148001194, + 0.000256980158155784, + 0.0002415281196590513, + 0.00022440051543526351, + 0.00021120885503478348, + 0.00019987139967270195, + 0.00019166592392139137, + 0.0001824002101784572, + 0.00017469454905949533, + 0.00016871507978066802, + 0.00016249077452812344, + 0.00015767784498166293, + 0.00015292088210117072, + 0.0001493355812272057, + 0.00014505717263091356, + 0.0001422565837856382, + 0.00013998440408613533, + 0.0001379159075440839, + 0.00013545199180953205, + 0.0001333056716248393, + 0.00013166821736376733, + 0.000130044951220043, + 0.0001288819476030767, + 0.00012778957898262888, + 0.00012630112178158015, + 0.00012525128840934485, + 0.00012363519635982811, + 0.00012233741290401667 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.264873631298542, + "first_to_last_ratio": 2.3520309299879743, + "imbalance_abs_log_ratio": 0.8552791804550044, + "last_quartile_mean": 0.11261485889554024, + "mean": 0.1660084358882159, + "normalized": [ + 2.1746021175238424, + 1.943362568695452, + 1.7490370652499811, + 1.5797937888156908, + 1.4558020518807187, + 1.3727770906872316, + 1.2902329816871188, + 1.1987379486864336, + 1.128268732118104, + 1.067704514212167, + 1.0238712059346726, + 0.9743741284397964, + 0.9332107750795899, + 0.9012687970282692, + 0.8680188148399602, + 0.8423083473020997, + 0.8168968752199287, + 0.7977442908994873, + 0.7748892254753884, + 0.7599285915530729, + 0.7477907763310051, + 0.7367409150028685, + 0.723578786768912, + 0.7121132849551643, + 0.7033659848486875, + 0.694694622945607, + 0.6884818276985868, + 0.6826464794318051, + 0.6746951840389442, + 0.6690870220520635, + 0.6604539434158253, + 0.6535212611815249 + ], + "peak_layer": 1, + "peak_normalized": 2.1746021175238424, + "population_cv": 0.3960989241805914, + "rest_mean": 0.17447124007675383, + "spike_contrast": 0.6895651868880185, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12030929327011108, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.3610022962093353, + 0.3226145803928375, + 0.2903549075126648, + 0.2622590959072113, + 0.24167542159557343, + 0.22789257764816284, + 0.2141895592212677, + 0.19900061190128326, + 0.1873021274805069, + 0.17724795639514923, + 0.1699712574481964, + 0.1617543250322342, + 0.15492086112499237, + 0.1496182233095169, + 0.1440984457731247, + 0.13983029127120972, + 0.13561177253723145, + 0.13243228197097778, + 0.1286381483078003, + 0.1261545568704605, + 0.12413957715034485, + 0.12230520695447922, + 0.12012018263339996, + 0.11821681261062622, + 0.11676468700170517, + 0.11532516777515411, + 0.11429379135370255, + 0.1133250743150711, + 0.1120050922036171, + 0.11107409000396729, + 0.1096409261226654, + 0.10849004238843918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010120303977600997, + "first_to_last_ratio": 2.7021675066537045, + "imbalance_abs_log_ratio": 0.994054231196686, + "last_quartile_mean": 3.7452541164384456e-05, + "mean": 6.280763545873924e-05, + "normalized": [ + 1.9944816746100333, + 1.8472480968114133, + 1.7373223217437128, + 1.6248488809553971, + 1.5471371990089653, + 1.475823179261819, + 1.3807460832170038, + 1.2829322622274897, + 1.184046410564159, + 1.1043973349513228, + 1.0901910170788929, + 1.0456150919711364, + 0.9888694060739084, + 0.9664075020831666, + 0.9198699716804313, + 0.9041232034205934, + 0.8692950074065211, + 0.8446819175937094, + 0.7923092109368705, + 0.7644112750281646, + 0.7503465778061283, + 0.730017034806627, + 0.7090250315296005, + 0.6754099655367306, + 0.6572034579968898, + 0.6267819521510305, + 0.6185439744898635, + 0.6068927943478243, + 0.594580659914726, + 0.5858446061910916, + 0.5555293303136664, + 0.5250675682911103 + ], + "peak_layer": 1, + "peak_normalized": 1.9944816746100333, + "population_cv": 0.40483316550850257, + "rest_mean": 6.624576713807915e-05, + "spike_contrast": 0.6678422833882899, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4241724390303715e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012526867794804275, + 0.0001160212850663811, + 0.0001091171070584096, + 0.00010205291619058698, + 9.717202920001e-05, + 9.269296424463391e-05, + 8.67213966557756e-05, + 8.057794184423983e-05, + 7.43671553209424e-05, + 6.936458521522582e-05, + 6.847231998108327e-05, + 6.567261152667925e-05, + 6.210854917299002e-05, + 6.0697770095430315e-05, + 5.777485785074532e-05, + 5.6785840570228174e-05, + 5.4598363931290805e-05, + 5.305247395881452e-05, + 4.9763068091124296e-05, + 4.801086470251903e-05, + 4.712749432655983e-05, + 4.585064380080439e-05, + 4.453218571143225e-05, + 4.242090290063061e-05, + 4.1277395212091506e-05, + 3.936669236281887e-05, + 3.8849284464959055e-05, + 3.811750138993375e-05, + 3.734420533874072e-05, + 3.679551446111873e-05, + 3.4891483664978296e-05, + 3.297825242043473e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021292950441420544, + "first_to_last_ratio": 1.6819067487661699, + "imbalance_abs_log_ratio": 0.5199281193278776, + "last_quartile_mean": 0.0001266000654140953, + "mean": 0.0001612693722563563, + "normalized": [ + 1.5193752902631465, + 1.4431842082185322, + 1.3854135927323608, + 1.3303138916593422, + 1.2819858609596173, + 1.2449680699964556, + 1.2025089225516594, + 1.1549255897117912, + 1.1161361523436344, + 1.0820631347545688, + 1.0617857510438193, + 1.0341003696475617, + 1.0077971864601107, + 0.9864740856164548, + 0.9634556759892069, + 0.9447446572547388, + 0.9240188125766456, + 0.9078710595764491, + 0.88634376010722, + 0.8724890232241534, + 0.8607832896405347, + 0.849628883322604, + 0.8359853832855236, + 0.8234683594489096, + 0.8139294954165563, + 0.8040629029670215, + 0.7969416676465471, + 0.7902390259734583, + 0.7809695092222964, + 0.7743360767166025, + 0.7639958481267083, + 0.7557044635457686 + ], + "peak_layer": 1, + "peak_normalized": 1.5193752902631465, + "population_cv": 0.21463654613558777, + "rest_mean": 0.00016614451305940747, + "spike_contrast": 0.8122062500591205, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013494361191987992, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024502869928255677, + 0.00023274141130968928, + 0.00022342478041537106, + 0.0002145388862118125, + 0.00020674505503848195, + 0.00020077521912753582, + 0.00019392785907257348, + 0.00018625412485562265, + 0.00017999857664108276, + 0.00017450364248361439, + 0.00017123352154158056, + 0.00016676871746312827, + 0.0001625268196221441, + 0.00015908805653452873, + 0.0001553758920636028, + 0.0001523583778180182, + 0.0001490159338572994, + 0.000146411795867607, + 0.00014294010179582983, + 0.0001407057570759207, + 0.00013881798076909035, + 0.00013701911666430533, + 0.0001348188379779458, + 0.0001328002254012972, + 0.00013126189878676087, + 0.0001296707196161151, + 0.00012852228246629238, + 0.00012744135165121406, + 0.0001259464625036344, + 0.00012487669300753623, + 0.00012320913083385676, + 0.00012187198444735259 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016320452232321259, + "first_to_last_ratio": 1.2889611434881496, + "imbalance_abs_log_ratio": 0.25383657880769345, + "last_quartile_mean": 0.0001266171002498595, + "mean": 0.0001437355554116948, + "normalized": [ + 1.188749445631, + 1.1713423911172207, + 1.1549430883999143, + 1.1417765084687417, + 1.1273283201397843, + 1.1145504039766105, + 1.0995991496765185, + 1.0853099095412984, + 1.0687879028680622, + 1.0554743211679045, + 1.0473286818228558, + 1.0363809984280803, + 1.0249947395076546, + 1.015899258153915, + 1.005968438235508, + 0.9970397006238613, + 0.9860146691908593, + 0.977410712347053, + 0.964005406402611, + 0.9549550784847994, + 0.9474810711889571, + 0.940343993567483, + 0.9294782140508817, + 0.9176125795270506, + 0.909513915953909, + 0.8990879268536105, + 0.8931112722873075, + 0.8859642730588357, + 0.8770245001904262, + 0.8704731011044752, + 0.8603223859779169, + 0.851727642054894 + ], + "peak_layer": 1, + "peak_normalized": 1.188749445631, + "population_cv": 0.0978236443065259, + "rest_mean": 0.0001456284474072702, + "spike_contrast": 0.9168122095142401, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001335139386355877, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00017086556181311607, + 0.00016836354916449636, + 0.00016600638628005981, + 0.00016411388060078025, + 0.0001620371622266248, + 0.00016020052134990692, + 0.0001580514945089817, + 0.00015599762264173478, + 0.00015362282283604145, + 0.0001517091877758503, + 0.00015053836978040636, + 0.0001489647984271869, + 0.00014732818817719817, + 0.0001460208441130817, + 0.00014459343219641596, + 0.0001433100551366806, + 0.00014172536612022668, + 0.00014048867160454392, + 0.00013856185250915587, + 0.00013726099859923124, + 0.0001361867180094123, + 0.00013516086619347334, + 0.00013359906733967364, + 0.0001318935537710786, + 0.0001307294878643006, + 0.00012923090253025293, + 0.00012837184476666152, + 0.0001273445668630302, + 0.00012605960364453495, + 0.0001251179346581921, + 0.00012365891598165035, + 0.00012242354569025338 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0004412600719660986, + "first_to_last_ratio": 3.098863205201153, + "imbalance_abs_log_ratio": 1.131035336237652, + "last_quartile_mean": 0.00014239417578210123, + "mean": 0.0002548141455918085, + "normalized": [ + 2.1785577639630773, + 1.9956115105027457, + 1.9014578504944815, + 1.7829105267225334, + 1.6475854563109211, + 1.565091756282257, + 1.45799103716316, + 1.3243443696030393, + 1.2370613678820481, + 1.1609137324491365, + 1.1300355744023234, + 1.0677196003316773, + 1.0048909110245325, + 0.948494062464312, + 0.8853730946340664, + 0.8440976906413151, + 0.7906677717627395, + 0.747069444517614, + 0.6991826523616375, + 0.6712814178889162, + 0.6509454457824981, + 0.6314362826751919, + 0.6113540983289655, + 0.5954000197507868, + 0.5831757760496624, + 0.5733601746441271, + 0.5653129211221045, + 0.559900286080882, + 0.5532114024202266, + 0.5496528914136888, + 0.545014697375157, + 0.5408984129541747 + ], + "peak_layer": 1, + "peak_normalized": 2.1785577639630773, + "population_cv": 0.4822008017525282, + "rest_mean": 0.00027300682214209465, + "spike_contrast": 0.573515676244785, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015657369222026317, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005551273352466524, + 0.0005085100419819355, + 0.000484518357552588, + 0.00045431082253344357, + 0.00041982808033935726, + 0.0003988075186498463, + 0.0003715167404152453, + 0.0003374616790097207, + 0.000315220735501498, + 0.0002958172408398241, + 0.0002879490493796766, + 0.0002720700576901436, + 0.0002560604189056903, + 0.00024168970412574708, + 0.00022560558863915503, + 0.00021508803183678538, + 0.0002014733327087015, + 0.0001903638622025028, + 0.0001781616301741451, + 0.0001710520009510219, + 0.00016587010759394616, + 0.0001608988968655467, + 0.00015578167221974581, + 0.00015171634731814265, + 0.00014860143710393459, + 0.00014610028301831335, + 0.0001440497289877385, + 0.0001426705130143091, + 0.00014096609083935618, + 0.0001400593318976462, + 0.0001388774544466287, + 0.00013782856694888324 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027412420604377985, + 0.002683233004063368, + 0.002711885841563344, + 0.0027249811682850122, + 0.0027012599166482687, + 0.002700963756069541, + 0.0026683746837079525, + 0.002738022478297353, + 0.0027370047755539417, + 0.002654933836311102, + 0.002663377905264497, + 0.002684312639757991, + 0.0027356327045708895, + 0.002652956871315837, + 0.002679797587916255, + 0.002698024269193411, + 0.0026484597474336624, + 0.0026680538430809975, + 0.002654345240443945, + 0.002591830212622881, + 0.002620960585772991, + 0.002651234855875373, + 0.0026270023081451654, + 0.002615687670186162, + 0.00258000404573977, + 0.002626793459057808, + 0.002573782578110695, + 0.002592637902125716, + 0.002581534907221794, + 0.002551236655563116, + 0.002563327783718705, + 0.002589041367173195 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024784023116808385, + "first_to_last_ratio": 4.996872150813687, + "imbalance_abs_log_ratio": 1.6088121468463858, + "last_quartile_mean": 4.959907391821616e-05, + "mean": 0.00012175017718618619, + "normalized": [ + 2.94187592163941, + 2.5796687009574604, + 2.293010133276413, + 2.014581184592976, + 1.8111575385173901, + 1.6842185280489494, + 1.5559062012549134, + 1.4047478191310576, + 1.2749623745204015, + 1.1699150302730006, + 1.0919472369387608, + 1.0017565099771182, + 0.923449750141626, + 0.8652102016053241, + 0.794546741948017, + 0.743905553038253, + 0.6942834150416333, + 0.6562793060663384, + 0.6082995898528493, + 0.5744438165059796, + 0.5512557468449423, + 0.5265581646918154, + 0.5020472770644135, + 0.47690127545745264, + 0.4572061982209032, + 0.43893553328289897, + 0.42961039869683454, + 0.41603121061550513, + 0.3999104909112459, + 0.3911954601706125, + 0.3717234123738856, + 0.35445927834161867 + ], + "peak_layer": 1, + "peak_normalized": 2.94187592163941, + "population_cv": 0.6874421460933116, + "rest_mean": 0.0001329603533290706, + "spike_contrast": 0.46040210094136547, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.121522601461038e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000358173914719373, + 0.0003140751214232296, + 0.00027917439001612365, + 0.0002452756161801517, + 0.00022050875122658908, + 0.0002050539042102173, + 0.00018943185568787158, + 0.0001710282958811149, + 0.00015522689500357956, + 0.00014243736222852021, + 0.00013294476957526058, + 0.00012196403258712962, + 0.00011243017070228234, + 0.0001053394953487441, + 9.673620661487803e-05, + 9.057063289219514e-05, + 8.45291287987493e-05, + 7.990212179720402e-05, + 7.406058284686878e-05, + 6.993863644311205e-05, + 6.711548485327512e-05, + 6.410854985006154e-05, + 6.112434493843466e-05, + 5.806281478726305e-05, + 5.566493564401753e-05, + 5.3440478950506076e-05, + 5.23051421623677e-05, + 5.0651873607421294e-05, + 4.8689173127058893e-05, + 4.762811659020372e-05, + 4.525739132077433e-05, + 4.315547994337976e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002960815872938838, + "first_to_last_ratio": 2.3434901925615312, + "imbalance_abs_log_ratio": 0.8516413534183918, + "last_quartile_mean": 0.00012634214908757713, + "mean": 0.00018583460405352525, + "normalized": [ + 2.1691817718266027, + 1.9356770461635995, + 1.7512361881653136, + 1.5771501673188, + 1.4517248962140685, + 1.3691574500653627, + 1.2898282592750647, + 1.2020702391870204, + 1.1274711846152252, + 1.06796916249687, + 1.023464646242978, + 0.9751293351351744, + 0.9332415616222968, + 0.9026335726010739, + 0.8668647658197504, + 0.8407791679025106, + 0.8171800119835181, + 0.7975248027052753, + 0.7770671945621679, + 0.7612080123915693, + 0.7492445405290613, + 0.7365475788483447, + 0.7249228575947372, + 0.7138182430319103, + 0.7048361054118131, + 0.6962681268232633, + 0.6911654119184109, + 0.6833348384348117, + 0.6760742523950837, + 0.6702828385523486, + 0.6624247014500871, + 0.654521068715886 + ], + "peak_layer": 1, + "peak_normalized": 2.1691817718266027, + "population_cv": 0.3943796352106793, + "rest_mean": 0.00019526833030222742, + "spike_contrast": 0.6908057343541223, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013489248231053353, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004031090356875211, + 0.0003597157774493098, + 0.0003254402836319059, + 0.0002930890768766403, + 0.00026978072128258646, + 0.00025443683261983097, + 0.00023969472385942936, + 0.00022338624694384634, + 0.00020952316117472947, + 0.0001984656264539808, + 0.0001901951472973451, + 0.00018121277389582247, + 0.0001734285760903731, + 0.00016774055256973952, + 0.0001610934705240652, + 0.0001562458637636155, + 0.00015186032396741211, + 0.00014820770593360066, + 0.00014440597442444414, + 0.00014145878958515823, + 0.00013923556252848357, + 0.0001368760276818648, + 0.00013471575221046805, + 0.00013265213056001812, + 0.00013098293857183307, + 0.00012939071166329086, + 0.00012844245065934956, + 0.00012698725913651288, + 0.00012563799100462347, + 0.0001245617459062487, + 0.00012310143210925162, + 0.00012163266364950687 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.26256811060011387, + "first_to_last_ratio": 2.3434901455089974, + "imbalance_abs_log_ratio": 0.8516413333404181, + "last_quartile_mean": 0.11204148270189762, + "mean": 0.16479998547583818, + "normalized": [ + 2.1691817568110046, + 1.935676923537455, + 1.7512361570514487, + 1.5771502517930829, + 1.4517248703109182, + 1.369157420355369, + 1.2898282606548555, + 1.2020702692325518, + 1.1274712340116186, + 1.0679692103147351, + 1.0234646460792782, + 0.975129361875517, + 0.9332415430463463, + 0.9026335787242323, + 0.8668647305888977, + 0.8407791967519535, + 0.8171800231493583, + 0.7975247749486774, + 0.7770672329681078, + 0.7612079835803747, + 0.7492445111174014, + 0.7365475542475244, + 0.7249228843083022, + 0.7138182179223472, + 0.7048361082067401, + 0.6962681206044983, + 0.6911654218524023, + 0.6833348067031066, + 0.676074287607254, + 0.6702828621875807, + 0.6624247142471237, + 0.6545210852099367 + ], + "peak_layer": 1, + "peak_normalized": 2.1691817568110046, + "population_cv": 0.3943796257753467, + "rest_mean": 0.17316590856622766, + "spike_contrast": 0.6908057237027376, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11962400078773498, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.35748112201690674, + 0.3189995288848877, + 0.28860369324684143, + 0.2599143385887146, + 0.2392442375421524, + 0.22563712298870087, + 0.2125636786222458, + 0.19810116291046143, + 0.1858072429895401, + 0.17600131034851074, + 0.16866695880889893, + 0.16070130467414856, + 0.1537981927394867, + 0.14875400066375732, + 0.1428592950105667, + 0.13856039941310883, + 0.13467125594615936, + 0.13143207132816315, + 0.12806066870689392, + 0.12544706463813782, + 0.12347548454999924, + 0.12138302624225616, + 0.11946728080511093, + 0.11763723194599152, + 0.11615698039531708, + 0.11474497616291046, + 0.11390405148267746, + 0.11261356621980667, + 0.11141703277826309, + 0.11046260595321655, + 0.10916758328676224, + 0.10786506533622742 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001010398200378404, + "first_to_last_ratio": 2.701478580108855, + "imbalance_abs_log_ratio": 0.9937992453824708, + "last_quartile_mean": 3.740167358046165e-05, + "mean": 6.270599271829269e-05, + "normalized": [ + 1.9907220064146427, + 1.8444741506775382, + 1.742884685905627, + 1.6239253294262157, + 1.5436270879110234, + 1.4787029723792133, + 1.3841644782908762, + 1.2821103017907618, + 1.1824374683113494, + 1.1042100658007115, + 1.0908679078034196, + 1.0498354286471192, + 0.9920033594422748, + 0.9660018389944104, + 0.9202838356691797, + 0.8989347700163286, + 0.8686518679586676, + 0.8410583182307008, + 0.7938538755561205, + 0.7653997098760733, + 0.7511903220519177, + 0.7265360595007238, + 0.7106247592487483, + 0.6758121071782397, + 0.6556340454607258, + 0.6278169583683687, + 0.6205082786340235, + 0.6073860029650574, + 0.5911469097961847, + 0.5867782134359579, + 0.5577710930141501, + 0.5246457912436486 + ], + "peak_layer": 1, + "peak_normalized": 1.9907220064146427, + "population_cv": 0.40468494452628273, + "rest_mean": 6.614368087159366e-05, + "spike_contrast": 0.6673725457789733, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4142476690467446e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001248301996383816, + 0.00011565958266146481, + 0.0001092893144232221, + 0.00010182984988205135, + 9.679466893430799e-05, + 9.272353781852871e-05, + 8.679540769662708e-05, + 8.039599924813956e-05, + 7.414591527776793e-05, + 6.92405883455649e-05, + 6.84039550833404e-05, + 6.583097274415195e-05, + 6.220455543370917e-05, + 6.0574104281840846e-05, + 5.770731149823405e-05, + 5.636859714286402e-05, + 5.446967770694755e-05, + 5.273939677863382e-05, + 4.9779395340010524e-05, + 4.799514863407239e-05, + 4.710413486463949e-05, + 4.5558164856629446e-05, + 4.456043097889051e-05, + 4.237746907165274e-05, + 4.111218368052505e-05, + 3.936788561986759e-05, + 3.890958760166541e-05, + 3.808674227911979e-05, + 3.7068453821120784e-05, + 3.679451037896797e-05, + 3.497559009701945e-05, + 3.2898435165407136e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021103879953443538, + "first_to_last_ratio": 1.6754332028061545, + "imbalance_abs_log_ratio": 0.5160717603794217, + "last_quartile_mean": 0.00012596073611348402, + "mean": 0.0001601102544555033, + "normalized": [ + 1.5150372860566723, + 1.439510593902249, + 1.3841055212355642, + 1.3272119947749268, + 1.2794362361966427, + 1.2428436489609036, + 1.2018556832108258, + 1.1546727794959446, + 1.1147996317367452, + 1.0814138049096294, + 1.0614905005088058, + 1.0346901524019803, + 1.0074919017055521, + 0.9875011592687566, + 0.9623166883068708, + 0.9431047552100263, + 0.9243559814545742, + 0.9073018837351503, + 0.8887802361951885, + 0.87382889642758, + 0.8622859040817829, + 0.849391060863955, + 0.8374811583413132, + 0.8253926631762389, + 0.8155827011133436, + 0.8058320881575772, + 0.7999867009830005, + 0.7910419816457831, + 0.7825317922950641, + 0.7756782882808636, + 0.7662211489344684, + 0.7568251764320209 + ], + "peak_layer": 1, + "peak_normalized": 1.5150372860566723, + "population_cv": 0.21304026877727222, + "rest_mean": 0.00016491277050658095, + "spike_contrast": 0.8136220583009953, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013417666777968407, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024257300538010895, + 0.00023048040748108178, + 0.0002216094871982932, + 0.00021250025019980967, + 0.0002048508613370359, + 0.00019899201288353652, + 0.0001924294192576781, + 0.00018487495253793895, + 0.00017849085270427167, + 0.00017314543947577477, + 0.00016995551413856447, + 0.00016566450358368456, + 0.00016130978474393487, + 0.00015810906188562512, + 0.00015407676983159035, + 0.00015100074233487248, + 0.00014799887139815837, + 0.0001452683354727924, + 0.00014230282977223396, + 0.00013990896695759147, + 0.00013806081551592797, + 0.00013599621888715774, + 0.00013408932136371732, + 0.00013215382932685316, + 0.00013058315380476415, + 0.00012902198068331927, + 0.00012808607425540686, + 0.0001266539329662919, + 0.00012529136438388377, + 0.00012419404811225832, + 0.00012267986312508583, + 0.00012117547157686204 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016164849512279034, + "first_to_last_ratio": 1.282986844747637, + "imbalance_abs_log_ratio": 0.24919083207168544, + "last_quartile_mean": 0.00012599388355738483, + "mean": 0.00014267631695474847, + "normalized": [ + 1.1849632230478688, + 1.1695181917643926, + 1.1537907432295644, + 1.136648454640294, + 1.1262413432306055, + 1.112705815659591, + 1.0977204646042988, + 1.082200264843305, + 1.06813080649915, + 1.0535985061590996, + 1.046252801833764, + 1.0365714691025223, + 1.024670983408149, + 1.0169605538784676, + 1.0046472025165352, + 0.995758555548206, + 0.9858673222379531, + 0.9760103584099877, + 0.9653106304982666, + 0.9567951741123486, + 0.9498417329681972, + 0.9401737612553472, + 0.9304537733640251, + 0.9205681496416466, + 0.911480005330001, + 0.9022600866995591, + 0.8969925816929462, + 0.8873573495747816, + 0.8789659160771423, + 0.8714788501373707, + 0.8627683840784355, + 0.8532965439561776 + ], + "peak_layer": 1, + "peak_normalized": 1.1849632230478688, + "population_cv": 0.09608600673281108, + "rest_mean": 0.00014451252193086676, + "spike_contrast": 0.918680321330362, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013276081008370966, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016906618839129806, + 0.00016686254821252078, + 0.00016461861378047615, + 0.00016217281518038362, + 0.00016068796685431153, + 0.00015875676763243973, + 0.0001566187129355967, + 0.00015440434799529612, + 0.00015239696949720383, + 0.0001503235544078052, + 0.00014927549636922777, + 0.00014789419947192073, + 0.00014619628200307488, + 0.0001450961863156408, + 0.00014333936269395053, + 0.00014207116328179836, + 0.00014065991854295135, + 0.00013925356324762106, + 0.00013772696547675878, + 0.0001365120115224272, + 0.00013551992014981806, + 0.00013414052955340594, + 0.00013275371748022735, + 0.0001313432730967179, + 0.00013004661013837904, + 0.00012873114610556513, + 0.0001279795978916809, + 0.00012660487846005708, + 0.0001254076196346432, + 0.00012433939264155924, + 0.00012309661542531103, + 0.00012174520816188306 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00043634913890855387, + "first_to_last_ratio": 3.0823770886184128, + "imbalance_abs_log_ratio": 1.125701081333854, + "last_quartile_mean": 0.00014156254292174708, + "mean": 0.0002526328457861382, + "normalized": [ + 2.161262991989335, + 1.9867412804887739, + 1.8904526133838582, + 1.7788259407438674, + 1.6404609794852394, + 1.5723911781894129, + 1.459249713339863, + 1.3282687448916313, + 1.2383315143134188, + 1.1700416770361293, + 1.131363959578303, + 1.0685850444189446, + 1.0069989316428685, + 0.9451411723514572, + 0.8866050347634621, + 0.8407661253267313, + 0.7895843736134068, + 0.7479926194272515, + 0.7047300318080335, + 0.6729704299307414, + 0.6547401608613147, + 0.6315820650248338, + 0.6127131155987415, + 0.5974089218754778, + 0.5839699651029305, + 0.5735248801032955, + 0.5690230706448541, + 0.5599717580649735, + 0.5552403508520525, + 0.5513694456095706, + 0.5470291494736481, + 0.5426627600655782 + ], + "peak_layer": 1, + "peak_normalized": 2.161262991989335, + "population_cv": 0.4790543297146424, + "rest_mean": 0.0002705939723118381, + "spike_contrast": 0.5751893185853856, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015564276254735886, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005460060201585293, + 0.0005019161035306752, + 0.0004775904235430062, + 0.00044938985956832767, + 0.0004144343256484717, + 0.0003972376580350101, + 0.000368654407793656, + 0.00033556431299075484, + 0.00031284321448765695, + 0.000295590958558023, + 0.0002858196967281401, + 0.0002699596807360649, + 0.0002544010058045387, + 0.00023877370404079556, + 0.0002239855530206114, + 0.00021240513888187706, + 0.00019947494729422033, + 0.0001889675040729344, + 0.0001780379534466192, + 0.00017001443484332412, + 0.00016540887008886784, + 0.00015955837443470955, + 0.00015479145804420114, + 0.00015092511603143066, + 0.00014752999413758516, + 0.00014489122258964926, + 0.00014375391765497625, + 0.00014146725879982114, + 0.00014027194993104786, + 0.00013929403212387115, + 0.00013819753075949848, + 0.00013709443737752736 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0023513585329055786, + 0.004408807028084993, + 0.005783648230135441, + 0.006751363631337881, + 0.002628475893288851, + 0.004723479505628347, + 0.006024772301316261, + 0.007445912808179855, + 0.003561305580660701, + 0.005543973762542009, + 0.006855612155050039, + 0.007876631803810596, + 0.0035210708156228065, + 0.0058924350887537, + 0.007390674669295549, + 0.008790239691734314, + 0.004162859171628952, + 0.006355749908834696, + 0.007784162648022175, + 0.00937180407345295, + 0.004737259820103645, + 0.006999304983764887, + 0.008728929795324802, + 0.010217384435236454, + 0.005580419674515724, + 0.008008211851119995, + 0.010209310799837112, + 0.012040933594107628, + 0.005235649179667234, + 0.007745372597128153, + 0.010180563665926456, + 0.011906592175364494 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024914481400628574, + "first_to_last_ratio": 5.005367240493075, + "imbalance_abs_log_ratio": 1.6105107847992817, + "last_quartile_mean": 4.977553135177004e-05, + "mean": 0.0001223312358433759, + "normalized": [ + 2.9462138255793877, + 2.588435651553335, + 2.2854731426364707, + 2.0157214492628985, + 1.8168708763555168, + 1.6875649797632875, + 1.5554268829788211, + 1.3974214212146046, + 1.275680469442624, + 1.1684152736282016, + 1.0925056707858531, + 1.0000365768115411, + 0.9220243844233116, + 0.8633153808206192, + 0.7954940059538663, + 0.747131175135411, + 0.6955592946052511, + 0.6567700236831091, + 0.6067846463225012, + 0.5734077695641915, + 0.5496986650820618, + 0.5277889932643854, + 0.50090777460584, + 0.47622023530363405, + 0.457550957846772, + 0.43796965221258716, + 0.42825768814458814, + 0.4157111150450942, + 0.4005890169587395, + 0.39033503431894295, + 0.37039430607646523, + 0.3543236606200867 + ], + "peak_layer": 1, + "peak_normalized": 2.9462138255793877, + "population_cv": 0.6883836085455175, + "rest_mean": 0.00013360307773971862, + "spike_contrast": 0.46004396487681354, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.14632896031253e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036041397834196687, + 0.00031664653215557337, + 0.0002795847540255636, + 0.0002465856960043311, + 0.0002222600596724078, + 0.0002064419095404446, + 0.00019027729285880923, + 0.00017094828945118934, + 0.00015605556836817414, + 0.00014293368440121412, + 0.00013364756887312979, + 0.00012233571032993495, + 0.00011279238242423162, + 0.00010561043745838106, + 9.73137648543343e-05, + 9.139747999142855e-05, + 8.508862811140716e-05, + 8.0343488662038e-05, + 7.422871567541733e-05, + 7.014568109298125e-05, + 6.724531704094261e-05, + 6.456507981056347e-05, + 6.127666711108759e-05, + 5.8256609918316826e-05, + 5.597277413471602e-05, + 5.357736881705932e-05, + 5.238929225015454e-05, + 5.0854454457294196e-05, + 4.9004549509845674e-05, + 4.775016714120284e-05, + 4.531079321168363e-05, + 4.334485129220411e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00029868123783671763, + "first_to_last_ratio": 2.352020548935129, + "imbalance_abs_log_ratio": 0.8552747667903616, + "last_quartile_mean": 0.00012698921273113228, + "mean": 0.0001871975605354237, + "normalized": [ + 2.1745956556501205, + 1.943389916860048, + 1.7490167059616495, + 1.579801534641211, + 1.4557872843351107, + 1.3727714128797337, + 1.290211227528517, + 1.198748061313539, + 1.1282731192017241, + 1.0677025479901412, + 1.0238711728275764, + 0.9743826669000581, + 0.9332092274716193, + 0.901273964826617, + 0.8680163418407979, + 0.8423074677807364, + 0.8168973316425937, + 0.7977426572048737, + 0.7748904888155362, + 0.7599280177062464, + 0.7477886704872306, + 0.7367391764207782, + 0.7235805599766263, + 0.7121146362854748, + 0.7033711688788506, + 0.6946931548498627, + 0.6884857337520279, + 0.6826539310873735, + 0.6746941940291191, + 0.6690858034871798, + 0.660454275348722, + 0.6535218920183048 + ], + "peak_layer": 1, + "peak_normalized": 2.1745956556501205, + "population_cv": 0.39609817473730136, + "rest_mean": 0.00019674051630621156, + "spike_contrast": 0.6895661448911525, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013566559937316924, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004070790018886328, + 0.00036379785160534084, + 0.00032741166069172323, + 0.00029573499341495335, + 0.00027251982828602195, + 0.00025697945966385305, + 0.00024152439436875284, + 0.000224402712774463, + 0.00021120997553225607, + 0.00019987131236121058, + 0.00019166618585586548, + 0.0001824020582716912, + 0.00017469449085183442, + 0.00016871628758963197, + 0.00016249054169747978, + 0.00015767790318932384, + 0.00015292118769139051, + 0.0001493354793637991, + 0.0001450576091883704, + 0.00014225667109712958, + 0.00013998421491123736, + 0.00013791577657684684, + 0.00013545251567848027, + 0.00013330612273421139, + 0.00013166936696507037, + 0.00013004486390855163, + 0.00012888284982182086, + 0.00012779115058947355, + 0.00012630110722966492, + 0.00012525123020168394, + 0.00012363542919047177, + 0.00012233770394232124 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.2648735139518976, + "first_to_last_ratio": 2.3520205903150884, + "imbalance_abs_log_ratio": 0.855274784383728, + "last_quartile_mean": 0.1126153040677309, + "mean": 0.1660086705815047, + "normalized": [ + 2.174595632268317, + 1.943389981095013, + 1.7490168198184282, + 1.5798016086666014, + 1.4557873374041121, + 1.3727712901980273, + 1.2902112306135503, + 1.1987480127171795, + 1.1282731510481394, + 1.0677025559476454, + 1.0238711946273023, + 0.9743826246747078, + 0.9332091864770874, + 0.9012739856841682, + 0.8680163310272434, + 0.8423074257801569, + 0.8168973360433722, + 0.7977426245274992, + 0.7748904637775177, + 0.7599280557788093, + 0.7477886868929706, + 0.7367391104705062, + 0.7235805912981054, + 0.7121146568868953, + 0.7033711391232357, + 0.6946931471398464, + 0.6884857463566807, + 0.682653951912613, + 0.6746941853139277, + 0.6690857619715078, + 0.6604542663637077, + 0.6535219080951254 + ], + "peak_layer": 1, + "peak_normalized": 2.174595632268317, + "population_cv": 0.39609818748801673, + "rest_mean": 0.1744714594549603, + "spike_contrast": 0.6895661390160055, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12030961066484451, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.36100172996520996, + 0.32261958718299866, + 0.2903519570827484, + 0.262260764837265, + 0.2416733205318451, + 0.2278919368982315, + 0.21418625116348267, + 0.19900256395339966, + 0.18730312585830688, + 0.17724788188934326, + 0.1699714958667755, + 0.16175596415996552, + 0.1549208164215088, + 0.14961929619312286, + 0.14409823715686798, + 0.1398303359746933, + 0.13561204075813293, + 0.13243219256401062, + 0.12863853573799133, + 0.12615464627742767, + 0.12413940578699112, + 0.12230508029460907, + 0.12012065201997757, + 0.11821720749139786, + 0.11676570773124695, + 0.11532508581876755, + 0.11429460346698761, + 0.11332647502422333, + 0.1120050847530365, + 0.1110740378499031, + 0.10964113473892212, + 0.10849030315876007 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010120356273546349, + "first_to_last_ratio": 2.7021644745482813, + "imbalance_abs_log_ratio": 0.9940531090948486, + "last_quartile_mean": 3.7452776723512216e-05, + "mean": 6.280785908074904e-05, + "normalized": [ + 1.9944873163456722, + 1.8472744197374709, + 1.7373237819064158, + 1.6248552595134786, + 1.5471235814302835, + 1.4758134067763151, + 1.3807453375978929, + 1.2829373095670376, + 1.18406084586495, + 1.104394561286083, + 1.0901943179184503, + 1.04561090576924, + 0.9888677388041794, + 0.9664019181449638, + 0.9198712145053942, + 0.9041143079769479, + 0.8692998477179782, + 0.8446721332610243, + 0.7923141515826202, + 0.7644051939108963, + 0.7503405467650113, + 0.7300062685920449, + 0.7090277780404898, + 0.6754004942706008, + 0.6572075474610624, + 0.6267776932652476, + 0.618543683654507, + 0.6069002486680706, + 0.5945805123210514, + 0.585842809953234, + 0.5555319861870183, + 0.5250728812043677 + ], + "peak_layer": 1, + "peak_normalized": 1.9944873163456722, + "population_cv": 0.404835319326031, + "rest_mean": 6.624604820562699e-05, + "spike_contrast": 0.6678381428743124, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.424163780640811e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001252694783033803, + 0.00011602335143834352, + 0.00010911758727161214, + 0.00010205368016613647, + 9.717151988297701e-05, + 9.269268048228696e-05, + 8.672165859024972e-05, + 8.057854574872181e-05, + 7.436832675011829e-05, + 6.936465797480196e-05, + 6.847277109045535e-05, + 6.567258242284879e-05, + 6.210866558831185e-05, + 6.069763549021445e-05, + 5.7775141613092273e-05, + 5.678548404830508e-05, + 5.459886233438738e-05, + 5.305204831529409e-05, + 4.9763555580284446e-05, + 4.801065369974822e-05, + 4.7127283323789015e-05, + 4.585013084579259e-05, + 4.4532516767503694e-05, + 4.242045906721614e-05, + 4.127779902773909e-05, + 3.936656503356062e-05, + 3.884940451825969e-05, + 3.811810529441573e-05, + 3.734432903002016e-05, + 3.679553265101276e-05, + 3.489177470328286e-05, + 3.297870352980681e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0002129299391526729, + "first_to_last_ratio": 1.6819035009332732, + "imbalance_abs_log_ratio": 0.5199261882838202, + "last_quartile_mean": 0.00012660056836466538, + "mean": 0.0001612697526525153, + "normalized": [ + 1.5193801883639617, + 1.4431946098051458, + 1.385412670946279, + 1.330312377973009, + 1.2819779644638467, + 1.2449644115522398, + 1.2025034693572292, + 1.15492638462487, + 1.1161420015875916, + 1.082061394531008, + 1.0617850512168836, + 1.0341029835254725, + 1.0077924632434987, + 0.9864749169321705, + 0.9634524108641764, + 0.9447429702319522, + 0.9240217763435827, + 0.9078672036949645, + 0.8863448276060427, + 0.8724873261680857, + 0.860780266694715, + 0.8496270597212868, + 0.835986840267667, + 0.8234693045553775, + 0.8139348844586144, + 0.8040606454444867, + 0.7969452920911189, + 0.7902469974302787, + 0.780967396404848, + 0.7743340697791917, + 0.7639952190801171, + 0.7557046210402876 + ], + "peak_layer": 1, + "peak_normalized": 1.5193801883639617, + "population_cv": 0.21463676643173402, + "rest_mean": 0.00016614488736700473, + "spike_contrast": 0.8122069076744362, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001349440251942724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024503006716258824, + 0.00023274363775271922, + 0.000223425158765167, + 0.0002145391481462866, + 0.00020674426923505962, + 0.000200775102712214, + 0.00019392743706703186, + 0.00018625469238031656, + 0.00017999994452111423, + 0.00017450377345085144, + 0.00017123381257988513, + 0.00016676953237038106, + 0.00016252644127234817, + 0.00015908856585156173, + 0.0001553757319925353, + 0.00015235846512950957, + 0.0001490167633164674, + 0.00014641151938121766, + 0.00014294061111286283, + 0.00014070581528358161, + 0.00013881782069802284, + 0.00013701914576813579, + 0.00013481939095072448, + 0.00013280069106258452, + 0.00013126307749189436, + 0.00012967066140845418, + 0.0001285231701331213, + 0.00012744293780997396, + 0.0001259464188478887, + 0.00012487666390370578, + 0.00012320932000875473, + 0.00012187229731353 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016320142822223715, + "first_to_last_ratio": 1.2889451135618213, + "imbalance_abs_log_ratio": 0.25382414241573026, + "last_quartile_mean": 0.0001266162744286703, + "mean": 0.00014373415069712792, + "normalized": [ + 1.1884743462891088, + 1.1713432082551107, + 1.1549462763142666, + 1.141829986160868, + 1.1273728485863819, + 1.1144091299418613, + 1.0997826147068885, + 1.0853573683237072, + 1.068690221812727, + 1.0554384700276902, + 1.0473609881004726, + 1.0364070219594264, + 1.025017513262844, + 1.015761575885501, + 1.006046000385858, + 0.9971413723222552, + 0.9860728003762654, + 0.9774913363845525, + 0.9641287247440273, + 0.9550391277708558, + 0.9473133601248869, + 0.9403037775179377, + 0.9293980025104189, + 0.9176260020019316, + 0.9094819029195437, + 0.8991813518536786, + 0.8931890476284186, + 0.8860794380468366, + 0.8769637206693581, + 0.8703930215998912, + 0.8602546600212342, + 0.8517047834951949 + ], + "peak_layer": 1, + "peak_normalized": 1.1884743462891088, + "population_cv": 0.09781347741247146, + "rest_mean": 0.00014562865706264146, + "spike_contrast": 0.9167413819240867, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013350381632335485, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00017082435078918934, + 0.00016836202121339738, + 0.00016600522212684155, + 0.0001641199633013457, + 0.00016204197891056538, + 0.00016017864982131869, + 0.0001580763200763613, + 0.0001560029195388779, + 0.00015360728139057755, + 0.00015170255210250616, + 0.00015054154209792614, + 0.00014896708307787776, + 0.00014733002171851695, + 0.00014599962742067873, + 0.00014460316742770374, + 0.00014332326827570796, + 0.00014173233648762107, + 0.00014049888704903424, + 0.00013857822341378778, + 0.00013727173791266978, + 0.0001361612812615931, + 0.0001351537648588419, + 0.00013358623255044222, + 0.00013189419405534863, + 0.00013072360889054835, + 0.00012924306793138385, + 0.00012838176917284727, + 0.00012735987547785044, + 0.00012604963558260351, + 0.00012510520173236728, + 0.00012364797294139862, + 0.00012241906370036304 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00044128536683274433, + "first_to_last_ratio": 3.0989416798064484, + "imbalance_abs_log_ratio": 1.1310606595922121, + "last_quartile_mean": 0.0001423987323505571, + "mean": 0.0002548205006860371, + "normalized": [ + 2.1788974669107217, + 1.9948922239742408, + 1.9013861017158324, + 1.7832746019711707, + 1.6474456862757656, + 1.56528080717771, + 1.458232784418858, + 1.3245892215128081, + 1.2376166576486232, + 1.160533346247024, + 1.1299807802432356, + 1.0675072615016112, + 1.0048234765069712, + 0.9484981041710848, + 0.8855756138701967, + 0.8439855543529297, + 0.7905268728467926, + 0.7470509271970999, + 0.6989608308202429, + 0.6713105901225432, + 0.6509228727076454, + 0.631461080612247, + 0.611226351598172, + 0.5954626643405772, + 0.5833123929329473, + 0.5733046444152264, + 0.5652547933675338, + 0.559916360487464, + 0.5532517426009346, + 0.5496220513891801, + 0.5450027610740823, + 0.5408933749885276 + ], + "peak_layer": 1, + "peak_normalized": 2.1788974669107217, + "population_cv": 0.4822174837001359, + "rest_mean": 0.0002730129350244102, + "spike_contrast": 0.5735309033794403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015658135525882245, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005552277434617281, + 0.0005083394353277981, + 0.0004845121584367007, + 0.0004544149269349873, + 0.00041980293462984264, + 0.0003988656389992684, + 0.00037158760824240744, + 0.0003375324886292219, + 0.00031537009635940194, + 0.0002957276883535087, + 0.00028794226818718016, + 0.00027202273486182094, + 0.00025604962138459086, + 0.00024169676180463284, + 0.0002256628213217482, + 0.00021506482153199613, + 0.0002014424535445869, + 0.00019036389130633324, + 0.00017810954886954278, + 0.00017106370069086552, + 0.0001658684923313558, + 0.00016090922872535884, + 0.00015575300494674593, + 0.00015173609426710755, + 0.00014863995602354407, + 0.00014608977653551847, + 0.00014403850946109742, + 0.00014267816732171923, + 0.00014097988605499268, + 0.00014005496632307768, + 0.0001388778764521703, + 0.00013783072063233703 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0036361459642648697, + 0.005152894649654627, + 0.006386642809957266, + 0.007284422870725393, + 0.0037505170330405235, + 0.0054016136564314365, + 0.006585344672203064, + 0.007922882214188576, + 0.0044892593286931515, + 0.006135277915745974, + 0.007316684816032648, + 0.0082859443500638, + 0.0044554853811860085, + 0.006464940495789051, + 0.007893112488090992, + 0.009196000173687935, + 0.004961193539202213, + 0.006907128728926182, + 0.008217990398406982, + 0.009676996618509293, + 0.005381465889513493, + 0.007502692751586437, + 0.0091153085231781, + 0.010543142445385456, + 0.006136673036962748, + 0.008469858206808567, + 0.010564134456217289, + 0.012301022186875343, + 0.005773921962827444, + 0.008156762458384037, + 0.010457510128617287, + 0.012218860909342766 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024784039669611957, + "first_to_last_ratio": 4.996847129630424, + "imbalance_abs_log_ratio": 1.6088071394647374, + "last_quartile_mean": 4.959935540682636e-05, + "mean": 0.0001217503483985638, + "normalized": [ + 2.9419062071052267, + 2.5796112881191693, + 2.2930860326581057, + 2.0145554032379356, + 1.8111535572920587, + 1.6841756414503508, + 1.5558994713918, + 1.4047664015764878, + 1.2749699043555138, + 1.1699192416770665, + 1.0919281315608147, + 1.001760718810264, + 0.9234449853785138, + 0.8652188455092179, + 0.7945394094365259, + 0.7439156225148, + 0.6942882953063699, + 0.6562804150522287, + 0.6082998698912725, + 0.574440737760942, + 0.551250788347098, + 0.5265559301830911, + 0.5020487822241618, + 0.47689842352430634, + 0.4572026867293029, + 0.43893712719402944, + 0.4296155316374904, + 0.41603615348749273, + 0.3999129464790325, + 0.391194073390914, + 0.3717285968379238, + 0.3544587798804922 + ], + "peak_layer": 1, + "peak_normalized": 2.9419062071052267, + "population_cv": 0.6874423943830519, + "rest_mean": 0.00013296059464833147, + "spike_contrast": 0.4603997057303065, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.121501864981837e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00035817810567095876, + 0.0003140685730613768, + 0.00027918402338400483, + 0.00024527282221242785, + 0.00022050857660360634, + 0.00020504897111095488, + 0.0001894313027150929, + 0.00017103079881053418, + 0.00015522803005296737, + 0.0001424380752723664, + 0.000132942630443722, + 0.00012196471652714536, + 0.00011242974869674072, + 0.00010534069588175043, + 9.673544991528615e-05, + 9.057198622031137e-05, + 8.452984184259549e-05, + 7.99023691797629e-05, + 7.406072109006345e-05, + 6.99383599567227e-05, + 6.711497553624213e-05, + 6.410836795112118e-05, + 6.112461414886639e-05, + 5.806254921481013e-05, + 5.566458639805205e-05, + 5.34407481609378e-05, + 5.230584065429866e-05, + 5.0652546633500606e-05, + 4.868954056291841e-05, + 4.762801472679712e-05, + 4.5258086174726486e-05, + 4.315547994337976e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00029608183103846386, + "first_to_last_ratio": 2.343484699021421, + "imbalance_abs_log_ratio": 0.8516390092453431, + "last_quartile_mean": 0.0001263425492652459, + "mean": 0.0001858347804954974, + "normalized": [ + 2.16920774571012, + 1.9356424765570717, + 1.751272112160408, + 1.5771369240348272, + 1.451727119925344, + 1.369130935687747, + 1.289826251583108, + 1.2020808437231323, + 1.127478492836466, + 1.067970341065492, + 1.0234512239081746, + 0.9751296621832141, + 0.9332398141881068, + 0.9026389800429313, + 0.8668613586828855, + 0.8407822065969367, + 0.8171794710239251, + 0.7975274126338607, + 0.777066300159633, + 0.7612061150730458, + 0.7492407752334779, + 0.7365489154759011, + 0.7249215428669998, + 0.7138135717041164, + 0.7048334785593375, + 0.6962720074755282, + 0.6911692191098191, + 0.683337870004401, + 0.6760758813563923, + 0.6702804011180051, + 0.662430102042749, + 0.6545204472768429 + ], + "peak_layer": 1, + "peak_normalized": 2.16920774571012, + "population_cv": 0.39438035124793563, + "rest_mean": 0.0001952685916977417, + "spike_contrast": 0.6908033638721538, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001348922000033781, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00040311424527317286, + 0.00035970969474874437, + 0.00032544726855121553, + 0.00029308689408935606, + 0.00026978139067068696, + 0.00025443214690312743, + 0.00023969457834027708, + 0.0002233884297311306, + 0.0002095247182296589, + 0.0001984660339076072, + 0.0001901928335428238, + 0.00018121300672646612, + 0.0001734284160193056, + 0.0001677417167229578, + 0.00016109299031086266, + 0.00015624657680746168, + 0.0001518603676231578, + 0.00014820833166595548, + 0.00014440594532061368, + 0.0001414585713064298, + 0.00013923499500378966, + 0.00013687640603166074, + 0.00013471563579514623, + 0.00013265138841234148, + 0.00013098257477395236, + 0.0001293915556743741, + 0.00012844328011851758, + 0.0001269879430765286, + 0.0001256384130101651, + 0.00012456141121219844, + 0.0001231025526067242, + 0.00012163266364950687 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.2625683266669512, + "first_to_last_ratio": 2.343484691137257, + "imbalance_abs_log_ratio": 0.8516390058810522, + "last_quartile_mean": 0.1120418356731534, + "mean": 0.16480014030821621, + "normalized": [ + 2.169207748902955, + 1.9356423730461585, + 1.7512721262865893, + 1.5771370154883375, + 1.4517271231757105, + 1.3691309069442503, + 1.2898263254837699, + 1.202080803995401, + 1.1274784933366124, + 1.0679703770106845, + 1.0234512066143264, + 0.9751297116000902, + 0.9332398524746174, + 0.9026389696372329, + 0.8668612939874407, + 0.8407822044491968, + 0.8171795266543764, + 0.7975273711861923, + 0.7770662316421936, + 0.7612060929585753, + 0.7492407781338722, + 0.7365489418998509, + 0.7249215250846839, + 0.7138135688166896, + 0.704833501980704, + 0.6962720326387369, + 0.6911692482611179, + 0.6833378266924969, + 0.6760758677045711, + 0.6702804240542598, + 0.6624300595814382, + 0.6545204702768674 + ], + "peak_layer": 1, + "peak_normalized": 2.169207748902955, + "population_cv": 0.3943803506949354, + "rest_mean": 0.17316613815448903, + "spike_contrast": 0.6908033707584418, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11962375193834304, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.35748574137687683, + 0.3189941346645355, + 0.28860989212989807, + 0.2599124014377594, + 0.23924483358860016, + 0.22563296556472778, + 0.21256355941295624, + 0.19810308516025543, + 0.18580861389636993, + 0.1760016679763794, + 0.16866490244865417, + 0.16070151329040527, + 0.15379805862903595, + 0.1487550288438797, + 0.1428588628768921, + 0.13856102526187897, + 0.13467130064964294, + 0.13143262267112732, + 0.12806062400341034, + 0.1254468709230423, + 0.12347498536109924, + 0.12138336896896362, + 0.11946716904640198, + 0.11763657629489899, + 0.11615666002035141, + 0.11474572867155075, + 0.11390478909015656, + 0.11261416971683502, + 0.11141739785671234, + 0.11046230792999268, + 0.10916856676340103, + 0.10786506533622742 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010104048214998329, + "first_to_last_ratio": 2.7014816335786884, + "imbalance_abs_log_ratio": 0.9938003756776099, + "last_quartile_mean": 3.740187639778014e-05, + "mean": 6.270627807225537e-05, + "normalized": [ + 1.9907250147127677, + 1.8444603036108247, + 1.7429056467158477, + 1.6239117898045463, + 1.5436487234069576, + 1.4786997242989472, + 1.3841625886805662, + 1.282123032540977, + 1.1824417181452251, + 1.1042125830371328, + 1.090862943651666, + 1.0498276343781445, + 0.9920003536086259, + 0.9660066676365643, + 0.9202805760437535, + 0.8989382213875909, + 0.8686500616317681, + 0.8410510099026279, + 0.7938571089189717, + 0.7653969230084752, + 0.751185801344563, + 0.7265337395686793, + 0.7106236720469997, + 0.6758027080339929, + 0.6556269427545545, + 0.6278192648368973, + 0.6205119527339269, + 0.6073894466988616, + 0.5911483388457922, + 0.586775311152339, + 0.5577767930941091, + 0.5246434037673043 + ], + "peak_layer": 1, + "peak_normalized": 1.9907250147127677, + "population_cv": 0.4046861823965828, + "rest_mean": 6.614403860617636e-05, + "spike_contrast": 0.6673673413246971, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.414237118908204e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012483095633797348, + 0.00011565924069145694, + 0.00010929112613666803, + 0.0001018294642562978, + 9.67964660958387e-05, + 9.272375609725714e-05, + 8.679568418301642e-05, + 8.039716340135783e-05, + 7.41465191822499e-05, + 6.924106128280982e-05, + 6.84039550833404e-05, + 6.583078356925398e-05, + 6.220465002115816e-05, + 6.0574682720471174e-05, + 5.770736970589496e-05, + 5.636907008010894e-05, + 5.446981231216341e-05, + 5.273917849990539e-05, + 4.977982462150976e-05, + 4.799519228981808e-05, + 4.7104065743042156e-05, + 4.555822670226917e-05, + 4.456056558410637e-05, + 4.237707253196277e-05, + 4.1111925384029746e-05, + 3.9368209399981424e-05, + 3.89099950552918e-05, + 3.808713154285215e-05, + 3.706871211761609e-05, + 3.679449582705274e-05, + 3.4976106690010056e-05, + 3.2898435165407136e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021103927974763792, + "first_to_last_ratio": 1.675431813331652, + "imbalance_abs_log_ratio": 0.5160709310565479, + "last_quartile_mean": 0.00012596112719620578, + "mean": 0.00016011051866371417, + "normalized": [ + 1.5150472374771589, + 1.439502583502474, + 1.3841101446264577, + 1.327205260326266, + 1.2794452130965426, + 1.2428385988073838, + 1.2018548814818522, + 1.154676418187547, + 1.1148050630726085, + 1.0814152014364051, + 1.0614838409967562, + 1.034687172582471, + 1.0074911480481314, + 0.9875049829338368, + 0.9623163727426111, + 0.9431041078017948, + 0.9243534563643369, + 0.9073040220081667, + 0.888778951337658, + 0.8738270000353379, + 0.8622815727966864, + 0.8493922949455803, + 0.8374788674952537, + 0.8253865750371285, + 0.8155793557629833, + 0.8058361207192702, + 0.7999903796437624, + 0.7910448570865577, + 0.7825332275919701, + 0.7756749178925374, + 0.766226246614208, + 0.7568239275482656 + ], + "peak_layer": 1, + "peak_normalized": 1.5150472374771589, + "population_cv": 0.21304076077436968, + "rest_mean": 0.0001649131305317429, + "spike_contrast": 0.8136187466924137, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001341764145763591, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024257499899249524, + 0.00023047950526233763, + 0.00022161059314385056, + 0.00021249952260404825, + 0.00020485263667069376, + 0.00019899153267033398, + 0.00019242960843257606, + 0.00018487584020476788, + 0.00017849201685748994, + 0.00017314594879280776, + 0.00016995472833514214, + 0.00016566429985687137, + 0.00016130993026308715, + 0.00015810993500053883, + 0.00015407697355840355, + 0.00015100088785402477, + 0.00014799871132709086, + 0.00014526891754940152, + 0.00014230285887606442, + 0.00013990889419801533, + 0.00013806034985464066, + 0.00013599664089269936, + 0.00013408917584456503, + 0.00013215307262726128, + 0.00013058283366262913, + 0.00012902283924631774, + 0.00012808687461074442, + 0.0001266546023543924, + 0.00012529180094134063, + 0.00012419371341820806, + 0.00012268088175915182, + 0.00012117547157686204 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016164389126060996, + "first_to_last_ratio": 1.2829777917904304, + "imbalance_abs_log_ratio": 0.24918377588932214, + "last_quartile_mean": 0.00012599118417710997, + "mean": 0.0001426773092134681, + "normalized": [ + 1.1847160153617624, + 1.1696203114063919, + 1.15376099486318, + 1.1365907777467705, + 1.126248503505431, + 1.112765799827437, + 1.097614510342432, + 1.0821504120145113, + 1.06828860482523, + 1.0536786878013262, + 1.046275103221197, + 1.0365937358276223, + 1.0246881313142864, + 1.0168969779036778, + 1.0047039605050023, + 0.9958410772863212, + 0.9859208451030141, + 0.9762312163738422, + 0.965237112554715, + 0.9568250330809598, + 0.9498426746244831, + 0.9400597234006037, + 0.9304691287067991, + 0.9205814319068928, + 0.9114724424808572, + 0.9022957305012854, + 0.8969570718532309, + 0.8872712168205663, + 0.8789155388108255, + 0.8714969614364647, + 0.8626996589435726, + 0.8532906096493081 + ], + "peak_layer": 1, + "peak_normalized": 1.1847160153617624, + "population_cv": 0.0960837578275494, + "rest_mean": 0.00014451401323269778, + "spike_contrast": 0.9186590596986468, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013275910750962794, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000169032093253918, + 0.00016687827883288264, + 0.0001646155142225325, + 0.00016216571384575218, + 0.0001606901059858501, + 0.00015876643010415137, + 0.00015660468488931656, + 0.00015439830895047635, + 0.00015242054359987378, + 0.00015033603995107114, + 0.000149279716424644, + 0.00014789840497542173, + 0.00014619974535889924, + 0.0001450881245546043, + 0.00014334845764096826, + 0.00014208392531145364, + 0.00014066853327676654, + 0.00013928604312241077, + 0.0001377174339722842, + 0.00013651722110807896, + 0.00013552099699154496, + 0.00013412519183475524, + 0.00013275683159008622, + 0.00013134608161635697, + 0.0001300464355153963, + 0.000128737126942724, + 0.00012797542149201035, + 0.00012659346975851804, + 0.00012540130410343409, + 0.00012434284144546837, + 0.00012308766599744558, + 0.00012174520816188306 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00043632716551655903, + "first_to_last_ratio": 3.082278345441195, + "imbalance_abs_log_ratio": 1.1256690460714511, + "last_quartile_mean": 0.00014155994904285762, + "mean": 0.00025262987765017897, + "normalized": [ + 2.1611427674158366, + 1.9871567751170711, + 1.8899978819847916, + 1.7784604477508343, + 1.6406551319992353, + 1.5724690971097588, + 1.4592296472822097, + 1.3280082080145534, + 1.2379389344592089, + 1.1701308820499392, + 1.131654892203279, + 1.0685236385646504, + 1.0072825277339317, + 0.9454673581714634, + 0.8865158004683366, + 0.8408935109532261, + 0.7895809204213243, + 0.7481431077843942, + 0.7047226439788088, + 0.6727720648712864, + 0.6548656488865329, + 0.6315662143568483, + 0.6127430670138975, + 0.5973169234484098, + 0.5839093169225268, + 0.5733930286828336, + 0.5691911560855093, + 0.5599482690481828, + 0.5552538441515218, + 0.5513421690146183, + 0.547054988271426, + 0.542669135783553 + ], + "peak_layer": 1, + "peak_normalized": 2.1611427674158366, + "population_cv": 0.47902914554115894, + "rest_mean": 0.0002705909153317205, + "spike_contrast": 0.5751866206559582, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001556402741698548, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000545969232916832, + 0.0005020151729695499, + 0.0004774699336849153, + 0.0004492922453209758, + 0.00041447850526310503, + 0.0003972526756115258, + 0.00036864500725641847, + 0.00033549455110915005, + 0.00031274036155082285, + 0.00029561002156697214, + 0.00028588983695954084, + 0.0002699409960769117, + 0.00025446966174058616, + 0.00023885330301709473, + 0.00022396037820726633, + 0.00021243482478894293, + 0.00019947173132095486, + 0.00018900330178439617, + 0.0001780339953256771, + 0.00016996232443489134, + 0.00016543862875550985, + 0.0001595524954609573, + 0.00015479720605071634, + 0.0001509001012891531, + 0.00014751293929293752, + 0.0001448562106816098, + 0.00014379469212144613, + 0.00014145966270007193, + 0.0001402737107127905, + 0.00013928550470154732, + 0.00013820243475493044, + 0.00013709443737752736 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.027978340163826942, + 0.0140878576785326, + 0.014187770895659924, + 0.014337530359625816, + 0.014477464370429516, + 0.009757626801729202, + 0.009825476445257664, + 0.00987118761986494, + 0.009999874979257584, + 0.007595349568873644, + 0.007658981718122959, + 0.007689184043556452, + 0.007755559403449297, + 0.00625597033649683, + 0.006315375678241253, + 0.00635211868211627, + 0.006408242508769035, + 0.005408597644418478, + 0.005475745070725679, + 0.005517482291907072, + 0.005585856735706329, + 0.004830927588045597, + 0.004899685736745596, + 0.004948681220412254, + 0.004989070352166891, + 0.004419817589223385, + 0.004463871940970421, + 0.004542885348200798, + 0.004592607729136944, + 0.004137600306421518, + 0.004193917382508516, + 0.004230021964758635 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024312340610777028, + "first_to_last_ratio": 2.286272500967691, + "imbalance_abs_log_ratio": 0.8269227625411849, + "last_quartile_mean": 0.00010634051977831405, + "mean": 0.00017363850702167838, + "normalized": [ + 1.061084599304074, + 1.6344608012282997, + 1.6065574901355, + 1.6039206242202213, + 1.1359733847422597, + 1.548756457352017, + 1.2967221414405534, + 1.313883475593909, + 1.1957449434685963, + 1.344067980756563, + 1.165161355052429, + 1.1171979507437393, + 1.0391889074947966, + 1.1329181602378293, + 1.0774500297071945, + 1.0521010358096583, + 0.862761425176938, + 0.9019434894037968, + 0.9280163978632505, + 0.8870730707939211, + 0.8354471012965897, + 0.8049103603266032, + 0.8255366455754076, + 0.7297241480789424, + 0.6373588767497196, + 0.6752584188072907, + 0.6298229328646214, + 0.6250051046248064, + 0.6032481952321226, + 0.5674884668477911, + 0.6733729976615692, + 0.4878430314089892 + ], + "peak_layer": 2, + "peak_normalized": 1.6344608012282997, + "population_cv": 0.3218023265359089, + "rest_mean": 0.00018114369622371332, + "spike_contrast": 0.7348336602687936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001331104853306897, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00018424514564685524, + 0.0002838053333107382, + 0.000278960244031623, + 0.00027850238257087767, + 0.00019724872254300863, + 0.00026892375899478793, + 0.00022516089666169137, + 0.0002281407651025802, + 0.0002076273667626083, + 0.00023338195751421154, + 0.00020231687813065946, + 0.00019398858421482146, + 0.0001804432104108855, + 0.00019671821792144328, + 0.00018708681454882026, + 0.00018268525309395045, + 0.000149808605783619, + 0.00015661212091799825, + 0.0001611393818166107, + 0.00015403004363179207, + 0.00014506578736472875, + 0.00013976343325339258, + 0.00014334495062939823, + 0.0001267082116100937, + 0.00011067004379583523, + 0.00011725086369551718, + 0.00010936151375062764, + 0.00010852495324797928, + 0.00010474711598362774, + 9.853785013547167e-05, + 0.000116923481982667, + 8.470833563478664e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00025287891548941843, + "first_to_last_ratio": 2.2739168136310908, + "imbalance_abs_log_ratio": 0.8215038131425536, + "last_quartile_mean": 0.0001112085165004828, + "mean": 0.0001807239029858465, + "normalized": [ + 1.0592123382591336, + 1.645736468094544, + 1.60181949161791, + 1.5891655894821897, + 1.1402836456293184, + 1.5483130487587196, + 1.300612463539172, + 1.3089009659072275, + 1.1885078058073624, + 1.3408206015264876, + 1.166937022076678, + 1.1148751106540256, + 1.0356001272424655, + 1.1351884917697168, + 1.0748875173492451, + 1.0458945475116934, + 0.8650051516349683, + 0.9033374117584585, + 0.9274894298403746, + 0.8851829368231718, + 0.8352656795149113, + 0.8098299287299446, + 0.8239713600606818, + 0.7303600865627311, + 0.6419462275110265, + 0.6820467100618508, + 0.635541252420758, + 0.625618271589476, + 0.6029839777576883, + 0.5727133544881818, + 0.6681570652306708, + 0.4937959207892163 + ], + "peak_layer": 2, + "peak_normalized": 1.645736468094544, + "population_cv": 0.3200470231318611, + "rest_mean": 0.00018847914530110925, + "spike_contrast": 0.7366629038009043, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001388455944834277, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001914249878609553, + 0.00029742391780018806, + 0.00028948707040399313, + 0.0002872002078220248, + 0.00020607651094906032, + 0.0002798171772155911, + 0.00023505176068283617, + 0.00023654969118069857, + 0.00021479176939465106, + 0.0002423183323116973, + 0.00021089341316837817, + 0.00020148458133917302, + 0.00018715769692789763, + 0.00020515569485723972, + 0.0001942578674061224, + 0.0001890181447379291, + 0.00015632710710633546, + 0.00016325466276612133, + 0.00016761950973886997, + 0.0001599737151991576, + 0.00015095247363206, + 0.0001463556254748255, + 0.00014891132013872266, + 0.00013199352542869747, + 0.0001160150277428329, + 0.0001232621434610337, + 0.00011485749564599246, + 0.00011306417582090944, + 0.00010897361789830029, + 0.0001035029927152209, + 0.00012075195263605565, + 8.924072608351707e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.22425555251538754, + "first_to_last_ratio": 2.2739168285843645, + "imbalance_abs_log_ratio": 0.8215038197185521, + "last_quartile_mean": 0.09862082451581955, + "mean": 0.1602677651681006, + "normalized": [ + 1.0592124025748084, + 1.645736496509649, + 1.6018195299533093, + 1.5891655927499475, + 1.1402836784663197, + 1.5483129508886688, + 1.300612441604666, + 1.3089009387606192, + 1.1885077887054891, + 1.3408205692725423, + 1.1669370180912673, + 1.1148751103389594, + 1.0356000505037584, + 1.1351884646301031, + 1.0748875233039092, + 1.0458946120331756, + 0.8650051766933494, + 0.9033374073089909, + 0.9274894888798537, + 0.8851828780404403, + 0.835265755835364, + 0.8098299455472033, + 0.8239713233255873, + 0.730360099613448, + 0.6419462245963619, + 0.6820467317825786, + 0.6355412485786012, + 0.6256182682629655, + 0.602983984145969, + 0.5727133411523797, + 0.6681570418174577, + 0.49379591603225714 + ], + "peak_layer": 2, + "peak_normalized": 1.645736496509649, + "population_cv": 0.3200470194234096, + "rest_mean": 0.16714519124340127, + "spike_contrast": 0.7366629183018027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1231296643614769, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.16975760459899902, + 0.26375851035118103, + 0.25672003626823425, + 0.254692018032074, + 0.18275071680545807, + 0.24814465641975403, + 0.20844624936580658, + 0.20977462828159332, + 0.19047948718070984, + 0.21489031612873077, + 0.1870223879814148, + 0.17867854237556458, + 0.16597330570220947, + 0.18193411827087402, + 0.1722698211669922, + 0.16762319207191467, + 0.13863244652748108, + 0.1447758674621582, + 0.14864666759967804, + 0.1418662816286087, + 0.13386617600917816, + 0.12978963553905487, + 0.13205604255199432, + 0.11705318093299866, + 0.10288328677415848, + 0.1093101054430008, + 0.1018567755818367, + 0.10026644170284271, + 0.09663889557123184, + 0.09178748726844788, + 0.10708403587341309, + 0.07913956791162491 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.143860511860112e-05, + "first_to_last_ratio": 0.9636580872932836, + "imbalance_abs_log_ratio": 0.037018728516889636, + "last_quartile_mean": 7.413273033307632e-05, + "mean": 7.686168260079285e-05, + "normalized": [ + 0.586022200160155, + 0.9787081875553917, + 0.912570839911241, + 1.015732496910769, + 0.8201270226617754, + 1.1119332047208261, + 1.0008210861718638, + 1.0096344009845781, + 0.9335569648148418, + 1.090167719321714, + 1.035494250546431, + 0.9697380161483544, + 1.0399314841024359, + 1.0649519568255064, + 1.06563466639597, + 1.1389203959735625, + 0.9952637999761301, + 1.0027093290728673, + 1.1427249020241628, + 1.1090175842590573, + 1.0432879501645442, + 1.0450477353718262, + 1.1125450117027875, + 1.0594965280199307, + 0.9180884623091344, + 0.995296932026832, + 0.9631737996002885, + 0.9446177686211702, + 0.9674620335911367, + 0.9355074959711648, + 1.1351899163905306, + 0.8566258576930204 + ], + "peak_layer": 19, + "peak_normalized": 1.1427249020241628, + "population_cv": 0.10846075768175395, + "rest_mean": 7.635363915512614e-05, + "spike_contrast": 1.0425844542348655, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.960511720739305e-05, + "top_five_layers": [ + 19, + 16, + 31, + 23, + 6 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.504265234572813e-05, + 7.522515807067975e-05, + 7.014173024799675e-05, + 7.807090878486633e-05, + 6.303634290816262e-05, + 8.546505705453455e-05, + 7.692479266552255e-05, + 7.760219887131825e-05, + 7.175475911935791e-05, + 8.379212522413582e-05, + 7.958983042044565e-05, + 7.453569560311735e-05, + 7.993088365765288e-05, + 8.185399929061532e-05, + 8.190647349692881e-05, + 8.753933798288926e-05, + 7.64976502978243e-05, + 7.706992619205266e-05, + 8.78317587194033e-05, + 8.52409575600177e-05, + 8.018886728677899e-05, + 8.032412733882666e-05, + 8.551208156859502e-05, + 8.143468585330993e-05, + 7.056582398945466e-05, + 7.650019688298926e-05, + 7.403115887427703e-05, + 7.260491111082956e-05, + 7.436075975419953e-05, + 7.190468022599816e-05, + 8.725260704522952e-05, + 6.584170478163287e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 8.107683152047684e-05, + "first_to_last_ratio": 1.03534602173519, + "imbalance_abs_log_ratio": 0.034735691361772646, + "last_quartile_mean": 7.83089226388256e-05, + "mean": 8.487996228723205e-05, + "normalized": [ + 0.5954853629443996, + 1.025163681929567, + 0.9520754208460104, + 1.0126519094835282, + 0.8443534832663374, + 1.1548844288399933, + 1.0326421147312754, + 1.0242956755731092, + 0.9710206925084107, + 1.15116021353655, + 1.072832454869962, + 1.0193061391332177, + 1.0419196505755806, + 1.0947619162703996, + 1.1089694127677372, + 1.1239932261258834, + 0.9835832115239759, + 1.0364664516448652, + 1.11612142168979, + 1.093437276508762, + 1.0179007506072324, + 1.033774483284387, + 1.0848380474290875, + 1.0276879808812729, + 0.8974590825111308, + 0.9692041024722606, + 0.9378977551023808, + 0.9155031739616566, + 0.918412101050971, + 0.8874638084461439, + 1.0536745962271137, + 0.8010599732570078 + ], + "peak_layer": 6, + "peak_normalized": 1.1548844288399933, + "population_cv": 0.11200209045145568, + "rest_mean": 8.468612051931107e-05, + "spike_contrast": 1.0146492401244358, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.592670783400536e-05, + "top_five_layers": [ + 6, + 10, + 16, + 19, + 15 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.054477514931932e-05, + 8.701585466042161e-05, + 8.081212581600994e-05, + 8.59538558870554e-05, + 7.166869181673974e-05, + 9.802654676605016e-05, + 8.76506237545982e-05, + 8.694217831362039e-05, + 8.242019976023585e-05, + 9.771043551154435e-05, + 9.106197830988094e-05, + 8.651866664877161e-05, + 8.843810064718127e-05, + 9.292335016652942e-05, + 9.41292819334194e-05, + 9.540450264466926e-05, + 8.348650590050966e-05, + 8.797523332759738e-05, + 9.473634418100119e-05, + 9.281091479351744e-05, + 8.639937732368708e-05, + 8.774673915468156e-05, + 9.20810125535354e-05, + 8.723011706024408e-05, + 7.617629307787865e-05, + 8.226600766647607e-05, + 7.960872608236969e-05, + 7.770787487970665e-05, + 7.795478450134397e-05, + 7.532789459219202e-05, + 8.943585999077186e-05, + 6.799394031986594e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.1427921979011444e-05, + "first_to_last_ratio": 0.40836765899304117, + "imbalance_abs_log_ratio": 0.8955873853644011, + "last_quartile_mean": 5.2472132665570825e-05, + "mean": 3.864622581772892e-05, + "normalized": [ + 0.2626641640048693, + 0.5716512870137274, + 0.5157707114648397, + 0.4843895386483896, + 0.5109475889421058, + 0.7512596968779521, + 0.675728069412672, + 0.6632970161544233, + 0.7004288748821966, + 0.9130902556041411, + 0.8717836320781894, + 0.8044820763577291, + 0.9241814793896349, + 0.9531622930600316, + 1.0319070449118461, + 1.0273708471578158, + 0.9893551980130566, + 1.0844069350216101, + 1.1887300698220302, + 1.2100415764612447, + 1.1128447755732793, + 1.2346303137099222, + 1.3033621630444512, + 1.3524689418392815, + 1.2055635543985064, + 1.318388624044801, + 1.3615446320871405, + 1.2933406942036905, + 1.3753836688642536, + 1.3573239762703455, + 1.6442457326403273, + 1.3062545680454958 + ], + "peak_layer": 31, + "peak_normalized": 1.6442457326403273, + "population_cv": 0.32680115045148705, + "rest_mean": 3.691592014042221e-05, + "spike_contrast": 1.2999777952883038, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.798987647518516e-05, + "top_five_layers": [ + 31, + 29, + 27, + 30, + 24 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.0150978596357163e-05, + 2.2092164726927876e-05, + 1.99325913854409e-05, + 1.8719827494351193e-05, + 1.974619590328075e-05, + 2.9033351893303916e-05, + 2.6114339561900124e-05, + 2.5633926270529628e-05, + 2.7068932467955165e-05, + 3.528749221004546e-05, + 3.369114710949361e-05, + 3.1090195989236236e-05, + 3.571612614905462e-05, + 3.683612521854229e-05, + 3.9879312680568546e-05, + 3.970400575781241e-05, + 3.823484439635649e-05, + 4.190823528915644e-05, + 4.5939930714666843e-05, + 4.676354001276195e-05, + 4.300725049688481e-05, + 4.771380190504715e-05, + 5.037002847529948e-05, + 5.226782013778575e-05, + 4.65904813609086e-05, + 5.09507444803603e-05, + 5.261856131255627e-05, + 4.998273652745411e-05, + 5.3153387852944434e-05, + 5.24554488947615e-05, + 6.354389188345522e-05, + 5.048180901212618e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031520502852799837, + "first_to_last_ratio": 1.31300997816315, + "imbalance_abs_log_ratio": 0.27232219480674863, + "last_quartile_mean": 0.00024006293460843153, + "mean": 0.00030979480334281106, + "normalized": [ + 0.607123837738274, + 1.0354820546705408, + 0.9347765087792952, + 1.1085716606000595, + 0.9306423427143409, + 1.2565833735558338, + 1.1285140374487672, + 1.138017377238765, + 1.0370737734279494, + 1.2142990636699202, + 1.1645184658462944, + 1.1206382096550715, + 1.1232577862254296, + 1.1815362250572234, + 1.1924463988431009, + 1.2309950595128432, + 1.0374639291264394, + 1.065463636492051, + 1.1630951913863559, + 1.1082935818936954, + 1.0270641612904527, + 0.9934767629460804, + 1.0585185268550952, + 0.9428720751183239, + 0.8088208526771234, + 0.8683362720246933, + 0.8018903045940483, + 0.8027332776454693, + 0.7640102071384416, + 0.7171383971343437, + 0.8334550561316563, + 0.6028915925620211 + ], + "peak_layer": 6, + "peak_normalized": 1.2565833735558338, + "population_cv": 0.17765693833110424, + "rest_mean": 0.0003117367305094376, + "spike_contrast": 0.9601319554288655, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00029930839664302764, + "top_five_layers": [ + 6, + 16, + 10, + 15, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00018808380991686136, + 0.00032078695949167013, + 0.00028958890470676124, + 0.00034342973958700895, + 0.0002883081615436822, + 0.00038928299909457564, + 0.00034960778430104256, + 0.0003525518695823848, + 0.00032128006569109857, + 0.00037618353962898254, + 0.00036076176911592484, + 0.00034716789377853274, + 0.0003479794249869883, + 0.00036603378248400986, + 0.0003694136976264417, + 0.00038135587237775326, + 0.0003214009338989854, + 0.0003300750977359712, + 0.0003603208460845053, + 0.000343343592248857, + 0.00031817913986742496, + 0.00030777393840253353, + 0.00032792353886179626, + 0.00029209686908870935, + 0.0002505684969946742, + 0.0002690060646273196, + 0.00024842144921422005, + 0.0002486825978849083, + 0.00023668639187235385, + 0.00022216574870981276, + 0.000258200045209378, + 0.0001867726823547855 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014031016267836094, + 0.014151837676763535, + 0.014265215955674648, + 0.014401593245565891, + 0.009722241200506687, + 0.009786579757928848, + 0.009839320555329323, + 0.00996283907443285, + 0.007562503684312105, + 0.007635751739144325, + 0.007660485804080963, + 0.007726665120571852, + 0.006238248664885759, + 0.006290202960371971, + 0.006327399518340826, + 0.006382675375789404, + 0.005390914157032967, + 0.0054517327807843685, + 0.005509276874363422, + 0.0055727302096784115, + 0.0048178439028561115, + 0.004884926602244377, + 0.004938251338899136, + 0.004975184798240662, + 0.004412420094013214, + 0.004455919377505779, + 0.004530393984168768, + 0.0045819394290447235, + 0.004127359017729759, + 0.004185030702501535, + 0.004221425391733646, + 0.004281661007553339 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 5.6193598993559135e-05, + "first_to_last_ratio": 1.5723739147519853, + "imbalance_abs_log_ratio": 0.4525865249799748, + "last_quartile_mean": 3.573806361600873e-05, + "mean": 4.4294749727669114e-05, + "normalized": [ + 1.5642840408451666, + 1.493254721227763, + 1.1816235069560812, + 1.0717429494942774, + 1.3553344243310417, + 1.3257825682631335, + 1.1255110934944355, + 1.0314977817441078, + 1.194075821513187, + 1.0893775754580661, + 1.0407197138445958, + 0.9875719064328349, + 1.135803276632048, + 1.0324555952346264, + 0.9635988909717662, + 0.90208003019059, + 0.9904506854292476, + 0.9196323607404276, + 0.8708711781381843, + 0.8075366439875266, + 0.9218591005239072, + 0.8776357461813653, + 0.8517186042296377, + 0.8109904231766266, + 0.8722342267309339, + 0.8500552841464633, + 0.7844009556175089, + 0.7665260139039981, + 0.8384393111065888, + 0.8154485842192296, + 0.7754690286134897, + 0.7520179566211438 + ], + "peak_layer": 1, + "peak_normalized": 1.5642840408451666, + "population_cv": 0.20749976292329902, + "rest_mean": 4.538663482957485e-05, + "spike_contrast": 0.8460325450777163, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.839857017737813e-05, + "top_five_layers": [ + 1, + 2, + 5, + 6, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.928957009222358e-05, + 6.614334415644407e-05, + 5.23397175129503e-05, + 4.747258572024293e-05, + 6.003419912303798e-05, + 5.872520705452189e-05, + 4.985423220205121e-05, + 4.5689936087001115e-05, + 5.289128966978751e-05, + 4.8253707063850015e-05, + 4.6098419261397794e-05, + 4.374425043351948e-05, + 5.03101218782831e-05, + 4.573236219584942e-05, + 4.2682371713453904e-05, + 3.9957409171620384e-05, + 4.3871765228686854e-05, + 4.073488526046276e-05, + 3.857502088067122e-05, + 3.576963354134932e-05, + 4.083351814188063e-05, + 3.887465572915971e-05, + 3.7726662412751466e-05, + 3.592261782614514e-05, + 3.863539677695371e-05, + 3.7652986065950245e-05, + 3.474484401522204e-05, + 3.395307794562541e-05, + 3.713845944730565e-05, + 3.612009095377289e-05, + 3.4349206543993205e-05, + 3.3310447179246694e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 9.100913848669734e-05, + "first_to_last_ratio": 0.8419197805282016, + "imbalance_abs_log_ratio": 0.17207054181035333, + "last_quartile_mean": 0.00010809716150106397, + "mean": 9.889523016681778e-05, + "normalized": [ + 1.0744641570554045, + 0.9868986027413778, + 0.8415148680287817, + 0.7657672453742986, + 1.0045876840265406, + 0.9858056114400382, + 0.8710937611903306, + 0.8319330217638002, + 1.0050985706445066, + 0.9363425314153265, + 0.9111662095698898, + 0.8822483642571924, + 1.0547921529408322, + 0.9937743096656301, + 0.9653849347701057, + 0.9236053864198286, + 1.0608213360421703, + 1.0216834776263237, + 0.9948854733452297, + 0.9444045194208119, + 1.0902032759381237, + 1.070297753084291, + 1.0352646449837608, + 1.0035839360241054, + 1.109605122267389, + 1.0988987221493531, + 1.0612028824132491, + 1.0392747076439328, + 1.1393068813062424, + 1.1142247321092786, + 1.0935193303224786, + 1.0883457940193757 + ], + "peak_layer": 29, + "peak_normalized": 1.1393068813062424, + "population_cv": 0.09089893356811851, + "rest_mean": 9.776359500527313e-05, + "spike_contrast": 1.0740814107080978, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010500606003915891, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00010625938011799008, + 9.759956446941942e-05, + 8.322180656250566e-05, + 7.573072798550129e-05, + 9.934893023455516e-05, + 9.749147284310311e-05, + 8.614701800979674e-05, + 8.227420767070726e-05, + 9.939945448422804e-05, + 9.259981015929952e-05, + 9.010999201564118e-05, + 8.725015504751354e-05, + 0.00010431391274323687, + 9.827953908825293e-05, + 9.547196532366797e-05, + 9.134016727330163e-05, + 0.00010491017019376159, + 0.00010103962267749012, + 9.838942787609994e-05, + 9.339710231870413e-05, + 0.0001078159039025195, + 0.00010584734263829887, + 0.00010238273534923792, + 9.924966434482485e-05, + 0.00010973465396091342, + 0.00010867584205698222, + 0.00010494790330994874, + 0.00010277931141899899, + 0.00011267201625742018, + 0.000110191511339508, + 0.00010814384586410597, + 0.0001076322078006342 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.08070781361311674, + "first_to_last_ratio": 0.8419197775247833, + "imbalance_abs_log_ratio": 0.1720705453776983, + "last_quartile_mean": 0.09586164355278015, + "mean": 0.0877012787386775, + "normalized": [ + 1.074464079725317, + 0.986898606287062, + 0.8415149217948403, + 0.7657672391640048, + 1.0045876578450608, + 0.9858055872189405, + 0.8710938052118494, + 0.8319330371629138, + 1.0050985716236776, + 0.9363425240221704, + 0.9111662153888755, + 0.8822483595926541, + 1.0547922001671663, + 0.993774279010469, + 0.9653849224854107, + 0.9236053602662656, + 1.060821305580311, + 1.021683475130349, + 0.9948854782496288, + 0.9444045803369281, + 1.0902032654624292, + 1.07029776221103, + 1.0352646575744078, + 1.0035839255047951, + 1.109605076031185, + 1.0988987393251362, + 1.0612029192639736, + 1.0392747445534645, + 1.1393068879878778, + 1.1142247084644445, + 1.0935193626414328, + 1.0883457447159286 + ], + "peak_layer": 29, + "peak_normalized": 1.1393068879878778, + "population_cv": 0.09089892559784572, + "rest_mean": 0.08669773351263117, + "spike_contrast": 1.0740813996684329, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0931204229593277, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.09423187375068665, + 0.08655226975679398, + 0.0738019347190857, + 0.0671587660908699, + 0.08810362219810486, + 0.08645641058683395, + 0.07639604061841965, + 0.07296159118413925, + 0.08814842998981476, + 0.0821184366941452, + 0.07991044223308563, + 0.07737430930137634, + 0.09250662475824356, + 0.08715527504682541, + 0.08466549217700958, + 0.08100137114524841, + 0.09303538501262665, + 0.08960294723510742, + 0.08725272864103317, + 0.08282548934221268, + 0.09561222046613693, + 0.09386648237705231, + 0.09079403430223465, + 0.0880155935883522, + 0.0973137840628624, + 0.09637482464313507, + 0.09306885302066803, + 0.09114572405815125, + 0.0999186709523201, + 0.09771893173456192, + 0.09590304642915726, + 0.09544931352138519 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0474835562490625e-05, + "first_to_last_ratio": 0.595870196340598, + "imbalance_abs_log_ratio": 0.5177324270116084, + "last_quartile_mean": 1.7579056020622374e-05, + "mean": 1.45315484587627e-05, + "normalized": [ + 0.6891108502390517, + 0.67218760206668, + 0.6248497232673742, + 0.6113105238885437, + 0.8536003812586072, + 0.8434482348779837, + 0.7260190066641083, + 0.7461464268756399, + 0.9515026546685825, + 0.8968707547028398, + 0.8666952712848448, + 0.9173542983424958, + 1.0474470628626384, + 1.0570493146141757, + 0.9856965747654381, + 1.0403130781822671, + 1.05710126231986, + 1.0899464822844398, + 1.0639055355381295, + 0.9944043245776518, + 1.1636416255463544, + 1.1162669452398895, + 1.1491653646620976, + 1.158233556446643, + 1.241320970071456, + 1.2666472916328237, + 1.1327368714456698, + 1.1012802199127885, + 1.2948691657915095, + 1.2555094531837419, + 1.2151558495322814, + 1.1702133232533922 + ], + "peak_layer": 29, + "peak_normalized": 1.2948691657915095, + "population_cv": 0.19478066534993646, + "rest_mean": 1.408557606434139e-05, + "spike_contrast": 1.202634475953173, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6939799388637765e-05, + "top_five_layers": [ + 29, + 26, + 30, + 25, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.0013847713707946e-05, + 9.767926712811459e-06, + 9.080034033104312e-06, + 8.883288501237985e-06, + 1.2404135304677766e-05, + 1.2256608897587284e-05, + 1.0550180377322249e-05, + 1.0842662959476002e-05, + 1.3826806934957858e-05, + 1.3032920833211392e-05, + 1.2594424333656207e-05, + 1.3330578440218233e-05, + 1.522102775197709e-05, + 1.5360563338617794e-05, + 1.4323697541840374e-05, + 1.5117359907890204e-05, + 1.5361318219220266e-05, + 1.5838610124774277e-05, + 1.546019484521821e-05, + 1.445023463020334e-05, + 1.690951467026025e-05, + 1.622108720766846e-05, + 1.669915218371898e-05, + 1.6830927052069455e-05, + 1.8038315829471685e-05, + 1.8406346498522907e-05, + 1.6460420738440007e-05, + 1.600330688233953e-05, + 1.8816454030456953e-05, + 1.8244496459374204e-05, + 1.7658096112427302e-05, + 1.7005011613946408e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.836043712610262e-05, + "first_to_last_ratio": 0.6391636816233681, + "imbalance_abs_log_ratio": 0.44759470463089907, + "last_quartile_mean": 0.00010695294349716278, + "mean": 9.121693358338234e-05, + "normalized": [ + 0.7254188545798665, + 0.6902023390007371, + 0.6496225556724757, + 0.6277900804246503, + 0.8680656002782756, + 0.8588952084862168, + 0.7971788633739051, + 0.7782428701770085, + 0.9734232055545582, + 0.9237343700472408, + 0.9096320021673064, + 0.8918620199754489, + 1.0844263684244682, + 1.0342037426828012, + 1.0147200800673386, + 0.9791745321037664, + 1.1290928536156495, + 1.0928814238713915, + 1.0660995437899934, + 1.013549283136708, + 1.170635644938427, + 1.1503062295054498, + 1.1129730585678779, + 1.0777737723208793, + 1.1919439416858, + 1.1807148042349604, + 1.1393605783805036, + 1.1153320412921235, + 1.222157091120308, + 1.1946826876057626, + 1.1714041050576325, + 1.1645002478604691 + ], + "peak_layer": 29, + "peak_normalized": 1.222157091120308, + "population_cv": 0.17156610618115575, + "rest_mean": 8.883977756744975e-05, + "spike_contrast": 1.1712498490939807, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010405357606941834, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.617048347834498e-05, + 6.295814091572538e-05, + 5.925657751504332e-05, + 5.7265086070401594e-05, + 7.91822822066024e-05, + 7.834578718757257e-05, + 7.271621143445373e-05, + 7.098892820067704e-05, + 8.879267988959327e-05, + 8.426021668128669e-05, + 8.297384192701429e-05, + 8.135291864164174e-05, + 9.891804802464321e-05, + 9.433689410798252e-05, + 9.255965414922684e-05, + 8.931729826144874e-05, + 0.00010299238783773035, + 9.968929225578904e-05, + 9.724633127916604e-05, + 9.245285764336586e-05, + 0.00010678179387468845, + 0.00010492740693734959, + 0.00010152198956348002, + 9.831121860770509e-05, + 0.00010872547136386856, + 0.00010770118387881666, + 0.0001039289782056585, + 0.00010173716873396188, + 0.00011148142220918089, + 0.00010897529136855155, + 0.00010685189045034349, + 0.00010622214176692069 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 5.421776268121903e-05, + "first_to_last_ratio": 0.5089233562442377, + "imbalance_abs_log_ratio": 0.6754578508927926, + "last_quartile_mean": 0.00010653423942130757, + "mean": 8.540443025140121e-05, + "normalized": [ + 0.5542697148172375, + 0.5395320644633284, + 0.526750599523528, + 0.5251350203104778, + 0.7537064875235374, + 0.7453106043033457, + 0.7183783488935762, + 0.7156010196903468, + 0.915261682615852, + 0.8828592813937833, + 0.8733821992497209, + 0.8643716402564832, + 1.0724322757212523, + 1.0352702478756863, + 1.026729960575756, + 1.0038080492636434, + 1.1695135624376918, + 1.1411903421843295, + 1.1203495535033974, + 1.0697851231751558, + 1.23599072599408, + 1.2140634563521424, + 1.1785251437606976, + 1.1385123493568967, + 1.2633504856351931, + 1.2489710766708417, + 1.2127527443953423, + 1.1862530196716095, + 1.298334269521613, + 1.274402551981099, + 1.2503128844914202, + 1.2448935143909348 + ], + "peak_layer": 29, + "peak_normalized": 1.298334269521613, + "population_cv": 0.24541141609846925, + "rest_mean": 8.214501078193145e-05, + "spike_contrast": 1.2539446328637511, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010300529538653791, + "top_five_layers": [ + 29, + 30, + 25, + 31, + 26 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.73370891995728e-05, + 4.6078428567852825e-05, + 4.498683483689092e-05, + 4.4848857214674354e-05, + 6.436987314373255e-05, + 6.365282752085477e-05, + 6.135269359219819e-05, + 6.111549737397581e-05, + 7.816740253474563e-05, + 7.540009391959757e-05, + 7.459070911863819e-05, + 7.382116746157408e-05, + 9.159046749118716e-05, + 8.841666567604989e-05, + 8.768728730501607e-05, + 8.572965452913195e-05, + 9.98816394712776e-05, + 9.746271098265424e-05, + 9.56828152993694e-05, + 9.136438893619925e-05, + 0.00010555908374954015, + 0.00010368639777880162, + 0.00010065126843983307, + 9.723399853101e-05, + 0.0001078957284335047, + 0.00010666766320355237, + 0.00010357445717090741, + 0.00010131126327905804, + 0.00011088349856436253, + 0.00010883962386287749, + 0.00010678225953597575, + 0.00010631942132022232 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00014064586594031425, + "first_to_last_ratio": 1.0563608173139223, + "imbalance_abs_log_ratio": 0.054829809986132017, + "last_quartile_mean": 0.00013314188072399702, + "mean": 0.0001347423383322166, + "normalized": [ + 1.1217990599984589, + 1.0213028410040081, + 0.9566177087867302, + 0.90173442925003, + 1.162761655685041, + 1.2034553361266669, + 1.0271109783027095, + 0.9557256445604886, + 1.16094674777681, + 1.05596774404836, + 1.0011734009390578, + 0.9818696062747876, + 1.1123781705815696, + 1.0319357929888284, + 0.9826152250905227, + 0.9109733958837348, + 1.0154471845455557, + 0.955267408663159, + 0.9085227032176484, + 0.853750823711138, + 0.9825495622515935, + 0.9600374683174735, + 0.9243229318330685, + 0.9067574744309754, + 0.9933509912573504, + 0.986398830187635, + 0.9577583846169437, + 0.9375602577665346, + 1.0292265528944584, + 1.007559112024919, + 0.9944595916547317, + 0.9986629853290112 + ], + "peak_layer": 6, + "peak_normalized": 1.2034553361266669, + "population_cv": 0.07980954026492859, + "rest_mean": 0.0001359050228428613, + "spike_contrast": 0.9452471975467027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012846384197473526, + "top_five_layers": [ + 6, + 5, + 9, + 1, + 13 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001511538284830749, + 0.0001376127329422161, + 0.00012889690697193146, + 0.00012150180555181578, + 0.00015667322441004217, + 0.0001621563860680908, + 0.00013839533494319767, + 0.00012877670815214515, + 0.00015642867947462946, + 0.0001422835630364716, + 0.00013490044511854649, + 0.00013229940668679774, + 0.000149884435813874, + 0.00013904544175602496, + 0.00013239987310953438, + 0.0001227466855198145, + 0.00013682372809853405, + 0.0001287149643758312, + 0.0001224164734594524, + 0.00011503638233989477, + 0.00013239102554507554, + 0.0001293576933676377, + 0.0001245454332092777, + 0.00012217862240504473, + 0.00013384643534664065, + 0.000132909684907645, + 0.00012905060430057347, + 0.00012632906145881861, + 0.00013868039241060615, + 0.00013576087076216936, + 0.00013399581075645983, + 0.0001345621858490631 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 8.038491115106986, + "forward": { + "activation_sha256": { + "attention_branch_output": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", + "mlp_branch_output": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", + "post_attention_state": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", + "post_mlp_state": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", + "pre_attention_input": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", + "pre_mlp_input": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd" + }, + "logits_sha256": "5e5a50bae6f74c41645e35a857f0cba662c8b3f5ba9a8a4635e860fdaa3b0c88", + "loss_nats": 5.571857452392578, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.571857452392578, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0023513585329055786, + 0.0023208872880786657, + 0.002778682392090559, + 0.002497476991266012, + 0.002628475893288851, + 0.002920303260907531, + 0.002932395786046982, + 0.0032228652853518724, + 0.003561305580660701, + 0.003354643704369664, + 0.0034162327647209167, + 0.0036349669098854065, + 0.0035210708156228065, + 0.0038089773152023554, + 0.003915763925760984, + 0.0037244565319269896, + 0.004162859171628952, + 0.004167158622294664, + 0.004440344404429197, + 0.0041700867004692554, + 0.004737259820103645, + 0.004552462603896856, + 0.004706811159849167, + 0.005186605267226696, + 0.005580419674515724, + 0.004905648995190859, + 0.005227713380008936, + 0.005349088925868273, + 0.005235649179667234, + 0.004925078712403774, + 0.005810525268316269, + 0.005564783234149218 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024914435198297724, + "first_to_last_ratio": 5.005386356076312, + "imbalance_abs_log_ratio": 1.61051460380912, + "last_quartile_mean": 4.977524895366514e-05, + "mean": 0.00012233095264946314, + "normalized": [ + 2.946204230178919, + 2.5883907308658176, + 2.285503889899197, + 2.0157125547168477, + 1.8168856694000293, + 1.687577213316311, + 1.555459627812154, + 1.3974018168018878, + 1.2756749767931916, + 1.1684180974434406, + 1.0925035606597346, + 1.0000212864934148, + 0.922022236502042, + 0.863306019148705, + 0.7954975128821132, + 0.747136294956448, + 0.6955650087701328, + 0.6567702950630189, + 0.6067855157189815, + 0.5734103460233209, + 0.549704279492861, + 0.527795032775898, + 0.5009080420334019, + 0.4762252632696638, + 0.45754883501456406, + 0.43797117166433, + 0.4282553488048722, + 0.41570642703193256, + 0.40058854659018955, + 0.39033614610916745, + 0.3703926654711839, + 0.3543213582962283 + ], + "peak_layer": 1, + "peak_normalized": 2.946204230178919, + "population_cv": 0.6883820030702144, + "rest_mean": 0.00013360270127628205, + "spike_contrast": 0.46004691130861425, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.146351006464101e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036041197017766535, + 0.0003166403039358556, + 0.00027958786813542247, + 0.0002465840370859951, + 0.00022226135479286313, + 0.0002064429281745106, + 0.00019028085807804018, + 0.0001709454954834655, + 0.00015605453518219292, + 0.00014293369895312935, + 0.00013364700134843588, + 0.00012233355664648116, + 0.0001127918585552834, + 0.00010560904775047675, + 9.731396858114749e-05, + 9.139789472101256e-05, + 8.508913015248254e-05, + 8.03433358669281e-05, + 7.42286501917988e-05, + 7.014583388809115e-05, + 6.724584818584844e-05, + 6.456566916313022e-05, + 6.127655797172338e-05, + 5.825709013151936e-05, + 5.597238487098366e-05, + 5.3577430662699044e-05, + 5.2388884796528146e-05, + 5.085376324132085e-05, + 4.900437852484174e-05, + 4.775019260705449e-05, + 4.5310487621463835e-05, + 4.334446930442937e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002986813742609229, + "first_to_last_ratio": 2.3520308544062645, + "imbalance_abs_log_ratio": 0.8552791483203458, + "last_quartile_mean": 0.0001269887143280357, + "mean": 0.00018719729814620223, + "normalized": [ + 2.1746021240914017, + 1.94336263487061, + 1.7490368812517914, + 1.5797937988337565, + 1.4558020735282082, + 1.372777068369229, + 1.290232936323773, + 1.1987380034727069, + 1.1282687150208124, + 1.0677045109732362, + 1.0238712087163733, + 0.9743741602296072, + 0.9332108464677616, + 0.9012687760530631, + 0.8680188022864366, + 0.8423083374767277, + 0.8168968442148058, + 0.7977443195284458, + 0.774889242886524, + 0.759928616461841, + 0.7477907292059667, + 0.7367409087088999, + 0.7235787757136496, + 0.7121132246295925, + 0.7033660136533255, + 0.6946945949961153, + 0.6884818791691059, + 0.6826464924874314, + 0.674695217464828, + 0.6690870522688999, + 0.6604539573176332, + 0.6535212533274407 + ], + "peak_layer": 1, + "peak_normalized": 2.1746021240914017, + "population_cv": 0.3960989175249597, + "rest_mean": 0.00019674027215741161, + "spike_contrast": 0.68956516628749, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013566523848567157, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004070796421729028, + 0.0003637922345660627, + 0.0003274149785283953, + 0.0002957331307698041, + 0.0002725222148001194, + 0.000256980158155784, + 0.0002415281196590513, + 0.00022440051543526351, + 0.00021120885503478348, + 0.00019987139967270195, + 0.00019166592392139137, + 0.0001824002101784572, + 0.00017469454905949533, + 0.00016871507978066802, + 0.00016249077452812344, + 0.00015767784498166293, + 0.00015292088210117072, + 0.0001493355812272057, + 0.00014505717263091356, + 0.0001422565837856382, + 0.00013998440408613533, + 0.0001379159075440839, + 0.00013545199180953205, + 0.0001333056716248393, + 0.00013166821736376733, + 0.000130044951220043, + 0.0001288819476030767, + 0.00012778957898262888, + 0.00012630112178158015, + 0.00012525128840934485, + 0.00012363519635982811, + 0.00012233741290401667 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.264873631298542, + "first_to_last_ratio": 2.3520309299879743, + "imbalance_abs_log_ratio": 0.8552791804550044, + "last_quartile_mean": 0.11261485889554024, + "mean": 0.1660084358882159, + "normalized": [ + 2.1746021175238424, + 1.943362568695452, + 1.7490370652499811, + 1.5797937888156908, + 1.4558020518807187, + 1.3727770906872316, + 1.2902329816871188, + 1.1987379486864336, + 1.128268732118104, + 1.067704514212167, + 1.0238712059346726, + 0.9743741284397964, + 0.9332107750795899, + 0.9012687970282692, + 0.8680188148399602, + 0.8423083473020997, + 0.8168968752199287, + 0.7977442908994873, + 0.7748892254753884, + 0.7599285915530729, + 0.7477907763310051, + 0.7367409150028685, + 0.723578786768912, + 0.7121132849551643, + 0.7033659848486875, + 0.694694622945607, + 0.6884818276985868, + 0.6826464794318051, + 0.6746951840389442, + 0.6690870220520635, + 0.6604539434158253, + 0.6535212611815249 + ], + "peak_layer": 1, + "peak_normalized": 2.1746021175238424, + "population_cv": 0.3960989241805914, + "rest_mean": 0.17447124007675383, + "spike_contrast": 0.6895651868880185, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12030929327011108, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.3610022962093353, + 0.3226145803928375, + 0.2903549075126648, + 0.2622590959072113, + 0.24167542159557343, + 0.22789257764816284, + 0.2141895592212677, + 0.19900061190128326, + 0.1873021274805069, + 0.17724795639514923, + 0.1699712574481964, + 0.1617543250322342, + 0.15492086112499237, + 0.1496182233095169, + 0.1440984457731247, + 0.13983029127120972, + 0.13561177253723145, + 0.13243228197097778, + 0.1286381483078003, + 0.1261545568704605, + 0.12413957715034485, + 0.12230520695447922, + 0.12012018263339996, + 0.11821681261062622, + 0.11676468700170517, + 0.11532516777515411, + 0.11429379135370255, + 0.1133250743150711, + 0.1120050922036171, + 0.11107409000396729, + 0.1096409261226654, + 0.10849004238843918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010120303977600997, + "first_to_last_ratio": 2.7021675066537045, + "imbalance_abs_log_ratio": 0.994054231196686, + "last_quartile_mean": 3.7452541164384456e-05, + "mean": 6.280763545873924e-05, + "normalized": [ + 1.9944816746100333, + 1.8472480968114133, + 1.7373223217437128, + 1.6248488809553971, + 1.5471371990089653, + 1.475823179261819, + 1.3807460832170038, + 1.2829322622274897, + 1.184046410564159, + 1.1043973349513228, + 1.0901910170788929, + 1.0456150919711364, + 0.9888694060739084, + 0.9664075020831666, + 0.9198699716804313, + 0.9041232034205934, + 0.8692950074065211, + 0.8446819175937094, + 0.7923092109368705, + 0.7644112750281646, + 0.7503465778061283, + 0.730017034806627, + 0.7090250315296005, + 0.6754099655367306, + 0.6572034579968898, + 0.6267819521510305, + 0.6185439744898635, + 0.6068927943478243, + 0.594580659914726, + 0.5858446061910916, + 0.5555293303136664, + 0.5250675682911103 + ], + "peak_layer": 1, + "peak_normalized": 1.9944816746100333, + "population_cv": 0.40483316550850257, + "rest_mean": 6.624576713807915e-05, + "spike_contrast": 0.6678422833882899, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4241724390303715e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012526867794804275, + 0.0001160212850663811, + 0.0001091171070584096, + 0.00010205291619058698, + 9.717202920001e-05, + 9.269296424463391e-05, + 8.67213966557756e-05, + 8.057794184423983e-05, + 7.43671553209424e-05, + 6.936458521522582e-05, + 6.847231998108327e-05, + 6.567261152667925e-05, + 6.210854917299002e-05, + 6.0697770095430315e-05, + 5.777485785074532e-05, + 5.6785840570228174e-05, + 5.4598363931290805e-05, + 5.305247395881452e-05, + 4.9763068091124296e-05, + 4.801086470251903e-05, + 4.712749432655983e-05, + 4.585064380080439e-05, + 4.453218571143225e-05, + 4.242090290063061e-05, + 4.1277395212091506e-05, + 3.936669236281887e-05, + 3.8849284464959055e-05, + 3.811750138993375e-05, + 3.734420533874072e-05, + 3.679551446111873e-05, + 3.4891483664978296e-05, + 3.297825242043473e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021292950441420544, + "first_to_last_ratio": 1.6819067487661699, + "imbalance_abs_log_ratio": 0.5199281193278776, + "last_quartile_mean": 0.0001266000654140953, + "mean": 0.0001612693722563563, + "normalized": [ + 1.5193752902631465, + 1.4431842082185322, + 1.3854135927323608, + 1.3303138916593422, + 1.2819858609596173, + 1.2449680699964556, + 1.2025089225516594, + 1.1549255897117912, + 1.1161361523436344, + 1.0820631347545688, + 1.0617857510438193, + 1.0341003696475617, + 1.0077971864601107, + 0.9864740856164548, + 0.9634556759892069, + 0.9447446572547388, + 0.9240188125766456, + 0.9078710595764491, + 0.88634376010722, + 0.8724890232241534, + 0.8607832896405347, + 0.849628883322604, + 0.8359853832855236, + 0.8234683594489096, + 0.8139294954165563, + 0.8040629029670215, + 0.7969416676465471, + 0.7902390259734583, + 0.7809695092222964, + 0.7743360767166025, + 0.7639958481267083, + 0.7557044635457686 + ], + "peak_layer": 1, + "peak_normalized": 1.5193752902631465, + "population_cv": 0.21463654613558777, + "rest_mean": 0.00016614451305940747, + "spike_contrast": 0.8122062500591205, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013494361191987992, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024502869928255677, + 0.00023274141130968928, + 0.00022342478041537106, + 0.0002145388862118125, + 0.00020674505503848195, + 0.00020077521912753582, + 0.00019392785907257348, + 0.00018625412485562265, + 0.00017999857664108276, + 0.00017450364248361439, + 0.00017123352154158056, + 0.00016676871746312827, + 0.0001625268196221441, + 0.00015908805653452873, + 0.0001553758920636028, + 0.0001523583778180182, + 0.0001490159338572994, + 0.000146411795867607, + 0.00014294010179582983, + 0.0001407057570759207, + 0.00013881798076909035, + 0.00013701911666430533, + 0.0001348188379779458, + 0.0001328002254012972, + 0.00013126189878676087, + 0.0001296707196161151, + 0.00012852228246629238, + 0.00012744135165121406, + 0.0001259464625036344, + 0.00012487669300753623, + 0.00012320913083385676, + 0.00012187198444735259 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016320452232321259, + "first_to_last_ratio": 1.2889611434881496, + "imbalance_abs_log_ratio": 0.25383657880769345, + "last_quartile_mean": 0.0001266171002498595, + "mean": 0.0001437355554116948, + "normalized": [ + 1.188749445631, + 1.1713423911172207, + 1.1549430883999143, + 1.1417765084687417, + 1.1273283201397843, + 1.1145504039766105, + 1.0995991496765185, + 1.0853099095412984, + 1.0687879028680622, + 1.0554743211679045, + 1.0473286818228558, + 1.0363809984280803, + 1.0249947395076546, + 1.015899258153915, + 1.005968438235508, + 0.9970397006238613, + 0.9860146691908593, + 0.977410712347053, + 0.964005406402611, + 0.9549550784847994, + 0.9474810711889571, + 0.940343993567483, + 0.9294782140508817, + 0.9176125795270506, + 0.909513915953909, + 0.8990879268536105, + 0.8931112722873075, + 0.8859642730588357, + 0.8770245001904262, + 0.8704731011044752, + 0.8603223859779169, + 0.851727642054894 + ], + "peak_layer": 1, + "peak_normalized": 1.188749445631, + "population_cv": 0.0978236443065259, + "rest_mean": 0.0001456284474072702, + "spike_contrast": 0.9168122095142401, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001335139386355877, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00017086556181311607, + 0.00016836354916449636, + 0.00016600638628005981, + 0.00016411388060078025, + 0.0001620371622266248, + 0.00016020052134990692, + 0.0001580514945089817, + 0.00015599762264173478, + 0.00015362282283604145, + 0.0001517091877758503, + 0.00015053836978040636, + 0.0001489647984271869, + 0.00014732818817719817, + 0.0001460208441130817, + 0.00014459343219641596, + 0.0001433100551366806, + 0.00014172536612022668, + 0.00014048867160454392, + 0.00013856185250915587, + 0.00013726099859923124, + 0.0001361867180094123, + 0.00013516086619347334, + 0.00013359906733967364, + 0.0001318935537710786, + 0.0001307294878643006, + 0.00012923090253025293, + 0.00012837184476666152, + 0.0001273445668630302, + 0.00012605960364453495, + 0.0001251179346581921, + 0.00012365891598165035, + 0.00012242354569025338 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0004412600719660986, + "first_to_last_ratio": 3.098863205201153, + "imbalance_abs_log_ratio": 1.131035336237652, + "last_quartile_mean": 0.00014239417578210123, + "mean": 0.0002548141455918085, + "normalized": [ + 2.1785577639630773, + 1.9956115105027457, + 1.9014578504944815, + 1.7829105267225334, + 1.6475854563109211, + 1.565091756282257, + 1.45799103716316, + 1.3243443696030393, + 1.2370613678820481, + 1.1609137324491365, + 1.1300355744023234, + 1.0677196003316773, + 1.0048909110245325, + 0.948494062464312, + 0.8853730946340664, + 0.8440976906413151, + 0.7906677717627395, + 0.747069444517614, + 0.6991826523616375, + 0.6712814178889162, + 0.6509454457824981, + 0.6314362826751919, + 0.6113540983289655, + 0.5954000197507868, + 0.5831757760496624, + 0.5733601746441271, + 0.5653129211221045, + 0.559900286080882, + 0.5532114024202266, + 0.5496528914136888, + 0.545014697375157, + 0.5408984129541747 + ], + "peak_layer": 1, + "peak_normalized": 2.1785577639630773, + "population_cv": 0.4822008017525282, + "rest_mean": 0.00027300682214209465, + "spike_contrast": 0.573515676244785, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015657369222026317, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005551273352466524, + 0.0005085100419819355, + 0.000484518357552588, + 0.00045431082253344357, + 0.00041982808033935726, + 0.0003988075186498463, + 0.0003715167404152453, + 0.0003374616790097207, + 0.000315220735501498, + 0.0002958172408398241, + 0.0002879490493796766, + 0.0002720700576901436, + 0.0002560604189056903, + 0.00024168970412574708, + 0.00022560558863915503, + 0.00021508803183678538, + 0.0002014733327087015, + 0.0001903638622025028, + 0.0001781616301741451, + 0.0001710520009510219, + 0.00016587010759394616, + 0.0001608988968655467, + 0.00015578167221974581, + 0.00015171634731814265, + 0.00014860143710393459, + 0.00014610028301831335, + 0.0001440497289877385, + 0.0001426705130143091, + 0.00014096609083935618, + 0.0001400593318976462, + 0.0001388774544466287, + 0.00013782856694888324 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027412420604377985, + 0.002683233004063368, + 0.002711885841563344, + 0.0027249811682850122, + 0.0027012599166482687, + 0.002700963756069541, + 0.0026683746837079525, + 0.002738022478297353, + 0.0027370047755539417, + 0.002654933836311102, + 0.002663377905264497, + 0.002684312639757991, + 0.0027356327045708895, + 0.002652956871315837, + 0.002679797587916255, + 0.002698024269193411, + 0.0026484597474336624, + 0.0026680538430809975, + 0.002654345240443945, + 0.002591830212622881, + 0.002620960585772991, + 0.002651234855875373, + 0.0026270023081451654, + 0.002615687670186162, + 0.00258000404573977, + 0.002626793459057808, + 0.002573782578110695, + 0.002592637902125716, + 0.002581534907221794, + 0.002551236655563116, + 0.002563327783718705, + 0.002589041367173195 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024784023116808385, + "first_to_last_ratio": 4.996872150813687, + "imbalance_abs_log_ratio": 1.6088121468463858, + "last_quartile_mean": 4.959907391821616e-05, + "mean": 0.00012175017718618619, + "normalized": [ + 2.94187592163941, + 2.5796687009574604, + 2.293010133276413, + 2.014581184592976, + 1.8111575385173901, + 1.6842185280489494, + 1.5559062012549134, + 1.4047478191310576, + 1.2749623745204015, + 1.1699150302730006, + 1.0919472369387608, + 1.0017565099771182, + 0.923449750141626, + 0.8652102016053241, + 0.794546741948017, + 0.743905553038253, + 0.6942834150416333, + 0.6562793060663384, + 0.6082995898528493, + 0.5744438165059796, + 0.5512557468449423, + 0.5265581646918154, + 0.5020472770644135, + 0.47690127545745264, + 0.4572061982209032, + 0.43893553328289897, + 0.42961039869683454, + 0.41603121061550513, + 0.3999104909112459, + 0.3911954601706125, + 0.3717234123738856, + 0.35445927834161867 + ], + "peak_layer": 1, + "peak_normalized": 2.94187592163941, + "population_cv": 0.6874421460933116, + "rest_mean": 0.0001329603533290706, + "spike_contrast": 0.46040210094136547, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.121522601461038e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000358173914719373, + 0.0003140751214232296, + 0.00027917439001612365, + 0.0002452756161801517, + 0.00022050875122658908, + 0.0002050539042102173, + 0.00018943185568787158, + 0.0001710282958811149, + 0.00015522689500357956, + 0.00014243736222852021, + 0.00013294476957526058, + 0.00012196403258712962, + 0.00011243017070228234, + 0.0001053394953487441, + 9.673620661487803e-05, + 9.057063289219514e-05, + 8.45291287987493e-05, + 7.990212179720402e-05, + 7.406058284686878e-05, + 6.993863644311205e-05, + 6.711548485327512e-05, + 6.410854985006154e-05, + 6.112434493843466e-05, + 5.806281478726305e-05, + 5.566493564401753e-05, + 5.3440478950506076e-05, + 5.23051421623677e-05, + 5.0651873607421294e-05, + 4.8689173127058893e-05, + 4.762811659020372e-05, + 4.525739132077433e-05, + 4.315547994337976e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002960815872938838, + "first_to_last_ratio": 2.3434901925615312, + "imbalance_abs_log_ratio": 0.8516413534183918, + "last_quartile_mean": 0.00012634214908757713, + "mean": 0.00018583460405352525, + "normalized": [ + 2.1691817718266027, + 1.9356770461635995, + 1.7512361881653136, + 1.5771501673188, + 1.4517248962140685, + 1.3691574500653627, + 1.2898282592750647, + 1.2020702391870204, + 1.1274711846152252, + 1.06796916249687, + 1.023464646242978, + 0.9751293351351744, + 0.9332415616222968, + 0.9026335726010739, + 0.8668647658197504, + 0.8407791679025106, + 0.8171800119835181, + 0.7975248027052753, + 0.7770671945621679, + 0.7612080123915693, + 0.7492445405290613, + 0.7365475788483447, + 0.7249228575947372, + 0.7138182430319103, + 0.7048361054118131, + 0.6962681268232633, + 0.6911654119184109, + 0.6833348384348117, + 0.6760742523950837, + 0.6702828385523486, + 0.6624247014500871, + 0.654521068715886 + ], + "peak_layer": 1, + "peak_normalized": 2.1691817718266027, + "population_cv": 0.3943796352106793, + "rest_mean": 0.00019526833030222742, + "spike_contrast": 0.6908057343541223, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013489248231053353, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004031090356875211, + 0.0003597157774493098, + 0.0003254402836319059, + 0.0002930890768766403, + 0.00026978072128258646, + 0.00025443683261983097, + 0.00023969472385942936, + 0.00022338624694384634, + 0.00020952316117472947, + 0.0001984656264539808, + 0.0001901951472973451, + 0.00018121277389582247, + 0.0001734285760903731, + 0.00016774055256973952, + 0.0001610934705240652, + 0.0001562458637636155, + 0.00015186032396741211, + 0.00014820770593360066, + 0.00014440597442444414, + 0.00014145878958515823, + 0.00013923556252848357, + 0.0001368760276818648, + 0.00013471575221046805, + 0.00013265213056001812, + 0.00013098293857183307, + 0.00012939071166329086, + 0.00012844245065934956, + 0.00012698725913651288, + 0.00012563799100462347, + 0.0001245617459062487, + 0.00012310143210925162, + 0.00012163266364950687 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.26256811060011387, + "first_to_last_ratio": 2.3434901455089974, + "imbalance_abs_log_ratio": 0.8516413333404181, + "last_quartile_mean": 0.11204148270189762, + "mean": 0.16479998547583818, + "normalized": [ + 2.1691817568110046, + 1.935676923537455, + 1.7512361570514487, + 1.5771502517930829, + 1.4517248703109182, + 1.369157420355369, + 1.2898282606548555, + 1.2020702692325518, + 1.1274712340116186, + 1.0679692103147351, + 1.0234646460792782, + 0.975129361875517, + 0.9332415430463463, + 0.9026335787242323, + 0.8668647305888977, + 0.8407791967519535, + 0.8171800231493583, + 0.7975247749486774, + 0.7770672329681078, + 0.7612079835803747, + 0.7492445111174014, + 0.7365475542475244, + 0.7249228843083022, + 0.7138182179223472, + 0.7048361082067401, + 0.6962681206044983, + 0.6911654218524023, + 0.6833348067031066, + 0.676074287607254, + 0.6702828621875807, + 0.6624247142471237, + 0.6545210852099367 + ], + "peak_layer": 1, + "peak_normalized": 2.1691817568110046, + "population_cv": 0.3943796257753467, + "rest_mean": 0.17316590856622766, + "spike_contrast": 0.6908057237027376, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11962400078773498, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.35748112201690674, + 0.3189995288848877, + 0.28860369324684143, + 0.2599143385887146, + 0.2392442375421524, + 0.22563712298870087, + 0.2125636786222458, + 0.19810116291046143, + 0.1858072429895401, + 0.17600131034851074, + 0.16866695880889893, + 0.16070130467414856, + 0.1537981927394867, + 0.14875400066375732, + 0.1428592950105667, + 0.13856039941310883, + 0.13467125594615936, + 0.13143207132816315, + 0.12806066870689392, + 0.12544706463813782, + 0.12347548454999924, + 0.12138302624225616, + 0.11946728080511093, + 0.11763723194599152, + 0.11615698039531708, + 0.11474497616291046, + 0.11390405148267746, + 0.11261356621980667, + 0.11141703277826309, + 0.11046260595321655, + 0.10916758328676224, + 0.10786506533622742 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001010398200378404, + "first_to_last_ratio": 2.701478580108855, + "imbalance_abs_log_ratio": 0.9937992453824708, + "last_quartile_mean": 3.740167358046165e-05, + "mean": 6.270599271829269e-05, + "normalized": [ + 1.9907220064146427, + 1.8444741506775382, + 1.742884685905627, + 1.6239253294262157, + 1.5436270879110234, + 1.4787029723792133, + 1.3841644782908762, + 1.2821103017907618, + 1.1824374683113494, + 1.1042100658007115, + 1.0908679078034196, + 1.0498354286471192, + 0.9920033594422748, + 0.9660018389944104, + 0.9202838356691797, + 0.8989347700163286, + 0.8686518679586676, + 0.8410583182307008, + 0.7938538755561205, + 0.7653997098760733, + 0.7511903220519177, + 0.7265360595007238, + 0.7106247592487483, + 0.6758121071782397, + 0.6556340454607258, + 0.6278169583683687, + 0.6205082786340235, + 0.6073860029650574, + 0.5911469097961847, + 0.5867782134359579, + 0.5577710930141501, + 0.5246457912436486 + ], + "peak_layer": 1, + "peak_normalized": 1.9907220064146427, + "population_cv": 0.40468494452628273, + "rest_mean": 6.614368087159366e-05, + "spike_contrast": 0.6673725457789733, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4142476690467446e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001248301996383816, + 0.00011565958266146481, + 0.0001092893144232221, + 0.00010182984988205135, + 9.679466893430799e-05, + 9.272353781852871e-05, + 8.679540769662708e-05, + 8.039599924813956e-05, + 7.414591527776793e-05, + 6.92405883455649e-05, + 6.84039550833404e-05, + 6.583097274415195e-05, + 6.220455543370917e-05, + 6.0574104281840846e-05, + 5.770731149823405e-05, + 5.636859714286402e-05, + 5.446967770694755e-05, + 5.273939677863382e-05, + 4.9779395340010524e-05, + 4.799514863407239e-05, + 4.710413486463949e-05, + 4.5558164856629446e-05, + 4.456043097889051e-05, + 4.237746907165274e-05, + 4.111218368052505e-05, + 3.936788561986759e-05, + 3.890958760166541e-05, + 3.808674227911979e-05, + 3.7068453821120784e-05, + 3.679451037896797e-05, + 3.497559009701945e-05, + 3.2898435165407136e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021103879953443538, + "first_to_last_ratio": 1.6754332028061545, + "imbalance_abs_log_ratio": 0.5160717603794217, + "last_quartile_mean": 0.00012596073611348402, + "mean": 0.0001601102544555033, + "normalized": [ + 1.5150372860566723, + 1.439510593902249, + 1.3841055212355642, + 1.3272119947749268, + 1.2794362361966427, + 1.2428436489609036, + 1.2018556832108258, + 1.1546727794959446, + 1.1147996317367452, + 1.0814138049096294, + 1.0614905005088058, + 1.0346901524019803, + 1.0074919017055521, + 0.9875011592687566, + 0.9623166883068708, + 0.9431047552100263, + 0.9243559814545742, + 0.9073018837351503, + 0.8887802361951885, + 0.87382889642758, + 0.8622859040817829, + 0.849391060863955, + 0.8374811583413132, + 0.8253926631762389, + 0.8155827011133436, + 0.8058320881575772, + 0.7999867009830005, + 0.7910419816457831, + 0.7825317922950641, + 0.7756782882808636, + 0.7662211489344684, + 0.7568251764320209 + ], + "peak_layer": 1, + "peak_normalized": 1.5150372860566723, + "population_cv": 0.21304026877727222, + "rest_mean": 0.00016491277050658095, + "spike_contrast": 0.8136220583009953, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013417666777968407, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024257300538010895, + 0.00023048040748108178, + 0.0002216094871982932, + 0.00021250025019980967, + 0.0002048508613370359, + 0.00019899201288353652, + 0.0001924294192576781, + 0.00018487495253793895, + 0.00017849085270427167, + 0.00017314543947577477, + 0.00016995551413856447, + 0.00016566450358368456, + 0.00016130978474393487, + 0.00015810906188562512, + 0.00015407676983159035, + 0.00015100074233487248, + 0.00014799887139815837, + 0.0001452683354727924, + 0.00014230282977223396, + 0.00013990896695759147, + 0.00013806081551592797, + 0.00013599621888715774, + 0.00013408932136371732, + 0.00013215382932685316, + 0.00013058315380476415, + 0.00012902198068331927, + 0.00012808607425540686, + 0.0001266539329662919, + 0.00012529136438388377, + 0.00012419404811225832, + 0.00012267986312508583, + 0.00012117547157686204 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016164849512279034, + "first_to_last_ratio": 1.282986844747637, + "imbalance_abs_log_ratio": 0.24919083207168544, + "last_quartile_mean": 0.00012599388355738483, + "mean": 0.00014267631695474847, + "normalized": [ + 1.1849632230478688, + 1.1695181917643926, + 1.1537907432295644, + 1.136648454640294, + 1.1262413432306055, + 1.112705815659591, + 1.0977204646042988, + 1.082200264843305, + 1.06813080649915, + 1.0535985061590996, + 1.046252801833764, + 1.0365714691025223, + 1.024670983408149, + 1.0169605538784676, + 1.0046472025165352, + 0.995758555548206, + 0.9858673222379531, + 0.9760103584099877, + 0.9653106304982666, + 0.9567951741123486, + 0.9498417329681972, + 0.9401737612553472, + 0.9304537733640251, + 0.9205681496416466, + 0.911480005330001, + 0.9022600866995591, + 0.8969925816929462, + 0.8873573495747816, + 0.8789659160771423, + 0.8714788501373707, + 0.8627683840784355, + 0.8532965439561776 + ], + "peak_layer": 1, + "peak_normalized": 1.1849632230478688, + "population_cv": 0.09608600673281108, + "rest_mean": 0.00014451252193086676, + "spike_contrast": 0.918680321330362, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013276081008370966, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016906618839129806, + 0.00016686254821252078, + 0.00016461861378047615, + 0.00016217281518038362, + 0.00016068796685431153, + 0.00015875676763243973, + 0.0001566187129355967, + 0.00015440434799529612, + 0.00015239696949720383, + 0.0001503235544078052, + 0.00014927549636922777, + 0.00014789419947192073, + 0.00014619628200307488, + 0.0001450961863156408, + 0.00014333936269395053, + 0.00014207116328179836, + 0.00014065991854295135, + 0.00013925356324762106, + 0.00013772696547675878, + 0.0001365120115224272, + 0.00013551992014981806, + 0.00013414052955340594, + 0.00013275371748022735, + 0.0001313432730967179, + 0.00013004661013837904, + 0.00012873114610556513, + 0.0001279795978916809, + 0.00012660487846005708, + 0.0001254076196346432, + 0.00012433939264155924, + 0.00012309661542531103, + 0.00012174520816188306 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00043634913890855387, + "first_to_last_ratio": 3.0823770886184128, + "imbalance_abs_log_ratio": 1.125701081333854, + "last_quartile_mean": 0.00014156254292174708, + "mean": 0.0002526328457861382, + "normalized": [ + 2.161262991989335, + 1.9867412804887739, + 1.8904526133838582, + 1.7788259407438674, + 1.6404609794852394, + 1.5723911781894129, + 1.459249713339863, + 1.3282687448916313, + 1.2383315143134188, + 1.1700416770361293, + 1.131363959578303, + 1.0685850444189446, + 1.0069989316428685, + 0.9451411723514572, + 0.8866050347634621, + 0.8407661253267313, + 0.7895843736134068, + 0.7479926194272515, + 0.7047300318080335, + 0.6729704299307414, + 0.6547401608613147, + 0.6315820650248338, + 0.6127131155987415, + 0.5974089218754778, + 0.5839699651029305, + 0.5735248801032955, + 0.5690230706448541, + 0.5599717580649735, + 0.5552403508520525, + 0.5513694456095706, + 0.5470291494736481, + 0.5426627600655782 + ], + "peak_layer": 1, + "peak_normalized": 2.161262991989335, + "population_cv": 0.4790543297146424, + "rest_mean": 0.0002705939723118381, + "spike_contrast": 0.5751893185853856, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015564276254735886, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005460060201585293, + 0.0005019161035306752, + 0.0004775904235430062, + 0.00044938985956832767, + 0.0004144343256484717, + 0.0003972376580350101, + 0.000368654407793656, + 0.00033556431299075484, + 0.00031284321448765695, + 0.000295590958558023, + 0.0002858196967281401, + 0.0002699596807360649, + 0.0002544010058045387, + 0.00023877370404079556, + 0.0002239855530206114, + 0.00021240513888187706, + 0.00019947494729422033, + 0.0001889675040729344, + 0.0001780379534466192, + 0.00017001443484332412, + 0.00016540887008886784, + 0.00015955837443470955, + 0.00015479145804420114, + 0.00015092511603143066, + 0.00014752999413758516, + 0.00014489122258964926, + 0.00014375391765497625, + 0.00014146725879982114, + 0.00014027194993104786, + 0.00013929403212387115, + 0.00013819753075949848, + 0.00013709443737752736 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0023513585329055786, + 0.004408807028084993, + 0.005783648230135441, + 0.006751363631337881, + 0.002628475893288851, + 0.004723479505628347, + 0.006024772301316261, + 0.007445912808179855, + 0.003561305580660701, + 0.005543973762542009, + 0.006855612155050039, + 0.007876631803810596, + 0.0035210708156228065, + 0.0058924350887537, + 0.007390674669295549, + 0.008790239691734314, + 0.004162859171628952, + 0.006355749908834696, + 0.007784162648022175, + 0.00937180407345295, + 0.004737259820103645, + 0.006999304983764887, + 0.008728929795324802, + 0.010217384435236454, + 0.005580419674515724, + 0.008008211851119995, + 0.010209310799837112, + 0.012040933594107628, + 0.005235649179667234, + 0.007745372597128153, + 0.010180563665926456, + 0.011906592175364494 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024914481400628574, + "first_to_last_ratio": 5.005367240493075, + "imbalance_abs_log_ratio": 1.6105107847992817, + "last_quartile_mean": 4.977553135177004e-05, + "mean": 0.0001223312358433759, + "normalized": [ + 2.9462138255793877, + 2.588435651553335, + 2.2854731426364707, + 2.0157214492628985, + 1.8168708763555168, + 1.6875649797632875, + 1.5554268829788211, + 1.3974214212146046, + 1.275680469442624, + 1.1684152736282016, + 1.0925056707858531, + 1.0000365768115411, + 0.9220243844233116, + 0.8633153808206192, + 0.7954940059538663, + 0.747131175135411, + 0.6955592946052511, + 0.6567700236831091, + 0.6067846463225012, + 0.5734077695641915, + 0.5496986650820618, + 0.5277889932643854, + 0.50090777460584, + 0.47622023530363405, + 0.457550957846772, + 0.43796965221258716, + 0.42825768814458814, + 0.4157111150450942, + 0.4005890169587395, + 0.39033503431894295, + 0.37039430607646523, + 0.3543236606200867 + ], + "peak_layer": 1, + "peak_normalized": 2.9462138255793877, + "population_cv": 0.6883836085455175, + "rest_mean": 0.00013360307773971862, + "spike_contrast": 0.46004396487681354, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.14632896031253e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036041397834196687, + 0.00031664653215557337, + 0.0002795847540255636, + 0.0002465856960043311, + 0.0002222600596724078, + 0.0002064419095404446, + 0.00019027729285880923, + 0.00017094828945118934, + 0.00015605556836817414, + 0.00014293368440121412, + 0.00013364756887312979, + 0.00012233571032993495, + 0.00011279238242423162, + 0.00010561043745838106, + 9.73137648543343e-05, + 9.139747999142855e-05, + 8.508862811140716e-05, + 8.0343488662038e-05, + 7.422871567541733e-05, + 7.014568109298125e-05, + 6.724531704094261e-05, + 6.456507981056347e-05, + 6.127666711108759e-05, + 5.8256609918316826e-05, + 5.597277413471602e-05, + 5.357736881705932e-05, + 5.238929225015454e-05, + 5.0854454457294196e-05, + 4.9004549509845674e-05, + 4.775016714120284e-05, + 4.531079321168363e-05, + 4.334485129220411e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00029868123783671763, + "first_to_last_ratio": 2.352020548935129, + "imbalance_abs_log_ratio": 0.8552747667903616, + "last_quartile_mean": 0.00012698921273113228, + "mean": 0.0001871975605354237, + "normalized": [ + 2.1745956556501205, + 1.943389916860048, + 1.7490167059616495, + 1.579801534641211, + 1.4557872843351107, + 1.3727714128797337, + 1.290211227528517, + 1.198748061313539, + 1.1282731192017241, + 1.0677025479901412, + 1.0238711728275764, + 0.9743826669000581, + 0.9332092274716193, + 0.901273964826617, + 0.8680163418407979, + 0.8423074677807364, + 0.8168973316425937, + 0.7977426572048737, + 0.7748904888155362, + 0.7599280177062464, + 0.7477886704872306, + 0.7367391764207782, + 0.7235805599766263, + 0.7121146362854748, + 0.7033711688788506, + 0.6946931548498627, + 0.6884857337520279, + 0.6826539310873735, + 0.6746941940291191, + 0.6690858034871798, + 0.660454275348722, + 0.6535218920183048 + ], + "peak_layer": 1, + "peak_normalized": 2.1745956556501205, + "population_cv": 0.39609817473730136, + "rest_mean": 0.00019674051630621156, + "spike_contrast": 0.6895661448911525, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013566559937316924, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004070790018886328, + 0.00036379785160534084, + 0.00032741166069172323, + 0.00029573499341495335, + 0.00027251982828602195, + 0.00025697945966385305, + 0.00024152439436875284, + 0.000224402712774463, + 0.00021120997553225607, + 0.00019987131236121058, + 0.00019166618585586548, + 0.0001824020582716912, + 0.00017469449085183442, + 0.00016871628758963197, + 0.00016249054169747978, + 0.00015767790318932384, + 0.00015292118769139051, + 0.0001493354793637991, + 0.0001450576091883704, + 0.00014225667109712958, + 0.00013998421491123736, + 0.00013791577657684684, + 0.00013545251567848027, + 0.00013330612273421139, + 0.00013166936696507037, + 0.00013004486390855163, + 0.00012888284982182086, + 0.00012779115058947355, + 0.00012630110722966492, + 0.00012525123020168394, + 0.00012363542919047177, + 0.00012233770394232124 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.2648735139518976, + "first_to_last_ratio": 2.3520205903150884, + "imbalance_abs_log_ratio": 0.855274784383728, + "last_quartile_mean": 0.1126153040677309, + "mean": 0.1660086705815047, + "normalized": [ + 2.174595632268317, + 1.943389981095013, + 1.7490168198184282, + 1.5798016086666014, + 1.4557873374041121, + 1.3727712901980273, + 1.2902112306135503, + 1.1987480127171795, + 1.1282731510481394, + 1.0677025559476454, + 1.0238711946273023, + 0.9743826246747078, + 0.9332091864770874, + 0.9012739856841682, + 0.8680163310272434, + 0.8423074257801569, + 0.8168973360433722, + 0.7977426245274992, + 0.7748904637775177, + 0.7599280557788093, + 0.7477886868929706, + 0.7367391104705062, + 0.7235805912981054, + 0.7121146568868953, + 0.7033711391232357, + 0.6946931471398464, + 0.6884857463566807, + 0.682653951912613, + 0.6746941853139277, + 0.6690857619715078, + 0.6604542663637077, + 0.6535219080951254 + ], + "peak_layer": 1, + "peak_normalized": 2.174595632268317, + "population_cv": 0.39609818748801673, + "rest_mean": 0.1744714594549603, + "spike_contrast": 0.6895661390160055, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12030961066484451, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.36100172996520996, + 0.32261958718299866, + 0.2903519570827484, + 0.262260764837265, + 0.2416733205318451, + 0.2278919368982315, + 0.21418625116348267, + 0.19900256395339966, + 0.18730312585830688, + 0.17724788188934326, + 0.1699714958667755, + 0.16175596415996552, + 0.1549208164215088, + 0.14961929619312286, + 0.14409823715686798, + 0.1398303359746933, + 0.13561204075813293, + 0.13243219256401062, + 0.12863853573799133, + 0.12615464627742767, + 0.12413940578699112, + 0.12230508029460907, + 0.12012065201997757, + 0.11821720749139786, + 0.11676570773124695, + 0.11532508581876755, + 0.11429460346698761, + 0.11332647502422333, + 0.1120050847530365, + 0.1110740378499031, + 0.10964113473892212, + 0.10849030315876007 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010120356273546349, + "first_to_last_ratio": 2.7021644745482813, + "imbalance_abs_log_ratio": 0.9940531090948486, + "last_quartile_mean": 3.7452776723512216e-05, + "mean": 6.280785908074904e-05, + "normalized": [ + 1.9944873163456722, + 1.8472744197374709, + 1.7373237819064158, + 1.6248552595134786, + 1.5471235814302835, + 1.4758134067763151, + 1.3807453375978929, + 1.2829373095670376, + 1.18406084586495, + 1.104394561286083, + 1.0901943179184503, + 1.04561090576924, + 0.9888677388041794, + 0.9664019181449638, + 0.9198712145053942, + 0.9041143079769479, + 0.8692998477179782, + 0.8446721332610243, + 0.7923141515826202, + 0.7644051939108963, + 0.7503405467650113, + 0.7300062685920449, + 0.7090277780404898, + 0.6754004942706008, + 0.6572075474610624, + 0.6267776932652476, + 0.618543683654507, + 0.6069002486680706, + 0.5945805123210514, + 0.585842809953234, + 0.5555319861870183, + 0.5250728812043677 + ], + "peak_layer": 1, + "peak_normalized": 1.9944873163456722, + "population_cv": 0.404835319326031, + "rest_mean": 6.624604820562699e-05, + "spike_contrast": 0.6678381428743124, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.424163780640811e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001252694783033803, + 0.00011602335143834352, + 0.00010911758727161214, + 0.00010205368016613647, + 9.717151988297701e-05, + 9.269268048228696e-05, + 8.672165859024972e-05, + 8.057854574872181e-05, + 7.436832675011829e-05, + 6.936465797480196e-05, + 6.847277109045535e-05, + 6.567258242284879e-05, + 6.210866558831185e-05, + 6.069763549021445e-05, + 5.7775141613092273e-05, + 5.678548404830508e-05, + 5.459886233438738e-05, + 5.305204831529409e-05, + 4.9763555580284446e-05, + 4.801065369974822e-05, + 4.7127283323789015e-05, + 4.585013084579259e-05, + 4.4532516767503694e-05, + 4.242045906721614e-05, + 4.127779902773909e-05, + 3.936656503356062e-05, + 3.884940451825969e-05, + 3.811810529441573e-05, + 3.734432903002016e-05, + 3.679553265101276e-05, + 3.489177470328286e-05, + 3.297870352980681e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0002129299391526729, + "first_to_last_ratio": 1.6819035009332732, + "imbalance_abs_log_ratio": 0.5199261882838202, + "last_quartile_mean": 0.00012660056836466538, + "mean": 0.0001612697526525153, + "normalized": [ + 1.5193801883639617, + 1.4431946098051458, + 1.385412670946279, + 1.330312377973009, + 1.2819779644638467, + 1.2449644115522398, + 1.2025034693572292, + 1.15492638462487, + 1.1161420015875916, + 1.082061394531008, + 1.0617850512168836, + 1.0341029835254725, + 1.0077924632434987, + 0.9864749169321705, + 0.9634524108641764, + 0.9447429702319522, + 0.9240217763435827, + 0.9078672036949645, + 0.8863448276060427, + 0.8724873261680857, + 0.860780266694715, + 0.8496270597212868, + 0.835986840267667, + 0.8234693045553775, + 0.8139348844586144, + 0.8040606454444867, + 0.7969452920911189, + 0.7902469974302787, + 0.780967396404848, + 0.7743340697791917, + 0.7639952190801171, + 0.7557046210402876 + ], + "peak_layer": 1, + "peak_normalized": 1.5193801883639617, + "population_cv": 0.21463676643173402, + "rest_mean": 0.00016614488736700473, + "spike_contrast": 0.8122069076744362, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001349440251942724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024503006716258824, + 0.00023274363775271922, + 0.000223425158765167, + 0.0002145391481462866, + 0.00020674426923505962, + 0.000200775102712214, + 0.00019392743706703186, + 0.00018625469238031656, + 0.00017999994452111423, + 0.00017450377345085144, + 0.00017123381257988513, + 0.00016676953237038106, + 0.00016252644127234817, + 0.00015908856585156173, + 0.0001553757319925353, + 0.00015235846512950957, + 0.0001490167633164674, + 0.00014641151938121766, + 0.00014294061111286283, + 0.00014070581528358161, + 0.00013881782069802284, + 0.00013701914576813579, + 0.00013481939095072448, + 0.00013280069106258452, + 0.00013126307749189436, + 0.00012967066140845418, + 0.0001285231701331213, + 0.00012744293780997396, + 0.0001259464188478887, + 0.00012487666390370578, + 0.00012320932000875473, + 0.00012187229731353 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016320142822223715, + "first_to_last_ratio": 1.2889451135618213, + "imbalance_abs_log_ratio": 0.25382414241573026, + "last_quartile_mean": 0.0001266162744286703, + "mean": 0.00014373415069712792, + "normalized": [ + 1.1884743462891088, + 1.1713432082551107, + 1.1549462763142666, + 1.141829986160868, + 1.1273728485863819, + 1.1144091299418613, + 1.0997826147068885, + 1.0853573683237072, + 1.068690221812727, + 1.0554384700276902, + 1.0473609881004726, + 1.0364070219594264, + 1.025017513262844, + 1.015761575885501, + 1.006046000385858, + 0.9971413723222552, + 0.9860728003762654, + 0.9774913363845525, + 0.9641287247440273, + 0.9550391277708558, + 0.9473133601248869, + 0.9403037775179377, + 0.9293980025104189, + 0.9176260020019316, + 0.9094819029195437, + 0.8991813518536786, + 0.8931890476284186, + 0.8860794380468366, + 0.8769637206693581, + 0.8703930215998912, + 0.8602546600212342, + 0.8517047834951949 + ], + "peak_layer": 1, + "peak_normalized": 1.1884743462891088, + "population_cv": 0.09781347741247146, + "rest_mean": 0.00014562865706264146, + "spike_contrast": 0.9167413819240867, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013350381632335485, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00017082435078918934, + 0.00016836202121339738, + 0.00016600522212684155, + 0.0001641199633013457, + 0.00016204197891056538, + 0.00016017864982131869, + 0.0001580763200763613, + 0.0001560029195388779, + 0.00015360728139057755, + 0.00015170255210250616, + 0.00015054154209792614, + 0.00014896708307787776, + 0.00014733002171851695, + 0.00014599962742067873, + 0.00014460316742770374, + 0.00014332326827570796, + 0.00014173233648762107, + 0.00014049888704903424, + 0.00013857822341378778, + 0.00013727173791266978, + 0.0001361612812615931, + 0.0001351537648588419, + 0.00013358623255044222, + 0.00013189419405534863, + 0.00013072360889054835, + 0.00012924306793138385, + 0.00012838176917284727, + 0.00012735987547785044, + 0.00012604963558260351, + 0.00012510520173236728, + 0.00012364797294139862, + 0.00012241906370036304 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00044128536683274433, + "first_to_last_ratio": 3.0989416798064484, + "imbalance_abs_log_ratio": 1.1310606595922121, + "last_quartile_mean": 0.0001423987323505571, + "mean": 0.0002548205006860371, + "normalized": [ + 2.1788974669107217, + 1.9948922239742408, + 1.9013861017158324, + 1.7832746019711707, + 1.6474456862757656, + 1.56528080717771, + 1.458232784418858, + 1.3245892215128081, + 1.2376166576486232, + 1.160533346247024, + 1.1299807802432356, + 1.0675072615016112, + 1.0048234765069712, + 0.9484981041710848, + 0.8855756138701967, + 0.8439855543529297, + 0.7905268728467926, + 0.7470509271970999, + 0.6989608308202429, + 0.6713105901225432, + 0.6509228727076454, + 0.631461080612247, + 0.611226351598172, + 0.5954626643405772, + 0.5833123929329473, + 0.5733046444152264, + 0.5652547933675338, + 0.559916360487464, + 0.5532517426009346, + 0.5496220513891801, + 0.5450027610740823, + 0.5408933749885276 + ], + "peak_layer": 1, + "peak_normalized": 2.1788974669107217, + "population_cv": 0.4822174837001359, + "rest_mean": 0.0002730129350244102, + "spike_contrast": 0.5735309033794403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015658135525882245, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005552277434617281, + 0.0005083394353277981, + 0.0004845121584367007, + 0.0004544149269349873, + 0.00041980293462984264, + 0.0003988656389992684, + 0.00037158760824240744, + 0.0003375324886292219, + 0.00031537009635940194, + 0.0002957276883535087, + 0.00028794226818718016, + 0.00027202273486182094, + 0.00025604962138459086, + 0.00024169676180463284, + 0.0002256628213217482, + 0.00021506482153199613, + 0.0002014424535445869, + 0.00019036389130633324, + 0.00017810954886954278, + 0.00017106370069086552, + 0.0001658684923313558, + 0.00016090922872535884, + 0.00015575300494674593, + 0.00015173609426710755, + 0.00014863995602354407, + 0.00014608977653551847, + 0.00014403850946109742, + 0.00014267816732171923, + 0.00014097988605499268, + 0.00014005496632307768, + 0.0001388778764521703, + 0.00013783072063233703 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0036361459642648697, + 0.005152894649654627, + 0.006386642809957266, + 0.007284422870725393, + 0.0037505170330405235, + 0.0054016136564314365, + 0.006585344672203064, + 0.007922882214188576, + 0.0044892593286931515, + 0.006135277915745974, + 0.007316684816032648, + 0.0082859443500638, + 0.0044554853811860085, + 0.006464940495789051, + 0.007893112488090992, + 0.009196000173687935, + 0.004961193539202213, + 0.006907128728926182, + 0.008217990398406982, + 0.009676996618509293, + 0.005381465889513493, + 0.007502692751586437, + 0.0091153085231781, + 0.010543142445385456, + 0.006136673036962748, + 0.008469858206808567, + 0.010564134456217289, + 0.012301022186875343, + 0.005773921962827444, + 0.008156762458384037, + 0.010457510128617287, + 0.012218860909342766 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024784039669611957, + "first_to_last_ratio": 4.996847129630424, + "imbalance_abs_log_ratio": 1.6088071394647374, + "last_quartile_mean": 4.959935540682636e-05, + "mean": 0.0001217503483985638, + "normalized": [ + 2.9419062071052267, + 2.5796112881191693, + 2.2930860326581057, + 2.0145554032379356, + 1.8111535572920587, + 1.6841756414503508, + 1.5558994713918, + 1.4047664015764878, + 1.2749699043555138, + 1.1699192416770665, + 1.0919281315608147, + 1.001760718810264, + 0.9234449853785138, + 0.8652188455092179, + 0.7945394094365259, + 0.7439156225148, + 0.6942882953063699, + 0.6562804150522287, + 0.6082998698912725, + 0.574440737760942, + 0.551250788347098, + 0.5265559301830911, + 0.5020487822241618, + 0.47689842352430634, + 0.4572026867293029, + 0.43893712719402944, + 0.4296155316374904, + 0.41603615348749273, + 0.3999129464790325, + 0.391194073390914, + 0.3717285968379238, + 0.3544587798804922 + ], + "peak_layer": 1, + "peak_normalized": 2.9419062071052267, + "population_cv": 0.6874423943830519, + "rest_mean": 0.00013296059464833147, + "spike_contrast": 0.4603997057303065, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.121501864981837e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00035817810567095876, + 0.0003140685730613768, + 0.00027918402338400483, + 0.00024527282221242785, + 0.00022050857660360634, + 0.00020504897111095488, + 0.0001894313027150929, + 0.00017103079881053418, + 0.00015522803005296737, + 0.0001424380752723664, + 0.000132942630443722, + 0.00012196471652714536, + 0.00011242974869674072, + 0.00010534069588175043, + 9.673544991528615e-05, + 9.057198622031137e-05, + 8.452984184259549e-05, + 7.99023691797629e-05, + 7.406072109006345e-05, + 6.99383599567227e-05, + 6.711497553624213e-05, + 6.410836795112118e-05, + 6.112461414886639e-05, + 5.806254921481013e-05, + 5.566458639805205e-05, + 5.34407481609378e-05, + 5.230584065429866e-05, + 5.0652546633500606e-05, + 4.868954056291841e-05, + 4.762801472679712e-05, + 4.5258086174726486e-05, + 4.315547994337976e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00029608183103846386, + "first_to_last_ratio": 2.343484699021421, + "imbalance_abs_log_ratio": 0.8516390092453431, + "last_quartile_mean": 0.0001263425492652459, + "mean": 0.0001858347804954974, + "normalized": [ + 2.16920774571012, + 1.9356424765570717, + 1.751272112160408, + 1.5771369240348272, + 1.451727119925344, + 1.369130935687747, + 1.289826251583108, + 1.2020808437231323, + 1.127478492836466, + 1.067970341065492, + 1.0234512239081746, + 0.9751296621832141, + 0.9332398141881068, + 0.9026389800429313, + 0.8668613586828855, + 0.8407822065969367, + 0.8171794710239251, + 0.7975274126338607, + 0.777066300159633, + 0.7612061150730458, + 0.7492407752334779, + 0.7365489154759011, + 0.7249215428669998, + 0.7138135717041164, + 0.7048334785593375, + 0.6962720074755282, + 0.6911692191098191, + 0.683337870004401, + 0.6760758813563923, + 0.6702804011180051, + 0.662430102042749, + 0.6545204472768429 + ], + "peak_layer": 1, + "peak_normalized": 2.16920774571012, + "population_cv": 0.39438035124793563, + "rest_mean": 0.0001952685916977417, + "spike_contrast": 0.6908033638721538, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001348922000033781, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00040311424527317286, + 0.00035970969474874437, + 0.00032544726855121553, + 0.00029308689408935606, + 0.00026978139067068696, + 0.00025443214690312743, + 0.00023969457834027708, + 0.0002233884297311306, + 0.0002095247182296589, + 0.0001984660339076072, + 0.0001901928335428238, + 0.00018121300672646612, + 0.0001734284160193056, + 0.0001677417167229578, + 0.00016109299031086266, + 0.00015624657680746168, + 0.0001518603676231578, + 0.00014820833166595548, + 0.00014440594532061368, + 0.0001414585713064298, + 0.00013923499500378966, + 0.00013687640603166074, + 0.00013471563579514623, + 0.00013265138841234148, + 0.00013098257477395236, + 0.0001293915556743741, + 0.00012844328011851758, + 0.0001269879430765286, + 0.0001256384130101651, + 0.00012456141121219844, + 0.0001231025526067242, + 0.00012163266364950687 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.2625683266669512, + "first_to_last_ratio": 2.343484691137257, + "imbalance_abs_log_ratio": 0.8516390058810522, + "last_quartile_mean": 0.1120418356731534, + "mean": 0.16480014030821621, + "normalized": [ + 2.169207748902955, + 1.9356423730461585, + 1.7512721262865893, + 1.5771370154883375, + 1.4517271231757105, + 1.3691309069442503, + 1.2898263254837699, + 1.202080803995401, + 1.1274784933366124, + 1.0679703770106845, + 1.0234512066143264, + 0.9751297116000902, + 0.9332398524746174, + 0.9026389696372329, + 0.8668612939874407, + 0.8407822044491968, + 0.8171795266543764, + 0.7975273711861923, + 0.7770662316421936, + 0.7612060929585753, + 0.7492407781338722, + 0.7365489418998509, + 0.7249215250846839, + 0.7138135688166896, + 0.704833501980704, + 0.6962720326387369, + 0.6911692482611179, + 0.6833378266924969, + 0.6760758677045711, + 0.6702804240542598, + 0.6624300595814382, + 0.6545204702768674 + ], + "peak_layer": 1, + "peak_normalized": 2.169207748902955, + "population_cv": 0.3943803506949354, + "rest_mean": 0.17316613815448903, + "spike_contrast": 0.6908033707584418, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11962375193834304, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.35748574137687683, + 0.3189941346645355, + 0.28860989212989807, + 0.2599124014377594, + 0.23924483358860016, + 0.22563296556472778, + 0.21256355941295624, + 0.19810308516025543, + 0.18580861389636993, + 0.1760016679763794, + 0.16866490244865417, + 0.16070151329040527, + 0.15379805862903595, + 0.1487550288438797, + 0.1428588628768921, + 0.13856102526187897, + 0.13467130064964294, + 0.13143262267112732, + 0.12806062400341034, + 0.1254468709230423, + 0.12347498536109924, + 0.12138336896896362, + 0.11946716904640198, + 0.11763657629489899, + 0.11615666002035141, + 0.11474572867155075, + 0.11390478909015656, + 0.11261416971683502, + 0.11141739785671234, + 0.11046230792999268, + 0.10916856676340103, + 0.10786506533622742 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010104048214998329, + "first_to_last_ratio": 2.7014816335786884, + "imbalance_abs_log_ratio": 0.9938003756776099, + "last_quartile_mean": 3.740187639778014e-05, + "mean": 6.270627807225537e-05, + "normalized": [ + 1.9907250147127677, + 1.8444603036108247, + 1.7429056467158477, + 1.6239117898045463, + 1.5436487234069576, + 1.4786997242989472, + 1.3841625886805662, + 1.282123032540977, + 1.1824417181452251, + 1.1042125830371328, + 1.090862943651666, + 1.0498276343781445, + 0.9920003536086259, + 0.9660066676365643, + 0.9202805760437535, + 0.8989382213875909, + 0.8686500616317681, + 0.8410510099026279, + 0.7938571089189717, + 0.7653969230084752, + 0.751185801344563, + 0.7265337395686793, + 0.7106236720469997, + 0.6758027080339929, + 0.6556269427545545, + 0.6278192648368973, + 0.6205119527339269, + 0.6073894466988616, + 0.5911483388457922, + 0.586775311152339, + 0.5577767930941091, + 0.5246434037673043 + ], + "peak_layer": 1, + "peak_normalized": 1.9907250147127677, + "population_cv": 0.4046861823965828, + "rest_mean": 6.614403860617636e-05, + "spike_contrast": 0.6673673413246971, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.414237118908204e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012483095633797348, + 0.00011565924069145694, + 0.00010929112613666803, + 0.0001018294642562978, + 9.67964660958387e-05, + 9.272375609725714e-05, + 8.679568418301642e-05, + 8.039716340135783e-05, + 7.41465191822499e-05, + 6.924106128280982e-05, + 6.84039550833404e-05, + 6.583078356925398e-05, + 6.220465002115816e-05, + 6.0574682720471174e-05, + 5.770736970589496e-05, + 5.636907008010894e-05, + 5.446981231216341e-05, + 5.273917849990539e-05, + 4.977982462150976e-05, + 4.799519228981808e-05, + 4.7104065743042156e-05, + 4.555822670226917e-05, + 4.456056558410637e-05, + 4.237707253196277e-05, + 4.1111925384029746e-05, + 3.9368209399981424e-05, + 3.89099950552918e-05, + 3.808713154285215e-05, + 3.706871211761609e-05, + 3.679449582705274e-05, + 3.4976106690010056e-05, + 3.2898435165407136e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021103927974763792, + "first_to_last_ratio": 1.675431813331652, + "imbalance_abs_log_ratio": 0.5160709310565479, + "last_quartile_mean": 0.00012596112719620578, + "mean": 0.00016011051866371417, + "normalized": [ + 1.5150472374771589, + 1.439502583502474, + 1.3841101446264577, + 1.327205260326266, + 1.2794452130965426, + 1.2428385988073838, + 1.2018548814818522, + 1.154676418187547, + 1.1148050630726085, + 1.0814152014364051, + 1.0614838409967562, + 1.034687172582471, + 1.0074911480481314, + 0.9875049829338368, + 0.9623163727426111, + 0.9431041078017948, + 0.9243534563643369, + 0.9073040220081667, + 0.888778951337658, + 0.8738270000353379, + 0.8622815727966864, + 0.8493922949455803, + 0.8374788674952537, + 0.8253865750371285, + 0.8155793557629833, + 0.8058361207192702, + 0.7999903796437624, + 0.7910448570865577, + 0.7825332275919701, + 0.7756749178925374, + 0.766226246614208, + 0.7568239275482656 + ], + "peak_layer": 1, + "peak_normalized": 1.5150472374771589, + "population_cv": 0.21304076077436968, + "rest_mean": 0.0001649131305317429, + "spike_contrast": 0.8136187466924137, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001341764145763591, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024257499899249524, + 0.00023047950526233763, + 0.00022161059314385056, + 0.00021249952260404825, + 0.00020485263667069376, + 0.00019899153267033398, + 0.00019242960843257606, + 0.00018487584020476788, + 0.00017849201685748994, + 0.00017314594879280776, + 0.00016995472833514214, + 0.00016566429985687137, + 0.00016130993026308715, + 0.00015810993500053883, + 0.00015407697355840355, + 0.00015100088785402477, + 0.00014799871132709086, + 0.00014526891754940152, + 0.00014230285887606442, + 0.00013990889419801533, + 0.00013806034985464066, + 0.00013599664089269936, + 0.00013408917584456503, + 0.00013215307262726128, + 0.00013058283366262913, + 0.00012902283924631774, + 0.00012808687461074442, + 0.0001266546023543924, + 0.00012529180094134063, + 0.00012419371341820806, + 0.00012268088175915182, + 0.00012117547157686204 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016164389126060996, + "first_to_last_ratio": 1.2829777917904304, + "imbalance_abs_log_ratio": 0.24918377588932214, + "last_quartile_mean": 0.00012599118417710997, + "mean": 0.0001426773092134681, + "normalized": [ + 1.1847160153617624, + 1.1696203114063919, + 1.15376099486318, + 1.1365907777467705, + 1.126248503505431, + 1.112765799827437, + 1.097614510342432, + 1.0821504120145113, + 1.06828860482523, + 1.0536786878013262, + 1.046275103221197, + 1.0365937358276223, + 1.0246881313142864, + 1.0168969779036778, + 1.0047039605050023, + 0.9958410772863212, + 0.9859208451030141, + 0.9762312163738422, + 0.965237112554715, + 0.9568250330809598, + 0.9498426746244831, + 0.9400597234006037, + 0.9304691287067991, + 0.9205814319068928, + 0.9114724424808572, + 0.9022957305012854, + 0.8969570718532309, + 0.8872712168205663, + 0.8789155388108255, + 0.8714969614364647, + 0.8626996589435726, + 0.8532906096493081 + ], + "peak_layer": 1, + "peak_normalized": 1.1847160153617624, + "population_cv": 0.0960837578275494, + "rest_mean": 0.00014451401323269778, + "spike_contrast": 0.9186590596986468, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013275910750962794, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000169032093253918, + 0.00016687827883288264, + 0.0001646155142225325, + 0.00016216571384575218, + 0.0001606901059858501, + 0.00015876643010415137, + 0.00015660468488931656, + 0.00015439830895047635, + 0.00015242054359987378, + 0.00015033603995107114, + 0.000149279716424644, + 0.00014789840497542173, + 0.00014619974535889924, + 0.0001450881245546043, + 0.00014334845764096826, + 0.00014208392531145364, + 0.00014066853327676654, + 0.00013928604312241077, + 0.0001377174339722842, + 0.00013651722110807896, + 0.00013552099699154496, + 0.00013412519183475524, + 0.00013275683159008622, + 0.00013134608161635697, + 0.0001300464355153963, + 0.000128737126942724, + 0.00012797542149201035, + 0.00012659346975851804, + 0.00012540130410343409, + 0.00012434284144546837, + 0.00012308766599744558, + 0.00012174520816188306 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00043632716551655903, + "first_to_last_ratio": 3.082278345441195, + "imbalance_abs_log_ratio": 1.1256690460714511, + "last_quartile_mean": 0.00014155994904285762, + "mean": 0.00025262987765017897, + "normalized": [ + 2.1611427674158366, + 1.9871567751170711, + 1.8899978819847916, + 1.7784604477508343, + 1.6406551319992353, + 1.5724690971097588, + 1.4592296472822097, + 1.3280082080145534, + 1.2379389344592089, + 1.1701308820499392, + 1.131654892203279, + 1.0685236385646504, + 1.0072825277339317, + 0.9454673581714634, + 0.8865158004683366, + 0.8408935109532261, + 0.7895809204213243, + 0.7481431077843942, + 0.7047226439788088, + 0.6727720648712864, + 0.6548656488865329, + 0.6315662143568483, + 0.6127430670138975, + 0.5973169234484098, + 0.5839093169225268, + 0.5733930286828336, + 0.5691911560855093, + 0.5599482690481828, + 0.5552538441515218, + 0.5513421690146183, + 0.547054988271426, + 0.542669135783553 + ], + "peak_layer": 1, + "peak_normalized": 2.1611427674158366, + "population_cv": 0.47902914554115894, + "rest_mean": 0.0002705909153317205, + "spike_contrast": 0.5751866206559582, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001556402741698548, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000545969232916832, + 0.0005020151729695499, + 0.0004774699336849153, + 0.0004492922453209758, + 0.00041447850526310503, + 0.0003972526756115258, + 0.00036864500725641847, + 0.00033549455110915005, + 0.00031274036155082285, + 0.00029561002156697214, + 0.00028588983695954084, + 0.0002699409960769117, + 0.00025446966174058616, + 0.00023885330301709473, + 0.00022396037820726633, + 0.00021243482478894293, + 0.00019947173132095486, + 0.00018900330178439617, + 0.0001780339953256771, + 0.00016996232443489134, + 0.00016543862875550985, + 0.0001595524954609573, + 0.00015479720605071634, + 0.0001509001012891531, + 0.00014751293929293752, + 0.0001448562106816098, + 0.00014379469212144613, + 0.00014145966270007193, + 0.0001402737107127905, + 0.00013928550470154732, + 0.00013820243475493044, + 0.00013709443737752736 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.027978340163826942, + 0.0140878576785326, + 0.014187770895659924, + 0.014337530359625816, + 0.014477464370429516, + 0.009757626801729202, + 0.009825476445257664, + 0.00987118761986494, + 0.009999874979257584, + 0.007595349568873644, + 0.007658981718122959, + 0.007689184043556452, + 0.007755559403449297, + 0.00625597033649683, + 0.006315375678241253, + 0.00635211868211627, + 0.006408242508769035, + 0.005408597644418478, + 0.005475745070725679, + 0.005517482291907072, + 0.005585856735706329, + 0.004830927588045597, + 0.004899685736745596, + 0.004948681220412254, + 0.004989070352166891, + 0.004419817589223385, + 0.004463871940970421, + 0.004542885348200798, + 0.004592607729136944, + 0.004137600306421518, + 0.004193917382508516, + 0.004230021964758635 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024312340610777028, + "first_to_last_ratio": 2.286272500967691, + "imbalance_abs_log_ratio": 0.8269227625411849, + "last_quartile_mean": 0.00010634051977831405, + "mean": 0.00017363850702167838, + "normalized": [ + 1.061084599304074, + 1.6344608012282997, + 1.6065574901355, + 1.6039206242202213, + 1.1359733847422597, + 1.548756457352017, + 1.2967221414405534, + 1.313883475593909, + 1.1957449434685963, + 1.344067980756563, + 1.165161355052429, + 1.1171979507437393, + 1.0391889074947966, + 1.1329181602378293, + 1.0774500297071945, + 1.0521010358096583, + 0.862761425176938, + 0.9019434894037968, + 0.9280163978632505, + 0.8870730707939211, + 0.8354471012965897, + 0.8049103603266032, + 0.8255366455754076, + 0.7297241480789424, + 0.6373588767497196, + 0.6752584188072907, + 0.6298229328646214, + 0.6250051046248064, + 0.6032481952321226, + 0.5674884668477911, + 0.6733729976615692, + 0.4878430314089892 + ], + "peak_layer": 2, + "peak_normalized": 1.6344608012282997, + "population_cv": 0.3218023265359089, + "rest_mean": 0.00018114369622371332, + "spike_contrast": 0.7348336602687936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001331104853306897, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00018424514564685524, + 0.0002838053333107382, + 0.000278960244031623, + 0.00027850238257087767, + 0.00019724872254300863, + 0.00026892375899478793, + 0.00022516089666169137, + 0.0002281407651025802, + 0.0002076273667626083, + 0.00023338195751421154, + 0.00020231687813065946, + 0.00019398858421482146, + 0.0001804432104108855, + 0.00019671821792144328, + 0.00018708681454882026, + 0.00018268525309395045, + 0.000149808605783619, + 0.00015661212091799825, + 0.0001611393818166107, + 0.00015403004363179207, + 0.00014506578736472875, + 0.00013976343325339258, + 0.00014334495062939823, + 0.0001267082116100937, + 0.00011067004379583523, + 0.00011725086369551718, + 0.00010936151375062764, + 0.00010852495324797928, + 0.00010474711598362774, + 9.853785013547167e-05, + 0.000116923481982667, + 8.470833563478664e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00025287891548941843, + "first_to_last_ratio": 2.2739168136310908, + "imbalance_abs_log_ratio": 0.8215038131425536, + "last_quartile_mean": 0.0001112085165004828, + "mean": 0.0001807239029858465, + "normalized": [ + 1.0592123382591336, + 1.645736468094544, + 1.60181949161791, + 1.5891655894821897, + 1.1402836456293184, + 1.5483130487587196, + 1.300612463539172, + 1.3089009659072275, + 1.1885078058073624, + 1.3408206015264876, + 1.166937022076678, + 1.1148751106540256, + 1.0356001272424655, + 1.1351884917697168, + 1.0748875173492451, + 1.0458945475116934, + 0.8650051516349683, + 0.9033374117584585, + 0.9274894298403746, + 0.8851829368231718, + 0.8352656795149113, + 0.8098299287299446, + 0.8239713600606818, + 0.7303600865627311, + 0.6419462275110265, + 0.6820467100618508, + 0.635541252420758, + 0.625618271589476, + 0.6029839777576883, + 0.5727133544881818, + 0.6681570652306708, + 0.4937959207892163 + ], + "peak_layer": 2, + "peak_normalized": 1.645736468094544, + "population_cv": 0.3200470231318611, + "rest_mean": 0.00018847914530110925, + "spike_contrast": 0.7366629038009043, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001388455944834277, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001914249878609553, + 0.00029742391780018806, + 0.00028948707040399313, + 0.0002872002078220248, + 0.00020607651094906032, + 0.0002798171772155911, + 0.00023505176068283617, + 0.00023654969118069857, + 0.00021479176939465106, + 0.0002423183323116973, + 0.00021089341316837817, + 0.00020148458133917302, + 0.00018715769692789763, + 0.00020515569485723972, + 0.0001942578674061224, + 0.0001890181447379291, + 0.00015632710710633546, + 0.00016325466276612133, + 0.00016761950973886997, + 0.0001599737151991576, + 0.00015095247363206, + 0.0001463556254748255, + 0.00014891132013872266, + 0.00013199352542869747, + 0.0001160150277428329, + 0.0001232621434610337, + 0.00011485749564599246, + 0.00011306417582090944, + 0.00010897361789830029, + 0.0001035029927152209, + 0.00012075195263605565, + 8.924072608351707e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.22425555251538754, + "first_to_last_ratio": 2.2739168285843645, + "imbalance_abs_log_ratio": 0.8215038197185521, + "last_quartile_mean": 0.09862082451581955, + "mean": 0.1602677651681006, + "normalized": [ + 1.0592124025748084, + 1.645736496509649, + 1.6018195299533093, + 1.5891655927499475, + 1.1402836784663197, + 1.5483129508886688, + 1.300612441604666, + 1.3089009387606192, + 1.1885077887054891, + 1.3408205692725423, + 1.1669370180912673, + 1.1148751103389594, + 1.0356000505037584, + 1.1351884646301031, + 1.0748875233039092, + 1.0458946120331756, + 0.8650051766933494, + 0.9033374073089909, + 0.9274894888798537, + 0.8851828780404403, + 0.835265755835364, + 0.8098299455472033, + 0.8239713233255873, + 0.730360099613448, + 0.6419462245963619, + 0.6820467317825786, + 0.6355412485786012, + 0.6256182682629655, + 0.602983984145969, + 0.5727133411523797, + 0.6681570418174577, + 0.49379591603225714 + ], + "peak_layer": 2, + "peak_normalized": 1.645736496509649, + "population_cv": 0.3200470194234096, + "rest_mean": 0.16714519124340127, + "spike_contrast": 0.7366629183018027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1231296643614769, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.16975760459899902, + 0.26375851035118103, + 0.25672003626823425, + 0.254692018032074, + 0.18275071680545807, + 0.24814465641975403, + 0.20844624936580658, + 0.20977462828159332, + 0.19047948718070984, + 0.21489031612873077, + 0.1870223879814148, + 0.17867854237556458, + 0.16597330570220947, + 0.18193411827087402, + 0.1722698211669922, + 0.16762319207191467, + 0.13863244652748108, + 0.1447758674621582, + 0.14864666759967804, + 0.1418662816286087, + 0.13386617600917816, + 0.12978963553905487, + 0.13205604255199432, + 0.11705318093299866, + 0.10288328677415848, + 0.1093101054430008, + 0.1018567755818367, + 0.10026644170284271, + 0.09663889557123184, + 0.09178748726844788, + 0.10708403587341309, + 0.07913956791162491 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.143860511860112e-05, + "first_to_last_ratio": 0.9636580872932836, + "imbalance_abs_log_ratio": 0.037018728516889636, + "last_quartile_mean": 7.413273033307632e-05, + "mean": 7.686168260079285e-05, + "normalized": [ + 0.586022200160155, + 0.9787081875553917, + 0.912570839911241, + 1.015732496910769, + 0.8201270226617754, + 1.1119332047208261, + 1.0008210861718638, + 1.0096344009845781, + 0.9335569648148418, + 1.090167719321714, + 1.035494250546431, + 0.9697380161483544, + 1.0399314841024359, + 1.0649519568255064, + 1.06563466639597, + 1.1389203959735625, + 0.9952637999761301, + 1.0027093290728673, + 1.1427249020241628, + 1.1090175842590573, + 1.0432879501645442, + 1.0450477353718262, + 1.1125450117027875, + 1.0594965280199307, + 0.9180884623091344, + 0.995296932026832, + 0.9631737996002885, + 0.9446177686211702, + 0.9674620335911367, + 0.9355074959711648, + 1.1351899163905306, + 0.8566258576930204 + ], + "peak_layer": 19, + "peak_normalized": 1.1427249020241628, + "population_cv": 0.10846075768175395, + "rest_mean": 7.635363915512614e-05, + "spike_contrast": 1.0425844542348655, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.960511720739305e-05, + "top_five_layers": [ + 19, + 16, + 31, + 23, + 6 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.504265234572813e-05, + 7.522515807067975e-05, + 7.014173024799675e-05, + 7.807090878486633e-05, + 6.303634290816262e-05, + 8.546505705453455e-05, + 7.692479266552255e-05, + 7.760219887131825e-05, + 7.175475911935791e-05, + 8.379212522413582e-05, + 7.958983042044565e-05, + 7.453569560311735e-05, + 7.993088365765288e-05, + 8.185399929061532e-05, + 8.190647349692881e-05, + 8.753933798288926e-05, + 7.64976502978243e-05, + 7.706992619205266e-05, + 8.78317587194033e-05, + 8.52409575600177e-05, + 8.018886728677899e-05, + 8.032412733882666e-05, + 8.551208156859502e-05, + 8.143468585330993e-05, + 7.056582398945466e-05, + 7.650019688298926e-05, + 7.403115887427703e-05, + 7.260491111082956e-05, + 7.436075975419953e-05, + 7.190468022599816e-05, + 8.725260704522952e-05, + 6.584170478163287e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 8.107683152047684e-05, + "first_to_last_ratio": 1.03534602173519, + "imbalance_abs_log_ratio": 0.034735691361772646, + "last_quartile_mean": 7.83089226388256e-05, + "mean": 8.487996228723205e-05, + "normalized": [ + 0.5954853629443996, + 1.025163681929567, + 0.9520754208460104, + 1.0126519094835282, + 0.8443534832663374, + 1.1548844288399933, + 1.0326421147312754, + 1.0242956755731092, + 0.9710206925084107, + 1.15116021353655, + 1.072832454869962, + 1.0193061391332177, + 1.0419196505755806, + 1.0947619162703996, + 1.1089694127677372, + 1.1239932261258834, + 0.9835832115239759, + 1.0364664516448652, + 1.11612142168979, + 1.093437276508762, + 1.0179007506072324, + 1.033774483284387, + 1.0848380474290875, + 1.0276879808812729, + 0.8974590825111308, + 0.9692041024722606, + 0.9378977551023808, + 0.9155031739616566, + 0.918412101050971, + 0.8874638084461439, + 1.0536745962271137, + 0.8010599732570078 + ], + "peak_layer": 6, + "peak_normalized": 1.1548844288399933, + "population_cv": 0.11200209045145568, + "rest_mean": 8.468612051931107e-05, + "spike_contrast": 1.0146492401244358, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.592670783400536e-05, + "top_five_layers": [ + 6, + 10, + 16, + 19, + 15 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.054477514931932e-05, + 8.701585466042161e-05, + 8.081212581600994e-05, + 8.59538558870554e-05, + 7.166869181673974e-05, + 9.802654676605016e-05, + 8.76506237545982e-05, + 8.694217831362039e-05, + 8.242019976023585e-05, + 9.771043551154435e-05, + 9.106197830988094e-05, + 8.651866664877161e-05, + 8.843810064718127e-05, + 9.292335016652942e-05, + 9.41292819334194e-05, + 9.540450264466926e-05, + 8.348650590050966e-05, + 8.797523332759738e-05, + 9.473634418100119e-05, + 9.281091479351744e-05, + 8.639937732368708e-05, + 8.774673915468156e-05, + 9.20810125535354e-05, + 8.723011706024408e-05, + 7.617629307787865e-05, + 8.226600766647607e-05, + 7.960872608236969e-05, + 7.770787487970665e-05, + 7.795478450134397e-05, + 7.532789459219202e-05, + 8.943585999077186e-05, + 6.799394031986594e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.1427921979011444e-05, + "first_to_last_ratio": 0.40836765899304117, + "imbalance_abs_log_ratio": 0.8955873853644011, + "last_quartile_mean": 5.2472132665570825e-05, + "mean": 3.864622581772892e-05, + "normalized": [ + 0.2626641640048693, + 0.5716512870137274, + 0.5157707114648397, + 0.4843895386483896, + 0.5109475889421058, + 0.7512596968779521, + 0.675728069412672, + 0.6632970161544233, + 0.7004288748821966, + 0.9130902556041411, + 0.8717836320781894, + 0.8044820763577291, + 0.9241814793896349, + 0.9531622930600316, + 1.0319070449118461, + 1.0273708471578158, + 0.9893551980130566, + 1.0844069350216101, + 1.1887300698220302, + 1.2100415764612447, + 1.1128447755732793, + 1.2346303137099222, + 1.3033621630444512, + 1.3524689418392815, + 1.2055635543985064, + 1.318388624044801, + 1.3615446320871405, + 1.2933406942036905, + 1.3753836688642536, + 1.3573239762703455, + 1.6442457326403273, + 1.3062545680454958 + ], + "peak_layer": 31, + "peak_normalized": 1.6442457326403273, + "population_cv": 0.32680115045148705, + "rest_mean": 3.691592014042221e-05, + "spike_contrast": 1.2999777952883038, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.798987647518516e-05, + "top_five_layers": [ + 31, + 29, + 27, + 30, + 24 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.0150978596357163e-05, + 2.2092164726927876e-05, + 1.99325913854409e-05, + 1.8719827494351193e-05, + 1.974619590328075e-05, + 2.9033351893303916e-05, + 2.6114339561900124e-05, + 2.5633926270529628e-05, + 2.7068932467955165e-05, + 3.528749221004546e-05, + 3.369114710949361e-05, + 3.1090195989236236e-05, + 3.571612614905462e-05, + 3.683612521854229e-05, + 3.9879312680568546e-05, + 3.970400575781241e-05, + 3.823484439635649e-05, + 4.190823528915644e-05, + 4.5939930714666843e-05, + 4.676354001276195e-05, + 4.300725049688481e-05, + 4.771380190504715e-05, + 5.037002847529948e-05, + 5.226782013778575e-05, + 4.65904813609086e-05, + 5.09507444803603e-05, + 5.261856131255627e-05, + 4.998273652745411e-05, + 5.3153387852944434e-05, + 5.24554488947615e-05, + 6.354389188345522e-05, + 5.048180901212618e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031520502852799837, + "first_to_last_ratio": 1.31300997816315, + "imbalance_abs_log_ratio": 0.27232219480674863, + "last_quartile_mean": 0.00024006293460843153, + "mean": 0.00030979480334281106, + "normalized": [ + 0.607123837738274, + 1.0354820546705408, + 0.9347765087792952, + 1.1085716606000595, + 0.9306423427143409, + 1.2565833735558338, + 1.1285140374487672, + 1.138017377238765, + 1.0370737734279494, + 1.2142990636699202, + 1.1645184658462944, + 1.1206382096550715, + 1.1232577862254296, + 1.1815362250572234, + 1.1924463988431009, + 1.2309950595128432, + 1.0374639291264394, + 1.065463636492051, + 1.1630951913863559, + 1.1082935818936954, + 1.0270641612904527, + 0.9934767629460804, + 1.0585185268550952, + 0.9428720751183239, + 0.8088208526771234, + 0.8683362720246933, + 0.8018903045940483, + 0.8027332776454693, + 0.7640102071384416, + 0.7171383971343437, + 0.8334550561316563, + 0.6028915925620211 + ], + "peak_layer": 6, + "peak_normalized": 1.2565833735558338, + "population_cv": 0.17765693833110424, + "rest_mean": 0.0003117367305094376, + "spike_contrast": 0.9601319554288655, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00029930839664302764, + "top_five_layers": [ + 6, + 16, + 10, + 15, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00018808380991686136, + 0.00032078695949167013, + 0.00028958890470676124, + 0.00034342973958700895, + 0.0002883081615436822, + 0.00038928299909457564, + 0.00034960778430104256, + 0.0003525518695823848, + 0.00032128006569109857, + 0.00037618353962898254, + 0.00036076176911592484, + 0.00034716789377853274, + 0.0003479794249869883, + 0.00036603378248400986, + 0.0003694136976264417, + 0.00038135587237775326, + 0.0003214009338989854, + 0.0003300750977359712, + 0.0003603208460845053, + 0.000343343592248857, + 0.00031817913986742496, + 0.00030777393840253353, + 0.00032792353886179626, + 0.00029209686908870935, + 0.0002505684969946742, + 0.0002690060646273196, + 0.00024842144921422005, + 0.0002486825978849083, + 0.00023668639187235385, + 0.00022216574870981276, + 0.000258200045209378, + 0.0001867726823547855 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014031016267836094, + 0.014151837676763535, + 0.014265215955674648, + 0.014401593245565891, + 0.009722241200506687, + 0.009786579757928848, + 0.009839320555329323, + 0.00996283907443285, + 0.007562503684312105, + 0.007635751739144325, + 0.007660485804080963, + 0.007726665120571852, + 0.006238248664885759, + 0.006290202960371971, + 0.006327399518340826, + 0.006382675375789404, + 0.005390914157032967, + 0.0054517327807843685, + 0.005509276874363422, + 0.0055727302096784115, + 0.0048178439028561115, + 0.004884926602244377, + 0.004938251338899136, + 0.004975184798240662, + 0.004412420094013214, + 0.004455919377505779, + 0.004530393984168768, + 0.0045819394290447235, + 0.004127359017729759, + 0.004185030702501535, + 0.004221425391733646, + 0.004281661007553339 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 5.6193598993559135e-05, + "first_to_last_ratio": 1.5723739147519853, + "imbalance_abs_log_ratio": 0.4525865249799748, + "last_quartile_mean": 3.573806361600873e-05, + "mean": 4.4294749727669114e-05, + "normalized": [ + 1.5642840408451666, + 1.493254721227763, + 1.1816235069560812, + 1.0717429494942774, + 1.3553344243310417, + 1.3257825682631335, + 1.1255110934944355, + 1.0314977817441078, + 1.194075821513187, + 1.0893775754580661, + 1.0407197138445958, + 0.9875719064328349, + 1.135803276632048, + 1.0324555952346264, + 0.9635988909717662, + 0.90208003019059, + 0.9904506854292476, + 0.9196323607404276, + 0.8708711781381843, + 0.8075366439875266, + 0.9218591005239072, + 0.8776357461813653, + 0.8517186042296377, + 0.8109904231766266, + 0.8722342267309339, + 0.8500552841464633, + 0.7844009556175089, + 0.7665260139039981, + 0.8384393111065888, + 0.8154485842192296, + 0.7754690286134897, + 0.7520179566211438 + ], + "peak_layer": 1, + "peak_normalized": 1.5642840408451666, + "population_cv": 0.20749976292329902, + "rest_mean": 4.538663482957485e-05, + "spike_contrast": 0.8460325450777163, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.839857017737813e-05, + "top_five_layers": [ + 1, + 2, + 5, + 6, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.928957009222358e-05, + 6.614334415644407e-05, + 5.23397175129503e-05, + 4.747258572024293e-05, + 6.003419912303798e-05, + 5.872520705452189e-05, + 4.985423220205121e-05, + 4.5689936087001115e-05, + 5.289128966978751e-05, + 4.8253707063850015e-05, + 4.6098419261397794e-05, + 4.374425043351948e-05, + 5.03101218782831e-05, + 4.573236219584942e-05, + 4.2682371713453904e-05, + 3.9957409171620384e-05, + 4.3871765228686854e-05, + 4.073488526046276e-05, + 3.857502088067122e-05, + 3.576963354134932e-05, + 4.083351814188063e-05, + 3.887465572915971e-05, + 3.7726662412751466e-05, + 3.592261782614514e-05, + 3.863539677695371e-05, + 3.7652986065950245e-05, + 3.474484401522204e-05, + 3.395307794562541e-05, + 3.713845944730565e-05, + 3.612009095377289e-05, + 3.4349206543993205e-05, + 3.3310447179246694e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 9.100913848669734e-05, + "first_to_last_ratio": 0.8419197805282016, + "imbalance_abs_log_ratio": 0.17207054181035333, + "last_quartile_mean": 0.00010809716150106397, + "mean": 9.889523016681778e-05, + "normalized": [ + 1.0744641570554045, + 0.9868986027413778, + 0.8415148680287817, + 0.7657672453742986, + 1.0045876840265406, + 0.9858056114400382, + 0.8710937611903306, + 0.8319330217638002, + 1.0050985706445066, + 0.9363425314153265, + 0.9111662095698898, + 0.8822483642571924, + 1.0547921529408322, + 0.9937743096656301, + 0.9653849347701057, + 0.9236053864198286, + 1.0608213360421703, + 1.0216834776263237, + 0.9948854733452297, + 0.9444045194208119, + 1.0902032759381237, + 1.070297753084291, + 1.0352646449837608, + 1.0035839360241054, + 1.109605122267389, + 1.0988987221493531, + 1.0612028824132491, + 1.0392747076439328, + 1.1393068813062424, + 1.1142247321092786, + 1.0935193303224786, + 1.0883457940193757 + ], + "peak_layer": 29, + "peak_normalized": 1.1393068813062424, + "population_cv": 0.09089893356811851, + "rest_mean": 9.776359500527313e-05, + "spike_contrast": 1.0740814107080978, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010500606003915891, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00010625938011799008, + 9.759956446941942e-05, + 8.322180656250566e-05, + 7.573072798550129e-05, + 9.934893023455516e-05, + 9.749147284310311e-05, + 8.614701800979674e-05, + 8.227420767070726e-05, + 9.939945448422804e-05, + 9.259981015929952e-05, + 9.010999201564118e-05, + 8.725015504751354e-05, + 0.00010431391274323687, + 9.827953908825293e-05, + 9.547196532366797e-05, + 9.134016727330163e-05, + 0.00010491017019376159, + 0.00010103962267749012, + 9.838942787609994e-05, + 9.339710231870413e-05, + 0.0001078159039025195, + 0.00010584734263829887, + 0.00010238273534923792, + 9.924966434482485e-05, + 0.00010973465396091342, + 0.00010867584205698222, + 0.00010494790330994874, + 0.00010277931141899899, + 0.00011267201625742018, + 0.000110191511339508, + 0.00010814384586410597, + 0.0001076322078006342 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.08070781361311674, + "first_to_last_ratio": 0.8419197775247833, + "imbalance_abs_log_ratio": 0.1720705453776983, + "last_quartile_mean": 0.09586164355278015, + "mean": 0.0877012787386775, + "normalized": [ + 1.074464079725317, + 0.986898606287062, + 0.8415149217948403, + 0.7657672391640048, + 1.0045876578450608, + 0.9858055872189405, + 0.8710938052118494, + 0.8319330371629138, + 1.0050985716236776, + 0.9363425240221704, + 0.9111662153888755, + 0.8822483595926541, + 1.0547922001671663, + 0.993774279010469, + 0.9653849224854107, + 0.9236053602662656, + 1.060821305580311, + 1.021683475130349, + 0.9948854782496288, + 0.9444045803369281, + 1.0902032654624292, + 1.07029776221103, + 1.0352646575744078, + 1.0035839255047951, + 1.109605076031185, + 1.0988987393251362, + 1.0612029192639736, + 1.0392747445534645, + 1.1393068879878778, + 1.1142247084644445, + 1.0935193626414328, + 1.0883457447159286 + ], + "peak_layer": 29, + "peak_normalized": 1.1393068879878778, + "population_cv": 0.09089892559784572, + "rest_mean": 0.08669773351263117, + "spike_contrast": 1.0740813996684329, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0931204229593277, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.09423187375068665, + 0.08655226975679398, + 0.0738019347190857, + 0.0671587660908699, + 0.08810362219810486, + 0.08645641058683395, + 0.07639604061841965, + 0.07296159118413925, + 0.08814842998981476, + 0.0821184366941452, + 0.07991044223308563, + 0.07737430930137634, + 0.09250662475824356, + 0.08715527504682541, + 0.08466549217700958, + 0.08100137114524841, + 0.09303538501262665, + 0.08960294723510742, + 0.08725272864103317, + 0.08282548934221268, + 0.09561222046613693, + 0.09386648237705231, + 0.09079403430223465, + 0.0880155935883522, + 0.0973137840628624, + 0.09637482464313507, + 0.09306885302066803, + 0.09114572405815125, + 0.0999186709523201, + 0.09771893173456192, + 0.09590304642915726, + 0.09544931352138519 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0474835562490625e-05, + "first_to_last_ratio": 0.595870196340598, + "imbalance_abs_log_ratio": 0.5177324270116084, + "last_quartile_mean": 1.7579056020622374e-05, + "mean": 1.45315484587627e-05, + "normalized": [ + 0.6891108502390517, + 0.67218760206668, + 0.6248497232673742, + 0.6113105238885437, + 0.8536003812586072, + 0.8434482348779837, + 0.7260190066641083, + 0.7461464268756399, + 0.9515026546685825, + 0.8968707547028398, + 0.8666952712848448, + 0.9173542983424958, + 1.0474470628626384, + 1.0570493146141757, + 0.9856965747654381, + 1.0403130781822671, + 1.05710126231986, + 1.0899464822844398, + 1.0639055355381295, + 0.9944043245776518, + 1.1636416255463544, + 1.1162669452398895, + 1.1491653646620976, + 1.158233556446643, + 1.241320970071456, + 1.2666472916328237, + 1.1327368714456698, + 1.1012802199127885, + 1.2948691657915095, + 1.2555094531837419, + 1.2151558495322814, + 1.1702133232533922 + ], + "peak_layer": 29, + "peak_normalized": 1.2948691657915095, + "population_cv": 0.19478066534993646, + "rest_mean": 1.408557606434139e-05, + "spike_contrast": 1.202634475953173, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6939799388637765e-05, + "top_five_layers": [ + 29, + 26, + 30, + 25, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.0013847713707946e-05, + 9.767926712811459e-06, + 9.080034033104312e-06, + 8.883288501237985e-06, + 1.2404135304677766e-05, + 1.2256608897587284e-05, + 1.0550180377322249e-05, + 1.0842662959476002e-05, + 1.3826806934957858e-05, + 1.3032920833211392e-05, + 1.2594424333656207e-05, + 1.3330578440218233e-05, + 1.522102775197709e-05, + 1.5360563338617794e-05, + 1.4323697541840374e-05, + 1.5117359907890204e-05, + 1.5361318219220266e-05, + 1.5838610124774277e-05, + 1.546019484521821e-05, + 1.445023463020334e-05, + 1.690951467026025e-05, + 1.622108720766846e-05, + 1.669915218371898e-05, + 1.6830927052069455e-05, + 1.8038315829471685e-05, + 1.8406346498522907e-05, + 1.6460420738440007e-05, + 1.600330688233953e-05, + 1.8816454030456953e-05, + 1.8244496459374204e-05, + 1.7658096112427302e-05, + 1.7005011613946408e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.836043712610262e-05, + "first_to_last_ratio": 0.6391636816233681, + "imbalance_abs_log_ratio": 0.44759470463089907, + "last_quartile_mean": 0.00010695294349716278, + "mean": 9.121693358338234e-05, + "normalized": [ + 0.7254188545798665, + 0.6902023390007371, + 0.6496225556724757, + 0.6277900804246503, + 0.8680656002782756, + 0.8588952084862168, + 0.7971788633739051, + 0.7782428701770085, + 0.9734232055545582, + 0.9237343700472408, + 0.9096320021673064, + 0.8918620199754489, + 1.0844263684244682, + 1.0342037426828012, + 1.0147200800673386, + 0.9791745321037664, + 1.1290928536156495, + 1.0928814238713915, + 1.0660995437899934, + 1.013549283136708, + 1.170635644938427, + 1.1503062295054498, + 1.1129730585678779, + 1.0777737723208793, + 1.1919439416858, + 1.1807148042349604, + 1.1393605783805036, + 1.1153320412921235, + 1.222157091120308, + 1.1946826876057626, + 1.1714041050576325, + 1.1645002478604691 + ], + "peak_layer": 29, + "peak_normalized": 1.222157091120308, + "population_cv": 0.17156610618115575, + "rest_mean": 8.883977756744975e-05, + "spike_contrast": 1.1712498490939807, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010405357606941834, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.617048347834498e-05, + 6.295814091572538e-05, + 5.925657751504332e-05, + 5.7265086070401594e-05, + 7.91822822066024e-05, + 7.834578718757257e-05, + 7.271621143445373e-05, + 7.098892820067704e-05, + 8.879267988959327e-05, + 8.426021668128669e-05, + 8.297384192701429e-05, + 8.135291864164174e-05, + 9.891804802464321e-05, + 9.433689410798252e-05, + 9.255965414922684e-05, + 8.931729826144874e-05, + 0.00010299238783773035, + 9.968929225578904e-05, + 9.724633127916604e-05, + 9.245285764336586e-05, + 0.00010678179387468845, + 0.00010492740693734959, + 0.00010152198956348002, + 9.831121860770509e-05, + 0.00010872547136386856, + 0.00010770118387881666, + 0.0001039289782056585, + 0.00010173716873396188, + 0.00011148142220918089, + 0.00010897529136855155, + 0.00010685189045034349, + 0.00010622214176692069 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 5.421776268121903e-05, + "first_to_last_ratio": 0.5089233562442377, + "imbalance_abs_log_ratio": 0.6754578508927926, + "last_quartile_mean": 0.00010653423942130757, + "mean": 8.540443025140121e-05, + "normalized": [ + 0.5542697148172375, + 0.5395320644633284, + 0.526750599523528, + 0.5251350203104778, + 0.7537064875235374, + 0.7453106043033457, + 0.7183783488935762, + 0.7156010196903468, + 0.915261682615852, + 0.8828592813937833, + 0.8733821992497209, + 0.8643716402564832, + 1.0724322757212523, + 1.0352702478756863, + 1.026729960575756, + 1.0038080492636434, + 1.1695135624376918, + 1.1411903421843295, + 1.1203495535033974, + 1.0697851231751558, + 1.23599072599408, + 1.2140634563521424, + 1.1785251437606976, + 1.1385123493568967, + 1.2633504856351931, + 1.2489710766708417, + 1.2127527443953423, + 1.1862530196716095, + 1.298334269521613, + 1.274402551981099, + 1.2503128844914202, + 1.2448935143909348 + ], + "peak_layer": 29, + "peak_normalized": 1.298334269521613, + "population_cv": 0.24541141609846925, + "rest_mean": 8.214501078193145e-05, + "spike_contrast": 1.2539446328637511, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010300529538653791, + "top_five_layers": [ + 29, + 30, + 25, + 31, + 26 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.73370891995728e-05, + 4.6078428567852825e-05, + 4.498683483689092e-05, + 4.4848857214674354e-05, + 6.436987314373255e-05, + 6.365282752085477e-05, + 6.135269359219819e-05, + 6.111549737397581e-05, + 7.816740253474563e-05, + 7.540009391959757e-05, + 7.459070911863819e-05, + 7.382116746157408e-05, + 9.159046749118716e-05, + 8.841666567604989e-05, + 8.768728730501607e-05, + 8.572965452913195e-05, + 9.98816394712776e-05, + 9.746271098265424e-05, + 9.56828152993694e-05, + 9.136438893619925e-05, + 0.00010555908374954015, + 0.00010368639777880162, + 0.00010065126843983307, + 9.723399853101e-05, + 0.0001078957284335047, + 0.00010666766320355237, + 0.00010357445717090741, + 0.00010131126327905804, + 0.00011088349856436253, + 0.00010883962386287749, + 0.00010678225953597575, + 0.00010631942132022232 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00014064586594031425, + "first_to_last_ratio": 1.0563608173139223, + "imbalance_abs_log_ratio": 0.054829809986132017, + "last_quartile_mean": 0.00013314188072399702, + "mean": 0.0001347423383322166, + "normalized": [ + 1.1217990599984589, + 1.0213028410040081, + 0.9566177087867302, + 0.90173442925003, + 1.162761655685041, + 1.2034553361266669, + 1.0271109783027095, + 0.9557256445604886, + 1.16094674777681, + 1.05596774404836, + 1.0011734009390578, + 0.9818696062747876, + 1.1123781705815696, + 1.0319357929888284, + 0.9826152250905227, + 0.9109733958837348, + 1.0154471845455557, + 0.955267408663159, + 0.9085227032176484, + 0.853750823711138, + 0.9825495622515935, + 0.9600374683174735, + 0.9243229318330685, + 0.9067574744309754, + 0.9933509912573504, + 0.986398830187635, + 0.9577583846169437, + 0.9375602577665346, + 1.0292265528944584, + 1.007559112024919, + 0.9944595916547317, + 0.9986629853290112 + ], + "peak_layer": 6, + "peak_normalized": 1.2034553361266669, + "population_cv": 0.07980954026492859, + "rest_mean": 0.0001359050228428613, + "spike_contrast": 0.9452471975467027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012846384197473526, + "top_five_layers": [ + 6, + 5, + 9, + 1, + 13 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001511538284830749, + 0.0001376127329422161, + 0.00012889690697193146, + 0.00012150180555181578, + 0.00015667322441004217, + 0.0001621563860680908, + 0.00013839533494319767, + 0.00012877670815214515, + 0.00015642867947462946, + 0.0001422835630364716, + 0.00013490044511854649, + 0.00013229940668679774, + 0.000149884435813874, + 0.00013904544175602496, + 0.00013239987310953438, + 0.0001227466855198145, + 0.00013682372809853405, + 0.0001287149643758312, + 0.0001224164734594524, + 0.00011503638233989477, + 0.00013239102554507554, + 0.0001293576933676377, + 0.0001245454332092777, + 0.00012217862240504473, + 0.00013384643534664065, + 0.000132909684907645, + 0.00012905060430057347, + 0.00012632906145881861, + 0.00013868039241060615, + 0.00013576087076216936, + 0.00013399581075645983, + 0.0001345621858490631 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 8.038491115106986, + "forward": { + "activation_sha256": { + "attention_branch_output": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", + "mlp_branch_output": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", + "post_attention_state": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", + "post_mlp_state": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", + "pre_attention_input": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", + "pre_mlp_input": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd" + }, + "logits_sha256": "5e5a50bae6f74c41645e35a857f0cba662c8b3f5ba9a8a4635e860fdaa3b0c88", + "loss_nats": 5.571857452392578, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.571857452392578, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e09a74368e31bc6c62d0aef79b955555c96aa595a56556be6a24c1ab400d8361", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0023513585329055786, + 0.0023208872880786657, + 0.002778682392090559, + 0.002497476991266012, + 0.002628475893288851, + 0.002920303260907531, + 0.002932395786046982, + 0.0032228652853518724, + 0.003561305580660701, + 0.003354643704369664, + 0.0034162327647209167, + 0.0036349669098854065, + 0.0035210708156228065, + 0.0038089773152023554, + 0.003915763925760984, + 0.0037244565319269896, + 0.004162859171628952, + 0.004167158622294664, + 0.004440344404429197, + 0.0041700867004692554, + 0.004737259820103645, + 0.004552462603896856, + 0.004706811159849167, + 0.005186605267226696, + 0.005580419674515724, + 0.004905648995190859, + 0.005227713380008936, + 0.005349088925868273, + 0.005235649179667234, + 0.004925078712403774, + 0.005810525268316269, + 0.005564783234149218 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024914435198297724, + "first_to_last_ratio": 5.005386356076312, + "imbalance_abs_log_ratio": 1.61051460380912, + "last_quartile_mean": 4.977524895366514e-05, + "mean": 0.00012233095264946314, + "normalized": [ + 2.946204230178919, + 2.5883907308658176, + 2.285503889899197, + 2.0157125547168477, + 1.8168856694000293, + 1.687577213316311, + 1.555459627812154, + 1.3974018168018878, + 1.2756749767931916, + 1.1684180974434406, + 1.0925035606597346, + 1.0000212864934148, + 0.922022236502042, + 0.863306019148705, + 0.7954975128821132, + 0.747136294956448, + 0.6955650087701328, + 0.6567702950630189, + 0.6067855157189815, + 0.5734103460233209, + 0.549704279492861, + 0.527795032775898, + 0.5009080420334019, + 0.4762252632696638, + 0.45754883501456406, + 0.43797117166433, + 0.4282553488048722, + 0.41570642703193256, + 0.40058854659018955, + 0.39033614610916745, + 0.3703926654711839, + 0.3543213582962283 + ], + "peak_layer": 1, + "peak_normalized": 2.946204230178919, + "population_cv": 0.6883820030702144, + "rest_mean": 0.00013360270127628205, + "spike_contrast": 0.46004691130861425, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.146351006464101e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036041197017766535, + 0.0003166403039358556, + 0.00027958786813542247, + 0.0002465840370859951, + 0.00022226135479286313, + 0.0002064429281745106, + 0.00019028085807804018, + 0.0001709454954834655, + 0.00015605453518219292, + 0.00014293369895312935, + 0.00013364700134843588, + 0.00012233355664648116, + 0.0001127918585552834, + 0.00010560904775047675, + 9.731396858114749e-05, + 9.139789472101256e-05, + 8.508913015248254e-05, + 8.03433358669281e-05, + 7.42286501917988e-05, + 7.014583388809115e-05, + 6.724584818584844e-05, + 6.456566916313022e-05, + 6.127655797172338e-05, + 5.825709013151936e-05, + 5.597238487098366e-05, + 5.3577430662699044e-05, + 5.2388884796528146e-05, + 5.085376324132085e-05, + 4.900437852484174e-05, + 4.775019260705449e-05, + 4.5310487621463835e-05, + 4.334446930442937e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002986813742609229, + "first_to_last_ratio": 2.3520308544062645, + "imbalance_abs_log_ratio": 0.8552791483203458, + "last_quartile_mean": 0.0001269887143280357, + "mean": 0.00018719729814620223, + "normalized": [ + 2.1746021240914017, + 1.94336263487061, + 1.7490368812517914, + 1.5797937988337565, + 1.4558020735282082, + 1.372777068369229, + 1.290232936323773, + 1.1987380034727069, + 1.1282687150208124, + 1.0677045109732362, + 1.0238712087163733, + 0.9743741602296072, + 0.9332108464677616, + 0.9012687760530631, + 0.8680188022864366, + 0.8423083374767277, + 0.8168968442148058, + 0.7977443195284458, + 0.774889242886524, + 0.759928616461841, + 0.7477907292059667, + 0.7367409087088999, + 0.7235787757136496, + 0.7121132246295925, + 0.7033660136533255, + 0.6946945949961153, + 0.6884818791691059, + 0.6826464924874314, + 0.674695217464828, + 0.6690870522688999, + 0.6604539573176332, + 0.6535212533274407 + ], + "peak_layer": 1, + "peak_normalized": 2.1746021240914017, + "population_cv": 0.3960989175249597, + "rest_mean": 0.00019674027215741161, + "spike_contrast": 0.68956516628749, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013566523848567157, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004070796421729028, + 0.0003637922345660627, + 0.0003274149785283953, + 0.0002957331307698041, + 0.0002725222148001194, + 0.000256980158155784, + 0.0002415281196590513, + 0.00022440051543526351, + 0.00021120885503478348, + 0.00019987139967270195, + 0.00019166592392139137, + 0.0001824002101784572, + 0.00017469454905949533, + 0.00016871507978066802, + 0.00016249077452812344, + 0.00015767784498166293, + 0.00015292088210117072, + 0.0001493355812272057, + 0.00014505717263091356, + 0.0001422565837856382, + 0.00013998440408613533, + 0.0001379159075440839, + 0.00013545199180953205, + 0.0001333056716248393, + 0.00013166821736376733, + 0.000130044951220043, + 0.0001288819476030767, + 0.00012778957898262888, + 0.00012630112178158015, + 0.00012525128840934485, + 0.00012363519635982811, + 0.00012233741290401667 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.264873631298542, + "first_to_last_ratio": 2.3520309299879743, + "imbalance_abs_log_ratio": 0.8552791804550044, + "last_quartile_mean": 0.11261485889554024, + "mean": 0.1660084358882159, + "normalized": [ + 2.1746021175238424, + 1.943362568695452, + 1.7490370652499811, + 1.5797937888156908, + 1.4558020518807187, + 1.3727770906872316, + 1.2902329816871188, + 1.1987379486864336, + 1.128268732118104, + 1.067704514212167, + 1.0238712059346726, + 0.9743741284397964, + 0.9332107750795899, + 0.9012687970282692, + 0.8680188148399602, + 0.8423083473020997, + 0.8168968752199287, + 0.7977442908994873, + 0.7748892254753884, + 0.7599285915530729, + 0.7477907763310051, + 0.7367409150028685, + 0.723578786768912, + 0.7121132849551643, + 0.7033659848486875, + 0.694694622945607, + 0.6884818276985868, + 0.6826464794318051, + 0.6746951840389442, + 0.6690870220520635, + 0.6604539434158253, + 0.6535212611815249 + ], + "peak_layer": 1, + "peak_normalized": 2.1746021175238424, + "population_cv": 0.3960989241805914, + "rest_mean": 0.17447124007675383, + "spike_contrast": 0.6895651868880185, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12030929327011108, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.3610022962093353, + 0.3226145803928375, + 0.2903549075126648, + 0.2622590959072113, + 0.24167542159557343, + 0.22789257764816284, + 0.2141895592212677, + 0.19900061190128326, + 0.1873021274805069, + 0.17724795639514923, + 0.1699712574481964, + 0.1617543250322342, + 0.15492086112499237, + 0.1496182233095169, + 0.1440984457731247, + 0.13983029127120972, + 0.13561177253723145, + 0.13243228197097778, + 0.1286381483078003, + 0.1261545568704605, + 0.12413957715034485, + 0.12230520695447922, + 0.12012018263339996, + 0.11821681261062622, + 0.11676468700170517, + 0.11532516777515411, + 0.11429379135370255, + 0.1133250743150711, + 0.1120050922036171, + 0.11107409000396729, + 0.1096409261226654, + 0.10849004238843918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010120303977600997, + "first_to_last_ratio": 2.7021675066537045, + "imbalance_abs_log_ratio": 0.994054231196686, + "last_quartile_mean": 3.7452541164384456e-05, + "mean": 6.280763545873924e-05, + "normalized": [ + 1.9944816746100333, + 1.8472480968114133, + 1.7373223217437128, + 1.6248488809553971, + 1.5471371990089653, + 1.475823179261819, + 1.3807460832170038, + 1.2829322622274897, + 1.184046410564159, + 1.1043973349513228, + 1.0901910170788929, + 1.0456150919711364, + 0.9888694060739084, + 0.9664075020831666, + 0.9198699716804313, + 0.9041232034205934, + 0.8692950074065211, + 0.8446819175937094, + 0.7923092109368705, + 0.7644112750281646, + 0.7503465778061283, + 0.730017034806627, + 0.7090250315296005, + 0.6754099655367306, + 0.6572034579968898, + 0.6267819521510305, + 0.6185439744898635, + 0.6068927943478243, + 0.594580659914726, + 0.5858446061910916, + 0.5555293303136664, + 0.5250675682911103 + ], + "peak_layer": 1, + "peak_normalized": 1.9944816746100333, + "population_cv": 0.40483316550850257, + "rest_mean": 6.624576713807915e-05, + "spike_contrast": 0.6678422833882899, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4241724390303715e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012526867794804275, + 0.0001160212850663811, + 0.0001091171070584096, + 0.00010205291619058698, + 9.717202920001e-05, + 9.269296424463391e-05, + 8.67213966557756e-05, + 8.057794184423983e-05, + 7.43671553209424e-05, + 6.936458521522582e-05, + 6.847231998108327e-05, + 6.567261152667925e-05, + 6.210854917299002e-05, + 6.0697770095430315e-05, + 5.777485785074532e-05, + 5.6785840570228174e-05, + 5.4598363931290805e-05, + 5.305247395881452e-05, + 4.9763068091124296e-05, + 4.801086470251903e-05, + 4.712749432655983e-05, + 4.585064380080439e-05, + 4.453218571143225e-05, + 4.242090290063061e-05, + 4.1277395212091506e-05, + 3.936669236281887e-05, + 3.8849284464959055e-05, + 3.811750138993375e-05, + 3.734420533874072e-05, + 3.679551446111873e-05, + 3.4891483664978296e-05, + 3.297825242043473e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021292950441420544, + "first_to_last_ratio": 1.6819067487661699, + "imbalance_abs_log_ratio": 0.5199281193278776, + "last_quartile_mean": 0.0001266000654140953, + "mean": 0.0001612693722563563, + "normalized": [ + 1.5193752902631465, + 1.4431842082185322, + 1.3854135927323608, + 1.3303138916593422, + 1.2819858609596173, + 1.2449680699964556, + 1.2025089225516594, + 1.1549255897117912, + 1.1161361523436344, + 1.0820631347545688, + 1.0617857510438193, + 1.0341003696475617, + 1.0077971864601107, + 0.9864740856164548, + 0.9634556759892069, + 0.9447446572547388, + 0.9240188125766456, + 0.9078710595764491, + 0.88634376010722, + 0.8724890232241534, + 0.8607832896405347, + 0.849628883322604, + 0.8359853832855236, + 0.8234683594489096, + 0.8139294954165563, + 0.8040629029670215, + 0.7969416676465471, + 0.7902390259734583, + 0.7809695092222964, + 0.7743360767166025, + 0.7639958481267083, + 0.7557044635457686 + ], + "peak_layer": 1, + "peak_normalized": 1.5193752902631465, + "population_cv": 0.21463654613558777, + "rest_mean": 0.00016614451305940747, + "spike_contrast": 0.8122062500591205, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013494361191987992, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024502869928255677, + 0.00023274141130968928, + 0.00022342478041537106, + 0.0002145388862118125, + 0.00020674505503848195, + 0.00020077521912753582, + 0.00019392785907257348, + 0.00018625412485562265, + 0.00017999857664108276, + 0.00017450364248361439, + 0.00017123352154158056, + 0.00016676871746312827, + 0.0001625268196221441, + 0.00015908805653452873, + 0.0001553758920636028, + 0.0001523583778180182, + 0.0001490159338572994, + 0.000146411795867607, + 0.00014294010179582983, + 0.0001407057570759207, + 0.00013881798076909035, + 0.00013701911666430533, + 0.0001348188379779458, + 0.0001328002254012972, + 0.00013126189878676087, + 0.0001296707196161151, + 0.00012852228246629238, + 0.00012744135165121406, + 0.0001259464625036344, + 0.00012487669300753623, + 0.00012320913083385676, + 0.00012187198444735259 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016320452232321259, + "first_to_last_ratio": 1.2889611434881496, + "imbalance_abs_log_ratio": 0.25383657880769345, + "last_quartile_mean": 0.0001266171002498595, + "mean": 0.0001437355554116948, + "normalized": [ + 1.188749445631, + 1.1713423911172207, + 1.1549430883999143, + 1.1417765084687417, + 1.1273283201397843, + 1.1145504039766105, + 1.0995991496765185, + 1.0853099095412984, + 1.0687879028680622, + 1.0554743211679045, + 1.0473286818228558, + 1.0363809984280803, + 1.0249947395076546, + 1.015899258153915, + 1.005968438235508, + 0.9970397006238613, + 0.9860146691908593, + 0.977410712347053, + 0.964005406402611, + 0.9549550784847994, + 0.9474810711889571, + 0.940343993567483, + 0.9294782140508817, + 0.9176125795270506, + 0.909513915953909, + 0.8990879268536105, + 0.8931112722873075, + 0.8859642730588357, + 0.8770245001904262, + 0.8704731011044752, + 0.8603223859779169, + 0.851727642054894 + ], + "peak_layer": 1, + "peak_normalized": 1.188749445631, + "population_cv": 0.0978236443065259, + "rest_mean": 0.0001456284474072702, + "spike_contrast": 0.9168122095142401, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001335139386355877, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00017086556181311607, + 0.00016836354916449636, + 0.00016600638628005981, + 0.00016411388060078025, + 0.0001620371622266248, + 0.00016020052134990692, + 0.0001580514945089817, + 0.00015599762264173478, + 0.00015362282283604145, + 0.0001517091877758503, + 0.00015053836978040636, + 0.0001489647984271869, + 0.00014732818817719817, + 0.0001460208441130817, + 0.00014459343219641596, + 0.0001433100551366806, + 0.00014172536612022668, + 0.00014048867160454392, + 0.00013856185250915587, + 0.00013726099859923124, + 0.0001361867180094123, + 0.00013516086619347334, + 0.00013359906733967364, + 0.0001318935537710786, + 0.0001307294878643006, + 0.00012923090253025293, + 0.00012837184476666152, + 0.0001273445668630302, + 0.00012605960364453495, + 0.0001251179346581921, + 0.00012365891598165035, + 0.00012242354569025338 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0004412600719660986, + "first_to_last_ratio": 3.098863205201153, + "imbalance_abs_log_ratio": 1.131035336237652, + "last_quartile_mean": 0.00014239417578210123, + "mean": 0.0002548141455918085, + "normalized": [ + 2.1785577639630773, + 1.9956115105027457, + 1.9014578504944815, + 1.7829105267225334, + 1.6475854563109211, + 1.565091756282257, + 1.45799103716316, + 1.3243443696030393, + 1.2370613678820481, + 1.1609137324491365, + 1.1300355744023234, + 1.0677196003316773, + 1.0048909110245325, + 0.948494062464312, + 0.8853730946340664, + 0.8440976906413151, + 0.7906677717627395, + 0.747069444517614, + 0.6991826523616375, + 0.6712814178889162, + 0.6509454457824981, + 0.6314362826751919, + 0.6113540983289655, + 0.5954000197507868, + 0.5831757760496624, + 0.5733601746441271, + 0.5653129211221045, + 0.559900286080882, + 0.5532114024202266, + 0.5496528914136888, + 0.545014697375157, + 0.5408984129541747 + ], + "peak_layer": 1, + "peak_normalized": 2.1785577639630773, + "population_cv": 0.4822008017525282, + "rest_mean": 0.00027300682214209465, + "spike_contrast": 0.573515676244785, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015657369222026317, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005551273352466524, + 0.0005085100419819355, + 0.000484518357552588, + 0.00045431082253344357, + 0.00041982808033935726, + 0.0003988075186498463, + 0.0003715167404152453, + 0.0003374616790097207, + 0.000315220735501498, + 0.0002958172408398241, + 0.0002879490493796766, + 0.0002720700576901436, + 0.0002560604189056903, + 0.00024168970412574708, + 0.00022560558863915503, + 0.00021508803183678538, + 0.0002014733327087015, + 0.0001903638622025028, + 0.0001781616301741451, + 0.0001710520009510219, + 0.00016587010759394616, + 0.0001608988968655467, + 0.00015578167221974581, + 0.00015171634731814265, + 0.00014860143710393459, + 0.00014610028301831335, + 0.0001440497289877385, + 0.0001426705130143091, + 0.00014096609083935618, + 0.0001400593318976462, + 0.0001388774544466287, + 0.00013782856694888324 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5fc2b38f630d3abe2bfa0bdec2dcf6dd2176078004f6dab96c8a9760a5eda637", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027412420604377985, + 0.002683233004063368, + 0.002711885841563344, + 0.0027249811682850122, + 0.0027012599166482687, + 0.002700963756069541, + 0.0026683746837079525, + 0.002738022478297353, + 0.0027370047755539417, + 0.002654933836311102, + 0.002663377905264497, + 0.002684312639757991, + 0.0027356327045708895, + 0.002652956871315837, + 0.002679797587916255, + 0.002698024269193411, + 0.0026484597474336624, + 0.0026680538430809975, + 0.002654345240443945, + 0.002591830212622881, + 0.002620960585772991, + 0.002651234855875373, + 0.0026270023081451654, + 0.002615687670186162, + 0.00258000404573977, + 0.002626793459057808, + 0.002573782578110695, + 0.002592637902125716, + 0.002581534907221794, + 0.002551236655563116, + 0.002563327783718705, + 0.002589041367173195 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024784023116808385, + "first_to_last_ratio": 4.996872150813687, + "imbalance_abs_log_ratio": 1.6088121468463858, + "last_quartile_mean": 4.959907391821616e-05, + "mean": 0.00012175017718618619, + "normalized": [ + 2.94187592163941, + 2.5796687009574604, + 2.293010133276413, + 2.014581184592976, + 1.8111575385173901, + 1.6842185280489494, + 1.5559062012549134, + 1.4047478191310576, + 1.2749623745204015, + 1.1699150302730006, + 1.0919472369387608, + 1.0017565099771182, + 0.923449750141626, + 0.8652102016053241, + 0.794546741948017, + 0.743905553038253, + 0.6942834150416333, + 0.6562793060663384, + 0.6082995898528493, + 0.5744438165059796, + 0.5512557468449423, + 0.5265581646918154, + 0.5020472770644135, + 0.47690127545745264, + 0.4572061982209032, + 0.43893553328289897, + 0.42961039869683454, + 0.41603121061550513, + 0.3999104909112459, + 0.3911954601706125, + 0.3717234123738856, + 0.35445927834161867 + ], + "peak_layer": 1, + "peak_normalized": 2.94187592163941, + "population_cv": 0.6874421460933116, + "rest_mean": 0.0001329603533290706, + "spike_contrast": 0.46040210094136547, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.121522601461038e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000358173914719373, + 0.0003140751214232296, + 0.00027917439001612365, + 0.0002452756161801517, + 0.00022050875122658908, + 0.0002050539042102173, + 0.00018943185568787158, + 0.0001710282958811149, + 0.00015522689500357956, + 0.00014243736222852021, + 0.00013294476957526058, + 0.00012196403258712962, + 0.00011243017070228234, + 0.0001053394953487441, + 9.673620661487803e-05, + 9.057063289219514e-05, + 8.45291287987493e-05, + 7.990212179720402e-05, + 7.406058284686878e-05, + 6.993863644311205e-05, + 6.711548485327512e-05, + 6.410854985006154e-05, + 6.112434493843466e-05, + 5.806281478726305e-05, + 5.566493564401753e-05, + 5.3440478950506076e-05, + 5.23051421623677e-05, + 5.0651873607421294e-05, + 4.8689173127058893e-05, + 4.762811659020372e-05, + 4.525739132077433e-05, + 4.315547994337976e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002960815872938838, + "first_to_last_ratio": 2.3434901925615312, + "imbalance_abs_log_ratio": 0.8516413534183918, + "last_quartile_mean": 0.00012634214908757713, + "mean": 0.00018583460405352525, + "normalized": [ + 2.1691817718266027, + 1.9356770461635995, + 1.7512361881653136, + 1.5771501673188, + 1.4517248962140685, + 1.3691574500653627, + 1.2898282592750647, + 1.2020702391870204, + 1.1274711846152252, + 1.06796916249687, + 1.023464646242978, + 0.9751293351351744, + 0.9332415616222968, + 0.9026335726010739, + 0.8668647658197504, + 0.8407791679025106, + 0.8171800119835181, + 0.7975248027052753, + 0.7770671945621679, + 0.7612080123915693, + 0.7492445405290613, + 0.7365475788483447, + 0.7249228575947372, + 0.7138182430319103, + 0.7048361054118131, + 0.6962681268232633, + 0.6911654119184109, + 0.6833348384348117, + 0.6760742523950837, + 0.6702828385523486, + 0.6624247014500871, + 0.654521068715886 + ], + "peak_layer": 1, + "peak_normalized": 2.1691817718266027, + "population_cv": 0.3943796352106793, + "rest_mean": 0.00019526833030222742, + "spike_contrast": 0.6908057343541223, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013489248231053353, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004031090356875211, + 0.0003597157774493098, + 0.0003254402836319059, + 0.0002930890768766403, + 0.00026978072128258646, + 0.00025443683261983097, + 0.00023969472385942936, + 0.00022338624694384634, + 0.00020952316117472947, + 0.0001984656264539808, + 0.0001901951472973451, + 0.00018121277389582247, + 0.0001734285760903731, + 0.00016774055256973952, + 0.0001610934705240652, + 0.0001562458637636155, + 0.00015186032396741211, + 0.00014820770593360066, + 0.00014440597442444414, + 0.00014145878958515823, + 0.00013923556252848357, + 0.0001368760276818648, + 0.00013471575221046805, + 0.00013265213056001812, + 0.00013098293857183307, + 0.00012939071166329086, + 0.00012844245065934956, + 0.00012698725913651288, + 0.00012563799100462347, + 0.0001245617459062487, + 0.00012310143210925162, + 0.00012163266364950687 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.26256811060011387, + "first_to_last_ratio": 2.3434901455089974, + "imbalance_abs_log_ratio": 0.8516413333404181, + "last_quartile_mean": 0.11204148270189762, + "mean": 0.16479998547583818, + "normalized": [ + 2.1691817568110046, + 1.935676923537455, + 1.7512361570514487, + 1.5771502517930829, + 1.4517248703109182, + 1.369157420355369, + 1.2898282606548555, + 1.2020702692325518, + 1.1274712340116186, + 1.0679692103147351, + 1.0234646460792782, + 0.975129361875517, + 0.9332415430463463, + 0.9026335787242323, + 0.8668647305888977, + 0.8407791967519535, + 0.8171800231493583, + 0.7975247749486774, + 0.7770672329681078, + 0.7612079835803747, + 0.7492445111174014, + 0.7365475542475244, + 0.7249228843083022, + 0.7138182179223472, + 0.7048361082067401, + 0.6962681206044983, + 0.6911654218524023, + 0.6833348067031066, + 0.676074287607254, + 0.6702828621875807, + 0.6624247142471237, + 0.6545210852099367 + ], + "peak_layer": 1, + "peak_normalized": 2.1691817568110046, + "population_cv": 0.3943796257753467, + "rest_mean": 0.17316590856622766, + "spike_contrast": 0.6908057237027376, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11962400078773498, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.35748112201690674, + 0.3189995288848877, + 0.28860369324684143, + 0.2599143385887146, + 0.2392442375421524, + 0.22563712298870087, + 0.2125636786222458, + 0.19810116291046143, + 0.1858072429895401, + 0.17600131034851074, + 0.16866695880889893, + 0.16070130467414856, + 0.1537981927394867, + 0.14875400066375732, + 0.1428592950105667, + 0.13856039941310883, + 0.13467125594615936, + 0.13143207132816315, + 0.12806066870689392, + 0.12544706463813782, + 0.12347548454999924, + 0.12138302624225616, + 0.11946728080511093, + 0.11763723194599152, + 0.11615698039531708, + 0.11474497616291046, + 0.11390405148267746, + 0.11261356621980667, + 0.11141703277826309, + 0.11046260595321655, + 0.10916758328676224, + 0.10786506533622742 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001010398200378404, + "first_to_last_ratio": 2.701478580108855, + "imbalance_abs_log_ratio": 0.9937992453824708, + "last_quartile_mean": 3.740167358046165e-05, + "mean": 6.270599271829269e-05, + "normalized": [ + 1.9907220064146427, + 1.8444741506775382, + 1.742884685905627, + 1.6239253294262157, + 1.5436270879110234, + 1.4787029723792133, + 1.3841644782908762, + 1.2821103017907618, + 1.1824374683113494, + 1.1042100658007115, + 1.0908679078034196, + 1.0498354286471192, + 0.9920033594422748, + 0.9660018389944104, + 0.9202838356691797, + 0.8989347700163286, + 0.8686518679586676, + 0.8410583182307008, + 0.7938538755561205, + 0.7653997098760733, + 0.7511903220519177, + 0.7265360595007238, + 0.7106247592487483, + 0.6758121071782397, + 0.6556340454607258, + 0.6278169583683687, + 0.6205082786340235, + 0.6073860029650574, + 0.5911469097961847, + 0.5867782134359579, + 0.5577710930141501, + 0.5246457912436486 + ], + "peak_layer": 1, + "peak_normalized": 1.9907220064146427, + "population_cv": 0.40468494452628273, + "rest_mean": 6.614368087159366e-05, + "spike_contrast": 0.6673725457789733, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4142476690467446e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001248301996383816, + 0.00011565958266146481, + 0.0001092893144232221, + 0.00010182984988205135, + 9.679466893430799e-05, + 9.272353781852871e-05, + 8.679540769662708e-05, + 8.039599924813956e-05, + 7.414591527776793e-05, + 6.92405883455649e-05, + 6.84039550833404e-05, + 6.583097274415195e-05, + 6.220455543370917e-05, + 6.0574104281840846e-05, + 5.770731149823405e-05, + 5.636859714286402e-05, + 5.446967770694755e-05, + 5.273939677863382e-05, + 4.9779395340010524e-05, + 4.799514863407239e-05, + 4.710413486463949e-05, + 4.5558164856629446e-05, + 4.456043097889051e-05, + 4.237746907165274e-05, + 4.111218368052505e-05, + 3.936788561986759e-05, + 3.890958760166541e-05, + 3.808674227911979e-05, + 3.7068453821120784e-05, + 3.679451037896797e-05, + 3.497559009701945e-05, + 3.2898435165407136e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021103879953443538, + "first_to_last_ratio": 1.6754332028061545, + "imbalance_abs_log_ratio": 0.5160717603794217, + "last_quartile_mean": 0.00012596073611348402, + "mean": 0.0001601102544555033, + "normalized": [ + 1.5150372860566723, + 1.439510593902249, + 1.3841055212355642, + 1.3272119947749268, + 1.2794362361966427, + 1.2428436489609036, + 1.2018556832108258, + 1.1546727794959446, + 1.1147996317367452, + 1.0814138049096294, + 1.0614905005088058, + 1.0346901524019803, + 1.0074919017055521, + 0.9875011592687566, + 0.9623166883068708, + 0.9431047552100263, + 0.9243559814545742, + 0.9073018837351503, + 0.8887802361951885, + 0.87382889642758, + 0.8622859040817829, + 0.849391060863955, + 0.8374811583413132, + 0.8253926631762389, + 0.8155827011133436, + 0.8058320881575772, + 0.7999867009830005, + 0.7910419816457831, + 0.7825317922950641, + 0.7756782882808636, + 0.7662211489344684, + 0.7568251764320209 + ], + "peak_layer": 1, + "peak_normalized": 1.5150372860566723, + "population_cv": 0.21304026877727222, + "rest_mean": 0.00016491277050658095, + "spike_contrast": 0.8136220583009953, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013417666777968407, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024257300538010895, + 0.00023048040748108178, + 0.0002216094871982932, + 0.00021250025019980967, + 0.0002048508613370359, + 0.00019899201288353652, + 0.0001924294192576781, + 0.00018487495253793895, + 0.00017849085270427167, + 0.00017314543947577477, + 0.00016995551413856447, + 0.00016566450358368456, + 0.00016130978474393487, + 0.00015810906188562512, + 0.00015407676983159035, + 0.00015100074233487248, + 0.00014799887139815837, + 0.0001452683354727924, + 0.00014230282977223396, + 0.00013990896695759147, + 0.00013806081551592797, + 0.00013599621888715774, + 0.00013408932136371732, + 0.00013215382932685316, + 0.00013058315380476415, + 0.00012902198068331927, + 0.00012808607425540686, + 0.0001266539329662919, + 0.00012529136438388377, + 0.00012419404811225832, + 0.00012267986312508583, + 0.00012117547157686204 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016164849512279034, + "first_to_last_ratio": 1.282986844747637, + "imbalance_abs_log_ratio": 0.24919083207168544, + "last_quartile_mean": 0.00012599388355738483, + "mean": 0.00014267631695474847, + "normalized": [ + 1.1849632230478688, + 1.1695181917643926, + 1.1537907432295644, + 1.136648454640294, + 1.1262413432306055, + 1.112705815659591, + 1.0977204646042988, + 1.082200264843305, + 1.06813080649915, + 1.0535985061590996, + 1.046252801833764, + 1.0365714691025223, + 1.024670983408149, + 1.0169605538784676, + 1.0046472025165352, + 0.995758555548206, + 0.9858673222379531, + 0.9760103584099877, + 0.9653106304982666, + 0.9567951741123486, + 0.9498417329681972, + 0.9401737612553472, + 0.9304537733640251, + 0.9205681496416466, + 0.911480005330001, + 0.9022600866995591, + 0.8969925816929462, + 0.8873573495747816, + 0.8789659160771423, + 0.8714788501373707, + 0.8627683840784355, + 0.8532965439561776 + ], + "peak_layer": 1, + "peak_normalized": 1.1849632230478688, + "population_cv": 0.09608600673281108, + "rest_mean": 0.00014451252193086676, + "spike_contrast": 0.918680321330362, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013276081008370966, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016906618839129806, + 0.00016686254821252078, + 0.00016461861378047615, + 0.00016217281518038362, + 0.00016068796685431153, + 0.00015875676763243973, + 0.0001566187129355967, + 0.00015440434799529612, + 0.00015239696949720383, + 0.0001503235544078052, + 0.00014927549636922777, + 0.00014789419947192073, + 0.00014619628200307488, + 0.0001450961863156408, + 0.00014333936269395053, + 0.00014207116328179836, + 0.00014065991854295135, + 0.00013925356324762106, + 0.00013772696547675878, + 0.0001365120115224272, + 0.00013551992014981806, + 0.00013414052955340594, + 0.00013275371748022735, + 0.0001313432730967179, + 0.00013004661013837904, + 0.00012873114610556513, + 0.0001279795978916809, + 0.00012660487846005708, + 0.0001254076196346432, + 0.00012433939264155924, + 0.00012309661542531103, + 0.00012174520816188306 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00043634913890855387, + "first_to_last_ratio": 3.0823770886184128, + "imbalance_abs_log_ratio": 1.125701081333854, + "last_quartile_mean": 0.00014156254292174708, + "mean": 0.0002526328457861382, + "normalized": [ + 2.161262991989335, + 1.9867412804887739, + 1.8904526133838582, + 1.7788259407438674, + 1.6404609794852394, + 1.5723911781894129, + 1.459249713339863, + 1.3282687448916313, + 1.2383315143134188, + 1.1700416770361293, + 1.131363959578303, + 1.0685850444189446, + 1.0069989316428685, + 0.9451411723514572, + 0.8866050347634621, + 0.8407661253267313, + 0.7895843736134068, + 0.7479926194272515, + 0.7047300318080335, + 0.6729704299307414, + 0.6547401608613147, + 0.6315820650248338, + 0.6127131155987415, + 0.5974089218754778, + 0.5839699651029305, + 0.5735248801032955, + 0.5690230706448541, + 0.5599717580649735, + 0.5552403508520525, + 0.5513694456095706, + 0.5470291494736481, + 0.5426627600655782 + ], + "peak_layer": 1, + "peak_normalized": 2.161262991989335, + "population_cv": 0.4790543297146424, + "rest_mean": 0.0002705939723118381, + "spike_contrast": 0.5751893185853856, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015564276254735886, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005460060201585293, + 0.0005019161035306752, + 0.0004775904235430062, + 0.00044938985956832767, + 0.0004144343256484717, + 0.0003972376580350101, + 0.000368654407793656, + 0.00033556431299075484, + 0.00031284321448765695, + 0.000295590958558023, + 0.0002858196967281401, + 0.0002699596807360649, + 0.0002544010058045387, + 0.00023877370404079556, + 0.0002239855530206114, + 0.00021240513888187706, + 0.00019947494729422033, + 0.0001889675040729344, + 0.0001780379534466192, + 0.00017001443484332412, + 0.00016540887008886784, + 0.00015955837443470955, + 0.00015479145804420114, + 0.00015092511603143066, + 0.00014752999413758516, + 0.00014489122258964926, + 0.00014375391765497625, + 0.00014146725879982114, + 0.00014027194993104786, + 0.00013929403212387115, + 0.00013819753075949848, + 0.00013709443737752736 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "fa136baf3a47ab2a864681a8138aae42720212f69227b3c003421051ed7bc3a1", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0023513585329055786, + 0.004408807028084993, + 0.005783648230135441, + 0.006751363631337881, + 0.002628475893288851, + 0.004723479505628347, + 0.006024772301316261, + 0.007445912808179855, + 0.003561305580660701, + 0.005543973762542009, + 0.006855612155050039, + 0.007876631803810596, + 0.0035210708156228065, + 0.0058924350887537, + 0.007390674669295549, + 0.008790239691734314, + 0.004162859171628952, + 0.006355749908834696, + 0.007784162648022175, + 0.00937180407345295, + 0.004737259820103645, + 0.006999304983764887, + 0.008728929795324802, + 0.010217384435236454, + 0.005580419674515724, + 0.008008211851119995, + 0.010209310799837112, + 0.012040933594107628, + 0.005235649179667234, + 0.007745372597128153, + 0.010180563665926456, + 0.011906592175364494 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024914481400628574, + "first_to_last_ratio": 5.005367240493075, + "imbalance_abs_log_ratio": 1.6105107847992817, + "last_quartile_mean": 4.977553135177004e-05, + "mean": 0.0001223312358433759, + "normalized": [ + 2.9462138255793877, + 2.588435651553335, + 2.2854731426364707, + 2.0157214492628985, + 1.8168708763555168, + 1.6875649797632875, + 1.5554268829788211, + 1.3974214212146046, + 1.275680469442624, + 1.1684152736282016, + 1.0925056707858531, + 1.0000365768115411, + 0.9220243844233116, + 0.8633153808206192, + 0.7954940059538663, + 0.747131175135411, + 0.6955592946052511, + 0.6567700236831091, + 0.6067846463225012, + 0.5734077695641915, + 0.5496986650820618, + 0.5277889932643854, + 0.50090777460584, + 0.47622023530363405, + 0.457550957846772, + 0.43796965221258716, + 0.42825768814458814, + 0.4157111150450942, + 0.4005890169587395, + 0.39033503431894295, + 0.37039430607646523, + 0.3543236606200867 + ], + "peak_layer": 1, + "peak_normalized": 2.9462138255793877, + "population_cv": 0.6883836085455175, + "rest_mean": 0.00013360307773971862, + "spike_contrast": 0.46004396487681354, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.14632896031253e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036041397834196687, + 0.00031664653215557337, + 0.0002795847540255636, + 0.0002465856960043311, + 0.0002222600596724078, + 0.0002064419095404446, + 0.00019027729285880923, + 0.00017094828945118934, + 0.00015605556836817414, + 0.00014293368440121412, + 0.00013364756887312979, + 0.00012233571032993495, + 0.00011279238242423162, + 0.00010561043745838106, + 9.73137648543343e-05, + 9.139747999142855e-05, + 8.508862811140716e-05, + 8.0343488662038e-05, + 7.422871567541733e-05, + 7.014568109298125e-05, + 6.724531704094261e-05, + 6.456507981056347e-05, + 6.127666711108759e-05, + 5.8256609918316826e-05, + 5.597277413471602e-05, + 5.357736881705932e-05, + 5.238929225015454e-05, + 5.0854454457294196e-05, + 4.9004549509845674e-05, + 4.775016714120284e-05, + 4.531079321168363e-05, + 4.334485129220411e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00029868123783671763, + "first_to_last_ratio": 2.352020548935129, + "imbalance_abs_log_ratio": 0.8552747667903616, + "last_quartile_mean": 0.00012698921273113228, + "mean": 0.0001871975605354237, + "normalized": [ + 2.1745956556501205, + 1.943389916860048, + 1.7490167059616495, + 1.579801534641211, + 1.4557872843351107, + 1.3727714128797337, + 1.290211227528517, + 1.198748061313539, + 1.1282731192017241, + 1.0677025479901412, + 1.0238711728275764, + 0.9743826669000581, + 0.9332092274716193, + 0.901273964826617, + 0.8680163418407979, + 0.8423074677807364, + 0.8168973316425937, + 0.7977426572048737, + 0.7748904888155362, + 0.7599280177062464, + 0.7477886704872306, + 0.7367391764207782, + 0.7235805599766263, + 0.7121146362854748, + 0.7033711688788506, + 0.6946931548498627, + 0.6884857337520279, + 0.6826539310873735, + 0.6746941940291191, + 0.6690858034871798, + 0.660454275348722, + 0.6535218920183048 + ], + "peak_layer": 1, + "peak_normalized": 2.1745956556501205, + "population_cv": 0.39609817473730136, + "rest_mean": 0.00019674051630621156, + "spike_contrast": 0.6895661448911525, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013566559937316924, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0004070790018886328, + 0.00036379785160534084, + 0.00032741166069172323, + 0.00029573499341495335, + 0.00027251982828602195, + 0.00025697945966385305, + 0.00024152439436875284, + 0.000224402712774463, + 0.00021120997553225607, + 0.00019987131236121058, + 0.00019166618585586548, + 0.0001824020582716912, + 0.00017469449085183442, + 0.00016871628758963197, + 0.00016249054169747978, + 0.00015767790318932384, + 0.00015292118769139051, + 0.0001493354793637991, + 0.0001450576091883704, + 0.00014225667109712958, + 0.00013998421491123736, + 0.00013791577657684684, + 0.00013545251567848027, + 0.00013330612273421139, + 0.00013166936696507037, + 0.00013004486390855163, + 0.00012888284982182086, + 0.00012779115058947355, + 0.00012630110722966492, + 0.00012525123020168394, + 0.00012363542919047177, + 0.00012233770394232124 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.2648735139518976, + "first_to_last_ratio": 2.3520205903150884, + "imbalance_abs_log_ratio": 0.855274784383728, + "last_quartile_mean": 0.1126153040677309, + "mean": 0.1660086705815047, + "normalized": [ + 2.174595632268317, + 1.943389981095013, + 1.7490168198184282, + 1.5798016086666014, + 1.4557873374041121, + 1.3727712901980273, + 1.2902112306135503, + 1.1987480127171795, + 1.1282731510481394, + 1.0677025559476454, + 1.0238711946273023, + 0.9743826246747078, + 0.9332091864770874, + 0.9012739856841682, + 0.8680163310272434, + 0.8423074257801569, + 0.8168973360433722, + 0.7977426245274992, + 0.7748904637775177, + 0.7599280557788093, + 0.7477886868929706, + 0.7367391104705062, + 0.7235805912981054, + 0.7121146568868953, + 0.7033711391232357, + 0.6946931471398464, + 0.6884857463566807, + 0.682653951912613, + 0.6746941853139277, + 0.6690857619715078, + 0.6604542663637077, + 0.6535219080951254 + ], + "peak_layer": 1, + "peak_normalized": 2.174595632268317, + "population_cv": 0.39609818748801673, + "rest_mean": 0.1744714594549603, + "spike_contrast": 0.6895661390160055, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12030961066484451, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.36100172996520996, + 0.32261958718299866, + 0.2903519570827484, + 0.262260764837265, + 0.2416733205318451, + 0.2278919368982315, + 0.21418625116348267, + 0.19900256395339966, + 0.18730312585830688, + 0.17724788188934326, + 0.1699714958667755, + 0.16175596415996552, + 0.1549208164215088, + 0.14961929619312286, + 0.14409823715686798, + 0.1398303359746933, + 0.13561204075813293, + 0.13243219256401062, + 0.12863853573799133, + 0.12615464627742767, + 0.12413940578699112, + 0.12230508029460907, + 0.12012065201997757, + 0.11821720749139786, + 0.11676570773124695, + 0.11532508581876755, + 0.11429460346698761, + 0.11332647502422333, + 0.1120050847530365, + 0.1110740378499031, + 0.10964113473892212, + 0.10849030315876007 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010120356273546349, + "first_to_last_ratio": 2.7021644745482813, + "imbalance_abs_log_ratio": 0.9940531090948486, + "last_quartile_mean": 3.7452776723512216e-05, + "mean": 6.280785908074904e-05, + "normalized": [ + 1.9944873163456722, + 1.8472744197374709, + 1.7373237819064158, + 1.6248552595134786, + 1.5471235814302835, + 1.4758134067763151, + 1.3807453375978929, + 1.2829373095670376, + 1.18406084586495, + 1.104394561286083, + 1.0901943179184503, + 1.04561090576924, + 0.9888677388041794, + 0.9664019181449638, + 0.9198712145053942, + 0.9041143079769479, + 0.8692998477179782, + 0.8446721332610243, + 0.7923141515826202, + 0.7644051939108963, + 0.7503405467650113, + 0.7300062685920449, + 0.7090277780404898, + 0.6754004942706008, + 0.6572075474610624, + 0.6267776932652476, + 0.618543683654507, + 0.6069002486680706, + 0.5945805123210514, + 0.585842809953234, + 0.5555319861870183, + 0.5250728812043677 + ], + "peak_layer": 1, + "peak_normalized": 1.9944873163456722, + "population_cv": 0.404835319326031, + "rest_mean": 6.624604820562699e-05, + "spike_contrast": 0.6678381428743124, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.424163780640811e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001252694783033803, + 0.00011602335143834352, + 0.00010911758727161214, + 0.00010205368016613647, + 9.717151988297701e-05, + 9.269268048228696e-05, + 8.672165859024972e-05, + 8.057854574872181e-05, + 7.436832675011829e-05, + 6.936465797480196e-05, + 6.847277109045535e-05, + 6.567258242284879e-05, + 6.210866558831185e-05, + 6.069763549021445e-05, + 5.7775141613092273e-05, + 5.678548404830508e-05, + 5.459886233438738e-05, + 5.305204831529409e-05, + 4.9763555580284446e-05, + 4.801065369974822e-05, + 4.7127283323789015e-05, + 4.585013084579259e-05, + 4.4532516767503694e-05, + 4.242045906721614e-05, + 4.127779902773909e-05, + 3.936656503356062e-05, + 3.884940451825969e-05, + 3.811810529441573e-05, + 3.734432903002016e-05, + 3.679553265101276e-05, + 3.489177470328286e-05, + 3.297870352980681e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0002129299391526729, + "first_to_last_ratio": 1.6819035009332732, + "imbalance_abs_log_ratio": 0.5199261882838202, + "last_quartile_mean": 0.00012660056836466538, + "mean": 0.0001612697526525153, + "normalized": [ + 1.5193801883639617, + 1.4431946098051458, + 1.385412670946279, + 1.330312377973009, + 1.2819779644638467, + 1.2449644115522398, + 1.2025034693572292, + 1.15492638462487, + 1.1161420015875916, + 1.082061394531008, + 1.0617850512168836, + 1.0341029835254725, + 1.0077924632434987, + 0.9864749169321705, + 0.9634524108641764, + 0.9447429702319522, + 0.9240217763435827, + 0.9078672036949645, + 0.8863448276060427, + 0.8724873261680857, + 0.860780266694715, + 0.8496270597212868, + 0.835986840267667, + 0.8234693045553775, + 0.8139348844586144, + 0.8040606454444867, + 0.7969452920911189, + 0.7902469974302787, + 0.780967396404848, + 0.7743340697791917, + 0.7639952190801171, + 0.7557046210402876 + ], + "peak_layer": 1, + "peak_normalized": 1.5193801883639617, + "population_cv": 0.21463676643173402, + "rest_mean": 0.00016614488736700473, + "spike_contrast": 0.8122069076744362, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001349440251942724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024503006716258824, + 0.00023274363775271922, + 0.000223425158765167, + 0.0002145391481462866, + 0.00020674426923505962, + 0.000200775102712214, + 0.00019392743706703186, + 0.00018625469238031656, + 0.00017999994452111423, + 0.00017450377345085144, + 0.00017123381257988513, + 0.00016676953237038106, + 0.00016252644127234817, + 0.00015908856585156173, + 0.0001553757319925353, + 0.00015235846512950957, + 0.0001490167633164674, + 0.00014641151938121766, + 0.00014294061111286283, + 0.00014070581528358161, + 0.00013881782069802284, + 0.00013701914576813579, + 0.00013481939095072448, + 0.00013280069106258452, + 0.00013126307749189436, + 0.00012967066140845418, + 0.0001285231701331213, + 0.00012744293780997396, + 0.0001259464188478887, + 0.00012487666390370578, + 0.00012320932000875473, + 0.00012187229731353 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016320142822223715, + "first_to_last_ratio": 1.2889451135618213, + "imbalance_abs_log_ratio": 0.25382414241573026, + "last_quartile_mean": 0.0001266162744286703, + "mean": 0.00014373415069712792, + "normalized": [ + 1.1884743462891088, + 1.1713432082551107, + 1.1549462763142666, + 1.141829986160868, + 1.1273728485863819, + 1.1144091299418613, + 1.0997826147068885, + 1.0853573683237072, + 1.068690221812727, + 1.0554384700276902, + 1.0473609881004726, + 1.0364070219594264, + 1.025017513262844, + 1.015761575885501, + 1.006046000385858, + 0.9971413723222552, + 0.9860728003762654, + 0.9774913363845525, + 0.9641287247440273, + 0.9550391277708558, + 0.9473133601248869, + 0.9403037775179377, + 0.9293980025104189, + 0.9176260020019316, + 0.9094819029195437, + 0.8991813518536786, + 0.8931890476284186, + 0.8860794380468366, + 0.8769637206693581, + 0.8703930215998912, + 0.8602546600212342, + 0.8517047834951949 + ], + "peak_layer": 1, + "peak_normalized": 1.1884743462891088, + "population_cv": 0.09781347741247146, + "rest_mean": 0.00014562865706264146, + "spike_contrast": 0.9167413819240867, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013350381632335485, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00017082435078918934, + 0.00016836202121339738, + 0.00016600522212684155, + 0.0001641199633013457, + 0.00016204197891056538, + 0.00016017864982131869, + 0.0001580763200763613, + 0.0001560029195388779, + 0.00015360728139057755, + 0.00015170255210250616, + 0.00015054154209792614, + 0.00014896708307787776, + 0.00014733002171851695, + 0.00014599962742067873, + 0.00014460316742770374, + 0.00014332326827570796, + 0.00014173233648762107, + 0.00014049888704903424, + 0.00013857822341378778, + 0.00013727173791266978, + 0.0001361612812615931, + 0.0001351537648588419, + 0.00013358623255044222, + 0.00013189419405534863, + 0.00013072360889054835, + 0.00012924306793138385, + 0.00012838176917284727, + 0.00012735987547785044, + 0.00012604963558260351, + 0.00012510520173236728, + 0.00012364797294139862, + 0.00012241906370036304 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00044128536683274433, + "first_to_last_ratio": 3.0989416798064484, + "imbalance_abs_log_ratio": 1.1310606595922121, + "last_quartile_mean": 0.0001423987323505571, + "mean": 0.0002548205006860371, + "normalized": [ + 2.1788974669107217, + 1.9948922239742408, + 1.9013861017158324, + 1.7832746019711707, + 1.6474456862757656, + 1.56528080717771, + 1.458232784418858, + 1.3245892215128081, + 1.2376166576486232, + 1.160533346247024, + 1.1299807802432356, + 1.0675072615016112, + 1.0048234765069712, + 0.9484981041710848, + 0.8855756138701967, + 0.8439855543529297, + 0.7905268728467926, + 0.7470509271970999, + 0.6989608308202429, + 0.6713105901225432, + 0.6509228727076454, + 0.631461080612247, + 0.611226351598172, + 0.5954626643405772, + 0.5833123929329473, + 0.5733046444152264, + 0.5652547933675338, + 0.559916360487464, + 0.5532517426009346, + 0.5496220513891801, + 0.5450027610740823, + 0.5408933749885276 + ], + "peak_layer": 1, + "peak_normalized": 2.1788974669107217, + "population_cv": 0.4822174837001359, + "rest_mean": 0.0002730129350244102, + "spike_contrast": 0.5735309033794403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015658135525882245, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0005552277434617281, + 0.0005083394353277981, + 0.0004845121584367007, + 0.0004544149269349873, + 0.00041980293462984264, + 0.0003988656389992684, + 0.00037158760824240744, + 0.0003375324886292219, + 0.00031537009635940194, + 0.0002957276883535087, + 0.00028794226818718016, + 0.00027202273486182094, + 0.00025604962138459086, + 0.00024169676180463284, + 0.0002256628213217482, + 0.00021506482153199613, + 0.0002014424535445869, + 0.00019036389130633324, + 0.00017810954886954278, + 0.00017106370069086552, + 0.0001658684923313558, + 0.00016090922872535884, + 0.00015575300494674593, + 0.00015173609426710755, + 0.00014863995602354407, + 0.00014608977653551847, + 0.00014403850946109742, + 0.00014267816732171923, + 0.00014097988605499268, + 0.00014005496632307768, + 0.0001388778764521703, + 0.00013783072063233703 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2a3a84f829820d08a03891b9dbdf9a92bd462faefce3ef1d7db1fc7c9dd54c0a", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0036361459642648697, + 0.005152894649654627, + 0.006386642809957266, + 0.007284422870725393, + 0.0037505170330405235, + 0.0054016136564314365, + 0.006585344672203064, + 0.007922882214188576, + 0.0044892593286931515, + 0.006135277915745974, + 0.007316684816032648, + 0.0082859443500638, + 0.0044554853811860085, + 0.006464940495789051, + 0.007893112488090992, + 0.009196000173687935, + 0.004961193539202213, + 0.006907128728926182, + 0.008217990398406982, + 0.009676996618509293, + 0.005381465889513493, + 0.007502692751586437, + 0.0091153085231781, + 0.010543142445385456, + 0.006136673036962748, + 0.008469858206808567, + 0.010564134456217289, + 0.012301022186875343, + 0.005773921962827444, + 0.008156762458384037, + 0.010457510128617287, + 0.012218860909342766 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024784039669611957, + "first_to_last_ratio": 4.996847129630424, + "imbalance_abs_log_ratio": 1.6088071394647374, + "last_quartile_mean": 4.959935540682636e-05, + "mean": 0.0001217503483985638, + "normalized": [ + 2.9419062071052267, + 2.5796112881191693, + 2.2930860326581057, + 2.0145554032379356, + 1.8111535572920587, + 1.6841756414503508, + 1.5558994713918, + 1.4047664015764878, + 1.2749699043555138, + 1.1699192416770665, + 1.0919281315608147, + 1.001760718810264, + 0.9234449853785138, + 0.8652188455092179, + 0.7945394094365259, + 0.7439156225148, + 0.6942882953063699, + 0.6562804150522287, + 0.6082998698912725, + 0.574440737760942, + 0.551250788347098, + 0.5265559301830911, + 0.5020487822241618, + 0.47689842352430634, + 0.4572026867293029, + 0.43893712719402944, + 0.4296155316374904, + 0.41603615348749273, + 0.3999129464790325, + 0.391194073390914, + 0.3717285968379238, + 0.3544587798804922 + ], + "peak_layer": 1, + "peak_normalized": 2.9419062071052267, + "population_cv": 0.6874423943830519, + "rest_mean": 0.00013296059464833147, + "spike_contrast": 0.4603997057303065, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.121501864981837e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00035817810567095876, + 0.0003140685730613768, + 0.00027918402338400483, + 0.00024527282221242785, + 0.00022050857660360634, + 0.00020504897111095488, + 0.0001894313027150929, + 0.00017103079881053418, + 0.00015522803005296737, + 0.0001424380752723664, + 0.000132942630443722, + 0.00012196471652714536, + 0.00011242974869674072, + 0.00010534069588175043, + 9.673544991528615e-05, + 9.057198622031137e-05, + 8.452984184259549e-05, + 7.99023691797629e-05, + 7.406072109006345e-05, + 6.99383599567227e-05, + 6.711497553624213e-05, + 6.410836795112118e-05, + 6.112461414886639e-05, + 5.806254921481013e-05, + 5.566458639805205e-05, + 5.34407481609378e-05, + 5.230584065429866e-05, + 5.0652546633500606e-05, + 4.868954056291841e-05, + 4.762801472679712e-05, + 4.5258086174726486e-05, + 4.315547994337976e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00029608183103846386, + "first_to_last_ratio": 2.343484699021421, + "imbalance_abs_log_ratio": 0.8516390092453431, + "last_quartile_mean": 0.0001263425492652459, + "mean": 0.0001858347804954974, + "normalized": [ + 2.16920774571012, + 1.9356424765570717, + 1.751272112160408, + 1.5771369240348272, + 1.451727119925344, + 1.369130935687747, + 1.289826251583108, + 1.2020808437231323, + 1.127478492836466, + 1.067970341065492, + 1.0234512239081746, + 0.9751296621832141, + 0.9332398141881068, + 0.9026389800429313, + 0.8668613586828855, + 0.8407822065969367, + 0.8171794710239251, + 0.7975274126338607, + 0.777066300159633, + 0.7612061150730458, + 0.7492407752334779, + 0.7365489154759011, + 0.7249215428669998, + 0.7138135717041164, + 0.7048334785593375, + 0.6962720074755282, + 0.6911692191098191, + 0.683337870004401, + 0.6760758813563923, + 0.6702804011180051, + 0.662430102042749, + 0.6545204472768429 + ], + "peak_layer": 1, + "peak_normalized": 2.16920774571012, + "population_cv": 0.39438035124793563, + "rest_mean": 0.0001952685916977417, + "spike_contrast": 0.6908033638721538, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001348922000033781, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00040311424527317286, + 0.00035970969474874437, + 0.00032544726855121553, + 0.00029308689408935606, + 0.00026978139067068696, + 0.00025443214690312743, + 0.00023969457834027708, + 0.0002233884297311306, + 0.0002095247182296589, + 0.0001984660339076072, + 0.0001901928335428238, + 0.00018121300672646612, + 0.0001734284160193056, + 0.0001677417167229578, + 0.00016109299031086266, + 0.00015624657680746168, + 0.0001518603676231578, + 0.00014820833166595548, + 0.00014440594532061368, + 0.0001414585713064298, + 0.00013923499500378966, + 0.00013687640603166074, + 0.00013471563579514623, + 0.00013265138841234148, + 0.00013098257477395236, + 0.0001293915556743741, + 0.00012844328011851758, + 0.0001269879430765286, + 0.0001256384130101651, + 0.00012456141121219844, + 0.0001231025526067242, + 0.00012163266364950687 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.2625683266669512, + "first_to_last_ratio": 2.343484691137257, + "imbalance_abs_log_ratio": 0.8516390058810522, + "last_quartile_mean": 0.1120418356731534, + "mean": 0.16480014030821621, + "normalized": [ + 2.169207748902955, + 1.9356423730461585, + 1.7512721262865893, + 1.5771370154883375, + 1.4517271231757105, + 1.3691309069442503, + 1.2898263254837699, + 1.202080803995401, + 1.1274784933366124, + 1.0679703770106845, + 1.0234512066143264, + 0.9751297116000902, + 0.9332398524746174, + 0.9026389696372329, + 0.8668612939874407, + 0.8407822044491968, + 0.8171795266543764, + 0.7975273711861923, + 0.7770662316421936, + 0.7612060929585753, + 0.7492407781338722, + 0.7365489418998509, + 0.7249215250846839, + 0.7138135688166896, + 0.704833501980704, + 0.6962720326387369, + 0.6911692482611179, + 0.6833378266924969, + 0.6760758677045711, + 0.6702804240542598, + 0.6624300595814382, + 0.6545204702768674 + ], + "peak_layer": 1, + "peak_normalized": 2.169207748902955, + "population_cv": 0.3943803506949354, + "rest_mean": 0.17316613815448903, + "spike_contrast": 0.6908033707584418, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11962375193834304, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.35748574137687683, + 0.3189941346645355, + 0.28860989212989807, + 0.2599124014377594, + 0.23924483358860016, + 0.22563296556472778, + 0.21256355941295624, + 0.19810308516025543, + 0.18580861389636993, + 0.1760016679763794, + 0.16866490244865417, + 0.16070151329040527, + 0.15379805862903595, + 0.1487550288438797, + 0.1428588628768921, + 0.13856102526187897, + 0.13467130064964294, + 0.13143262267112732, + 0.12806062400341034, + 0.1254468709230423, + 0.12347498536109924, + 0.12138336896896362, + 0.11946716904640198, + 0.11763657629489899, + 0.11615666002035141, + 0.11474572867155075, + 0.11390478909015656, + 0.11261416971683502, + 0.11141739785671234, + 0.11046230792999268, + 0.10916856676340103, + 0.10786506533622742 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00010104048214998329, + "first_to_last_ratio": 2.7014816335786884, + "imbalance_abs_log_ratio": 0.9938003756776099, + "last_quartile_mean": 3.740187639778014e-05, + "mean": 6.270627807225537e-05, + "normalized": [ + 1.9907250147127677, + 1.8444603036108247, + 1.7429056467158477, + 1.6239117898045463, + 1.5436487234069576, + 1.4786997242989472, + 1.3841625886805662, + 1.282123032540977, + 1.1824417181452251, + 1.1042125830371328, + 1.090862943651666, + 1.0498276343781445, + 0.9920003536086259, + 0.9660066676365643, + 0.9202805760437535, + 0.8989382213875909, + 0.8686500616317681, + 0.8410510099026279, + 0.7938571089189717, + 0.7653969230084752, + 0.751185801344563, + 0.7265337395686793, + 0.7106236720469997, + 0.6758027080339929, + 0.6556269427545545, + 0.6278192648368973, + 0.6205119527339269, + 0.6073894466988616, + 0.5911483388457922, + 0.586775311152339, + 0.5577767930941091, + 0.5246434037673043 + ], + "peak_layer": 1, + "peak_normalized": 1.9907250147127677, + "population_cv": 0.4046861823965828, + "rest_mean": 6.614403860617636e-05, + "spike_contrast": 0.6673673413246971, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.414237118908204e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012483095633797348, + 0.00011565924069145694, + 0.00010929112613666803, + 0.0001018294642562978, + 9.67964660958387e-05, + 9.272375609725714e-05, + 8.679568418301642e-05, + 8.039716340135783e-05, + 7.41465191822499e-05, + 6.924106128280982e-05, + 6.84039550833404e-05, + 6.583078356925398e-05, + 6.220465002115816e-05, + 6.0574682720471174e-05, + 5.770736970589496e-05, + 5.636907008010894e-05, + 5.446981231216341e-05, + 5.273917849990539e-05, + 4.977982462150976e-05, + 4.799519228981808e-05, + 4.7104065743042156e-05, + 4.555822670226917e-05, + 4.456056558410637e-05, + 4.237707253196277e-05, + 4.1111925384029746e-05, + 3.9368209399981424e-05, + 3.89099950552918e-05, + 3.808713154285215e-05, + 3.706871211761609e-05, + 3.679449582705274e-05, + 3.4976106690010056e-05, + 3.2898435165407136e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00021103927974763792, + "first_to_last_ratio": 1.675431813331652, + "imbalance_abs_log_ratio": 0.5160709310565479, + "last_quartile_mean": 0.00012596112719620578, + "mean": 0.00016011051866371417, + "normalized": [ + 1.5150472374771589, + 1.439502583502474, + 1.3841101446264577, + 1.327205260326266, + 1.2794452130965426, + 1.2428385988073838, + 1.2018548814818522, + 1.154676418187547, + 1.1148050630726085, + 1.0814152014364051, + 1.0614838409967562, + 1.034687172582471, + 1.0074911480481314, + 0.9875049829338368, + 0.9623163727426111, + 0.9431041078017948, + 0.9243534563643369, + 0.9073040220081667, + 0.888778951337658, + 0.8738270000353379, + 0.8622815727966864, + 0.8493922949455803, + 0.8374788674952537, + 0.8253865750371285, + 0.8155793557629833, + 0.8058361207192702, + 0.7999903796437624, + 0.7910448570865577, + 0.7825332275919701, + 0.7756749178925374, + 0.766226246614208, + 0.7568239275482656 + ], + "peak_layer": 1, + "peak_normalized": 1.5150472374771589, + "population_cv": 0.21304076077436968, + "rest_mean": 0.0001649131305317429, + "spike_contrast": 0.8136187466924137, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001341764145763591, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024257499899249524, + 0.00023047950526233763, + 0.00022161059314385056, + 0.00021249952260404825, + 0.00020485263667069376, + 0.00019899153267033398, + 0.00019242960843257606, + 0.00018487584020476788, + 0.00017849201685748994, + 0.00017314594879280776, + 0.00016995472833514214, + 0.00016566429985687137, + 0.00016130993026308715, + 0.00015810993500053883, + 0.00015407697355840355, + 0.00015100088785402477, + 0.00014799871132709086, + 0.00014526891754940152, + 0.00014230285887606442, + 0.00013990889419801533, + 0.00013806034985464066, + 0.00013599664089269936, + 0.00013408917584456503, + 0.00013215307262726128, + 0.00013058283366262913, + 0.00012902283924631774, + 0.00012808687461074442, + 0.0001266546023543924, + 0.00012529180094134063, + 0.00012419371341820806, + 0.00012268088175915182, + 0.00012117547157686204 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00016164389126060996, + "first_to_last_ratio": 1.2829777917904304, + "imbalance_abs_log_ratio": 0.24918377588932214, + "last_quartile_mean": 0.00012599118417710997, + "mean": 0.0001426773092134681, + "normalized": [ + 1.1847160153617624, + 1.1696203114063919, + 1.15376099486318, + 1.1365907777467705, + 1.126248503505431, + 1.112765799827437, + 1.097614510342432, + 1.0821504120145113, + 1.06828860482523, + 1.0536786878013262, + 1.046275103221197, + 1.0365937358276223, + 1.0246881313142864, + 1.0168969779036778, + 1.0047039605050023, + 0.9958410772863212, + 0.9859208451030141, + 0.9762312163738422, + 0.965237112554715, + 0.9568250330809598, + 0.9498426746244831, + 0.9400597234006037, + 0.9304691287067991, + 0.9205814319068928, + 0.9114724424808572, + 0.9022957305012854, + 0.8969570718532309, + 0.8872712168205663, + 0.8789155388108255, + 0.8714969614364647, + 0.8626996589435726, + 0.8532906096493081 + ], + "peak_layer": 1, + "peak_normalized": 1.1847160153617624, + "population_cv": 0.0960837578275494, + "rest_mean": 0.00014451401323269778, + "spike_contrast": 0.9186590596986468, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013275910750962794, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000169032093253918, + 0.00016687827883288264, + 0.0001646155142225325, + 0.00016216571384575218, + 0.0001606901059858501, + 0.00015876643010415137, + 0.00015660468488931656, + 0.00015439830895047635, + 0.00015242054359987378, + 0.00015033603995107114, + 0.000149279716424644, + 0.00014789840497542173, + 0.00014619974535889924, + 0.0001450881245546043, + 0.00014334845764096826, + 0.00014208392531145364, + 0.00014066853327676654, + 0.00013928604312241077, + 0.0001377174339722842, + 0.00013651722110807896, + 0.00013552099699154496, + 0.00013412519183475524, + 0.00013275683159008622, + 0.00013134608161635697, + 0.0001300464355153963, + 0.000128737126942724, + 0.00012797542149201035, + 0.00012659346975851804, + 0.00012540130410343409, + 0.00012434284144546837, + 0.00012308766599744558, + 0.00012174520816188306 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00043632716551655903, + "first_to_last_ratio": 3.082278345441195, + "imbalance_abs_log_ratio": 1.1256690460714511, + "last_quartile_mean": 0.00014155994904285762, + "mean": 0.00025262987765017897, + "normalized": [ + 2.1611427674158366, + 1.9871567751170711, + 1.8899978819847916, + 1.7784604477508343, + 1.6406551319992353, + 1.5724690971097588, + 1.4592296472822097, + 1.3280082080145534, + 1.2379389344592089, + 1.1701308820499392, + 1.131654892203279, + 1.0685236385646504, + 1.0072825277339317, + 0.9454673581714634, + 0.8865158004683366, + 0.8408935109532261, + 0.7895809204213243, + 0.7481431077843942, + 0.7047226439788088, + 0.6727720648712864, + 0.6548656488865329, + 0.6315662143568483, + 0.6127430670138975, + 0.5973169234484098, + 0.5839093169225268, + 0.5733930286828336, + 0.5691911560855093, + 0.5599482690481828, + 0.5552538441515218, + 0.5513421690146183, + 0.547054988271426, + 0.542669135783553 + ], + "peak_layer": 1, + "peak_normalized": 2.1611427674158366, + "population_cv": 0.47902914554115894, + "rest_mean": 0.0002705909153317205, + "spike_contrast": 0.5751866206559582, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001556402741698548, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.000545969232916832, + 0.0005020151729695499, + 0.0004774699336849153, + 0.0004492922453209758, + 0.00041447850526310503, + 0.0003972526756115258, + 0.00036864500725641847, + 0.00033549455110915005, + 0.00031274036155082285, + 0.00029561002156697214, + 0.00028588983695954084, + 0.0002699409960769117, + 0.00025446966174058616, + 0.00023885330301709473, + 0.00022396037820726633, + 0.00021243482478894293, + 0.00019947173132095486, + 0.00018900330178439617, + 0.0001780339953256771, + 0.00016996232443489134, + 0.00016543862875550985, + 0.0001595524954609573, + 0.00015479720605071634, + 0.0001509001012891531, + 0.00014751293929293752, + 0.0001448562106816098, + 0.00014379469212144613, + 0.00014145966270007193, + 0.0001402737107127905, + 0.00013928550470154732, + 0.00013820243475493044, + 0.00013709443737752736 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ba6f6cfe37248da2b23ae1596694e5b761bdaaa45b4ddda2715989bf1ef9864e", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.027978340163826942, + 0.0140878576785326, + 0.014187770895659924, + 0.014337530359625816, + 0.014477464370429516, + 0.009757626801729202, + 0.009825476445257664, + 0.00987118761986494, + 0.009999874979257584, + 0.007595349568873644, + 0.007658981718122959, + 0.007689184043556452, + 0.007755559403449297, + 0.00625597033649683, + 0.006315375678241253, + 0.00635211868211627, + 0.006408242508769035, + 0.005408597644418478, + 0.005475745070725679, + 0.005517482291907072, + 0.005585856735706329, + 0.004830927588045597, + 0.004899685736745596, + 0.004948681220412254, + 0.004989070352166891, + 0.004419817589223385, + 0.004463871940970421, + 0.004542885348200798, + 0.004592607729136944, + 0.004137600306421518, + 0.004193917382508516, + 0.004230021964758635 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00024312340610777028, + "first_to_last_ratio": 2.286272500967691, + "imbalance_abs_log_ratio": 0.8269227625411849, + "last_quartile_mean": 0.00010634051977831405, + "mean": 0.00017363850702167838, + "normalized": [ + 1.061084599304074, + 1.6344608012282997, + 1.6065574901355, + 1.6039206242202213, + 1.1359733847422597, + 1.548756457352017, + 1.2967221414405534, + 1.313883475593909, + 1.1957449434685963, + 1.344067980756563, + 1.165161355052429, + 1.1171979507437393, + 1.0391889074947966, + 1.1329181602378293, + 1.0774500297071945, + 1.0521010358096583, + 0.862761425176938, + 0.9019434894037968, + 0.9280163978632505, + 0.8870730707939211, + 0.8354471012965897, + 0.8049103603266032, + 0.8255366455754076, + 0.7297241480789424, + 0.6373588767497196, + 0.6752584188072907, + 0.6298229328646214, + 0.6250051046248064, + 0.6032481952321226, + 0.5674884668477911, + 0.6733729976615692, + 0.4878430314089892 + ], + "peak_layer": 2, + "peak_normalized": 1.6344608012282997, + "population_cv": 0.3218023265359089, + "rest_mean": 0.00018114369622371332, + "spike_contrast": 0.7348336602687936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001331104853306897, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00018424514564685524, + 0.0002838053333107382, + 0.000278960244031623, + 0.00027850238257087767, + 0.00019724872254300863, + 0.00026892375899478793, + 0.00022516089666169137, + 0.0002281407651025802, + 0.0002076273667626083, + 0.00023338195751421154, + 0.00020231687813065946, + 0.00019398858421482146, + 0.0001804432104108855, + 0.00019671821792144328, + 0.00018708681454882026, + 0.00018268525309395045, + 0.000149808605783619, + 0.00015661212091799825, + 0.0001611393818166107, + 0.00015403004363179207, + 0.00014506578736472875, + 0.00013976343325339258, + 0.00014334495062939823, + 0.0001267082116100937, + 0.00011067004379583523, + 0.00011725086369551718, + 0.00010936151375062764, + 0.00010852495324797928, + 0.00010474711598362774, + 9.853785013547167e-05, + 0.000116923481982667, + 8.470833563478664e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00025287891548941843, + "first_to_last_ratio": 2.2739168136310908, + "imbalance_abs_log_ratio": 0.8215038131425536, + "last_quartile_mean": 0.0001112085165004828, + "mean": 0.0001807239029858465, + "normalized": [ + 1.0592123382591336, + 1.645736468094544, + 1.60181949161791, + 1.5891655894821897, + 1.1402836456293184, + 1.5483130487587196, + 1.300612463539172, + 1.3089009659072275, + 1.1885078058073624, + 1.3408206015264876, + 1.166937022076678, + 1.1148751106540256, + 1.0356001272424655, + 1.1351884917697168, + 1.0748875173492451, + 1.0458945475116934, + 0.8650051516349683, + 0.9033374117584585, + 0.9274894298403746, + 0.8851829368231718, + 0.8352656795149113, + 0.8098299287299446, + 0.8239713600606818, + 0.7303600865627311, + 0.6419462275110265, + 0.6820467100618508, + 0.635541252420758, + 0.625618271589476, + 0.6029839777576883, + 0.5727133544881818, + 0.6681570652306708, + 0.4937959207892163 + ], + "peak_layer": 2, + "peak_normalized": 1.645736468094544, + "population_cv": 0.3200470231318611, + "rest_mean": 0.00018847914530110925, + "spike_contrast": 0.7366629038009043, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001388455944834277, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001914249878609553, + 0.00029742391780018806, + 0.00028948707040399313, + 0.0002872002078220248, + 0.00020607651094906032, + 0.0002798171772155911, + 0.00023505176068283617, + 0.00023654969118069857, + 0.00021479176939465106, + 0.0002423183323116973, + 0.00021089341316837817, + 0.00020148458133917302, + 0.00018715769692789763, + 0.00020515569485723972, + 0.0001942578674061224, + 0.0001890181447379291, + 0.00015632710710633546, + 0.00016325466276612133, + 0.00016761950973886997, + 0.0001599737151991576, + 0.00015095247363206, + 0.0001463556254748255, + 0.00014891132013872266, + 0.00013199352542869747, + 0.0001160150277428329, + 0.0001232621434610337, + 0.00011485749564599246, + 0.00011306417582090944, + 0.00010897361789830029, + 0.0001035029927152209, + 0.00012075195263605565, + 8.924072608351707e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.22425555251538754, + "first_to_last_ratio": 2.2739168285843645, + "imbalance_abs_log_ratio": 0.8215038197185521, + "last_quartile_mean": 0.09862082451581955, + "mean": 0.1602677651681006, + "normalized": [ + 1.0592124025748084, + 1.645736496509649, + 1.6018195299533093, + 1.5891655927499475, + 1.1402836784663197, + 1.5483129508886688, + 1.300612441604666, + 1.3089009387606192, + 1.1885077887054891, + 1.3408205692725423, + 1.1669370180912673, + 1.1148751103389594, + 1.0356000505037584, + 1.1351884646301031, + 1.0748875233039092, + 1.0458946120331756, + 0.8650051766933494, + 0.9033374073089909, + 0.9274894888798537, + 0.8851828780404403, + 0.835265755835364, + 0.8098299455472033, + 0.8239713233255873, + 0.730360099613448, + 0.6419462245963619, + 0.6820467317825786, + 0.6355412485786012, + 0.6256182682629655, + 0.602983984145969, + 0.5727133411523797, + 0.6681570418174577, + 0.49379591603225714 + ], + "peak_layer": 2, + "peak_normalized": 1.645736496509649, + "population_cv": 0.3200470194234096, + "rest_mean": 0.16714519124340127, + "spike_contrast": 0.7366629183018027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1231296643614769, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.16975760459899902, + 0.26375851035118103, + 0.25672003626823425, + 0.254692018032074, + 0.18275071680545807, + 0.24814465641975403, + 0.20844624936580658, + 0.20977462828159332, + 0.19047948718070984, + 0.21489031612873077, + 0.1870223879814148, + 0.17867854237556458, + 0.16597330570220947, + 0.18193411827087402, + 0.1722698211669922, + 0.16762319207191467, + 0.13863244652748108, + 0.1447758674621582, + 0.14864666759967804, + 0.1418662816286087, + 0.13386617600917816, + 0.12978963553905487, + 0.13205604255199432, + 0.11705318093299866, + 0.10288328677415848, + 0.1093101054430008, + 0.1018567755818367, + 0.10026644170284271, + 0.09663889557123184, + 0.09178748726844788, + 0.10708403587341309, + 0.07913956791162491 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.143860511860112e-05, + "first_to_last_ratio": 0.9636580872932836, + "imbalance_abs_log_ratio": 0.037018728516889636, + "last_quartile_mean": 7.413273033307632e-05, + "mean": 7.686168260079285e-05, + "normalized": [ + 0.586022200160155, + 0.9787081875553917, + 0.912570839911241, + 1.015732496910769, + 0.8201270226617754, + 1.1119332047208261, + 1.0008210861718638, + 1.0096344009845781, + 0.9335569648148418, + 1.090167719321714, + 1.035494250546431, + 0.9697380161483544, + 1.0399314841024359, + 1.0649519568255064, + 1.06563466639597, + 1.1389203959735625, + 0.9952637999761301, + 1.0027093290728673, + 1.1427249020241628, + 1.1090175842590573, + 1.0432879501645442, + 1.0450477353718262, + 1.1125450117027875, + 1.0594965280199307, + 0.9180884623091344, + 0.995296932026832, + 0.9631737996002885, + 0.9446177686211702, + 0.9674620335911367, + 0.9355074959711648, + 1.1351899163905306, + 0.8566258576930204 + ], + "peak_layer": 19, + "peak_normalized": 1.1427249020241628, + "population_cv": 0.10846075768175395, + "rest_mean": 7.635363915512614e-05, + "spike_contrast": 1.0425844542348655, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.960511720739305e-05, + "top_five_layers": [ + 19, + 16, + 31, + 23, + 6 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.504265234572813e-05, + 7.522515807067975e-05, + 7.014173024799675e-05, + 7.807090878486633e-05, + 6.303634290816262e-05, + 8.546505705453455e-05, + 7.692479266552255e-05, + 7.760219887131825e-05, + 7.175475911935791e-05, + 8.379212522413582e-05, + 7.958983042044565e-05, + 7.453569560311735e-05, + 7.993088365765288e-05, + 8.185399929061532e-05, + 8.190647349692881e-05, + 8.753933798288926e-05, + 7.64976502978243e-05, + 7.706992619205266e-05, + 8.78317587194033e-05, + 8.52409575600177e-05, + 8.018886728677899e-05, + 8.032412733882666e-05, + 8.551208156859502e-05, + 8.143468585330993e-05, + 7.056582398945466e-05, + 7.650019688298926e-05, + 7.403115887427703e-05, + 7.260491111082956e-05, + 7.436075975419953e-05, + 7.190468022599816e-05, + 8.725260704522952e-05, + 6.584170478163287e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 8.107683152047684e-05, + "first_to_last_ratio": 1.03534602173519, + "imbalance_abs_log_ratio": 0.034735691361772646, + "last_quartile_mean": 7.83089226388256e-05, + "mean": 8.487996228723205e-05, + "normalized": [ + 0.5954853629443996, + 1.025163681929567, + 0.9520754208460104, + 1.0126519094835282, + 0.8443534832663374, + 1.1548844288399933, + 1.0326421147312754, + 1.0242956755731092, + 0.9710206925084107, + 1.15116021353655, + 1.072832454869962, + 1.0193061391332177, + 1.0419196505755806, + 1.0947619162703996, + 1.1089694127677372, + 1.1239932261258834, + 0.9835832115239759, + 1.0364664516448652, + 1.11612142168979, + 1.093437276508762, + 1.0179007506072324, + 1.033774483284387, + 1.0848380474290875, + 1.0276879808812729, + 0.8974590825111308, + 0.9692041024722606, + 0.9378977551023808, + 0.9155031739616566, + 0.918412101050971, + 0.8874638084461439, + 1.0536745962271137, + 0.8010599732570078 + ], + "peak_layer": 6, + "peak_normalized": 1.1548844288399933, + "population_cv": 0.11200209045145568, + "rest_mean": 8.468612051931107e-05, + "spike_contrast": 1.0146492401244358, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.592670783400536e-05, + "top_five_layers": [ + 6, + 10, + 16, + 19, + 15 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.054477514931932e-05, + 8.701585466042161e-05, + 8.081212581600994e-05, + 8.59538558870554e-05, + 7.166869181673974e-05, + 9.802654676605016e-05, + 8.76506237545982e-05, + 8.694217831362039e-05, + 8.242019976023585e-05, + 9.771043551154435e-05, + 9.106197830988094e-05, + 8.651866664877161e-05, + 8.843810064718127e-05, + 9.292335016652942e-05, + 9.41292819334194e-05, + 9.540450264466926e-05, + 8.348650590050966e-05, + 8.797523332759738e-05, + 9.473634418100119e-05, + 9.281091479351744e-05, + 8.639937732368708e-05, + 8.774673915468156e-05, + 9.20810125535354e-05, + 8.723011706024408e-05, + 7.617629307787865e-05, + 8.226600766647607e-05, + 7.960872608236969e-05, + 7.770787487970665e-05, + 7.795478450134397e-05, + 7.532789459219202e-05, + 8.943585999077186e-05, + 6.799394031986594e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.1427921979011444e-05, + "first_to_last_ratio": 0.40836765899304117, + "imbalance_abs_log_ratio": 0.8955873853644011, + "last_quartile_mean": 5.2472132665570825e-05, + "mean": 3.864622581772892e-05, + "normalized": [ + 0.2626641640048693, + 0.5716512870137274, + 0.5157707114648397, + 0.4843895386483896, + 0.5109475889421058, + 0.7512596968779521, + 0.675728069412672, + 0.6632970161544233, + 0.7004288748821966, + 0.9130902556041411, + 0.8717836320781894, + 0.8044820763577291, + 0.9241814793896349, + 0.9531622930600316, + 1.0319070449118461, + 1.0273708471578158, + 0.9893551980130566, + 1.0844069350216101, + 1.1887300698220302, + 1.2100415764612447, + 1.1128447755732793, + 1.2346303137099222, + 1.3033621630444512, + 1.3524689418392815, + 1.2055635543985064, + 1.318388624044801, + 1.3615446320871405, + 1.2933406942036905, + 1.3753836688642536, + 1.3573239762703455, + 1.6442457326403273, + 1.3062545680454958 + ], + "peak_layer": 31, + "peak_normalized": 1.6442457326403273, + "population_cv": 0.32680115045148705, + "rest_mean": 3.691592014042221e-05, + "spike_contrast": 1.2999777952883038, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.798987647518516e-05, + "top_five_layers": [ + 31, + 29, + 27, + 30, + 24 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.0150978596357163e-05, + 2.2092164726927876e-05, + 1.99325913854409e-05, + 1.8719827494351193e-05, + 1.974619590328075e-05, + 2.9033351893303916e-05, + 2.6114339561900124e-05, + 2.5633926270529628e-05, + 2.7068932467955165e-05, + 3.528749221004546e-05, + 3.369114710949361e-05, + 3.1090195989236236e-05, + 3.571612614905462e-05, + 3.683612521854229e-05, + 3.9879312680568546e-05, + 3.970400575781241e-05, + 3.823484439635649e-05, + 4.190823528915644e-05, + 4.5939930714666843e-05, + 4.676354001276195e-05, + 4.300725049688481e-05, + 4.771380190504715e-05, + 5.037002847529948e-05, + 5.226782013778575e-05, + 4.65904813609086e-05, + 5.09507444803603e-05, + 5.261856131255627e-05, + 4.998273652745411e-05, + 5.3153387852944434e-05, + 5.24554488947615e-05, + 6.354389188345522e-05, + 5.048180901212618e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031520502852799837, + "first_to_last_ratio": 1.31300997816315, + "imbalance_abs_log_ratio": 0.27232219480674863, + "last_quartile_mean": 0.00024006293460843153, + "mean": 0.00030979480334281106, + "normalized": [ + 0.607123837738274, + 1.0354820546705408, + 0.9347765087792952, + 1.1085716606000595, + 0.9306423427143409, + 1.2565833735558338, + 1.1285140374487672, + 1.138017377238765, + 1.0370737734279494, + 1.2142990636699202, + 1.1645184658462944, + 1.1206382096550715, + 1.1232577862254296, + 1.1815362250572234, + 1.1924463988431009, + 1.2309950595128432, + 1.0374639291264394, + 1.065463636492051, + 1.1630951913863559, + 1.1082935818936954, + 1.0270641612904527, + 0.9934767629460804, + 1.0585185268550952, + 0.9428720751183239, + 0.8088208526771234, + 0.8683362720246933, + 0.8018903045940483, + 0.8027332776454693, + 0.7640102071384416, + 0.7171383971343437, + 0.8334550561316563, + 0.6028915925620211 + ], + "peak_layer": 6, + "peak_normalized": 1.2565833735558338, + "population_cv": 0.17765693833110424, + "rest_mean": 0.0003117367305094376, + "spike_contrast": 0.9601319554288655, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00029930839664302764, + "top_five_layers": [ + 6, + 16, + 10, + 15, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00018808380991686136, + 0.00032078695949167013, + 0.00028958890470676124, + 0.00034342973958700895, + 0.0002883081615436822, + 0.00038928299909457564, + 0.00034960778430104256, + 0.0003525518695823848, + 0.00032128006569109857, + 0.00037618353962898254, + 0.00036076176911592484, + 0.00034716789377853274, + 0.0003479794249869883, + 0.00036603378248400986, + 0.0003694136976264417, + 0.00038135587237775326, + 0.0003214009338989854, + 0.0003300750977359712, + 0.0003603208460845053, + 0.000343343592248857, + 0.00031817913986742496, + 0.00030777393840253353, + 0.00032792353886179626, + 0.00029209686908870935, + 0.0002505684969946742, + 0.0002690060646273196, + 0.00024842144921422005, + 0.0002486825978849083, + 0.00023668639187235385, + 0.00022216574870981276, + 0.000258200045209378, + 0.0001867726823547855 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "abe86edb39bd04cd7b48202254a77e2ca437d13373a8258957c24114bc65afbd", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014031016267836094, + 0.014151837676763535, + 0.014265215955674648, + 0.014401593245565891, + 0.009722241200506687, + 0.009786579757928848, + 0.009839320555329323, + 0.00996283907443285, + 0.007562503684312105, + 0.007635751739144325, + 0.007660485804080963, + 0.007726665120571852, + 0.006238248664885759, + 0.006290202960371971, + 0.006327399518340826, + 0.006382675375789404, + 0.005390914157032967, + 0.0054517327807843685, + 0.005509276874363422, + 0.0055727302096784115, + 0.0048178439028561115, + 0.004884926602244377, + 0.004938251338899136, + 0.004975184798240662, + 0.004412420094013214, + 0.004455919377505779, + 0.004530393984168768, + 0.0045819394290447235, + 0.004127359017729759, + 0.004185030702501535, + 0.004221425391733646, + 0.004281661007553339 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 5.6193598993559135e-05, + "first_to_last_ratio": 1.5723739147519853, + "imbalance_abs_log_ratio": 0.4525865249799748, + "last_quartile_mean": 3.573806361600873e-05, + "mean": 4.4294749727669114e-05, + "normalized": [ + 1.5642840408451666, + 1.493254721227763, + 1.1816235069560812, + 1.0717429494942774, + 1.3553344243310417, + 1.3257825682631335, + 1.1255110934944355, + 1.0314977817441078, + 1.194075821513187, + 1.0893775754580661, + 1.0407197138445958, + 0.9875719064328349, + 1.135803276632048, + 1.0324555952346264, + 0.9635988909717662, + 0.90208003019059, + 0.9904506854292476, + 0.9196323607404276, + 0.8708711781381843, + 0.8075366439875266, + 0.9218591005239072, + 0.8776357461813653, + 0.8517186042296377, + 0.8109904231766266, + 0.8722342267309339, + 0.8500552841464633, + 0.7844009556175089, + 0.7665260139039981, + 0.8384393111065888, + 0.8154485842192296, + 0.7754690286134897, + 0.7520179566211438 + ], + "peak_layer": 1, + "peak_normalized": 1.5642840408451666, + "population_cv": 0.20749976292329902, + "rest_mean": 4.538663482957485e-05, + "spike_contrast": 0.8460325450777163, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.839857017737813e-05, + "top_five_layers": [ + 1, + 2, + 5, + 6, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.928957009222358e-05, + 6.614334415644407e-05, + 5.23397175129503e-05, + 4.747258572024293e-05, + 6.003419912303798e-05, + 5.872520705452189e-05, + 4.985423220205121e-05, + 4.5689936087001115e-05, + 5.289128966978751e-05, + 4.8253707063850015e-05, + 4.6098419261397794e-05, + 4.374425043351948e-05, + 5.03101218782831e-05, + 4.573236219584942e-05, + 4.2682371713453904e-05, + 3.9957409171620384e-05, + 4.3871765228686854e-05, + 4.073488526046276e-05, + 3.857502088067122e-05, + 3.576963354134932e-05, + 4.083351814188063e-05, + 3.887465572915971e-05, + 3.7726662412751466e-05, + 3.592261782614514e-05, + 3.863539677695371e-05, + 3.7652986065950245e-05, + 3.474484401522204e-05, + 3.395307794562541e-05, + 3.713845944730565e-05, + 3.612009095377289e-05, + 3.4349206543993205e-05, + 3.3310447179246694e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 9.100913848669734e-05, + "first_to_last_ratio": 0.8419197805282016, + "imbalance_abs_log_ratio": 0.17207054181035333, + "last_quartile_mean": 0.00010809716150106397, + "mean": 9.889523016681778e-05, + "normalized": [ + 1.0744641570554045, + 0.9868986027413778, + 0.8415148680287817, + 0.7657672453742986, + 1.0045876840265406, + 0.9858056114400382, + 0.8710937611903306, + 0.8319330217638002, + 1.0050985706445066, + 0.9363425314153265, + 0.9111662095698898, + 0.8822483642571924, + 1.0547921529408322, + 0.9937743096656301, + 0.9653849347701057, + 0.9236053864198286, + 1.0608213360421703, + 1.0216834776263237, + 0.9948854733452297, + 0.9444045194208119, + 1.0902032759381237, + 1.070297753084291, + 1.0352646449837608, + 1.0035839360241054, + 1.109605122267389, + 1.0988987221493531, + 1.0612028824132491, + 1.0392747076439328, + 1.1393068813062424, + 1.1142247321092786, + 1.0935193303224786, + 1.0883457940193757 + ], + "peak_layer": 29, + "peak_normalized": 1.1393068813062424, + "population_cv": 0.09089893356811851, + "rest_mean": 9.776359500527313e-05, + "spike_contrast": 1.0740814107080978, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010500606003915891, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00010625938011799008, + 9.759956446941942e-05, + 8.322180656250566e-05, + 7.573072798550129e-05, + 9.934893023455516e-05, + 9.749147284310311e-05, + 8.614701800979674e-05, + 8.227420767070726e-05, + 9.939945448422804e-05, + 9.259981015929952e-05, + 9.010999201564118e-05, + 8.725015504751354e-05, + 0.00010431391274323687, + 9.827953908825293e-05, + 9.547196532366797e-05, + 9.134016727330163e-05, + 0.00010491017019376159, + 0.00010103962267749012, + 9.838942787609994e-05, + 9.339710231870413e-05, + 0.0001078159039025195, + 0.00010584734263829887, + 0.00010238273534923792, + 9.924966434482485e-05, + 0.00010973465396091342, + 0.00010867584205698222, + 0.00010494790330994874, + 0.00010277931141899899, + 0.00011267201625742018, + 0.000110191511339508, + 0.00010814384586410597, + 0.0001076322078006342 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.08070781361311674, + "first_to_last_ratio": 0.8419197775247833, + "imbalance_abs_log_ratio": 0.1720705453776983, + "last_quartile_mean": 0.09586164355278015, + "mean": 0.0877012787386775, + "normalized": [ + 1.074464079725317, + 0.986898606287062, + 0.8415149217948403, + 0.7657672391640048, + 1.0045876578450608, + 0.9858055872189405, + 0.8710938052118494, + 0.8319330371629138, + 1.0050985716236776, + 0.9363425240221704, + 0.9111662153888755, + 0.8822483595926541, + 1.0547922001671663, + 0.993774279010469, + 0.9653849224854107, + 0.9236053602662656, + 1.060821305580311, + 1.021683475130349, + 0.9948854782496288, + 0.9444045803369281, + 1.0902032654624292, + 1.07029776221103, + 1.0352646575744078, + 1.0035839255047951, + 1.109605076031185, + 1.0988987393251362, + 1.0612029192639736, + 1.0392747445534645, + 1.1393068879878778, + 1.1142247084644445, + 1.0935193626414328, + 1.0883457447159286 + ], + "peak_layer": 29, + "peak_normalized": 1.1393068879878778, + "population_cv": 0.09089892559784572, + "rest_mean": 0.08669773351263117, + "spike_contrast": 1.0740813996684329, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0931204229593277, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.09423187375068665, + 0.08655226975679398, + 0.0738019347190857, + 0.0671587660908699, + 0.08810362219810486, + 0.08645641058683395, + 0.07639604061841965, + 0.07296159118413925, + 0.08814842998981476, + 0.0821184366941452, + 0.07991044223308563, + 0.07737430930137634, + 0.09250662475824356, + 0.08715527504682541, + 0.08466549217700958, + 0.08100137114524841, + 0.09303538501262665, + 0.08960294723510742, + 0.08725272864103317, + 0.08282548934221268, + 0.09561222046613693, + 0.09386648237705231, + 0.09079403430223465, + 0.0880155935883522, + 0.0973137840628624, + 0.09637482464313507, + 0.09306885302066803, + 0.09114572405815125, + 0.0999186709523201, + 0.09771893173456192, + 0.09590304642915726, + 0.09544931352138519 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0474835562490625e-05, + "first_to_last_ratio": 0.595870196340598, + "imbalance_abs_log_ratio": 0.5177324270116084, + "last_quartile_mean": 1.7579056020622374e-05, + "mean": 1.45315484587627e-05, + "normalized": [ + 0.6891108502390517, + 0.67218760206668, + 0.6248497232673742, + 0.6113105238885437, + 0.8536003812586072, + 0.8434482348779837, + 0.7260190066641083, + 0.7461464268756399, + 0.9515026546685825, + 0.8968707547028398, + 0.8666952712848448, + 0.9173542983424958, + 1.0474470628626384, + 1.0570493146141757, + 0.9856965747654381, + 1.0403130781822671, + 1.05710126231986, + 1.0899464822844398, + 1.0639055355381295, + 0.9944043245776518, + 1.1636416255463544, + 1.1162669452398895, + 1.1491653646620976, + 1.158233556446643, + 1.241320970071456, + 1.2666472916328237, + 1.1327368714456698, + 1.1012802199127885, + 1.2948691657915095, + 1.2555094531837419, + 1.2151558495322814, + 1.1702133232533922 + ], + "peak_layer": 29, + "peak_normalized": 1.2948691657915095, + "population_cv": 0.19478066534993646, + "rest_mean": 1.408557606434139e-05, + "spike_contrast": 1.202634475953173, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6939799388637765e-05, + "top_five_layers": [ + 29, + 26, + 30, + 25, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.0013847713707946e-05, + 9.767926712811459e-06, + 9.080034033104312e-06, + 8.883288501237985e-06, + 1.2404135304677766e-05, + 1.2256608897587284e-05, + 1.0550180377322249e-05, + 1.0842662959476002e-05, + 1.3826806934957858e-05, + 1.3032920833211392e-05, + 1.2594424333656207e-05, + 1.3330578440218233e-05, + 1.522102775197709e-05, + 1.5360563338617794e-05, + 1.4323697541840374e-05, + 1.5117359907890204e-05, + 1.5361318219220266e-05, + 1.5838610124774277e-05, + 1.546019484521821e-05, + 1.445023463020334e-05, + 1.690951467026025e-05, + 1.622108720766846e-05, + 1.669915218371898e-05, + 1.6830927052069455e-05, + 1.8038315829471685e-05, + 1.8406346498522907e-05, + 1.6460420738440007e-05, + 1.600330688233953e-05, + 1.8816454030456953e-05, + 1.8244496459374204e-05, + 1.7658096112427302e-05, + 1.7005011613946408e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.836043712610262e-05, + "first_to_last_ratio": 0.6391636816233681, + "imbalance_abs_log_ratio": 0.44759470463089907, + "last_quartile_mean": 0.00010695294349716278, + "mean": 9.121693358338234e-05, + "normalized": [ + 0.7254188545798665, + 0.6902023390007371, + 0.6496225556724757, + 0.6277900804246503, + 0.8680656002782756, + 0.8588952084862168, + 0.7971788633739051, + 0.7782428701770085, + 0.9734232055545582, + 0.9237343700472408, + 0.9096320021673064, + 0.8918620199754489, + 1.0844263684244682, + 1.0342037426828012, + 1.0147200800673386, + 0.9791745321037664, + 1.1290928536156495, + 1.0928814238713915, + 1.0660995437899934, + 1.013549283136708, + 1.170635644938427, + 1.1503062295054498, + 1.1129730585678779, + 1.0777737723208793, + 1.1919439416858, + 1.1807148042349604, + 1.1393605783805036, + 1.1153320412921235, + 1.222157091120308, + 1.1946826876057626, + 1.1714041050576325, + 1.1645002478604691 + ], + "peak_layer": 29, + "peak_normalized": 1.222157091120308, + "population_cv": 0.17156610618115575, + "rest_mean": 8.883977756744975e-05, + "spike_contrast": 1.1712498490939807, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010405357606941834, + "top_five_layers": [ + 29, + 30, + 25, + 26, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.617048347834498e-05, + 6.295814091572538e-05, + 5.925657751504332e-05, + 5.7265086070401594e-05, + 7.91822822066024e-05, + 7.834578718757257e-05, + 7.271621143445373e-05, + 7.098892820067704e-05, + 8.879267988959327e-05, + 8.426021668128669e-05, + 8.297384192701429e-05, + 8.135291864164174e-05, + 9.891804802464321e-05, + 9.433689410798252e-05, + 9.255965414922684e-05, + 8.931729826144874e-05, + 0.00010299238783773035, + 9.968929225578904e-05, + 9.724633127916604e-05, + 9.245285764336586e-05, + 0.00010678179387468845, + 0.00010492740693734959, + 0.00010152198956348002, + 9.831121860770509e-05, + 0.00010872547136386856, + 0.00010770118387881666, + 0.0001039289782056585, + 0.00010173716873396188, + 0.00011148142220918089, + 0.00010897529136855155, + 0.00010685189045034349, + 0.00010622214176692069 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 5.421776268121903e-05, + "first_to_last_ratio": 0.5089233562442377, + "imbalance_abs_log_ratio": 0.6754578508927926, + "last_quartile_mean": 0.00010653423942130757, + "mean": 8.540443025140121e-05, + "normalized": [ + 0.5542697148172375, + 0.5395320644633284, + 0.526750599523528, + 0.5251350203104778, + 0.7537064875235374, + 0.7453106043033457, + 0.7183783488935762, + 0.7156010196903468, + 0.915261682615852, + 0.8828592813937833, + 0.8733821992497209, + 0.8643716402564832, + 1.0724322757212523, + 1.0352702478756863, + 1.026729960575756, + 1.0038080492636434, + 1.1695135624376918, + 1.1411903421843295, + 1.1203495535033974, + 1.0697851231751558, + 1.23599072599408, + 1.2140634563521424, + 1.1785251437606976, + 1.1385123493568967, + 1.2633504856351931, + 1.2489710766708417, + 1.2127527443953423, + 1.1862530196716095, + 1.298334269521613, + 1.274402551981099, + 1.2503128844914202, + 1.2448935143909348 + ], + "peak_layer": 29, + "peak_normalized": 1.298334269521613, + "population_cv": 0.24541141609846925, + "rest_mean": 8.214501078193145e-05, + "spike_contrast": 1.2539446328637511, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010300529538653791, + "top_five_layers": [ + 29, + 30, + 25, + 31, + 26 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.73370891995728e-05, + 4.6078428567852825e-05, + 4.498683483689092e-05, + 4.4848857214674354e-05, + 6.436987314373255e-05, + 6.365282752085477e-05, + 6.135269359219819e-05, + 6.111549737397581e-05, + 7.816740253474563e-05, + 7.540009391959757e-05, + 7.459070911863819e-05, + 7.382116746157408e-05, + 9.159046749118716e-05, + 8.841666567604989e-05, + 8.768728730501607e-05, + 8.572965452913195e-05, + 9.98816394712776e-05, + 9.746271098265424e-05, + 9.56828152993694e-05, + 9.136438893619925e-05, + 0.00010555908374954015, + 0.00010368639777880162, + 0.00010065126843983307, + 9.723399853101e-05, + 0.0001078957284335047, + 0.00010666766320355237, + 0.00010357445717090741, + 0.00010131126327905804, + 0.00011088349856436253, + 0.00010883962386287749, + 0.00010678225953597575, + 0.00010631942132022232 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00014064586594031425, + "first_to_last_ratio": 1.0563608173139223, + "imbalance_abs_log_ratio": 0.054829809986132017, + "last_quartile_mean": 0.00013314188072399702, + "mean": 0.0001347423383322166, + "normalized": [ + 1.1217990599984589, + 1.0213028410040081, + 0.9566177087867302, + 0.90173442925003, + 1.162761655685041, + 1.2034553361266669, + 1.0271109783027095, + 0.9557256445604886, + 1.16094674777681, + 1.05596774404836, + 1.0011734009390578, + 0.9818696062747876, + 1.1123781705815696, + 1.0319357929888284, + 0.9826152250905227, + 0.9109733958837348, + 1.0154471845455557, + 0.955267408663159, + 0.9085227032176484, + 0.853750823711138, + 0.9825495622515935, + 0.9600374683174735, + 0.9243229318330685, + 0.9067574744309754, + 0.9933509912573504, + 0.986398830187635, + 0.9577583846169437, + 0.9375602577665346, + 1.0292265528944584, + 1.007559112024919, + 0.9944595916547317, + 0.9986629853290112 + ], + "peak_layer": 6, + "peak_normalized": 1.2034553361266669, + "population_cv": 0.07980954026492859, + "rest_mean": 0.0001359050228428613, + "spike_contrast": 0.9452471975467027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012846384197473526, + "top_five_layers": [ + 6, + 5, + 9, + 1, + 13 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001511538284830749, + 0.0001376127329422161, + 0.00012889690697193146, + 0.00012150180555181578, + 0.00015667322441004217, + 0.0001621563860680908, + 0.00013839533494319767, + 0.00012877670815214515, + 0.00015642867947462946, + 0.0001422835630364716, + 0.00013490044511854649, + 0.00013229940668679774, + 0.000149884435813874, + 0.00013904544175602496, + 0.00013239987310953438, + 0.0001227466855198145, + 0.00013682372809853405, + 0.0001287149643758312, + 0.0001224164734594524, + 0.00011503638233989477, + 0.00013239102554507554, + 0.0001293576933676377, + 0.0001245454332092777, + 0.00012217862240504473, + 0.00013384643534664065, + 0.000132909684907645, + 0.00012905060430057347, + 0.00012632906145881861, + 0.00013868039241060615, + 0.00013576087076216936, + 0.00013399581075645983, + 0.0001345621858490631 + ] + } + } + } + } + } + }, + "step": 0 + }, + { + "modes": { + "learned": { + "bits_per_byte": 5.118322357706281, + "forward": { + "activation_sha256": { + "attention_branch_output": "18380f2ab4362bd0b1746896f121e69ee1546f1794cea323af41740b6024651d", + "mlp_branch_output": "28d398d2ea80931917268ed413eca333d2f81168391c83a3adc104502478e55b", + "post_attention_state": "1ff52033329506a86a2df82471984832a6abc7bac6dc80a52bb5d468ca4544c7", + "post_mlp_state": "6cc11327437b0ddd855bc16b53a520d7ca542f4fd424dbb567b2c52308727f3b", + "pre_attention_input": "a1eaf8973066bfbad01efbb61e56182f134640c86d8dd38e1601245f8a0a6387", + "pre_mlp_input": "51c96475782d47da4b4460ab98a6f5efa352f41865c65bc22f8b46e27bda20aa" + }, + "logits_sha256": "994050c6c37c378cc5716dc8662ae2251d60efe8d70e351d49deded85b14b9c5", + "loss_nats": 3.54775071144104, + "mixer_summary_sha256": "be87753ea159fa25b47c87030938167aea73f40104b512c76f39174a23668c39" + }, + "loss_nats": 3.54775071144104, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.692298173904419, + "group": 1, + "latest_source_mass": 0.5198995471000671, + "layer": 1, + "max_source_mass": 0.5198995471000671, + "mixer_index": 1, + "normalized_entropy": 0.9987751423083904, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48010045289993286, + "median": 0.4801749289035797, + "p05": 0.47152742743492126, + "p95": 0.4886074662208557 + }, + { + "label": "current_group_partial", + "mean": 0.5198995471000671, + "median": 0.5198251008987427, + "p05": 0.5113925337791443, + "p95": 0.5284726023674011 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6927915811538696, + "group": 1, + "latest_source_mass": 0.4869963526725769, + "layer": 2, + "max_source_mass": 0.5130037069320679, + "mixer_index": 2, + "normalized_entropy": 0.9994869785003115, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5130037069320679, + "median": 0.512967050075531, + "p05": 0.5081846714019775, + "p95": 0.5176204442977905 + }, + { + "label": "current_group_partial", + "mean": 0.4869963526725769, + "median": 0.487032949924469, + "p05": 0.48237958550453186, + "p95": 0.49181535840034485 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6928644776344299, + "group": 1, + "latest_source_mass": 0.4918276369571686, + "layer": 2, + "max_source_mass": 0.5081723928451538, + "mixer_index": 3, + "normalized_entropy": 0.9995921458913142, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5081723928451538, + "median": 0.5079053044319153, + "p05": 0.49492087960243225, + "p95": 0.5215321183204651 + }, + { + "label": "current_group_partial", + "mean": 0.4918276369571686, + "median": 0.4920946955680847, + "p05": 0.4784679412841797, + "p95": 0.5050790905952454 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.692646861076355, + "group": 1, + "latest_source_mass": 0.48500266671180725, + "layer": 3, + "max_source_mass": 0.5149973630905151, + "mixer_index": 4, + "normalized_entropy": 0.9992781915621641, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5149973630905151, + "median": 0.5156484842300415, + "p05": 0.5058209896087646, + "p95": 0.5222364664077759 + }, + { + "label": "current_group_partial", + "mean": 0.48500266671180725, + "median": 0.4843515157699585, + "p05": 0.4777635335922241, + "p95": 0.49417904019355774 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6925361156463623, + "group": 1, + "latest_source_mass": 0.4838656783103943, + "layer": 3, + "max_source_mass": 0.5161343812942505, + "mixer_index": 5, + "normalized_entropy": 0.9991184196795125, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5161343812942505, + "median": 0.5157076120376587, + "p05": 0.5059124827384949, + "p95": 0.5276854634284973 + }, + { + "label": "current_group_partial", + "mean": 0.4838656783103943, + "median": 0.4842924475669861, + "p05": 0.47231459617614746, + "p95": 0.49408748745918274 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6928029656410217, + "group": 1, + "latest_source_mass": 0.4892655909061432, + "layer": 4, + "max_source_mass": 0.5107344388961792, + "mixer_index": 6, + "normalized_entropy": 0.999503402843469, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5107344388961792, + "median": 0.5119034051895142, + "p05": 0.4968262016773224, + "p95": 0.5211055874824524 + }, + { + "label": "current_group_partial", + "mean": 0.4892655909061432, + "median": 0.48809656500816345, + "p05": 0.4788943827152252, + "p95": 0.5031737685203552 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6924350261688232, + "group": 1, + "latest_source_mass": 0.48286139965057373, + "layer": 4, + "max_source_mass": 0.5171386003494263, + "mixer_index": 7, + "normalized_entropy": 0.9989725783915809, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5171386003494263, + "median": 0.5181499719619751, + "p05": 0.5028476715087891, + "p95": 0.5284117460250854 + }, + { + "label": "current_group_partial", + "mean": 0.48286139965057373, + "median": 0.4818500280380249, + "p05": 0.47158828377723694, + "p95": 0.4971523582935333 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6929639577865601, + "group": 2, + "latest_source_mass": 0.49272024631500244, + "layer": 5, + "max_source_mass": 0.5072797536849976, + "mixer_index": 8, + "normalized_entropy": 0.9997356654134593, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5072797536849976, + "median": 0.5079033970832825, + "p05": 0.49702075123786926, + "p95": 0.5173761248588562 + }, + { + "label": "completed_group_1", + "mean": 0.49272024631500244, + "median": 0.49209660291671753, + "p05": 0.4826239049434662, + "p95": 0.5029792189598083 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0978727340698242, + "group": 2, + "latest_source_mass": 0.3277629613876343, + "layer": 5, + "max_source_mass": 0.34929555654525757, + "mixer_index": 9, + "normalized_entropy": 0.9993268283944082, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34929555654525757, + "median": 0.3491864800453186, + "p05": 0.34007197618484497, + "p95": 0.3589988350868225 + }, + { + "label": "completed_group_1", + "mean": 0.32294148206710815, + "median": 0.3218739628791809, + "p05": 0.31213808059692383, + "p95": 0.3344322443008423 + }, + { + "label": "current_group_partial", + "mean": 0.3277629613876343, + "median": 0.328006386756897, + "p05": 0.3200114071369171, + "p95": 0.3352474272251129 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.098320722579956, + "group": 2, + "latest_source_mass": 0.3298703134059906, + "layer": 6, + "max_source_mass": 0.3432672321796417, + "mixer_index": 10, + "normalized_entropy": 0.9997346051094084, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3432672321796417, + "median": 0.34342849254608154, + "p05": 0.3353024423122406, + "p95": 0.3504941165447235 + }, + { + "label": "completed_group_1", + "mean": 0.3268624544143677, + "median": 0.3270367980003357, + "p05": 0.3202572166919708, + "p95": 0.3326030373573303 + }, + { + "label": "current_group_partial", + "mean": 0.3298703134059906, + "median": 0.32962095737457275, + "p05": 0.32568222284317017, + "p95": 0.33469292521476746 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.098150610923767, + "group": 2, + "latest_source_mass": 0.32557153701782227, + "layer": 6, + "max_source_mass": 0.3442404866218567, + "mixer_index": 11, + "normalized_entropy": 0.9995797628070386, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3442404866218567, + "median": 0.34447744488716125, + "p05": 0.3311336636543274, + "p95": 0.3566625714302063 + }, + { + "label": "completed_group_1", + "mean": 0.33018797636032104, + "median": 0.3304833471775055, + "p05": 0.3209139108657837, + "p95": 0.338434636592865 + }, + { + "label": "current_group_partial", + "mean": 0.32557153701782227, + "median": 0.32545968890190125, + "p05": 0.3193055987358093, + "p95": 0.3318970203399658 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0984500646591187, + "group": 2, + "latest_source_mass": 0.3301700949668884, + "layer": 7, + "max_source_mass": 0.34007447957992554, + "mixer_index": 12, + "normalized_entropy": 0.9998523373435156, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34007447957992554, + "median": 0.34058719873428345, + "p05": 0.3314547538757324, + "p95": 0.34787896275520325 + }, + { + "label": "completed_group_1", + "mean": 0.32975539565086365, + "median": 0.3295257091522217, + "p05": 0.324510782957077, + "p95": 0.3352199196815491 + }, + { + "label": "current_group_partial", + "mean": 0.3301700949668884, + "median": 0.3300544023513794, + "p05": 0.3263310492038727, + "p95": 0.3340227007865906 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0982636213302612, + "group": 2, + "latest_source_mass": 0.3256300091743469, + "layer": 7, + "max_source_mass": 0.3434537351131439, + "mixer_index": 13, + "normalized_entropy": 0.9996826293120467, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3434537351131439, + "median": 0.3430320620536804, + "p05": 0.3340873420238495, + "p95": 0.35414862632751465 + }, + { + "label": "completed_group_1", + "mean": 0.33091625571250916, + "median": 0.33138126134872437, + "p05": 0.3248554766178131, + "p95": 0.335583359003067 + }, + { + "label": "current_group_partial", + "mean": 0.3256300091743469, + "median": 0.3252079486846924, + "p05": 0.3196634352207184, + "p95": 0.3317061960697174 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0984963178634644, + "group": 2, + "latest_source_mass": 0.32797664403915405, + "layer": 8, + "max_source_mass": 0.33822712302207947, + "mixer_index": 14, + "normalized_entropy": 0.9998944388244683, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33822712302207947, + "median": 0.3383355736732483, + "p05": 0.33158448338508606, + "p95": 0.3443746566772461 + }, + { + "label": "completed_group_1", + "mean": 0.33379626274108887, + "median": 0.3336103558540344, + "p05": 0.3301801085472107, + "p95": 0.337948203086853 + }, + { + "label": "current_group_partial", + "mean": 0.32797664403915405, + "median": 0.3281283378601074, + "p05": 0.3242723345756531, + "p95": 0.3310714364051819 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0980933904647827, + "group": 2, + "latest_source_mass": 0.3236028254032135, + "layer": 8, + "max_source_mass": 0.34460341930389404, + "mixer_index": 15, + "normalized_entropy": 0.9995276785007056, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34460341930389404, + "median": 0.34293144941329956, + "p05": 0.33235934376716614, + "p95": 0.35933688282966614 + }, + { + "label": "completed_group_1", + "mean": 0.33179372549057007, + "median": 0.3316723704338074, + "p05": 0.32370349764823914, + "p95": 0.33980825543403625 + }, + { + "label": "current_group_partial", + "mean": 0.3236028254032135, + "median": 0.3241952061653137, + "p05": 0.3152182102203369, + "p95": 0.3305118680000305 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0985530614852905, + "group": 3, + "latest_source_mass": 0.32961612939834595, + "layer": 9, + "max_source_mass": 0.3358767628669739, + "mixer_index": 16, + "normalized_entropy": 0.9999460890949153, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3358767628669739, + "median": 0.335875540971756, + "p05": 0.3299671709537506, + "p95": 0.3413305878639221 + }, + { + "label": "completed_group_1", + "mean": 0.3345070481300354, + "median": 0.3343847990036011, + "p05": 0.33154726028442383, + "p95": 0.337911993265152 + }, + { + "label": "completed_group_2", + "mean": 0.32961612939834595, + "median": 0.3299309015274048, + "p05": 0.3261342942714691, + "p95": 0.3323304057121277 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3860312700271606, + "group": 3, + "latest_source_mass": 0.24687285721302032, + "layer": 9, + "max_source_mass": 0.25749218463897705, + "mixer_index": 17, + "normalized_entropy": 0.9998102198926082, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25749218463897705, + "median": 0.2577459216117859, + "p05": 0.25051605701446533, + "p95": 0.26390716433525085 + }, + { + "label": "completed_group_1", + "mean": 0.24992282688617706, + "median": 0.2488851249217987, + "p05": 0.24565663933753967, + "p95": 0.25723880529403687 + }, + { + "label": "completed_group_2", + "mean": 0.24571210145950317, + "median": 0.24514061212539673, + "p05": 0.24144421517848969, + "p95": 0.251026451587677 + }, + { + "label": "current_group_partial", + "mean": 0.24687285721302032, + "median": 0.24738571047782898, + "p05": 0.24093922972679138, + "p95": 0.25144800543785095 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862019777297974, + "group": 3, + "latest_source_mass": 0.2477031946182251, + "layer": 10, + "max_source_mass": 0.25314557552337646, + "mixer_index": 18, + "normalized_entropy": 0.999933359470626, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25314557552337646, + "median": 0.2529510259628296, + "p05": 0.2480011284351349, + "p95": 0.25820392370224 + }, + { + "label": "completed_group_1", + "mean": 0.25236955285072327, + "median": 0.25230497121810913, + "p05": 0.2500428557395935, + "p95": 0.2550833523273468 + }, + { + "label": "completed_group_2", + "mean": 0.24678170680999756, + "median": 0.24716459214687347, + "p05": 0.24367791414260864, + "p95": 0.2489454448223114 + }, + { + "label": "current_group_partial", + "mean": 0.2477031946182251, + "median": 0.247479647397995, + "p05": 0.24608317017555237, + "p95": 0.24990878999233246 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.385892391204834, + "group": 3, + "latest_source_mass": 0.24460072815418243, + "layer": 10, + "max_source_mass": 0.25815320014953613, + "mixer_index": 19, + "normalized_entropy": 0.9997100399984806, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25815320014953613, + "median": 0.2583469748497009, + "p05": 0.24714797735214233, + "p95": 0.26989418268203735 + }, + { + "label": "completed_group_1", + "mean": 0.25234687328338623, + "median": 0.25168517231941223, + "p05": 0.24759340286254883, + "p95": 0.25932571291923523 + }, + { + "label": "completed_group_2", + "mean": 0.2448991984128952, + "median": 0.24570998549461365, + "p05": 0.23882809281349182, + "p95": 0.24954037368297577 + }, + { + "label": "current_group_partial", + "mean": 0.24460072815418243, + "median": 0.24506908655166626, + "p05": 0.23951184749603271, + "p95": 0.2482105791568756 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3861613273620605, + "group": 3, + "latest_source_mass": 0.24515381455421448, + "layer": 11, + "max_source_mass": 0.25378289818763733, + "mixer_index": 20, + "normalized_entropy": 0.9999040364286539, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25378289818763733, + "median": 0.25346964597702026, + "p05": 0.2502344846725464, + "p95": 0.25824105739593506 + }, + { + "label": "completed_group_1", + "mean": 0.25339174270629883, + "median": 0.2532607913017273, + "p05": 0.2513120174407959, + "p95": 0.25618526339530945 + }, + { + "label": "completed_group_2", + "mean": 0.24767157435417175, + "median": 0.24778661131858826, + "p05": 0.24526026844978333, + "p95": 0.24950486421585083 + }, + { + "label": "current_group_partial", + "mean": 0.24515381455421448, + "median": 0.2451559603214264, + "p05": 0.24296040832996368, + "p95": 0.2471310794353485 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3855845928192139, + "group": 3, + "latest_source_mass": 0.24013002216815948, + "layer": 11, + "max_source_mass": 0.2592235505580902, + "mixer_index": 21, + "normalized_entropy": 0.9994880103962168, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2592235505580902, + "median": 0.260243684053421, + "p05": 0.2400972694158554, + "p95": 0.27410486340522766 + }, + { + "label": "completed_group_1", + "mean": 0.2543467879295349, + "median": 0.2532445192337036, + "p05": 0.2488831728696823, + "p95": 0.26358044147491455 + }, + { + "label": "completed_group_2", + "mean": 0.2462996244430542, + "median": 0.24627387523651123, + "p05": 0.23948119580745697, + "p95": 0.2534843385219574 + }, + { + "label": "current_group_partial", + "mean": 0.24013002216815948, + "median": 0.24106061458587646, + "p05": 0.2340238392353058, + "p95": 0.24541442096233368 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3861030340194702, + "group": 3, + "latest_source_mass": 0.24375030398368835, + "layer": 12, + "max_source_mass": 0.25586679577827454, + "mixer_index": 22, + "normalized_entropy": 0.999861986670518, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25138330459594727, + "median": 0.2519329786300659, + "p05": 0.24518054723739624, + "p95": 0.25645288825035095 + }, + { + "label": "completed_group_1", + "mean": 0.25586679577827454, + "median": 0.2557966709136963, + "p05": 0.25358980894088745, + "p95": 0.25876402854919434 + }, + { + "label": "completed_group_2", + "mean": 0.24899962544441223, + "median": 0.24906687438488007, + "p05": 0.2462620884180069, + "p95": 0.2515370845794678 + }, + { + "label": "current_group_partial", + "mean": 0.24375030398368835, + "median": 0.24418899416923523, + "p05": 0.2408682256937027, + "p95": 0.24600279331207275 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3856689929962158, + "group": 3, + "latest_source_mass": 0.24451105296611786, + "layer": 12, + "max_source_mass": 0.2574368715286255, + "mixer_index": 23, + "normalized_entropy": 0.9995488922546222, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2574368715286255, + "median": 0.25917017459869385, + "p05": 0.23716294765472412, + "p95": 0.27229955792427063 + }, + { + "label": "completed_group_1", + "mean": 0.25204822421073914, + "median": 0.2513560652732849, + "p05": 0.24607181549072266, + "p95": 0.2608553171157837 + }, + { + "label": "completed_group_2", + "mean": 0.2460038810968399, + "median": 0.2455829679965973, + "p05": 0.2395436018705368, + "p95": 0.2544419467449188 + }, + { + "label": "current_group_partial", + "mean": 0.24451105296611786, + "median": 0.2424975037574768, + "p05": 0.2388056218624115, + "p95": 0.2537745237350464 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.386252999305725, + "group": 4, + "latest_source_mass": 0.24712973833084106, + "layer": 13, + "max_source_mass": 0.25257009267807007, + "mixer_index": 24, + "normalized_entropy": 0.9999701637579107, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25002047419548035, + "median": 0.24998526275157928, + "p05": 0.24717272818088531, + "p95": 0.2532656788825989 + }, + { + "label": "completed_group_1", + "mean": 0.25257009267807007, + "median": 0.2525791525840759, + "p05": 0.25114181637763977, + "p95": 0.2538524568080902 + }, + { + "label": "completed_group_2", + "mean": 0.25027966499328613, + "median": 0.25028353929519653, + "p05": 0.24859410524368286, + "p95": 0.2516433000564575 + }, + { + "label": "completed_group_3", + "mean": 0.24712973833084106, + "median": 0.24733176827430725, + "p05": 0.2456049621105194, + "p95": 0.24814137816429138 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.608945608139038, + "group": 4, + "latest_source_mass": 0.19259600341320038, + "layer": 13, + "max_source_mass": 0.2066430151462555, + "mixer_index": 25, + "normalized_entropy": 0.9996941141430441, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20495271682739258, + "median": 0.20551061630249023, + "p05": 0.19701695442199707, + "p95": 0.21219341456890106 + }, + { + "label": "completed_group_1", + "mean": 0.2066430151462555, + "median": 0.2066696584224701, + "p05": 0.20182059705257416, + "p95": 0.21241214871406555 + }, + { + "label": "completed_group_2", + "mean": 0.2000001221895218, + "median": 0.1994028240442276, + "p05": 0.1966598778963089, + "p95": 0.20397058129310608 + }, + { + "label": "completed_group_3", + "mean": 0.19580815732479095, + "median": 0.19627662003040314, + "p05": 0.1906815767288208, + "p95": 0.20030993223190308 + }, + { + "label": "current_group_partial", + "mean": 0.19259600341320038, + "median": 0.1924200803041458, + "p05": 0.18821945786476135, + "p95": 0.19660082459449768 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093038320541382, + "group": 4, + "latest_source_mass": 0.19651609659194946, + "layer": 14, + "max_source_mass": 0.20420849323272705, + "mixer_index": 26, + "normalized_entropy": 0.9999166911758905, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20154938101768494, + "median": 0.2015763819217682, + "p05": 0.19716835021972656, + "p95": 0.20529012382030487 + }, + { + "label": "completed_group_1", + "mean": 0.20420849323272705, + "median": 0.20418605208396912, + "p05": 0.20256520807743073, + "p95": 0.20618648827075958 + }, + { + "label": "completed_group_2", + "mean": 0.2010107934474945, + "median": 0.20109188556671143, + "p05": 0.19940726459026337, + "p95": 0.2023579627275467 + }, + { + "label": "completed_group_3", + "mean": 0.19671522080898285, + "median": 0.19701939821243286, + "p05": 0.19455352425575256, + "p95": 0.19804880023002625 + }, + { + "label": "current_group_partial", + "mean": 0.19651609659194946, + "median": 0.1964191049337387, + "p05": 0.19523656368255615, + "p95": 0.19822117686271667 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6085759401321411, + "group": 4, + "latest_source_mass": 0.18957461416721344, + "layer": 14, + "max_source_mass": 0.209174245595932, + "mixer_index": 27, + "normalized_entropy": 0.99946442649617, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20549985766410828, + "median": 0.20732679963111877, + "p05": 0.1906183660030365, + "p95": 0.21469318866729736 + }, + { + "label": "completed_group_1", + "mean": 0.209174245595932, + "median": 0.2091023325920105, + "p05": 0.2042047083377838, + "p95": 0.21456730365753174 + }, + { + "label": "completed_group_2", + "mean": 0.20157773792743683, + "median": 0.20102596282958984, + "p05": 0.19761772453784943, + "p95": 0.2060481607913971 + }, + { + "label": "completed_group_3", + "mean": 0.19417354464530945, + "median": 0.19437026977539062, + "p05": 0.18892596662044525, + "p95": 0.1989399641752243 + }, + { + "label": "current_group_partial", + "mean": 0.18957461416721344, + "median": 0.18890255689620972, + "p05": 0.18614861369132996, + "p95": 0.19485054910182953 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093413829803467, + "group": 4, + "latest_source_mass": 0.1961907148361206, + "layer": 15, + "max_source_mass": 0.20404505729675293, + "mixer_index": 28, + "normalized_entropy": 0.999940022878169, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20104943215847015, + "median": 0.20089785754680634, + "p05": 0.19876199960708618, + "p95": 0.20387773215770721 + }, + { + "label": "completed_group_1", + "mean": 0.20404505729675293, + "median": 0.20400074124336243, + "p05": 0.20308127999305725, + "p95": 0.20513048768043518 + }, + { + "label": "completed_group_2", + "mean": 0.2002137303352356, + "median": 0.20016872882843018, + "p05": 0.1987290233373642, + "p95": 0.2015499770641327 + }, + { + "label": "completed_group_3", + "mean": 0.19850105047225952, + "median": 0.19858190417289734, + "p05": 0.19726355373859406, + "p95": 0.19943322241306305 + }, + { + "label": "current_group_partial", + "mean": 0.1961907148361206, + "median": 0.19618821144104004, + "p05": 0.1951756328344345, + "p95": 0.19696630537509918 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6080443859100342, + "group": 4, + "latest_source_mass": 0.18532700836658478, + "layer": 15, + "max_source_mass": 0.21283750236034393, + "mixer_index": 29, + "normalized_entropy": 0.9991341532883623, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20538949966430664, + "median": 0.20632551610469818, + "p05": 0.19240552186965942, + "p95": 0.21609632670879364 + }, + { + "label": "completed_group_1", + "mean": 0.21283750236034393, + "median": 0.2127622365951538, + "p05": 0.20815539360046387, + "p95": 0.21858084201812744 + }, + { + "label": "completed_group_2", + "mean": 0.20341980457305908, + "median": 0.20320799946784973, + "p05": 0.19931574165821075, + "p95": 0.2073775827884674 + }, + { + "label": "completed_group_3", + "mean": 0.19302619993686676, + "median": 0.1941618025302887, + "p05": 0.18753090500831604, + "p95": 0.1975058764219284 + }, + { + "label": "current_group_partial", + "mean": 0.18532700836658478, + "median": 0.18489490449428558, + "p05": 0.1818811148405075, + "p95": 0.19045278429985046 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093816757202148, + "group": 4, + "latest_source_mass": 0.1984887570142746, + "layer": 16, + "max_source_mass": 0.20253092050552368, + "mixer_index": 30, + "normalized_entropy": 0.9999650581650581, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2010989934206009, + "median": 0.20145544409751892, + "p05": 0.1965426653623581, + "p95": 0.20464390516281128 + }, + { + "label": "completed_group_1", + "mean": 0.20253092050552368, + "median": 0.20251227915287018, + "p05": 0.2015175074338913, + "p95": 0.20370690524578094 + }, + { + "label": "completed_group_2", + "mean": 0.19950580596923828, + "median": 0.19937199354171753, + "p05": 0.19786886870861053, + "p95": 0.20136688649654388 + }, + { + "label": "completed_group_3", + "mean": 0.19837552309036255, + "median": 0.19810429215431213, + "p05": 0.19704630970954895, + "p95": 0.20048341155052185 + }, + { + "label": "current_group_partial", + "mean": 0.1984887570142746, + "median": 0.19826334714889526, + "p05": 0.19740399718284607, + "p95": 0.19980067014694214 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076874732971191, + "group": 4, + "latest_source_mass": 0.18312416970729828, + "layer": 16, + "max_source_mass": 0.21552258729934692, + "mixer_index": 31, + "normalized_entropy": 0.9989123910133733, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20581099390983582, + "median": 0.2059018313884735, + "p05": 0.19574317336082458, + "p95": 0.21629589796066284 + }, + { + "label": "completed_group_1", + "mean": 0.21552258729934692, + "median": 0.2158341407775879, + "p05": 0.20979133248329163, + "p95": 0.2216476947069168 + }, + { + "label": "completed_group_2", + "mean": 0.20295079052448273, + "median": 0.2030434012413025, + "p05": 0.19869419932365417, + "p95": 0.20675882697105408 + }, + { + "label": "completed_group_3", + "mean": 0.19259145855903625, + "median": 0.1934264600276947, + "p05": 0.18681757152080536, + "p95": 0.19656985998153687 + }, + { + "label": "current_group_partial", + "mean": 0.18312416970729828, + "median": 0.18330088257789612, + "p05": 0.17835915088653564, + "p95": 0.18824028968811035 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.609355092048645, + "group": 5, + "latest_source_mass": 0.19643527269363403, + "layer": 17, + "max_source_mass": 0.2033483237028122, + "mixer_index": 32, + "normalized_entropy": 0.999948540801223, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20059311389923096, + "median": 0.20052847266197205, + "p05": 0.1975974291563034, + "p95": 0.20357753336429596 + }, + { + "label": "completed_group_1", + "mean": 0.2033483237028122, + "median": 0.20335550606250763, + "p05": 0.20221826434135437, + "p95": 0.20440047979354858 + }, + { + "label": "completed_group_2", + "mean": 0.2011984884738922, + "median": 0.2013639658689499, + "p05": 0.19996295869350433, + "p95": 0.20205698907375336 + }, + { + "label": "completed_group_3", + "mean": 0.1984248161315918, + "median": 0.1985226571559906, + "p05": 0.1971665471792221, + "p95": 0.1996813714504242 + }, + { + "label": "completed_group_4", + "mean": 0.19643527269363403, + "median": 0.19629330933094025, + "p05": 0.19542984664440155, + "p95": 0.19760140776634216 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7901467084884644, + "group": 5, + "latest_source_mass": 0.15958338975906372, + "layer": 17, + "max_source_mass": 0.18124164640903473, + "mixer_index": 33, + "normalized_entropy": 0.9990999010931498, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17125868797302246, + "median": 0.1713503748178482, + "p05": 0.16346795856952667, + "p95": 0.17907589673995972 + }, + { + "label": "completed_group_1", + "mean": 0.18124164640903473, + "median": 0.18149954080581665, + "p05": 0.17634934186935425, + "p95": 0.18695694208145142 + }, + { + "label": "completed_group_2", + "mean": 0.17160002887248993, + "median": 0.17129477858543396, + "p05": 0.16841214895248413, + "p95": 0.17520035803318024 + }, + { + "label": "completed_group_3", + "mean": 0.1620369851589203, + "median": 0.1626986563205719, + "p05": 0.15807151794433594, + "p95": 0.16494593024253845 + }, + { + "label": "completed_group_4", + "mean": 0.15427924692630768, + "median": 0.15451020002365112, + "p05": 0.1496276557445526, + "p95": 0.15884152054786682 + }, + { + "label": "current_group_partial", + "mean": 0.15958338975906372, + "median": 0.16020870208740234, + "p05": 0.1548137664794922, + "p95": 0.16307921707630157 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7916979789733887, + "group": 5, + "latest_source_mass": 0.1646067053079605, + "layer": 18, + "max_source_mass": 0.16927622258663177, + "mixer_index": 34, + "normalized_entropy": 0.9999656816354414, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16689208149909973, + "median": 0.1665830761194229, + "p05": 0.16523399949073792, + "p95": 0.16929879784584045 + }, + { + "label": "completed_group_1", + "mean": 0.16927622258663177, + "median": 0.16933584213256836, + "p05": 0.16805575788021088, + "p95": 0.17042185366153717 + }, + { + "label": "completed_group_2", + "mean": 0.1683787703514099, + "median": 0.16833901405334473, + "p05": 0.16767027974128723, + "p95": 0.16930325329303741 + }, + { + "label": "completed_group_3", + "mean": 0.16569086909294128, + "median": 0.1657993346452713, + "p05": 0.16471867263317108, + "p95": 0.16646946966648102 + }, + { + "label": "completed_group_4", + "mean": 0.1651553511619568, + "median": 0.16505351662635803, + "p05": 0.16424953937530518, + "p95": 0.16621217131614685 + }, + { + "label": "current_group_partial", + "mean": 0.1646067053079605, + "median": 0.16458091139793396, + "p05": 0.16394029557704926, + "p95": 0.1652415692806244 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.789665699005127, + "group": 5, + "latest_source_mass": 0.15269872546195984, + "layer": 18, + "max_source_mass": 0.18269017338752747, + "mixer_index": 35, + "normalized_entropy": 0.9988314445890273, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1720278114080429, + "median": 0.17178001999855042, + "p05": 0.1651601344347, + "p95": 0.18030256032943726 + }, + { + "label": "completed_group_1", + "mean": 0.18269017338752747, + "median": 0.1828061193227768, + "p05": 0.1778087317943573, + "p95": 0.18853498995304108 + }, + { + "label": "completed_group_2", + "mean": 0.17356479167938232, + "median": 0.17310965061187744, + "p05": 0.1708383560180664, + "p95": 0.17728117108345032 + }, + { + "label": "completed_group_3", + "mean": 0.16241811215877533, + "median": 0.16282695531845093, + "p05": 0.15827099978923798, + "p95": 0.16520199179649353 + }, + { + "label": "completed_group_4", + "mean": 0.15660037100315094, + "median": 0.15704499185085297, + "p05": 0.15247587859630585, + "p95": 0.16102959215641022 + }, + { + "label": "current_group_partial", + "mean": 0.15269872546195984, + "median": 0.15324696898460388, + "p05": 0.14927825331687927, + "p95": 0.15543338656425476 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917176485061646, + "group": 5, + "latest_source_mass": 0.16697046160697937, + "layer": 19, + "max_source_mass": 0.16854418814182281, + "mixer_index": 36, + "normalized_entropy": 0.9999766594107029, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16644492745399475, + "median": 0.16678380966186523, + "p05": 0.16287505626678467, + "p95": 0.1690630316734314 + }, + { + "label": "completed_group_1", + "mean": 0.16854418814182281, + "median": 0.16858378052711487, + "p05": 0.1675834059715271, + "p95": 0.1694023609161377 + }, + { + "label": "completed_group_2", + "mean": 0.16665583848953247, + "median": 0.16633233428001404, + "p05": 0.16533653438091278, + "p95": 0.16844478249549866 + }, + { + "label": "completed_group_3", + "mean": 0.16677430272102356, + "median": 0.1665923297405243, + "p05": 0.16590546071529388, + "p95": 0.16807952523231506 + }, + { + "label": "completed_group_4", + "mean": 0.16461026668548584, + "median": 0.1646481156349182, + "p05": 0.16389033198356628, + "p95": 0.1651972234249115 + }, + { + "label": "current_group_partial", + "mean": 0.16697046160697937, + "median": 0.166935995221138, + "p05": 0.16631023585796356, + "p95": 0.16765466332435608 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7893871068954468, + "group": 5, + "latest_source_mass": 0.15126986801624298, + "layer": 19, + "max_source_mass": 0.18359911441802979, + "mixer_index": 37, + "normalized_entropy": 0.9986759593721415, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17303554713726044, + "median": 0.17261022329330444, + "p05": 0.16660651564598083, + "p95": 0.18199719488620758 + }, + { + "label": "completed_group_1", + "mean": 0.18359911441802979, + "median": 0.18389560282230377, + "p05": 0.17790013551712036, + "p95": 0.1901787668466568 + }, + { + "label": "completed_group_2", + "mean": 0.17276406288146973, + "median": 0.17272299528121948, + "p05": 0.16975931823253632, + "p95": 0.1759810596704483 + }, + { + "label": "completed_group_3", + "mean": 0.16385790705680847, + "median": 0.16339454054832458, + "p05": 0.16003964841365814, + "p95": 0.1676909327507019 + }, + { + "label": "completed_group_4", + "mean": 0.1554735004901886, + "median": 0.15558910369873047, + "p05": 0.15120893716812134, + "p95": 0.15934500098228455 + }, + { + "label": "current_group_partial", + "mean": 0.15126986801624298, + "median": 0.15185734629631042, + "p05": 0.14759114384651184, + "p95": 0.1540675163269043 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917413711547852, + "group": 5, + "latest_source_mass": 0.16742712259292603, + "layer": 20, + "max_source_mass": 0.16742712259292603, + "mixer_index": 38, + "normalized_entropy": 0.999989899272988, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16618916392326355, + "median": 0.16628703474998474, + "p05": 0.1636679470539093, + "p95": 0.16814759373664856 + }, + { + "label": "completed_group_1", + "mean": 0.16740113496780396, + "median": 0.16743052005767822, + "p05": 0.16664837300777435, + "p95": 0.16812929511070251 + }, + { + "label": "completed_group_2", + "mean": 0.16675519943237305, + "median": 0.16677126288414001, + "p05": 0.1660051941871643, + "p95": 0.16743934154510498 + }, + { + "label": "completed_group_3", + "mean": 0.1668717861175537, + "median": 0.16686314344406128, + "p05": 0.16629263758659363, + "p95": 0.1675407886505127 + }, + { + "label": "completed_group_4", + "mean": 0.1653555929660797, + "median": 0.16525551676750183, + "p05": 0.16465890407562256, + "p95": 0.166346937417984 + }, + { + "label": "current_group_partial", + "mean": 0.16742712259292603, + "median": 0.1672172248363495, + "p05": 0.16667790710926056, + "p95": 0.16856777667999268 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7882882356643677, + "group": 5, + "latest_source_mass": 0.14739990234375, + "layer": 20, + "max_source_mass": 0.18687990307807922, + "mixer_index": 39, + "normalized_entropy": 0.9980626676608648, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17288538813591003, + "median": 0.17220303416252136, + "p05": 0.16650697588920593, + "p95": 0.18239524960517883 + }, + { + "label": "completed_group_1", + "mean": 0.18687990307807922, + "median": 0.1869410276412964, + "p05": 0.1804148554801941, + "p95": 0.19480778276920319 + }, + { + "label": "completed_group_2", + "mean": 0.17607632279396057, + "median": 0.17579063773155212, + "p05": 0.1729249656200409, + "p95": 0.18001750111579895 + }, + { + "label": "completed_group_3", + "mean": 0.16286307573318481, + "median": 0.162565678358078, + "p05": 0.15797676146030426, + "p95": 0.16752192378044128 + }, + { + "label": "completed_group_4", + "mean": 0.15389540791511536, + "median": 0.15402349829673767, + "p05": 0.14864680171012878, + "p95": 0.1582801640033722 + }, + { + "label": "current_group_partial", + "mean": 0.14739990234375, + "median": 0.14793044328689575, + "p05": 0.14291448891162872, + "p95": 0.15072257816791534 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917335033416748, + "group": 6, + "latest_source_mass": 0.16607257723808289, + "layer": 21, + "max_source_mass": 0.16799162328243256, + "mixer_index": 40, + "normalized_entropy": 0.9999855081628835, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16660980880260468, + "median": 0.16617761552333832, + "p05": 0.1644040048122406, + "p95": 0.1693679392337799 + }, + { + "label": "completed_group_1", + "mean": 0.16799162328243256, + "median": 0.1680111587047577, + "p05": 0.16704340279102325, + "p95": 0.16876636445522308 + }, + { + "label": "completed_group_2", + "mean": 0.16742658615112305, + "median": 0.16741225123405457, + "p05": 0.1667558252811432, + "p95": 0.16810807585716248 + }, + { + "label": "completed_group_3", + "mean": 0.1664734035730362, + "median": 0.1665918529033661, + "p05": 0.16557857394218445, + "p95": 0.16731621325016022 + }, + { + "label": "completed_group_4", + "mean": 0.16542600095272064, + "median": 0.16541361808776855, + "p05": 0.16449882090091705, + "p95": 0.166628897190094 + }, + { + "label": "completed_group_5", + "mean": 0.16607257723808289, + "median": 0.166090726852417, + "p05": 0.16493722796440125, + "p95": 0.16713958978652954 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.941649079322815, + "group": 6, + "latest_source_mass": 0.13102038204669952, + "layer": 21, + "max_source_mass": 0.16432973742485046, + "mixer_index": 41, + "normalized_entropy": 0.9978102433277473, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15037035942077637, + "median": 0.15097856521606445, + "p05": 0.1399948000907898, + "p95": 0.1612456738948822 + }, + { + "label": "completed_group_1", + "mean": 0.16432973742485046, + "median": 0.16473636031150818, + "p05": 0.15782266855239868, + "p95": 0.17067503929138184 + }, + { + "label": "completed_group_2", + "mean": 0.15334899723529816, + "median": 0.15358540415763855, + "p05": 0.1502012014389038, + "p95": 0.1558559238910675 + }, + { + "label": "completed_group_3", + "mean": 0.1418340653181076, + "median": 0.14079436659812927, + "p05": 0.13714224100112915, + "p95": 0.14798131585121155 + }, + { + "label": "completed_group_4", + "mean": 0.13212506473064423, + "median": 0.13160677254199982, + "p05": 0.1273195594549179, + "p95": 0.13741742074489594 + }, + { + "label": "completed_group_5", + "mean": 0.12697139382362366, + "median": 0.12719248235225677, + "p05": 0.1226799264550209, + "p95": 0.1304648220539093 + }, + { + "label": "current_group_partial", + "mean": 0.13102038204669952, + "median": 0.1311827003955841, + "p05": 0.12728165090084076, + "p95": 0.134886234998703 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458887577056885, + "group": 6, + "latest_source_mass": 0.14201194047927856, + "layer": 22, + "max_source_mass": 0.14392223954200745, + "mixer_index": 42, + "normalized_entropy": 0.9999890070208868, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1431310474872589, + "median": 0.1429322063922882, + "p05": 0.14122292399406433, + "p95": 0.14551937580108643 + }, + { + "label": "completed_group_1", + "mean": 0.14324922859668732, + "median": 0.14312732219696045, + "p05": 0.14240022003650665, + "p95": 0.14425528049468994 + }, + { + "label": "completed_group_2", + "mean": 0.14309078454971313, + "median": 0.14308927953243256, + "p05": 0.14254748821258545, + "p95": 0.14363758265972137 + }, + { + "label": "completed_group_3", + "mean": 0.142366424202919, + "median": 0.1424175649881363, + "p05": 0.14179164171218872, + "p95": 0.14290453493595123 + }, + { + "label": "completed_group_4", + "mean": 0.14222830533981323, + "median": 0.14226765930652618, + "p05": 0.14123091101646423, + "p95": 0.1430286169052124 + }, + { + "label": "completed_group_5", + "mean": 0.14392223954200745, + "median": 0.14401790499687195, + "p05": 0.14277607202529907, + "p95": 0.1448441743850708 + }, + { + "label": "current_group_partial", + "mean": 0.14201194047927856, + "median": 0.1420515477657318, + "p05": 0.14140468835830688, + "p95": 0.1425478160381317 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9409761428833008, + "group": 6, + "latest_source_mass": 0.12605468928813934, + "layer": 22, + "max_source_mass": 0.16567249596118927, + "mixer_index": 43, + "normalized_entropy": 0.9974644224069606, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1505419909954071, + "median": 0.15057668089866638, + "p05": 0.1423991620540619, + "p95": 0.1602301448583603 + }, + { + "label": "completed_group_1", + "mean": 0.16567249596118927, + "median": 0.16589392721652985, + "p05": 0.15891215205192566, + "p95": 0.17315104603767395 + }, + { + "label": "completed_group_2", + "mean": 0.1550290882587433, + "median": 0.15488041937351227, + "p05": 0.15190331637859344, + "p95": 0.15830960869789124 + }, + { + "label": "completed_group_3", + "mean": 0.14243589341640472, + "median": 0.14207899570465088, + "p05": 0.13828127086162567, + "p95": 0.14758256077766418 + }, + { + "label": "completed_group_4", + "mean": 0.13249853253364563, + "median": 0.13208040595054626, + "p05": 0.12817804515361786, + "p95": 0.13691219687461853 + }, + { + "label": "completed_group_5", + "mean": 0.12776729464530945, + "median": 0.1281416267156601, + "p05": 0.1238994225859642, + "p95": 0.13073651492595673 + }, + { + "label": "current_group_partial", + "mean": 0.12605468928813934, + "median": 0.12603645026683807, + "p05": 0.12259864807128906, + "p95": 0.12950734794139862 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458999633789062, + "group": 6, + "latest_source_mass": 0.14296774566173553, + "layer": 23, + "max_source_mass": 0.1435922086238861, + "mixer_index": 44, + "normalized_entropy": 0.9999947655977786, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14249610900878906, + "median": 0.14253944158554077, + "p05": 0.14135295152664185, + "p95": 0.14354844391345978 + }, + { + "label": "completed_group_1", + "mean": 0.14256572723388672, + "median": 0.1425473988056183, + "p05": 0.14203670620918274, + "p95": 0.14315003156661987 + }, + { + "label": "completed_group_2", + "mean": 0.1435922086238861, + "median": 0.14354602992534637, + "p05": 0.142880380153656, + "p95": 0.14443112909793854 + }, + { + "label": "completed_group_3", + "mean": 0.14210084080696106, + "median": 0.14210137724876404, + "p05": 0.14155790209770203, + "p95": 0.14259196817874908 + }, + { + "label": "completed_group_4", + "mean": 0.14355877041816711, + "median": 0.14355827867984772, + "p05": 0.1432723104953766, + "p95": 0.14383304119110107 + }, + { + "label": "completed_group_5", + "mean": 0.1427185833454132, + "median": 0.1427343189716339, + "p05": 0.14239519834518433, + "p95": 0.14299128949642181 + }, + { + "label": "current_group_partial", + "mean": 0.14296774566173553, + "median": 0.14299029111862183, + "p05": 0.14251580834388733, + "p95": 0.14344699680805206 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.941575527191162, + "group": 6, + "latest_source_mass": 0.126667320728302, + "layer": 23, + "max_source_mass": 0.1640411913394928, + "mixer_index": 45, + "normalized_entropy": 0.997772445009213, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15046924352645874, + "median": 0.15056449174880981, + "p05": 0.1412365734577179, + "p95": 0.16024990379810333 + }, + { + "label": "completed_group_1", + "mean": 0.1640411913394928, + "median": 0.16397136449813843, + "p05": 0.15763643383979797, + "p95": 0.17164170742034912 + }, + { + "label": "completed_group_2", + "mean": 0.15395452082157135, + "median": 0.1541147232055664, + "p05": 0.1505676656961441, + "p95": 0.157239630818367 + }, + { + "label": "completed_group_3", + "mean": 0.14198558032512665, + "median": 0.14145654439926147, + "p05": 0.13773861527442932, + "p95": 0.14743509888648987 + }, + { + "label": "completed_group_4", + "mean": 0.13400861620903015, + "median": 0.13384035229682922, + "p05": 0.1295107901096344, + "p95": 0.13838393986225128 + }, + { + "label": "completed_group_5", + "mean": 0.1288735270500183, + "median": 0.12916186451911926, + "p05": 0.12452569603919983, + "p95": 0.1320379376411438 + }, + { + "label": "current_group_partial", + "mean": 0.126667320728302, + "median": 0.12697717547416687, + "p05": 0.12317797541618347, + "p95": 0.1303042471408844 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458911418914795, + "group": 6, + "latest_source_mass": 0.14433911442756653, + "layer": 24, + "max_source_mass": 0.14433911442756653, + "mixer_index": 46, + "normalized_entropy": 0.9999902322500127, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14260169863700867, + "median": 0.1426181197166443, + "p05": 0.14112016558647156, + "p95": 0.14391711354255676 + }, + { + "label": "completed_group_1", + "mean": 0.14355596899986267, + "median": 0.14354628324508667, + "p05": 0.1428137570619583, + "p95": 0.1442246437072754 + }, + { + "label": "completed_group_2", + "mean": 0.1428615301847458, + "median": 0.14287462830543518, + "p05": 0.14239956438541412, + "p95": 0.1433551013469696 + }, + { + "label": "completed_group_3", + "mean": 0.14244115352630615, + "median": 0.14245545864105225, + "p05": 0.14187632501125336, + "p95": 0.1429358869791031 + }, + { + "label": "completed_group_4", + "mean": 0.14214617013931274, + "median": 0.14210420846939087, + "p05": 0.1417112946510315, + "p95": 0.1426859349012375 + }, + { + "label": "completed_group_5", + "mean": 0.14205437898635864, + "median": 0.14205071330070496, + "p05": 0.14159706234931946, + "p95": 0.14274871349334717 + }, + { + "label": "current_group_partial", + "mean": 0.14433911442756653, + "median": 0.14430615305900574, + "p05": 0.14394080638885498, + "p95": 0.14513593912124634 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9400205612182617, + "group": 6, + "latest_source_mass": 0.12250205874443054, + "layer": 24, + "max_source_mass": 0.16689133644104004, + "mixer_index": 47, + "normalized_entropy": 0.9969733505732982, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1507154405117035, + "median": 0.15137067437171936, + "p05": 0.13938520848751068, + "p95": 0.16102933883666992 + }, + { + "label": "completed_group_1", + "mean": 0.16689133644104004, + "median": 0.16768822073936462, + "p05": 0.15714310109615326, + "p95": 0.17547596991062164 + }, + { + "label": "completed_group_2", + "mean": 0.15648441016674042, + "median": 0.1563897281885147, + "p05": 0.15373846888542175, + "p95": 0.15962360799312592 + }, + { + "label": "completed_group_3", + "mean": 0.1427520215511322, + "median": 0.14218726754188538, + "p05": 0.13820181787014008, + "p95": 0.14905300736427307 + }, + { + "label": "completed_group_4", + "mean": 0.13353285193443298, + "median": 0.1328810751438141, + "p05": 0.1281396746635437, + "p95": 0.1398344486951828 + }, + { + "label": "completed_group_5", + "mean": 0.12712189555168152, + "median": 0.12679056823253632, + "p05": 0.12242501974105835, + "p95": 0.1326143443584442 + }, + { + "label": "current_group_partial", + "mean": 0.12250205874443054, + "median": 0.12238125503063202, + "p05": 0.11881453543901443, + "p95": 0.12615910172462463 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458441734313965, + "group": 7, + "latest_source_mass": 0.14612418413162231, + "layer": 25, + "max_source_mass": 0.14612418413162231, + "mixer_index": 48, + "normalized_entropy": 0.9999660952362324, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1422245353460312, + "median": 0.14217790961265564, + "p05": 0.14090952277183533, + "p95": 0.14379017055034637 + }, + { + "label": "completed_group_1", + "mean": 0.14142528176307678, + "median": 0.14123442769050598, + "p05": 0.1402144730091095, + "p95": 0.14290855824947357 + }, + { + "label": "completed_group_2", + "mean": 0.14198407530784607, + "median": 0.14185458421707153, + "p05": 0.14110887050628662, + "p95": 0.14297308027744293 + }, + { + "label": "completed_group_3", + "mean": 0.14165858924388885, + "median": 0.14167557656764984, + "p05": 0.1411055624485016, + "p95": 0.14217206835746765 + }, + { + "label": "completed_group_4", + "mean": 0.14288201928138733, + "median": 0.14292356371879578, + "p05": 0.1422932744026184, + "p95": 0.14346298575401306 + }, + { + "label": "completed_group_5", + "mean": 0.14370128512382507, + "median": 0.14391976594924927, + "p05": 0.14265118539333344, + "p95": 0.14442476630210876 + }, + { + "label": "completed_group_6", + "mean": 0.14612418413162231, + "median": 0.14619040489196777, + "p05": 0.14509475231170654, + "p95": 0.14701899886131287 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.0741422176361084, + "group": 7, + "latest_source_mass": 0.11380299180746078, + "layer": 25, + "max_source_mass": 0.14654947817325592, + "mixer_index": 49, + "normalized_entropy": 0.9974515638273503, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13300859928131104, + "median": 0.13330551981925964, + "p05": 0.12400922924280167, + "p95": 0.14149883389472961 + }, + { + "label": "completed_group_1", + "mean": 0.14654947817325592, + "median": 0.14719218015670776, + "p05": 0.1369621753692627, + "p95": 0.15508656203746796 + }, + { + "label": "completed_group_2", + "mean": 0.1379268318414688, + "median": 0.1377311646938324, + "p05": 0.13488081097602844, + "p95": 0.14173594117164612 + }, + { + "label": "completed_group_3", + "mean": 0.12711459398269653, + "median": 0.12693974375724792, + "p05": 0.12290681898593903, + "p95": 0.131547749042511 + }, + { + "label": "completed_group_4", + "mean": 0.1185426190495491, + "median": 0.118228018283844, + "p05": 0.11395476758480072, + "p95": 0.12371979653835297 + }, + { + "label": "completed_group_5", + "mean": 0.1134917140007019, + "median": 0.11304019391536713, + "p05": 0.10924124717712402, + "p95": 0.1182163655757904 + }, + { + "label": "completed_group_6", + "mean": 0.10956315696239471, + "median": 0.10938596725463867, + "p05": 0.10588769614696503, + "p95": 0.11350928992033005 + }, + { + "label": "current_group_partial", + "mean": 0.11380299180746078, + "median": 0.11401359736919403, + "p05": 0.1090584248304367, + "p95": 0.11826062947511673 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0794055461883545, + "group": 7, + "latest_source_mass": 0.1260453760623932, + "layer": 26, + "max_source_mass": 0.1263071894645691, + "mixer_index": 50, + "normalized_entropy": 0.9999826898276486, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12428364902734756, + "median": 0.1242905929684639, + "p05": 0.12320776283740997, + "p95": 0.12528444826602936 + }, + { + "label": "completed_group_1", + "mean": 0.123402900993824, + "median": 0.12333650141954422, + "p05": 0.12251013517379761, + "p95": 0.12441572546958923 + }, + { + "label": "completed_group_2", + "mean": 0.12436956912279129, + "median": 0.1244126409292221, + "p05": 0.12380266189575195, + "p95": 0.12485788762569427 + }, + { + "label": "completed_group_3", + "mean": 0.12428067624568939, + "median": 0.12426625192165375, + "p05": 0.12392986565828323, + "p95": 0.1246793270111084 + }, + { + "label": "completed_group_4", + "mean": 0.12555459141731262, + "median": 0.12557968497276306, + "p05": 0.1250462681055069, + "p95": 0.12604334950447083 + }, + { + "label": "completed_group_5", + "mean": 0.12575605511665344, + "median": 0.1258038580417633, + "p05": 0.12512780725955963, + "p95": 0.12622345983982086 + }, + { + "label": "completed_group_6", + "mean": 0.1263071894645691, + "median": 0.12632015347480774, + "p05": 0.12580350041389465, + "p95": 0.12684133648872375 + }, + { + "label": "current_group_partial", + "mean": 0.1260453760623932, + "median": 0.12603351473808289, + "p05": 0.12576976418495178, + "p95": 0.12635180354118347 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.071772813796997, + "group": 7, + "latest_source_mass": 0.1069779098033905, + "layer": 26, + "max_source_mass": 0.1495765745639801, + "mixer_index": 51, + "normalized_entropy": 0.9963121214378339, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.134201318025589, + "median": 0.13547956943511963, + "p05": 0.12102854251861572, + "p95": 0.1446569263935089 + }, + { + "label": "completed_group_1", + "mean": 0.1495765745639801, + "median": 0.15066714584827423, + "p05": 0.13745656609535217, + "p95": 0.15931519865989685 + }, + { + "label": "completed_group_2", + "mean": 0.14145472645759583, + "median": 0.14118221402168274, + "p05": 0.13830184936523438, + "p95": 0.14542292058467865 + }, + { + "label": "completed_group_3", + "mean": 0.12840478122234344, + "median": 0.12756571173667908, + "p05": 0.1237710565328598, + "p95": 0.1351458877325058 + }, + { + "label": "completed_group_4", + "mean": 0.11872729659080505, + "median": 0.1178978681564331, + "p05": 0.11296786367893219, + "p95": 0.1259407103061676 + }, + { + "label": "completed_group_5", + "mean": 0.11272042989730835, + "median": 0.11211050301790237, + "p05": 0.10776206851005554, + "p95": 0.11910594999790192 + }, + { + "label": "completed_group_6", + "mean": 0.10793697834014893, + "median": 0.10766834765672684, + "p05": 0.10393505543470383, + "p95": 0.11274770647287369 + }, + { + "label": "current_group_partial", + "mean": 0.1069779098033905, + "median": 0.10693258047103882, + "p05": 0.10282129794359207, + "p95": 0.11074373126029968 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079404592514038, + "group": 7, + "latest_source_mass": 0.12623761594295502, + "layer": 27, + "max_source_mass": 0.1266230344772339, + "mixer_index": 52, + "normalized_entropy": 0.9999822312072463, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12455718219280243, + "median": 0.12448663264513016, + "p05": 0.12370531260967255, + "p95": 0.12576276063919067 + }, + { + "label": "completed_group_1", + "mean": 0.12374450266361237, + "median": 0.12364563345909119, + "p05": 0.12273170799016953, + "p95": 0.12496861815452576 + }, + { + "label": "completed_group_2", + "mean": 0.12391386926174164, + "median": 0.12387685477733612, + "p05": 0.12311351299285889, + "p95": 0.12478289753198624 + }, + { + "label": "completed_group_3", + "mean": 0.12458804249763489, + "median": 0.12460290640592575, + "p05": 0.12419690191745758, + "p95": 0.12499304115772247 + }, + { + "label": "completed_group_4", + "mean": 0.1251734495162964, + "median": 0.12516582012176514, + "p05": 0.12470955401659012, + "p95": 0.12564992904663086 + }, + { + "label": "completed_group_5", + "mean": 0.1251622885465622, + "median": 0.12524060904979706, + "p05": 0.12443183362483978, + "p95": 0.12575514614582062 + }, + { + "label": "completed_group_6", + "mean": 0.1266230344772339, + "median": 0.12661367654800415, + "p05": 0.12584306299686432, + "p95": 0.12732140719890594 + }, + { + "label": "current_group_partial", + "mean": 0.12623761594295502, + "median": 0.12622185051441193, + "p05": 0.12569603323936462, + "p95": 0.1268007904291153 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0732176303863525, + "group": 7, + "latest_source_mass": 0.1073671281337738, + "layer": 27, + "max_source_mass": 0.1473538726568222, + "mixer_index": 53, + "normalized_entropy": 0.9970069313473197, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13281872868537903, + "median": 0.13317491114139557, + "p05": 0.12417874485254288, + "p95": 0.14101722836494446 + }, + { + "label": "completed_group_1", + "mean": 0.1473538726568222, + "median": 0.14751407504081726, + "p05": 0.13885104656219482, + "p95": 0.15571141242980957 + }, + { + "label": "completed_group_2", + "mean": 0.13968363404273987, + "median": 0.13911551237106323, + "p05": 0.1365838497877121, + "p95": 0.14400556683540344 + }, + { + "label": "completed_group_3", + "mean": 0.12848341464996338, + "median": 0.12823450565338135, + "p05": 0.12424321472644806, + "p95": 0.13310366868972778 + }, + { + "label": "completed_group_4", + "mean": 0.12016645073890686, + "median": 0.11972066760063171, + "p05": 0.11583883315324783, + "p95": 0.12517021596431732 + }, + { + "label": "completed_group_5", + "mean": 0.11505335569381714, + "median": 0.11486441642045975, + "p05": 0.11096000671386719, + "p95": 0.11946934461593628 + }, + { + "label": "completed_group_6", + "mean": 0.10907343029975891, + "median": 0.1089182049036026, + "p05": 0.10512679070234299, + "p95": 0.11236356198787689 + }, + { + "label": "current_group_partial", + "mean": 0.1073671281337738, + "median": 0.10785426199436188, + "p05": 0.10343525558710098, + "p95": 0.11085459589958191 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0794081687927246, + "group": 7, + "latest_source_mass": 0.12505511939525604, + "layer": 28, + "max_source_mass": 0.12684063613414764, + "mixer_index": 54, + "normalized_entropy": 0.9999839510337549, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12450858950614929, + "median": 0.12455325573682785, + "p05": 0.12283766269683838, + "p95": 0.12648260593414307 + }, + { + "label": "completed_group_1", + "mean": 0.12424901872873306, + "median": 0.12403430044651031, + "p05": 0.12319816648960114, + "p95": 0.1257738173007965 + }, + { + "label": "completed_group_2", + "mean": 0.12409762293100357, + "median": 0.12406422197818756, + "p05": 0.12322939932346344, + "p95": 0.12511804699897766 + }, + { + "label": "completed_group_3", + "mean": 0.12468484044075012, + "median": 0.12464898824691772, + "p05": 0.12426739186048508, + "p95": 0.12527169287204742 + }, + { + "label": "completed_group_4", + "mean": 0.12504075467586517, + "median": 0.1250670850276947, + "p05": 0.1244644820690155, + "p95": 0.12563633918762207 + }, + { + "label": "completed_group_5", + "mean": 0.1255234032869339, + "median": 0.12571458518505096, + "p05": 0.12471026182174683, + "p95": 0.12614558637142181 + }, + { + "label": "completed_group_6", + "mean": 0.12684063613414764, + "median": 0.12695147097110748, + "p05": 0.12606506049633026, + "p95": 0.1274568736553192 + }, + { + "label": "current_group_partial", + "mean": 0.12505511939525604, + "median": 0.12517717480659485, + "p05": 0.12436046451330185, + "p95": 0.1254468858242035 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.072478771209717, + "group": 7, + "latest_source_mass": 0.10563355684280396, + "layer": 28, + "max_source_mass": 0.14877822995185852, + "mixer_index": 55, + "normalized_entropy": 0.996651615190637, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13302236795425415, + "median": 0.13431280851364136, + "p05": 0.11974859982728958, + "p95": 0.14355170726776123 + }, + { + "label": "completed_group_1", + "mean": 0.14877822995185852, + "median": 0.1492476761341095, + "p05": 0.13942936062812805, + "p95": 0.15665946900844574 + }, + { + "label": "completed_group_2", + "mean": 0.14031866192817688, + "median": 0.14013901352882385, + "p05": 0.13727493584156036, + "p95": 0.14390599727630615 + }, + { + "label": "completed_group_3", + "mean": 0.12882456183433533, + "median": 0.12811559438705444, + "p05": 0.12456346303224564, + "p95": 0.13553638756275177 + }, + { + "label": "completed_group_4", + "mean": 0.11896270513534546, + "median": 0.1182594895362854, + "p05": 0.11376718431711197, + "p95": 0.12510910630226135 + }, + { + "label": "completed_group_5", + "mean": 0.1144586130976677, + "median": 0.11415413022041321, + "p05": 0.10992536693811417, + "p95": 0.11974817514419556 + }, + { + "label": "completed_group_6", + "mean": 0.11000129580497742, + "median": 0.10966727137565613, + "p05": 0.10602983832359314, + "p95": 0.11363138258457184 + }, + { + "label": "current_group_partial", + "mean": 0.10563355684280396, + "median": 0.10599470138549805, + "p05": 0.10157415270805359, + "p95": 0.10950814187526703 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079291820526123, + "group": 8, + "latest_source_mass": 0.12825238704681396, + "layer": 29, + "max_source_mass": 0.12825238704681396, + "mixer_index": 56, + "normalized_entropy": 0.9999279993446741, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12426379323005676, + "median": 0.12415257841348648, + "p05": 0.12234964966773987, + "p95": 0.12625235319137573 + }, + { + "label": "completed_group_1", + "mean": 0.12249694764614105, + "median": 0.12249137461185455, + "p05": 0.12120498716831207, + "p95": 0.12380178272724152 + }, + { + "label": "completed_group_2", + "mean": 0.12262053787708282, + "median": 0.12276404350996017, + "p05": 0.1215757429599762, + "p95": 0.12327249348163605 + }, + { + "label": "completed_group_3", + "mean": 0.12315946817398071, + "median": 0.12308797240257263, + "p05": 0.12251235544681549, + "p95": 0.12412448227405548 + }, + { + "label": "completed_group_4", + "mean": 0.1255876123905182, + "median": 0.1256473809480667, + "p05": 0.12454833835363388, + "p95": 0.12639300525188446 + }, + { + "label": "completed_group_5", + "mean": 0.1262608915567398, + "median": 0.12635794281959534, + "p05": 0.12536443769931793, + "p95": 0.12692558765411377 + }, + { + "label": "completed_group_6", + "mean": 0.1273583620786667, + "median": 0.12732446193695068, + "p05": 0.12652143836021423, + "p95": 0.1282241940498352 + }, + { + "label": "completed_group_7", + "mean": 0.12825238704681396, + "median": 0.12817136943340302, + "p05": 0.1276511698961258, + "p95": 0.1291390061378479 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1905503273010254, + "group": 8, + "latest_source_mass": 0.10074523091316223, + "layer": 29, + "max_source_mass": 0.13242119550704956, + "mixer_index": 57, + "normalized_entropy": 0.9969624179048253, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11892962455749512, + "median": 0.12071715295314789, + "p05": 0.10431190580129623, + "p95": 0.12862780690193176 + }, + { + "label": "completed_group_1", + "mean": 0.13242119550704956, + "median": 0.13316701352596283, + "p05": 0.12286566942930222, + "p95": 0.14010177552700043 + }, + { + "label": "completed_group_2", + "mean": 0.12570279836654663, + "median": 0.12560519576072693, + "p05": 0.12152561545372009, + "p95": 0.1298297792673111 + }, + { + "label": "completed_group_3", + "mean": 0.11684690415859222, + "median": 0.11606287956237793, + "p05": 0.11148497462272644, + "p95": 0.12368319928646088 + }, + { + "label": "completed_group_4", + "mean": 0.10851434618234634, + "median": 0.10789132863283157, + "p05": 0.10347767174243927, + "p95": 0.11500974744558334 + }, + { + "label": "completed_group_5", + "mean": 0.10355254262685776, + "median": 0.10266663134098053, + "p05": 0.09956449270248413, + "p95": 0.10929413884878159 + }, + { + "label": "completed_group_6", + "mean": 0.09824738651514053, + "median": 0.09788262844085693, + "p05": 0.09488938748836517, + "p95": 0.10226631164550781 + }, + { + "label": "completed_group_7", + "mean": 0.0950399786233902, + "median": 0.0952744334936142, + "p05": 0.09167876094579697, + "p95": 0.09785640239715576 + }, + { + "label": "current_group_partial", + "mean": 0.10074523091316223, + "median": 0.10040169954299927, + "p05": 0.09587707370519638, + "p95": 0.10640941560268402 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1971964836120605, + "group": 8, + "latest_source_mass": 0.11228709667921066, + "layer": 30, + "max_source_mass": 0.11228709667921066, + "mixer_index": 58, + "normalized_entropy": 0.9999872139951242, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11044953763484955, + "median": 0.11043476313352585, + "p05": 0.10951471328735352, + "p95": 0.11141420900821686 + }, + { + "label": "completed_group_1", + "mean": 0.11010265350341797, + "median": 0.11030832678079605, + "p05": 0.10908854007720947, + "p95": 0.11079797893762589 + }, + { + "label": "completed_group_2", + "mean": 0.1108412891626358, + "median": 0.11078473180532455, + "p05": 0.10996625572443008, + "p95": 0.11167524755001068 + }, + { + "label": "completed_group_3", + "mean": 0.11044170707464218, + "median": 0.1103954166173935, + "p05": 0.1099170595407486, + "p95": 0.11091860383749008 + }, + { + "label": "completed_group_4", + "mean": 0.11131919920444489, + "median": 0.11133146286010742, + "p05": 0.11095823347568512, + "p95": 0.11168018728494644 + }, + { + "label": "completed_group_5", + "mean": 0.11133328080177307, + "median": 0.11128850281238556, + "p05": 0.11106889694929123, + "p95": 0.11170809715986252 + }, + { + "label": "completed_group_6", + "mean": 0.11097346991300583, + "median": 0.11084912717342377, + "p05": 0.1105891764163971, + "p95": 0.11148890852928162 + }, + { + "label": "completed_group_7", + "mean": 0.11225177347660065, + "median": 0.11221592128276825, + "p05": 0.11170009523630142, + "p95": 0.11293211579322815 + }, + { + "label": "current_group_partial", + "mean": 0.11228709667921066, + "median": 0.11222326010465622, + "p05": 0.1117643192410469, + "p95": 0.11299698054790497 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1902995109558105, + "group": 8, + "latest_source_mass": 0.09534209221601486, + "layer": 30, + "max_source_mass": 0.13237422704696655, + "mixer_index": 59, + "normalized_entropy": 0.9968482664667785, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11926669627428055, + "median": 0.12010449916124344, + "p05": 0.10881246626377106, + "p95": 0.12763431668281555 + }, + { + "label": "completed_group_1", + "mean": 0.13237422704696655, + "median": 0.1329837143421173, + "p05": 0.12264501303434372, + "p95": 0.14073863625526428 + }, + { + "label": "completed_group_2", + "mean": 0.12644407153129578, + "median": 0.1259550154209137, + "p05": 0.12286148220300674, + "p95": 0.13101284205913544 + }, + { + "label": "completed_group_3", + "mean": 0.11611802130937576, + "median": 0.11590427160263062, + "p05": 0.11185136437416077, + "p95": 0.1205722838640213 + }, + { + "label": "completed_group_4", + "mean": 0.11051423847675323, + "median": 0.11005840450525284, + "p05": 0.10631610453128815, + "p95": 0.11548890173435211 + }, + { + "label": "completed_group_5", + "mean": 0.10466913878917694, + "median": 0.10406064987182617, + "p05": 0.10079286992549896, + "p95": 0.10976865142583847 + }, + { + "label": "completed_group_6", + "mean": 0.09992007911205292, + "median": 0.09962707757949829, + "p05": 0.0964651107788086, + "p95": 0.10393689572811127 + }, + { + "label": "completed_group_7", + "mean": 0.09535142779350281, + "median": 0.09555896371603012, + "p05": 0.09170880168676376, + "p95": 0.0982663482427597 + }, + { + "label": "current_group_partial", + "mean": 0.09534209221601486, + "median": 0.0955871120095253, + "p05": 0.09114285558462143, + "p95": 0.09938932955265045 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.197113037109375, + "group": 8, + "latest_source_mass": 0.11361998319625854, + "layer": 31, + "max_source_mass": 0.11361998319625854, + "mixer_index": 60, + "normalized_entropy": 0.9999492358550897, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11037027835845947, + "median": 0.11050257831811905, + "p05": 0.10850981622934341, + "p95": 0.1116580218076706 + }, + { + "label": "completed_group_1", + "mean": 0.10917964577674866, + "median": 0.1092953234910965, + "p05": 0.10749757289886475, + "p95": 0.11104606091976166 + }, + { + "label": "completed_group_2", + "mean": 0.10961419343948364, + "median": 0.10983817279338837, + "p05": 0.10842632502317429, + "p95": 0.11050552129745483 + }, + { + "label": "completed_group_3", + "mean": 0.10947251319885254, + "median": 0.10951714217662811, + "p05": 0.10870970040559769, + "p95": 0.1103382408618927 + }, + { + "label": "completed_group_4", + "mean": 0.11109665781259537, + "median": 0.11102215945720673, + "p05": 0.11052000522613525, + "p95": 0.11176691204309464 + }, + { + "label": "completed_group_5", + "mean": 0.11174711585044861, + "median": 0.11183997988700867, + "p05": 0.11095673590898514, + "p95": 0.11240804195404053 + }, + { + "label": "completed_group_6", + "mean": 0.1118166595697403, + "median": 0.11173175275325775, + "p05": 0.11096500605344772, + "p95": 0.11265994608402252 + }, + { + "label": "completed_group_7", + "mean": 0.11308293789625168, + "median": 0.11292627453804016, + "p05": 0.11227726191282272, + "p95": 0.11403826624155045 + }, + { + "label": "current_group_partial", + "mean": 0.11361998319625854, + "median": 0.11346748471260071, + "p05": 0.11268991231918335, + "p95": 0.11467830836772919 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1894307136535645, + "group": 8, + "latest_source_mass": 0.09481187164783478, + "layer": 31, + "max_source_mass": 0.13425534963607788, + "mixer_index": 61, + "normalized_entropy": 0.9964528597745325, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1198490560054779, + "median": 0.12130686640739441, + "p05": 0.10625117272138596, + "p95": 0.12946197390556335 + }, + { + "label": "completed_group_1", + "mean": 0.13425534963607788, + "median": 0.13473033905029297, + "p05": 0.12351738661527634, + "p95": 0.14344483613967896 + }, + { + "label": "completed_group_2", + "mean": 0.12682178616523743, + "median": 0.1263064742088318, + "p05": 0.12210550904273987, + "p95": 0.13173425197601318 + }, + { + "label": "completed_group_3", + "mean": 0.11624304950237274, + "median": 0.1158868670463562, + "p05": 0.11147508770227432, + "p95": 0.12220768630504608 + }, + { + "label": "completed_group_4", + "mean": 0.10967147350311279, + "median": 0.10917282849550247, + "p05": 0.104550302028656, + "p95": 0.11562594771385193 + }, + { + "label": "completed_group_5", + "mean": 0.10377678275108337, + "median": 0.10315801948308945, + "p05": 0.09931983798742294, + "p95": 0.10953669250011444 + }, + { + "label": "completed_group_6", + "mean": 0.09994889795780182, + "median": 0.09950269758701324, + "p05": 0.09602029621601105, + "p95": 0.1042325347661972 + }, + { + "label": "completed_group_7", + "mean": 0.0946216881275177, + "median": 0.09491042792797089, + "p05": 0.09054946899414062, + "p95": 0.098105289041996 + }, + { + "label": "current_group_partial", + "mean": 0.09481187164783478, + "median": 0.09540842473506927, + "p05": 0.09017777442932129, + "p95": 0.09939083456993103 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.196977138519287, + "group": 8, + "latest_source_mass": 0.11403195559978485, + "layer": 32, + "max_source_mass": 0.11424341797828674, + "mixer_index": 62, + "normalized_entropy": 0.999887385741319, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10978135466575623, + "median": 0.10953371971845627, + "p05": 0.108054518699646, + "p95": 0.11213957518339157 + }, + { + "label": "completed_group_1", + "mean": 0.1075383648276329, + "median": 0.10733462870121002, + "p05": 0.10584845393896103, + "p95": 0.10943715274333954 + }, + { + "label": "completed_group_2", + "mean": 0.10839901119470596, + "median": 0.10859507322311401, + "p05": 0.10721821337938309, + "p95": 0.10926944017410278 + }, + { + "label": "completed_group_3", + "mean": 0.11003260314464569, + "median": 0.11011333763599396, + "p05": 0.10928525030612946, + "p95": 0.11081099510192871 + }, + { + "label": "completed_group_4", + "mean": 0.11053797602653503, + "median": 0.11058371514081955, + "p05": 0.1097012460231781, + "p95": 0.11142666637897491 + }, + { + "label": "completed_group_5", + "mean": 0.11152469366788864, + "median": 0.1117575541138649, + "p05": 0.11029430478811264, + "p95": 0.1124267652630806 + }, + { + "label": "completed_group_6", + "mean": 0.11391061544418335, + "median": 0.11399351060390472, + "p05": 0.11294781416654587, + "p95": 0.11480505764484406 + }, + { + "label": "completed_group_7", + "mean": 0.11424341797828674, + "median": 0.11410531401634216, + "p05": 0.11359494924545288, + "p95": 0.11528046429157257 + }, + { + "label": "current_group_partial", + "mean": 0.11403195559978485, + "median": 0.11387525498867035, + "p05": 0.11328872293233871, + "p95": 0.11511529237031937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1905651092529297, + "group": 8, + "latest_source_mass": 0.0946686640381813, + "layer": 32, + "max_source_mass": 0.1320299208164215, + "mixer_index": 63, + "normalized_entropy": 0.9969691454610601, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11943811178207397, + "median": 0.12088920176029205, + "p05": 0.10572259873151779, + "p95": 0.1288197934627533 + }, + { + "label": "completed_group_1", + "mean": 0.1320299208164215, + "median": 0.13276371359825134, + "p05": 0.12282373756170273, + "p95": 0.13998818397521973 + }, + { + "label": "completed_group_2", + "mean": 0.12506362795829773, + "median": 0.12487159669399261, + "p05": 0.12049838155508041, + "p95": 0.12907996773719788 + }, + { + "label": "completed_group_3", + "mean": 0.11625386774539948, + "median": 0.11589770764112473, + "p05": 0.11143262684345245, + "p95": 0.12191615998744965 + }, + { + "label": "completed_group_4", + "mean": 0.11043961346149445, + "median": 0.10992902517318726, + "p05": 0.10569487512111664, + "p95": 0.11613331735134125 + }, + { + "label": "completed_group_5", + "mean": 0.1047937273979187, + "median": 0.10408039391040802, + "p05": 0.10086895525455475, + "p95": 0.11024832725524902 + }, + { + "label": "completed_group_6", + "mean": 0.1010652706027031, + "median": 0.10061101615428925, + "p05": 0.09771151095628738, + "p95": 0.10514533519744873 + }, + { + "label": "completed_group_7", + "mean": 0.09624720364809036, + "median": 0.09628060460090637, + "p05": 0.09263256937265396, + "p95": 0.09951217472553253 + }, + { + "label": "current_group_partial", + "mean": 0.0946686640381813, + "median": 0.09522657096385956, + "p05": 0.09038722515106201, + "p95": 0.09931783378124237 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1935477256774902, + "group": null, + "latest_source_mass": 0.11779685318470001, + "layer": null, + "max_source_mass": 0.11986880004405975, + "mixer_index": null, + "normalized_entropy": 0.9983265926948683, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10072024166584015, + "median": 0.1001858189702034, + "p05": 0.09205758571624756, + "p95": 0.11032550781965256 + }, + { + "label": "completed_group_1", + "mean": 0.09480114281177521, + "median": 0.0942176878452301, + "p05": 0.09034405648708344, + "p95": 0.0996023491024971 + }, + { + "label": "completed_group_2", + "mean": 0.10205776989459991, + "median": 0.10174158215522766, + "p05": 0.09974265098571777, + "p95": 0.10526342689990997 + }, + { + "label": "completed_group_3", + "mean": 0.11059840023517609, + "median": 0.11073200404644012, + "p05": 0.1057434156537056, + "p95": 0.11464740335941315 + }, + { + "label": "completed_group_4", + "mean": 0.1157325878739357, + "median": 0.11606095731258392, + "p05": 0.11147730052471161, + "p95": 0.12005195766687393 + }, + { + "label": "completed_group_5", + "mean": 0.11904929578304291, + "median": 0.11922121047973633, + "p05": 0.11581429839134216, + "p95": 0.1222524642944336 + }, + { + "label": "completed_group_6", + "mean": 0.11986880004405975, + "median": 0.11976201832294464, + "p05": 0.11818123608827591, + "p95": 0.12199589610099792 + }, + { + "label": "completed_group_7", + "mean": 0.11937490850687027, + "median": 0.11958470195531845, + "p05": 0.11761733889579773, + "p95": 0.12075771391391754 + }, + { + "label": "completed_group_8", + "mean": 0.11779685318470001, + "median": 0.1180996373295784, + "p05": 0.11525309085845947, + "p95": 0.11991780996322632 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "334f6a2e9305f6b7249f6d6caf2b255325d80c7e8a5aad5effed4c164d06179a", + "optimizer_state_before": "334f6a2e9305f6b7249f6d6caf2b255325d80c7e8a5aad5effed4c164d06179a", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "18380f2ab4362bd0b1746896f121e69ee1546f1794cea323af41740b6024651d", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.003087003482505679, + 0.0026560353580862284, + 0.0032712947577238083, + 0.002734958194196224, + 0.003094236133620143, + 0.0030397549271583557, + 0.0035086015705019236, + 0.0038121570833027363, + 0.0047028325498104095, + 0.004038033075630665, + 0.004860103130340576, + 0.005031581036746502, + 0.005196469835937023, + 0.005135253071784973, + 0.006614624988287687, + 0.005866772960871458, + 0.006203304044902325, + 0.007167549338191748, + 0.007609309628605843, + 0.008147990331053734, + 0.00901782512664795, + 0.008936969563364983, + 0.009206709451973438, + 0.01152731105685234, + 0.010577243752777576, + 0.011347576975822449, + 0.011137762106955051, + 0.012936098501086235, + 0.013407763093709946, + 0.013788831420242786, + 0.015265416353940964, + 0.0202538650482893 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.47538081591847e-05, + "first_to_last_ratio": 3.523320813580708, + "imbalance_abs_log_ratio": 1.2594039578247644, + "last_quartile_mean": 4.187472256944602e-06, + "mean": 8.003732475003744e-06, + "normalized": [ + 2.2864072450452606, + 2.1505052861894143, + 2.035100368661317, + 1.914613292131888, + 1.7534318989662239, + 1.6431636846734412, + 1.536642318353803, + 1.4270637477194381, + 1.2842766020276717, + 1.1810344805085975, + 1.099480510026868, + 1.0224360977234737, + 0.928328885424462, + 0.8663541927168132, + 0.8170363190541032, + 0.770456622754232, + 0.7301180367940027, + 0.6940574801022341, + 0.6637352609099907, + 0.6396770492722799, + 0.6174712393642938, + 0.5998406679830636, + 0.583621712740686, + 0.5696275372587853, + 0.5553044486825874, + 0.5452271173697439, + 0.5355955147189773, + 0.527231951874486, + 0.5148029686402454, + 0.5078768736288382, + 0.5021984211528733, + 0.49728216752990523 + ], + "peak_layer": 1, + "peak_normalized": 2.2864072450452606, + "population_cv": 0.5439316214178195, + "rest_mean": 8.618577541912802e-06, + "spike_contrast": 0.5434271596349021, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.68356911369483e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8299791918252595e-05, + 1.7212068996741436e-05, + 1.6288398910546675e-05, + 1.5324052583309822e-05, + 1.403399983246345e-05, + 1.3151442544767633e-05, + 1.2298874025873374e-05, + 1.1421836461522616e-05, + 1.0279006346536335e-05, + 9.452684025745839e-06, + 8.799947863735724e-06, + 8.183304998965468e-06, + 7.4300960477557965e-06, + 6.93406718710321e-06, + 6.539340120070847e-06, + 6.166528692119755e-06, + 5.8436694416741375e-06, + 5.5550503930135164e-06, + 5.312359462550376e-06, + 5.1198039727751166e-06, + 4.942074610880809e-06, + 4.800964234163985e-06, + 4.671152055379935e-06, + 4.559146418614546e-06, + 4.444508249434875e-06, + 4.363851985544898e-06, + 4.286763214622624e-06, + 4.219823495077435e-06, + 4.1203452383342665e-06, + 4.064910626766505e-06, + 4.019461812276859e-06, + 3.980113433499355e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.9807127854583086e-05, + "first_to_last_ratio": 3.5566643349459857, + "imbalance_abs_log_ratio": 1.2688231210537682, + "last_quartile_mean": 1.681551088950073e-05, + "mean": 3.247700539077414e-05, + "normalized": [ + 2.2579241602152, + 2.135019037063823, + 2.0280333310485936, + 1.9173080473526447, + 1.758657494805108, + 1.6519298761064487, + 1.544128685861779, + 1.4391773217546358, + 1.2962904826236146, + 1.192609616061844, + 1.110941110824676, + 1.032164886192474, + 0.9359594784290148, + 0.8726072754652816, + 0.821823275119541, + 0.7740717447349674, + 0.7320077127739505, + 0.6952135796200956, + 0.6640023209515697, + 0.639026268143393, + 0.6157178693827154, + 0.5971561937295294, + 0.5802935929104301, + 0.5658032894829641, + 0.5509069802052773, + 0.5405054676570583, + 0.530569722062962, + 0.5218335672124204, + 0.509106468388458, + 0.5020150600957286, + 0.49619577359580697, + 0.49100031012799433 + ], + "peak_layer": 1, + "peak_normalized": 2.2579241602152, + "population_cv": 0.5433134428802157, + "rest_mean": 3.499111301573942e-05, + "spike_contrast": 0.5401607033037161, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.890082421596162e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.333061512326822e-05, + 6.933902477612719e-05, + 6.586444942513481e-05, + 6.226842378964648e-05, + 5.7115928939310834e-05, + 5.364973549148999e-05, + 5.014867565478198e-05, + 4.674016963690519e-05, + 4.209963299217634e-05, + 3.8732388929929584e-05, + 3.6080040445085615e-05, + 3.3521624573040754e-05, + 3.0397161026485264e-05, + 2.8339671189314686e-05, + 2.669035893632099e-05, + 2.5139532226603478e-05, + 2.3773418433847837e-05, + 2.257845517306123e-05, + 2.156480695703067e-05, + 2.0753659555339254e-05, + 1.9996672563138418e-05, + 1.9393844922888093e-05, + 1.8846198145183735e-05, + 1.8375596482655965e-05, + 1.789180896594189e-05, + 1.755399898684118e-05, + 1.7231315723620355e-05, + 1.6947591575444676e-05, + 1.6534253518329933e-05, + 1.6303945812978782e-05, + 1.6114952813950367e-05, + 1.5946219718898647e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05303755821660161, + "first_to_last_ratio": 3.5566642959691026, + "imbalance_abs_log_ratio": 1.2688231100949372, + "last_quartile_mean": 0.014912163140252233, + "mean": 0.028800932981539518, + "normalized": [ + 2.257924127684546, + 2.135019140736797, + 2.0280331746484235, + 1.9173079402069289, + 1.7586574931696988, + 1.651929831377606, + 1.5441287257423313, + 1.4391773748670262, + 1.2962904745959671, + 1.192609691122949, + 1.1109411677627854, + 1.03216488937858, + 0.9359594736307031, + 0.8726072729963495, + 0.8218232514089269, + 0.7740717507159982, + 0.7320077303494418, + 0.695213596207834, + 0.6640023013431785, + 0.6390262678237983, + 0.6157178297675131, + 0.5971562020352453, + 0.5802936008847488, + 0.56580333779038, + 0.5509069924034854, + 0.5405054586699878, + 0.5305697328218089, + 0.5218335956665606, + 0.5091064501799054, + 0.5020150788892077, + 0.49619576248172176, + 0.4910002826395651 + ], + "peak_layer": 1, + "peak_normalized": 2.257924127684546, + "population_cv": 0.5433134335599328, + "rest_mean": 0.031030468711698497, + "spike_contrast": 0.5401607108938693, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01676144003868103, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06503032147884369, + 0.06149054318666458, + 0.05840924754738808, + 0.05522025749087334, + 0.05065097659826279, + 0.04757712036371231, + 0.0444723479449749, + 0.0414496511220932, + 0.0373343750834465, + 0.034348271787166595, + 0.031996142119169235, + 0.029727311804890633, + 0.026956506073474884, + 0.02513190358877182, + 0.023669276386499405, + 0.02229398861527443, + 0.021082505583763123, + 0.0200228001922369, + 0.01912388578057289, + 0.018404552713036537, + 0.017733247950673103, + 0.017198655754327774, + 0.01671299710869789, + 0.0162956640124321, + 0.015866635367274284, + 0.015567061491310596, + 0.015280903317034245, + 0.015029294416308403, + 0.014662740752100945, + 0.014458502642810345, + 0.014290900900959969, + 0.014141266234219074 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.542342338709204e-06, + "first_to_last_ratio": 3.9100869045545728, + "imbalance_abs_log_ratio": 1.3635595999790664, + "last_quartile_mean": 1.1616985631235366e-06, + "mean": 2.356269877878958e-06, + "normalized": [ + 2.626358068851995, + 2.359482633900792, + 2.1673808318573355, + 1.970612227511214, + 1.7715197752462144, + 1.611066095373113, + 1.5253639505266257, + 1.390362687339307, + 1.2470742362409657, + 1.1482697848345025, + 1.0649746685773553, + 0.9926077747670571, + 0.8935238671869759, + 0.8352705716146634, + 0.7976779264251802, + 0.7534193926689576, + 0.7091631265992269, + 0.6729593670129498, + 0.6388931190640394, + 0.6146294043377577, + 0.5903414687897984, + 0.573600345637639, + 0.5583270911267058, + 0.5429263154370998, + 0.5273495286768174, + 0.5154150806600781, + 0.5059577182706787, + 0.4967911022387073, + 0.4840893077120886, + 0.476822723468287, + 0.4716374894346956, + 0.4661323186111764 + ], + "peak_layer": 1, + "peak_normalized": 2.626358068851995, + "population_cv": 0.6048322607097205, + "rest_mean": 2.5489128524390658e-06, + "spike_contrast": 0.5162976890304795, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3159978152543771e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.188408406160306e-06, + 5.559577857638942e-06, + 5.106934167997679e-06, + 4.64329423266463e-06, + 4.174178684479557e-06, + 3.796106511799735e-06, + 3.5941691294283373e-06, + 3.2760697195044486e-06, + 2.938443458333495e-06, + 2.7056335056840908e-06, + 2.509367732272949e-06, + 2.338851800232078e-06, + 2.10538337341859e-06, + 1.9681228877743706e-06, + 1.8795444702845998e-06, + 1.7752594203557237e-06, + 1.6709797137082205e-06, + 1.5856738855291042e-06, + 1.5054046116347308e-06, + 1.448232751499745e-06, + 1.391003820572223e-06, + 1.3515572163669276e-06, + 1.3155693068256369e-06, + 1.279280922972248e-06, + 1.2425778095348505e-06, + 1.2144570291638956e-06, + 1.1921729310415685e-06, + 1.1705739098033519e-06, + 1.1406450539652724e-06, + 1.123523020396533e-06, + 1.1113052096334286e-06, + 1.0983335414493922e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.8821781547303544e-05, + "first_to_last_ratio": 2.980254400081657, + "imbalance_abs_log_ratio": 1.0920086660284927, + "last_quartile_mean": 1.6381749674110324e-05, + "mean": 2.848307332214972e-05, + "normalized": [ + 2.0810926090025332, + 1.969923093544882, + 1.8754316812286433, + 1.775123579435819, + 1.6405766968825957, + 1.5475620321943906, + 1.4549157242056538, + 1.367878426084163, + 1.2540203154379537, + 1.1689725777559958, + 1.0996704285612149, + 1.0350754283963208, + 0.960189941905928, + 0.9068543441283098, + 0.8632891395831829, + 0.821927178126067, + 0.7877167521462786, + 0.7537929393542872, + 0.7245378305968498, + 0.700636091646331, + 0.6787484361888465, + 0.6601340974768426, + 0.6429525692553493, + 0.6278595238949346, + 0.6122509835140441, + 0.6009361488079162, + 0.5899975897039497, + 0.5801705507723549, + 0.5656037931304417, + 0.5573685175022867, + 0.5505152181183216, + 0.5442757614173119 + ], + "peak_layer": 1, + "peak_normalized": 2.0810926090025332, + "population_cv": 0.46429465779931894, + "rest_mean": 3.0358793824066027e-05, + "spike_contrast": 0.6045754886760992, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.835418261180166e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.927591337240301e-05, + 5.6109463912434876e-05, + 5.341805808711797e-05, + 5.05609750689473e-05, + 4.672866634791717e-05, + 4.407932283356786e-05, + 4.14404712500982e-05, + 3.896138150594197e-05, + 3.571835259208456e-05, + 3.32959316438064e-05, + 3.132199344690889e-05, + 2.9482129320967942e-05, + 2.734916051849723e-05, + 2.5829998776316643e-05, + 2.4589127860963345e-05, + 2.341101208003238e-05, + 2.243659400846809e-05, + 2.147033956134692e-05, + 2.063706415356137e-05, + 1.9956269170506857e-05, + 1.9332841475261375e-05, + 1.880264790088404e-05, + 1.831326517276466e-05, + 1.7883368855109438e-05, + 1.7438789654988796e-05, + 1.7116508388426155e-05, + 1.6804944607429206e-05, + 1.6525040337000974e-05, + 1.6110134311020374e-05, + 1.5875568351475522e-05, + 1.5680365322623402e-05, + 1.5502646419918165e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.9543975162814604e-05, + "first_to_last_ratio": 2.361814271071357, + "imbalance_abs_log_ratio": 0.8594300826222705, + "last_quartile_mean": 1.6743050309742102e-05, + "mean": 2.5007048577663227e-05, + "normalized": [ + 1.912859565038877, + 1.8105236978529957, + 1.7273211150778174, + 1.6308285226915125, + 1.5141405100255678, + 1.4329864048795495, + 1.3476327823895244, + 1.2742127313731153, + 1.1867378921860672, + 1.112686902495014, + 1.0470750994574345, + 0.9973623109170946, + 0.9478955259042676, + 0.9058401245160093, + 0.8736388301292974, + 0.8429558183060917, + 0.8275675048043188, + 0.803939236955949, + 0.7837442531107164, + 0.7644177737357688, + 0.7470428108125197, + 0.7311443054887553, + 0.7174787440702515, + 0.7037016009216458, + 0.6932620899019373, + 0.6852674844312424, + 0.6794925843554572, + 0.6746814041897898, + 0.6624963737585712, + 0.6587300903027569, + 0.6539322941255763, + 0.6484036157945076 + ], + "peak_layer": 1, + "peak_normalized": 1.912859565038877, + "population_cv": 0.375701127391813, + "rest_mean": 2.631053676951625e-05, + "spike_contrast": 0.6829283833720613, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7968212341656907e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.783497206517495e-05, + 4.527585406322032e-05, + 4.3195203033974394e-05, + 4.078220808878541e-05, + 3.786418528761715e-05, + 3.583476063795388e-05, + 3.370031845406629e-05, + 3.186429967172444e-05, + 2.967681211885065e-05, + 2.782501542242244e-05, + 2.618425787659362e-05, + 2.494108775863424e-05, + 2.370406946283765e-05, + 2.265238799736835e-05, + 2.1847128664376214e-05, + 2.107983709720429e-05, + 2.0695020793937147e-05, + 2.0104147552046925e-05, + 1.9599130610004067e-05, + 1.9115832401439548e-05, + 1.868133585958276e-05, + 1.8283761164639145e-05, + 1.794202580640558e-05, + 1.759750011842698e-05, + 1.7336438759230077e-05, + 1.713651727186516e-05, + 1.6992104065138847e-05, + 1.6871790649020113e-05, + 1.6567079001106322e-05, + 1.6472895367769524e-05, + 1.6352916645701043e-05, + 1.6214660718105733e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011525819263624726, + "first_to_last_ratio": 5.424369947393303, + "imbalance_abs_log_ratio": 1.6909017539525537, + "last_quartile_mean": 2.12482175356854e-05, + "mean": 5.662517071414186e-05, + "normalized": [ + 2.5055056176095984, + 2.3706297508444365, + 2.241830077392606, + 2.124145849270733, + 1.9443711953183194, + 1.8164591538959356, + 1.7002866499817122, + 1.5804407783115777, + 1.4040231414989057, + 1.2897275398797217, + 1.1905083194532244, + 1.0938772119880942, + 0.9633505879752084, + 0.8774870943542636, + 0.8097317824087507, + 0.7409024588364089, + 0.6759871710557561, + 0.6280736730006866, + 0.5867444951930028, + 0.5522911890332279, + 0.5129096169838441, + 0.48683288837526706, + 0.4612191855365385, + 0.4407179437979303, + 0.4173472207950482, + 0.4025342509602604, + 0.3881747341410948, + 0.376294400049804, + 0.36321827476366064, + 0.3558481534629913, + 0.35107774164810374, + 0.34745185218328734 + ], + "peak_layer": 1, + "peak_normalized": 2.5055056176095984, + "population_cv": 0.6753260599958857, + "rest_mean": 6.224778412110431e-05, + "spike_contrast": 0.4219115376934434, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.626305831654463e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014187468332238495, + 0.0001342373143415898, + 0.00012694401084445417, + 0.00012028012133669108, + 0.00011010035086655989, + 0.00010285730968462303, + 9.627902181819081e-05, + 8.949272887548432e-05, + 7.950305007398129e-05, + 7.303104212041944e-05, + 6.741273682564497e-05, + 6.194098386913538e-05, + 5.4549891501665115e-05, + 4.968785651726648e-05, + 4.5851200411561877e-05, + 4.195372821413912e-05, + 3.8277888961602e-05, + 3.556477895472199e-05, + 3.3224507205886766e-05, + 3.127358286292292e-05, + 2.9043594622635283e-05, + 2.7566995413508266e-05, + 2.611661511764396e-05, + 2.495572880434338e-05, + 2.363235762459226e-05, + 2.2793570678913966e-05, + 2.1980460587656125e-05, + 2.1307734641595744e-05, + 2.0567296814988367e-05, + 2.0149962438154034e-05, + 1.9879837054759264e-05, + 1.967452044482343e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "28d398d2ea80931917268ed413eca333d2f81168391c83a3adc104502478e55b", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0049620093777775764, + 0.004914142191410065, + 0.00555405905470252, + 0.005823355168104172, + 0.006141432095319033, + 0.006136075593531132, + 0.006357426289469004, + 0.0064498125575482845, + 0.006525229662656784, + 0.007023691199719906, + 0.006682637147605419, + 0.006716069765388966, + 0.006977807730436325, + 0.006669311318546534, + 0.006923797074705362, + 0.007155857048928738, + 0.007240671198815107, + 0.007543859537690878, + 0.007281317841261625, + 0.007711030542850494, + 0.00788753479719162, + 0.007908415980637074, + 0.008078642189502716, + 0.008181036449968815, + 0.007894126698374748, + 0.008452145382761955, + 0.008351319469511509, + 0.008498509414494038, + 0.009082214906811714, + 0.009139914996922016, + 0.008129997178912163, + 0.00888112559914589 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3918628951614664e-05, + "first_to_last_ratio": 3.3693491917247846, + "imbalance_abs_log_ratio": 1.2147196075319966, + "last_quartile_mean": 4.130954721404123e-06, + "mean": 7.606227299561397e-06, + "normalized": [ + 2.2824447028233354, + 2.156224576681686, + 2.02967801794237, + 1.9096018794072827, + 1.736504358227357, + 1.6206331020107865, + 1.508932291267265, + 1.395174357296542, + 1.2454912604425945, + 1.159982779559482, + 1.0770364956371037, + 0.9985646101779827, + 0.9126193313903731, + 0.8598667474559943, + 0.8109240384824798, + 0.7673321935183389, + 0.7307233755519057, + 0.6989382867571854, + 0.6733294103254632, + 0.6494885311058664, + 0.6312572107399844, + 0.6141046126011543, + 0.5996497467158993, + 0.586684752686662, + 0.5738065084191857, + 0.5636248604676733, + 0.5548170991634841, + 0.5473936870898765, + 0.5344900934778194, + 0.5284722547322606, + 0.5232893301307341, + 0.5189194977138724 + ], + "peak_layer": 1, + "peak_normalized": 2.2824447028233354, + "population_cv": 0.5333395699341017, + "rest_mean": 8.168101329728313e-06, + "spike_contrast": 0.5597515691951044, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.572107536660042e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.7360793208354153e-05, + 1.6400734239141457e-05, + 1.543819234939292e-05, + 1.4524865946441423e-05, + 1.3208246855356265e-05, + 1.2326903743087314e-05, + 1.14772819870268e-05, + 1.0612013284116983e-05, + 9.473489626543596e-06, + 8.823092684906442e-06, + 8.192184395738877e-06, + 7.595309398311656e-06, + 6.941590072528925e-06, + 6.54034192848485e-06, + 6.168072559376014e-06, + 5.836503078171518e-06, + 5.55804808755056e-06, + 5.316283477441175e-06, + 5.121496542415116e-06, + 4.9401573960494716e-06, + 4.8014858293754514e-06, + 4.671019269153476e-06, + 4.56107227364555e-06, + 4.462457582121715e-06, + 4.364502729004016e-06, + 4.2870588004007e-06, + 4.220064965920756e-06, + 4.163600806350587e-06, + 4.065453140356112e-06, + 4.019680091005284e-06, + 3.980257588409586e-06, + 3.947019649785943e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.6411598961858544e-05, + "first_to_last_ratio": 3.4027524402802842, + "imbalance_abs_log_ratio": 1.2245846453792368, + "last_quartile_mean": 1.6578225995544926e-05, + "mean": 3.084055111912676e-05, + "normalized": [ + 2.25484903989412, + 2.1412529403387204, + 2.024538426739007, + 1.9138206908201378, + 1.744005847409706, + 1.6283901394849352, + 1.51851794721373, + 1.4077224750694408, + 1.2571797277143097, + 1.1712619710004104, + 1.087585943008638, + 1.0075084298767483, + 0.9193230520257301, + 0.8658498478469501, + 0.8153148724555359, + 0.7704877885294069, + 0.7322822621292467, + 0.6994579345140977, + 0.6731224584255281, + 0.6484140168699147, + 0.6289011036610557, + 0.6111628530369178, + 0.595980107097907, + 0.5826992445616984, + 0.5692429152040753, + 0.5587822615864424, + 0.549587741163871, + 0.541809636108114, + 0.5286857205338498, + 0.5225713358473244, + 0.5170881598743383, + 0.5126031099580918 + ], + "peak_layer": 1, + "peak_normalized": 2.25484903989412, + "population_cv": 0.5330474913622731, + "rest_mean": 3.3138759034853945e-05, + "spike_contrast": 0.5561532450510837, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.843022837419994e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.95407870807685e-05, + 6.603742076549679e-05, + 6.243788084248081e-05, + 5.902328484808095e-05, + 5.378610148909502e-05, + 5.0220449338667095e-05, + 4.683193037635647e-05, + 4.3414936953922734e-05, + 3.8772115658503026e-05, + 3.612236469052732e-05, + 3.3541749871801585e-05, + 3.107211523456499e-05, + 2.8352429580991156e-05, + 2.670328649401199e-05, + 2.514476000214927e-05, + 2.37622680288041e-05, + 2.2583988538826816e-05, + 2.1571668185060844e-05, + 2.0759467588504776e-05, + 1.9997445633634925e-05, + 1.9395656636334024e-05, + 1.8848599211196415e-05, + 1.838035495893564e-05, + 1.7970765838981606e-05, + 1.7555765225552022e-05, + 1.7233152902917936e-05, + 1.694958882580977e-05, + 1.6709707779227756e-05, + 1.6304958990076557e-05, + 1.6116387996589765e-05, + 1.594728382769972e-05, + 1.5808962416485883e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05002636928111315, + "first_to_last_ratio": 3.4027524331610692, + "imbalance_abs_log_ratio": 1.2245846432870438, + "last_quartile_mean": 0.014701736392453313, + "mean": 0.027349708892870694, + "normalized": [ + 2.2548490514458424, + 2.1412530488236308, + 2.024538393384759, + 1.9138206338036392, + 1.744005792683758, + 1.6283900755277658, + 1.5185179045510238, + 1.4077225055522984, + 1.257179674815174, + 1.1712619883598294, + 1.087585969825964, + 1.0075084701933625, + 0.9193230998011659, + 0.8658498362871361, + 0.8153148842057143, + 0.7704877914133448, + 0.7322822512666904, + 0.6994579400171268, + 0.6731224452510738, + 0.6484140411218751, + 0.6289011442302842, + 0.6111628528689982, + 0.5959801242324752, + 0.5826992208035262, + 0.5692429135850658, + 0.5587822620006379, + 0.5495877458659506, + 0.5418096383169221, + 0.5286857163419914, + 0.5225713057341705, + 0.5170881921276606, + 0.5126030855611432 + ], + "peak_layer": 1, + "peak_normalized": 2.2548490514458424, + "population_cv": 0.5330474880442257, + "rest_mean": 0.02938778260377822, + "spike_contrast": 0.5561532516532492, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01634411085397005, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06166946515440941, + 0.05856264755129814, + 0.05537053570151329, + 0.05234243720769882, + 0.04769805073738098, + 0.04453599452972412, + 0.041531022638082504, + 0.03850080072879791, + 0.03438349813222885, + 0.03203367441892624, + 0.029745159670710564, + 0.027555063366889954, + 0.025143219158053398, + 0.02368074096739292, + 0.022298624739050865, + 0.021072616800665855, + 0.020027706399559975, + 0.01912997104227543, + 0.01840970292687416, + 0.01773393526673317, + 0.01720026321709156, + 0.016715126112103462, + 0.016299882903695107, + 0.015936654061079025, + 0.0155686279758811, + 0.015282532200217247, + 0.015031064860522747, + 0.014818335883319378, + 0.014459400437772274, + 0.014292173087596893, + 0.014142211526632309, + 0.014019545167684555 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.3751225575761055e-06, + "first_to_last_ratio": 3.8220970734227353, + "imbalance_abs_log_ratio": 1.3407992440886183, + "last_quartile_mean": 1.1446916374779903e-06, + "mean": 2.2650369189136654e-06, + "normalized": [ + 2.6260982509869395, + 2.400216776938017, + 2.1919812201941786, + 1.9790918569249247, + 1.7559460501631168, + 1.6114596120900806, + 1.5126018597778093, + 1.3753240033827305, + 1.2117365805847247, + 1.134432397143863, + 1.0401833788995027, + 0.9674957594857626, + 0.875746003990826, + 0.8237594917355141, + 0.7832661047995432, + 0.740719511828896, + 0.7019638739171157, + 0.6704099413729608, + 0.6414011471084737, + 0.6173397326477368, + 0.5968863232373988, + 0.5799016861412486, + 0.5669030002035256, + 0.5521403649934858, + 0.5377798684577518, + 0.5258530838590796, + 0.5176403099124206, + 0.5093490356054753, + 0.496021692149111, + 0.49050001495252427, + 0.4851144585852832, + 0.4807366079299788 + ], + "peak_layer": 1, + "peak_normalized": 2.6260982509869395, + "population_cv": 0.6051676860135061, + "rest_mean": 2.4467758268858434e-06, + "spike_contrast": 0.5246278803962507, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2836468158639037e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.948209491180023e-06, + 5.436579613160575e-06, + 4.9649183893052395e-06, + 4.482716121856356e-06, + 3.977282631240087e-06, + 3.6500155147223268e-06, + 3.426099056014209e-06, + 3.1151596431300277e-06, + 2.744628091022605e-06, + 2.569531261542579e-06, + 2.3560537556477357e-06, + 2.1914136141276686e-06, + 1.983597030630335e-06, + 1.865845661086496e-06, + 1.7741266447046655e-06, + 1.677757040852157e-06, + 1.5899740901659243e-06, + 1.5185032680165023e-06, + 1.452797278034268e-06, + 1.3982972859594156e-06, + 1.351969558527344e-06, + 1.313498728450213e-06, + 1.2840562249039067e-06, + 1.2506183111327118e-06, + 1.2180912563053425e-06, + 1.191076648865419e-06, + 1.1724744126695441e-06, + 1.1536943702594726e-06, + 1.123507445299765e-06, + 1.1110006425951724e-06, + 1.0988021585944807e-06, + 1.088886165234726e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.6260539875220275e-05, + "first_to_last_ratio": 2.865320598644344, + "imbalance_abs_log_ratio": 1.0526802457017466, + "last_quartile_mean": 1.6144978644661023e-05, + "mean": 2.7227252871853125e-05, + "normalized": [ + 2.069229242257896, + 1.9687734840713744, + 1.8639501074620208, + 1.7653990360295826, + 1.6236824582203917, + 1.5253078926597279, + 1.434250504279973, + 1.341829571296541, + 1.2243737990511798, + 1.1518364329038495, + 1.0836884644807132, + 1.0178401497737837, + 0.9492199847186634, + 0.9036392473516051, + 0.8600860560490738, + 0.8201864427615304, + 0.7888244492232437, + 0.7582210568106187, + 0.7331773989358682, + 0.7093267256759077, + 0.6906736225590872, + 0.672715786894029, + 0.6570059626140338, + 0.6429919843026688, + 0.6287389013300317, + 0.6172847926207785, + 0.6070179898741197, + 0.5981761279825236, + 0.5831187474359685, + 0.5759696583600498, + 0.5694268489156044, + 0.564037073097561 + ], + "peak_layer": 1, + "peak_normalized": 2.069229242257896, + "population_cv": 0.4529330483375927, + "rest_mean": 2.894950139980884e-05, + "spike_contrast": 0.6192545623950071, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7927110820892268e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.633942782878876e-05, + 5.360429349821061e-05, + 5.075024091638625e-05, + 4.806696597370319e-05, + 4.42084128735587e-05, + 4.152994370087981e-05, + 3.905070116161369e-05, + 3.653433304862119e-05, + 3.333633503643796e-05, + 3.1361341825686395e-05, + 2.9505859856726602e-05, + 2.7712991141015664e-05, + 2.5844652554951608e-05, + 2.4603614292573184e-05, + 2.3417780539602973e-05, + 2.2331423679133877e-05, + 2.147752275050152e-05, + 2.0644276446546428e-05, + 1.996240644075442e-05, + 1.9313018128741533e-05, + 1.8805145373335108e-05, + 1.8316202840651385e-05, + 1.7888467482407577e-05, + 1.7506905351183377e-05, + 1.7118833056883886e-05, + 1.6806969142635353e-05, + 1.6527432308066636e-05, + 1.628669269848615e-05, + 1.587672159075737e-05, + 1.5682071534683928e-05, + 1.5503928807447664e-05, + 1.53571800183272e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.756828982659499e-05, + "first_to_last_ratio": 2.2614511370462873, + "imbalance_abs_log_ratio": 0.8160067032669138, + "last_quartile_mean": 1.661247029005608e-05, + "mean": 2.4095109154131933e-05, + "normalized": [ + 1.889963326736216, + 1.799730688213, + 1.7036000060988148, + 1.610459564444366, + 1.4904922646790375, + 1.4035903358019606, + 1.3280490559101101, + 1.2474476942829602, + 1.158111146500897, + 1.0893113074839385, + 1.0361619537088214, + 0.9872898290164548, + 0.9417149709880597, + 0.9072412957378696, + 0.8752606702463853, + 0.8478439926779969, + 0.8346944083690043, + 0.8136049467473944, + 0.7934463022141092, + 0.7733479003439614, + 0.7587235786610493, + 0.7446988149112661, + 0.7306767688756092, + 0.718906877438168, + 0.7112476796011702, + 0.7050319654256372, + 0.7000888963818952, + 0.6958494133286793, + 0.6835587021757187, + 0.6787900952804858, + 0.67292504171933, + 0.6681405059996335 + ], + "peak_layer": 1, + "peak_normalized": 1.889963326736216, + "population_cv": 0.3606025174079256, + "rest_mean": 2.5287144429360826e-05, + "spike_contrast": 0.6983041804986536, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7658118667895906e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.553887265501544e-05, + 4.336470738053322e-05, + 4.104842810193077e-05, + 3.880419899360277e-05, + 3.591357381083071e-05, + 3.3819662348832935e-05, + 3.199948696419597e-05, + 3.0057388357818127e-05, + 2.7904814487555996e-05, + 2.6247074856655672e-05, + 2.496643537597265e-05, + 2.378885619691573e-05, + 2.2690725018037483e-05, + 2.1860078049940057e-05, + 2.108950138790533e-05, + 2.042889354925137e-05, + 2.0112052879994735e-05, + 1.9603900000220165e-05, + 1.9118175259791315e-05, + 1.8633902072906494e-05, + 1.828152744565159e-05, + 1.794359923223965e-05, + 1.7605736502446234e-05, + 1.7322139683528803e-05, + 1.713759047561325e-05, + 1.69878221640829e-05, + 1.6868718375917524e-05, + 1.6766567568993196e-05, + 1.6470421542180702e-05, + 1.635552143852692e-05, + 1.621420233277604e-05, + 1.609891842235811e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010847938028746285, + "first_to_last_ratio": 5.225746258439172, + "imbalance_abs_log_ratio": 1.653597612234075, + "last_quartile_mean": 2.075863903883146e-05, + "mean": 5.315361454449885e-05, + "normalized": [ + 2.5242915739433354, + 2.39631132323169, + 2.26527797895308, + 2.1515636583598874, + 1.938662398543429, + 1.8109880868616515, + 1.684831424407678, + 1.5549978812909424, + 1.3759063748543887, + 1.26834355048635, + 1.1636757169651435, + 1.0626666952243982, + 0.9347756253977458, + 0.8624265202529392, + 0.7881968824941187, + 0.7298512980179852, + 0.6687342618334785, + 0.625193525578969, + 0.5885755667414656, + 0.5509759337416824, + 0.518584792950607, + 0.49089751607579774, + 0.469366663303134, + 0.45058077490001597, + 0.42852582187652194, + 0.4135415510225047, + 0.4009779970762889, + 0.3909110972534533, + 0.3790735154078651, + 0.37379596447854513, + 0.37033105096913865, + 0.36716697750576865 + ], + "peak_layer": 1, + "peak_normalized": 2.5242915739433354, + "population_cv": 0.6748031786716517, + "rest_mean": 5.8354881629283125e-05, + "spike_contrast": 0.4295574181078448, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5066772286663765e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00013417522131931037, + 0.00012737260840367526, + 0.0001204077125294134, + 0.00011436338536441326, + 0.00010304691386409104, + 9.626056271372363e-05, + 8.955488010542467e-05, + 8.265375799965113e-05, + 7.313439709832892e-05, + 6.741704419255257e-05, + 6.185357051435858e-05, + 5.64845759072341e-05, + 4.9686703277984634e-05, + 4.584108683047816e-05, + 4.189551327726804e-05, + 3.879423456965014e-05, + 3.5545643186196685e-05, + 3.32312956743408e-05, + 3.128491880488582e-05, + 2.9286362405400723e-05, + 2.7564656193135306e-05, + 2.609297735034488e-05, + 2.4948534701252356e-05, + 2.394999683019705e-05, + 2.277769635838922e-05, + 2.1981228201184422e-05, + 2.1313429897418246e-05, + 2.077833778457716e-05, + 2.0149127522017807e-05, + 1.9868606614181772e-05, + 1.9684433937072754e-05, + 1.9516251995810308e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "1ff52033329506a86a2df82471984832a6abc7bac6dc80a52bb5d468ca4544c7", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.003087003482505679, + 0.006683253683149815, + 0.010629965923726559, + 0.01484723947942257, + 0.003094236133620143, + 0.007696541491895914, + 0.012912816368043423, + 0.018475037068128586, + 0.0047028325498104095, + 0.0095678074285388, + 0.01631263643503189, + 0.023062339052557945, + 0.005196469835937023, + 0.011978921480476856, + 0.01994149386882782, + 0.027569252997636795, + 0.006203304044902325, + 0.014378104358911514, + 0.024205319583415985, + 0.03444278985261917, + 0.00901782512664795, + 0.01929495297372341, + 0.030904190614819527, + 0.04446368291974068, + 0.010577243752777576, + 0.023803669959306717, + 0.03807948902249336, + 0.05412018671631813, + 0.013407763093709946, + 0.030261604115366936, + 0.04958754777908325, + 0.07224015146493912 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4753848404325254e-05, + "first_to_last_ratio": 3.523327698234082, + "imbalance_abs_log_ratio": 1.2594059118468395, + "last_quartile_mean": 4.187475497019477e-06, + "mean": 8.003761109875995e-06, + "normalized": [ + 2.2863863380761895, + 2.1504887289697443, + 2.035107178272848, + 1.9146241690765207, + 1.7534335800921985, + 1.6431624649344594, + 1.536645343249951, + 1.4270675055525637, + 1.2842963248514263, + 1.1810293460813548, + 1.0994791900092338, + 1.022432212510507, + 0.9283341434863113, + 0.8663573430191046, + 0.817050781874415, + 0.7704539231259797, + 0.7301217881386737, + 0.6940567583066531, + 0.6637388520338441, + 0.6396683404264909, + 0.6174723824373465, + 0.5998404537149282, + 0.5836225791994625, + 0.569622544846661, + 0.5553030869663969, + 0.5452192009689635, + 0.5355922917469447, + 0.5272338155115143, + 0.5148020927255124, + 0.507880056479642, + 0.5021977039665544, + 0.4972794793476046 + ], + "peak_layer": 1, + "peak_normalized": 2.2863863380761895, + "population_cv": 0.5439307229766922, + "rest_mean": 8.61860972791975e-06, + "spike_contrast": 0.5434262276974213, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.683578572439728e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8299690054845996e-05, + 1.7211998056154698e-05, + 1.6288511687889695e-05, + 1.53241944644833e-05, + 1.4034063497092575e-05, + 1.3151479834050406e-05, + 1.2298942237976007e-05, + 1.1421907402109355e-05, + 1.0279200978402514e-05, + 9.452676749788225e-06, + 8.799968782113865e-06, + 8.183303179976065e-06, + 7.43016471460578e-06, + 6.9341172093118075e-06, + 6.539479272760218e-06, + 6.166529146867106e-06, + 5.843720373377437e-06, + 5.555064490181394e-06, + 5.312407211022219e-06, + 5.119752586324466e-06, + 4.942101440974511e-06, + 4.800979695573915e-06, + 4.671175702242181e-06, + 4.5591227717523e-06, + 4.444513251655735e-06, + 4.3638042370730545e-06, + 4.286752755433554e-06, + 4.2198535084025934e-06, + 4.1203529690392315e-06, + 4.064950644533383e-06, + 4.019470452476526e-06, + 3.980106157541741e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.980720561638009e-05, + "first_to_last_ratio": 3.556659196689387, + "imbalance_abs_log_ratio": 1.2688216763685727, + "last_quartile_mean": 1.6815557046356844e-05, + "mean": 3.2477058425683936e-05, + "normalized": [ + 2.257907031008062, + 2.1350155505860586, + 2.0280560071945923, + 1.9172997636195261, + 1.7586560791437353, + 1.6519288587646246, + 1.5441298608627463, + 1.4391799003264687, + 1.2963020318458602, + 1.1925989312182288, + 1.1109307833791746, + 1.032159280080384, + 0.9359660712347485, + 0.8726045623068648, + 0.8218273098956514, + 0.7740706487041483, + 0.732006853458854, + 0.6952173170733782, + 0.6640029168915916, + 0.6390278010042643, + 0.6157213445930976, + 0.597151914077787, + 0.5802953336984127, + 0.5658058940601672, + 0.5509085449477206, + 0.5405036888786843, + 0.530568183543385, + 0.5218326590517988, + 0.5091077653403295, + 0.502017712830421, + 0.49619905192622005, + 0.49100034845301377 + ], + "peak_layer": 1, + "peak_normalized": 2.257907031008062, + "population_cv": 0.5433125364543616, + "rest_mean": 3.4991164014812695e-05, + "spike_contrast": 0.5401617458735403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.8900888244388624e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.333017856581137e-05, + 6.933902477612719e-05, + 6.586529343621805e-05, + 6.226825644262135e-05, + 5.7115976233035326e-05, + 5.36497900611721e-05, + 5.0148795708082616e-05, + 4.6740329707972705e-05, + 4.2100076825590804e-05, + 3.873210516758263e-05, + 3.6079763958696276e-05, + 3.3521497243782505e-05, + 3.039742477994878e-05, + 2.8339629352558404e-05, + 2.669053355930373e-05, + 2.513953768357169e-05, + 2.377342934778426e-05, + 2.2578613425139338e-05, + 2.1564861526712775e-05, + 2.0753743228851818e-05, + 1.99968180822907e-05, + 1.9393737602513283e-05, + 1.8846285456675105e-05, + 1.8375711078988388e-05, + 1.7891889001475647e-05, + 1.7553969883010723e-05, + 1.7231293895747513e-05, + 1.6947589756455272e-05, + 1.6534322639927268e-05, + 1.63040585903218e-05, + 1.6115085600176826e-05, + 1.59462470037397e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.053037628065794706, + "first_to_last_ratio": 3.5566592064140465, + "imbalance_abs_log_ratio": 1.2688216791027844, + "last_quartile_mean": 0.014912204118445516, + "mean": 0.02880098018795252, + "normalized": [ + 2.2579069748415974, + 2.135015511975382, + 2.028055978471918, + 1.9172997531400104, + 1.758656162783437, + 1.651928934615846, + 1.5441298165159716, + 1.4391799311213385, + 1.2963020605748452, + 1.192598940844177, + 1.110930680689234, + 1.0321593172240922, + 0.9359660883361088, + 0.8726045492848026, + 0.8218273369249041, + 0.7740706759881554, + 0.7320068539128299, + 0.6952173071872988, + 0.6640028945037896, + 0.6390278073432092, + 0.6157213476800849, + 0.5971519249410595, + 0.5802953660162322, + 0.5658058380749674, + 0.5509085470093485, + 0.5405036996661936, + 0.5305681841221567, + 0.5218326756792125, + 0.5091077499343003, + 0.5020176837263095, + 0.4961990559232735, + 0.49100035094791344 + ], + "peak_layer": 1, + "peak_normalized": 2.2579069748415974, + "population_cv": 0.5433125339470698, + "rest_mean": 0.03103051417403751, + "spike_contrast": 0.5401617443102993, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.016761496663093567, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06502993404865265, + 0.06149053946137428, + 0.058410000056028366, + 0.0552201122045517, + 0.05065102130174637, + 0.04757717251777649, + 0.044472452253103256, + 0.04144979268312454, + 0.03733476996421814, + 0.0343480184674263, + 0.03199589252471924, + 0.02972720004618168, + 0.026956740766763687, + 0.025131866335868835, + 0.02366943284869194, + 0.022293994203209877, + 0.02108251489698887, + 0.020022939890623093, + 0.01912393420934677, + 0.018404627218842506, + 0.01773337833583355, + 0.017198560759425163, + 0.01671307533979416, + 0.016295762732625008, + 0.015866706147789955, + 0.015567036345601082, + 0.015280883759260178, + 0.015029292553663254, + 0.01466280221939087, + 0.014458601363003254, + 0.014291019178926945, + 0.014141291379928589 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.542383834404973e-06, + "first_to_last_ratio": 3.910089142398143, + "imbalance_abs_log_ratio": 1.363560172304691, + "last_quartile_mean": 1.1617085107218372e-06, + "mean": 2.3562732494042393e-06, + "normalized": [ + 2.6263801721086546, + 2.3595531746074276, + 2.1673972230391225, + 1.9706080568572362, + 1.7715820865194194, + 1.611051245513112, + 1.5253714176420914, + 1.390321713055932, + 1.2471349819967512, + 1.1481946109719883, + 1.064921325760361, + 0.9926023015917457, + 0.8935298259557028, + 0.835274104810251, + 0.7976860970261026, + 0.7533694870596088, + 0.7091585414827071, + 0.6729707074821024, + 0.6388703482831869, + 0.6146359069144856, + 0.590298599573816, + 0.5736151091802485, + 0.5583379201398495, + 0.5429116429881082, + 0.5273414403230551, + 0.5154169485904055, + 0.5059683327262104, + 0.4967915493601712, + 0.4840866850990917, + 0.476824743116495, + 0.4716594914143145, + 0.46613420881024553 + ], + "peak_layer": 1, + "peak_normalized": 2.6263801721086546, + "population_cv": 0.60484183936143, + "rest_mean": 2.548919993656724e-06, + "spike_contrast": 0.5162895790043583, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3159808304408217e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.188469342305325e-06, + 5.559752025874332e-06, + 5.106980097480118e-06, + 4.643291049433174e-06, + 4.174331479589455e-06, + 3.7960769532219274e-06, + 3.5941918667958817e-06, + 3.2759778605395695e-06, + 2.9385907964751823e-06, + 2.705460246943403e-06, + 2.509245632609236e-06, + 2.3388422505377093e-06, + 2.1054004264442483e-06, + 1.9681340290844673e-06, + 1.87956641184428e-06, + 1.7751443692759494e-06, + 1.6709713008822291e-06, + 1.5857028756727232e-06, + 1.505353111497243e-06, + 1.4482501455859165e-06, + 1.3909047993365675e-06, + 1.3515939372155117e-06, + 1.3155967053535278e-06, + 1.2792481811629841e-06, + 1.2425605291355168e-06, + 1.2144631682531326e-06, + 1.192199647448433e-06, + 1.1705766382874572e-06, + 1.1406405064917635e-06, + 1.1235293868594454e-06, + 1.1113586424471578e-06, + 1.0983395668517915e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.882185112364823e-05, + "first_to_last_ratio": 2.9802495883434017, + "imbalance_abs_log_ratio": 1.0920070514877542, + "last_quartile_mean": 1.6381799468945246e-05, + "mean": 2.8483126868650288e-05, + "normalized": [ + 2.0810870362663727, + 1.9699227110363493, + 1.8754345417339926, + 1.775115005630423, + 1.6405724631848402, + 1.547564615006236, + 1.4549152880868532, + 1.3678859447538592, + 1.2540251105018627, + 1.1689693583685432, + 1.0996615918776176, + 1.0350748874830915, + 0.9601920962526763, + 0.9068520006787653, + 0.8632928171993581, + 0.8219285067442073, + 0.7877145049439734, + 0.7537945876465386, + 0.7245395338868796, + 0.7006371373881553, + 0.6787509919040607, + 0.6601298549520191, + 0.6429542981965809, + 0.627861217348151, + 0.6122530256203341, + 0.600934635913477, + 0.589995714201328, + 0.5801691407779421, + 0.5656053481725521, + 0.5573713652661699, + 0.5505186535239659, + 0.5442760154528234 + ], + "peak_layer": 1, + "peak_normalized": 2.0810870362663727, + "population_cv": 0.46429372489568066, + "rest_mean": 3.0358846911608622e-05, + "spike_contrast": 0.6045762768959777, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.835423863667529e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.927586607867852e-05, + 5.610955849988386e-05, + 5.3418239986058325e-05, + 5.056082591181621e-05, + 4.6728633606107906e-05, + 4.407947926665656e-05, + 4.1440536733716726e-05, + 3.896166890626773e-05, + 3.5718556318897754e-05, + 3.329590253997594e-05, + 3.1321800634032115e-05, + 2.948216933873482e-05, + 2.734927329584025e-05, + 2.5829980586422607e-05, + 2.458927883708384e-05, + 2.3411093934555538e-05, + 2.243657218059525e-05, + 2.147042687283829e-05, + 2.063715146505274e-05, + 1.995633647311479e-05, + 1.9332950614625588e-05, + 1.8802562408382073e-05, + 1.8313348846277222e-05, + 1.7883450709632598e-05, + 1.7438880604458973e-05, + 1.7116497474489734e-05, + 1.6804922779556364e-05, + 1.6525031242053956e-05, + 1.611020888958592e-05, + 1.5875679309829138e-05, + 1.568049265188165e-05, + 1.5502682799706236e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.9545781419292325e-05, + "first_to_last_ratio": 2.3619145181348626, + "imbalance_abs_log_ratio": 0.8594725266608655, + "last_quartile_mean": 1.6743104424676858e-05, + "mean": 2.5006789769577153e-05, + "normalized": [ + 1.9129420639284915, + 1.8107688022127852, + 1.7275839797255208, + 1.6309198865482473, + 1.5143239186683517, + 1.4329543911459017, + 1.3474859747283041, + 1.2742350840712526, + 1.1868783419088809, + 1.1125469739554936, + 1.0468013052560716, + 0.9972033675681935, + 0.9479134830208824, + 0.9058158937226485, + 0.8735998635837403, + 0.8429661427640184, + 0.8275111858046021, + 0.8037572699643292, + 0.7836623125837494, + 0.7643368697661114, + 0.747091058421628, + 0.731126704498011, + 0.7175431249051513, + 0.7036933175686139, + 0.6932477338397589, + 0.6853092007703565, + 0.6793487543844439, + 0.6746621277382263, + 0.6626876984583827, + 0.658797136409654, + 0.6539462632483479, + 0.6483397688298493 + ], + "peak_layer": 1, + "peak_normalized": 1.9129420639284915, + "population_cv": 0.37575052857690755, + "rest_mean": 2.631019742693752e-05, + "spike_contrast": 0.6829438840103251, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.796838841983117e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.783654003404081e-05, + 4.528151475824416e-05, + 4.320132939028554e-05, + 4.078407073393464e-05, + 3.7868379877181724e-05, + 3.5833589208777994e-05, + 3.3696298487484455e-05, + 3.1864528864389285e-05, + 2.9680017178179696e-05, + 2.7821228286484256e-05, + 2.6177140171057545e-05, + 2.4936854970292188e-05, + 2.370427318965085e-05, + 2.2651547624263912e-05, + 2.1845928131369874e-05, + 2.107987711497117e-05, + 2.0693398255389184e-05, + 2.009938907576725e-05, + 1.9596878701122478e-05, + 1.9113611415377818e-05, + 1.8682349036680534e-05, + 1.8283131794305518e-05, + 1.7943450075108558e-05, + 1.759711085469462e-05, + 1.7335900338366628e-05, + 1.7137383110821247e-05, + 1.6988331481115893e-05, + 1.6871133993845433e-05, + 1.6571691958233714e-05, + 1.647440149099566e-05, + 1.6353096725651994e-05, + 1.6212896298384294e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.000115249800728634, + "first_to_last_ratio": 5.424078025809064, + "imbalance_abs_log_ratio": 1.6908479358239088, + "last_quartile_mean": 2.1247813947411487e-05, + "mean": 5.662298849529179e-05, + "normalized": [ + 2.5049859005845536, + 2.3700562634212203, + 2.2418900056421593, + 2.1239131488521026, + 1.9444163187103436, + 1.8169448512252133, + 1.7002382001047527, + 1.580666293949324, + 1.404181335399843, + 1.2897692783756878, + 1.1905349262448084, + 1.0938543492760489, + 0.9632283128394009, + 0.8775672358607169, + 0.8099566359606734, + 0.7410335544511957, + 0.6760490100055481, + 0.6280269475546286, + 0.5865953700288571, + 0.5523741532147923, + 0.5129543450327941, + 0.48678309680551346, + 0.4611919541417525, + 0.44078321206045507, + 0.41741618217688703, + 0.4024211697379105, + 0.38818779883207655, + 0.3763044690378674, + 0.36334143229992716, + 0.35587172989953436, + 0.3510644158471216, + 0.34739810242629 + ], + "peak_layer": 1, + "peak_normalized": 2.5049859005845536, + "population_cv": 0.6752679783118715, + "rest_mean": 6.224517144599101e-05, + "spike_contrast": 0.42193153222019925, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6263200561515986e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014183978782966733, + 0.00013419966853689402, + 0.00012694251199718565, + 0.00012026230979245156, + 0.0001100986628443934, + 0.00010288084740750492, + 9.627256804378703e-05, + 8.950204937718809e-05, + 7.950894359964877e-05, + 7.303059101104736e-05, + 6.741164543200284e-05, + 6.19373022345826e-05, + 5.454086567624472e-05, + 4.9690479499986395e-05, + 4.586216527968645e-05, + 4.195953442831524e-05, + 3.827991531579755e-05, + 3.556076262611896e-05, + 3.321478288853541e-05, + 3.127707532257773e-05, + 2.9045007977401838e-05, + 2.75631136901211e-05, + 2.6114066713489592e-05, + 2.495846274541691e-05, + 2.3635351681150496e-05, + 2.2786289264331572e-05, + 2.1980353267281316e-05, + 2.1307483621058054e-05, + 2.0573477740981616e-05, + 2.0150520867900923e-05, + 1.98783163796179e-05, + 1.9670718756970018e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "6cc11327437b0ddd855bc16b53a520d7ca542f4fd424dbb567b2c52308727f3b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.005864492617547512, + 0.009892112575471401, + 0.014509912580251694, + 0.019168438389897346, + 0.006793031003326178, + 0.01223041582852602, + 0.017751524224877357, + 0.023574935272336006, + 0.008267601951956749, + 0.014755276031792164, + 0.02151178941130638, + 0.02830074541270733, + 0.009282662533223629, + 0.01676790602505207, + 0.02516039088368416, + 0.0331411249935627, + 0.010437680408358574, + 0.01991143450140953, + 0.029664477333426476, + 0.04036801680922508, + 0.01357664167881012, + 0.025029508396983147, + 0.03699437528848648, + 0.05095698684453964, + 0.015438608825206757, + 0.029965175315737724, + 0.04443887993693352, + 0.060807984322309494, + 0.019359532743692398, + 0.037317242473363876, + 0.05564062297344208, + 0.07935244590044022 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3918705690230126e-05, + "first_to_last_ratio": 3.3693622509367107, + "imbalance_abs_log_ratio": 1.2147234834100593, + "last_quartile_mean": 4.1309614857709676e-06, + "mean": 7.606254285974501e-06, + "normalized": [ + 2.2824394746096446, + 2.1562501675642154, + 2.0296925788947253, + 1.9095870929636571, + 1.7365187636180548, + 1.6206280695592366, + 1.5089330358597304, + 1.395172874910497, + 1.24547847149719, + 1.1599616848108578, + 1.0770525233380077, + 0.9985747583354554, + 0.9126325944161369, + 0.8598702731715868, + 0.8109135685663204, + 0.7673206825431794, + 0.7307283758200082, + 0.6989328774642551, + 0.673329950916721, + 0.6494877214192842, + 0.6312478565559216, + 0.6141082928304196, + 0.5996495921413664, + 0.5866836875352973, + 0.57380070608062, + 0.5636230401273656, + 0.554819016803077, + 0.5473940766317459, + 0.534486881856006, + 0.5284700210369294, + 0.5232880116136952, + 0.5189232765087916 + ], + "peak_layer": 1, + "peak_normalized": 2.2824394746096446, + "population_cv": 0.5333416755025421, + "rest_mean": 8.168133886270138e-06, + "spike_contrast": 0.5597489595589675, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.572104444378056e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.7360815036226995e-05, + 1.640098707866855e-05, + 1.5438357877428643e-05, + 1.4524805010296404e-05, + 1.320840328844497e-05, + 1.2326909200055525e-05, + 1.147732837125659e-05, + 1.0612039659463335e-05, + 9.473425961914472e-06, + 8.82296353665879e-06, + 8.192335371859372e-06, + 7.595413535455009e-06, + 6.9417155827977695e-06, + 6.540391950693447e-06, + 6.168014806462452e-06, + 5.836436230310937e-06, + 5.558105840464123e-06, + 5.3162611948209815e-06, + 5.121518825035309e-06, + 4.940168764733244e-06, + 4.801431714440696e-06, + 4.671063834393863e-06, + 4.561087280308129e-06, + 4.46246531282668e-06, + 4.36447407992091e-06, + 4.287060164642753e-06, + 4.220094524498563e-06, + 4.1636185414972715e-06, + 4.065443135914393e-06, + 4.019677362521179e-06, + 3.980261681135744e-06, + 3.947062396036927e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.641166717396118e-05, + "first_to_last_ratio": 3.40273807382454, + "imbalance_abs_log_ratio": 1.2245804233600555, + "last_quartile_mean": 1.6578316035520402e-05, + "mean": 3.084058886315688e-05, + "normalized": [ + 2.254829765809383, + 2.1412750915381364, + 2.0245423189025695, + 1.913826251974467, + 1.7440086673720954, + 1.6283807150646488, + 1.518515381022133, + 1.407719100788124, + 1.2571699318719751, + 1.1712526341901288, + 1.0875882687575642, + 1.0075046017085245, + 0.9193296533476374, + 0.8658469008110146, + 0.8153151132261776, + 0.770486727612793, + 0.7322862023214354, + 0.6994539525278819, + 0.6731242297658762, + 0.648415700488765, + 0.6288998621420432, + 0.6111632256967778, + 0.5959791417905044, + 0.5826975877435189, + 0.5692453444999938, + 0.5587842318415797, + 0.5495883661230995, + 0.5418092679201958, + 0.5286894380532074, + 0.5225725836741036, + 0.517092481391006, + 0.5126072600226392 + ], + "peak_layer": 1, + "peak_normalized": 2.254829765809383, + "population_cv": 0.5330470009230729, + "rest_mean": 3.313880080424025e-05, + "spike_contrast": 0.5561530270868602, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.8430244381306693e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.95402777637355e-05, + 6.603818474104628e-05, + 6.243807729333639e-05, + 5.902352859266102e-05, + 5.3786254284204915e-05, + 5.022022014600225e-05, + 4.6831908548483625e-05, + 4.3414886022219434e-05, + 3.877186099998653e-05, + 3.6122120945947245e-05, + 3.3541862649144605e-05, + 3.1072035199031234e-05, + 2.835266786860302e-05, + 2.6703228286351077e-05, + 2.5144798200926743e-05, + 2.3762264390825294e-05, + 2.2584137695957907e-05, + 2.1571571778622456e-05, + 2.0759547624038532e-05, + 1.9997522031189874e-05, + 1.9395642084418796e-05, + 1.8848633771995082e-05, + 1.8380347682978027e-05, + 1.797073673515115e-05, + 1.755586163199041e-05, + 1.7233234757441096e-05, + 1.6949628843576647e-05, + 1.6709716874174774e-05, + 1.630509359529242e-05, + 1.611644620425068e-05, + 1.594743662280962e-05, + 1.580910975462757e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05002643074840307, + "first_to_last_ratio": 3.402738130192313, + "imbalance_abs_log_ratio": 1.2245804399254718, + "last_quartile_mean": 0.014701816253364086, + "mean": 0.027349742653314024, + "normalized": [ + 2.254829786736978, + 2.1412751957662834, + 2.02454215992989, + 1.9138263077398268, + 1.7440085434241568, + 1.628380710143593, + 1.5185154852595628, + 1.4077191333516668, + 1.2571699503960287, + 1.171252642415901, + 1.0875882368603997, + 1.0075045704457566, + 0.9193296608188956, + 0.8658469286585574, + 0.8153151036674042, + 0.7704867722207724, + 0.7322861827710381, + 0.6994539437935281, + 0.673124202328064, + 0.6484156924895115, + 0.6288998911838509, + 0.6111632562306977, + 0.5959791161376584, + 0.5826976161606044, + 0.5692453437279956, + 0.5587841942711541, + 0.5495883614451207, + 0.5418092759796765, + 0.5286894564849702, + 0.5225725676043702, + 0.5170924573716015, + 0.5126072541844844 + ], + "peak_layer": 1, + "peak_normalized": 2.254829786736978, + "population_cv": 0.5330470020894653, + "rest_mean": 0.029387819925668062, + "spike_contrast": 0.5561530397267354, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.016344125382602215, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.061669014394283295, + 0.05856332555413246, + 0.05537070706486702, + 0.05234265699982643, + 0.047698184847831726, + 0.044535793364048004, + 0.04153100773692131, + 0.03850075602531433, + 0.034383274614810944, + 0.03203345835208893, + 0.029745258390903473, + 0.027554990723729134, + 0.02514342963695526, + 0.023680690675973892, + 0.02229865826666355, + 0.021072614938020706, + 0.02002783864736557, + 0.019129885360598564, + 0.01840977370738983, + 0.017734002321958542, + 0.017200250178575516, + 0.016715157777071, + 0.01629987545311451, + 0.015936629846692085, + 0.015568713657557964, + 0.015282603912055492, + 0.015031100250780582, + 0.01481834426522255, + 0.0144595205783844, + 0.014292225241661072, + 0.014142345637083054, + 0.014019676484167576 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.375096921194199e-06, + "first_to_last_ratio": 3.8221077500661207, + "imbalance_abs_log_ratio": 1.3408020374840417, + "last_quartile_mean": 1.1446817325122538e-06, + "mean": 2.265028140158165e-06, + "normalized": [ + 2.626026515434601, + 2.4001979720003934, + 2.1920710272657535, + 1.9791200058879264, + 1.7560134880541989, + 1.6114191789724852, + 1.5125696765745171, + 1.375271110839599, + 1.211780627728412, + 1.1344335816532227, + 1.0401368166449354, + 0.9675242038709502, + 0.8757726873912133, + 0.8237780432734694, + 0.7832771211345639, + 0.7406910125458439, + 0.7019739226432762, + 0.6704065668591555, + 0.6413840580656591, + 0.6173312336057731, + 0.5969185512142089, + 0.5799188408093214, + 0.5668593216879733, + 0.5521486786148365, + 0.5377886785355955, + 0.5258358481305954, + 0.517639304639018, + 0.5093446353167698, + 0.49603515883647503, + 0.4905101977434218, + 0.48510625014282244, + 0.48071568388301256 + ], + "peak_layer": 1, + "peak_normalized": 2.626026515434601, + "population_cv": 0.6051676351147116, + "rest_mean": 2.446764428727706e-06, + "spike_contrast": 0.5246325174631261, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.283652181882644e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.9480239542608615e-06, + 5.436515948531451e-06, + 4.9651025619823486e-06, + 4.4827625060861465e-06, + 3.977419964940054e-06, + 3.6499097859632457e-06, + 3.426012881391216e-06, + 3.1150277663982706e-06, + 2.744717221503379e-06, + 2.569523985584965e-06, + 2.3559391593153123e-06, + 2.1914695480518276e-06, + 1.983649781323038e-06, + 1.8658804492588388e-06, + 1.7741447209118633e-06, + 1.677685986578581e-06, + 1.5899906884442316e-06, + 1.5184897392828134e-06, + 1.4527529401675565e-06, + 1.39827261591563e-06, + 1.352037315882626e-06, + 1.3135324934410164e-06, + 1.2839523151342291e-06, + 1.2506322946137516e-06, + 1.2181064903415972e-06, + 1.1910329931197339e-06, + 1.1724675914592808e-06, + 1.153679932031082e-06, + 1.123533593272441e-06, + 1.1110194009233965e-06, + 1.0987793075400987e-06, + 1.0888345514104003e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.626059489964973e-05, + "first_to_last_ratio": 2.8653088341201625, + "imbalance_abs_log_ratio": 1.0526761398616047, + "last_quartile_mean": 1.614506413716299e-05, + "mean": 2.7227290900100343e-05, + "normalized": [ + 2.069219136955177, + 1.9687798201242246, + 1.8639507108542548, + 1.7654020485254056, + 1.6236839316541174, + 1.525302822734854, + 1.4342547809805677, + 1.3418262274018482, + 1.2243702183610123, + 1.1518310829140999, + 1.0836883538565836, + 1.0178339180148281, + 0.949221197634969, + 0.9036382524726367, + 0.8600878611114346, + 0.8201852972104389, + 0.7888274227376761, + 0.7582177263486256, + 0.7331791140207194, + 0.7093278728046818, + 0.6906726578980285, + 0.6727165175040963, + 0.657004042862766, + 0.6429903513549862, + 0.6287412299367678, + 0.6172859346889895, + 0.6070185450137164, + 0.5981756933578195, + 0.58312260952694, + 0.5759704572874644, + 0.569432199896746, + 0.5640419639535237 + ], + "peak_layer": 1, + "peak_normalized": 2.069219136955177, + "population_cv": 0.45293224377011354, + "rest_mean": 2.89495433039351e-05, + "spike_contrast": 0.6192542566623438, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.792712791939266e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.6339231377933174e-05, + 5.360454088076949e-05, + 5.075032822787762e-05, + 4.806711513083428e-05, + 4.42085147369653e-05, + 4.1529863665346056e-05, + 3.905087214661762e-05, + 3.6534293030854315e-05, + 3.333628410473466e-05, + 3.1361239962279797e-05, + 2.9505898055504076e-05, + 2.771286017377861e-05, + 2.5844721676548943e-05, + 2.4603621568530798e-05, + 2.3417862394126132e-05, + 2.2331423679133877e-05, + 2.1477633708855137e-05, + 2.0644214600906707e-05, + 1.9962481019319966e-05, + 1.9313076336402446e-05, + 1.8805145373335108e-05, + 1.8316248315386474e-05, + 1.7888440197566524e-05, + 1.7506885342299938e-05, + 1.7118920368375257e-05, + 1.680702371231746e-05, + 1.652747050684411e-05, + 1.628670361242257e-05, + 1.5876848920015618e-05, + 1.5682115190429613e-05, + 1.550409615447279e-05, + 1.53573346324265e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.757111198865459e-05, + "first_to_last_ratio": 2.261484300797815, + "imbalance_abs_log_ratio": 0.8160213679695097, + "last_quartile_mean": 1.6613474599580513e-05, + "mean": 2.4096830372855038e-05, + "normalized": [ + 1.8905505844846502, + 1.7997195920380655, + 1.703326591013662, + 1.610437077165417, + 1.490662081122157, + 1.4031283462782311, + 1.3280828236310434, + 1.2474718200193673, + 1.158269301434096, + 1.0891577101716936, + 1.0361895464823103, + 0.9872929826228773, + 0.9419073791196091, + 0.9071102148715466, + 0.8752487270925183, + 0.8477505951103422, + 0.8347755693998784, + 0.8134040107759765, + 0.7935380337199553, + 0.7733807536170327, + 0.7587708376785838, + 0.7447349978405977, + 0.7306079701092049, + 0.7189107072064382, + 0.711201857826888, + 0.7050007791285537, + 0.7000690842179099, + 0.6958731578478594, + 0.6835994787579611, + 0.6787755788074042, + 0.6729650681227183, + 0.6680867422854508 + ], + "peak_layer": 1, + "peak_normalized": 1.8905505844846502, + "population_cv": 0.3606253067246935, + "rest_mean": 2.5288975208510505e-05, + "spike_contrast": 0.6982982946012236, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.765924826031551e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.5556276745628566e-05, + 4.3367537728045136e-05, + 4.104477193322964e-05, + 3.8806429074611515e-05, + 3.592023131204769e-05, + 3.381094575161114e-05, + 3.200258652213961e-05, + 3.0060116841923445e-05, + 2.791061888274271e-05, + 2.6245248591294512e-05, + 2.4968883735709824e-05, + 2.379063153057359e-05, + 2.269698234158568e-05, + 2.1858480977243744e-05, + 2.1090720110805705e-05, + 2.0428102288860828e-05, + 2.0115445295232348e-05, + 1.9600458472268656e-05, + 1.9121751392958686e-05, + 1.8636024833540432e-05, + 1.8283972167409956e-05, + 1.7945752915693447e-05, + 1.7605336324777454e-05, + 1.7323469364782795e-05, + 1.7137710528913885e-05, + 1.69882841873914e-05, + 1.6869445971678942e-05, + 1.6768337445682846e-05, + 1.647258068260271e-05, + 1.6356339983758517e-05, + 1.6216325093409978e-05, + 1.6098772903205827e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001084787909348961, + "first_to_last_ratio": 5.225248440781252, + "imbalance_abs_log_ratio": 1.6535023451953625, + "last_quartile_mean": 2.0760503957717447e-05, + "mean": 5.3153509043113445e-05, + "normalized": [ + 2.5237742282686555, + 2.39597413894643, + 2.265748981789298, + 2.151554924730914, + 1.9389326261416564, + 1.811160187739653, + 1.6845666093621592, + 1.5551563330371112, + 1.3755697660543504, + 1.2684162903360312, + 1.1635545557382814, + 1.0626373891730778, + 0.9348691942291281, + 0.8623306324991865, + 0.788381531623569, + 0.7297638393679164, + 0.6688462612899458, + 0.6250625348617511, + 0.5885533959524297, + 0.5510274355176712, + 0.5186065946717524, + 0.49092990570860817, + 0.46938426076169515, + 0.4505875210977826, + 0.4285538100318043, + 0.4136111569238702, + 0.40105822090753923, + 0.39087833614351924, + 0.37910335602991646, + 0.3738905758036781, + 0.37037507613467424, + 0.36714032912594324 + ], + "peak_layer": 1, + "peak_normalized": 2.5237742282686555, + "population_cv": 0.6747695646712838, + "rest_mean": 5.835455609754987e-05, + "spike_contrast": 0.42957836757855616, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5067854949156754e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00013414745626505464, + 0.00012735443306155503, + 0.00012043250899296254, + 0.00011436269414843991, + 0.00010306107287760824, + 9.626951941754669e-05, + 8.954062650445849e-05, + 8.266201621154323e-05, + 7.311635999940336e-05, + 6.742077675880864e-05, + 6.184700760059059e-05, + 5.648290607496165e-05, + 4.969157816958614e-05, + 4.583589907269925e-05, + 4.190524487057701e-05, + 3.878950883517973e-05, + 3.555152579792775e-05, + 3.32242670992855e-05, + 3.12836782541126e-05, + 2.9289041776792146e-05, + 2.756576031970326e-05, + 2.6094647182617337e-05, + 2.4949420549091883e-05, + 2.3950307877385058e-05, + 2.2779138816986233e-05, + 2.198488436988555e-05, + 2.1317651771823876e-05, + 2.0776555174961686e-05, + 2.015067366301082e-05, + 1.9873596102115698e-05, + 1.968673495866824e-05, + 1.951479680428747e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "a1eaf8973066bfbad01efbb61e56182f134640c86d8dd38e1601245f8a0a6387", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.02856997773051262, + 0.015031684190034866, + 0.015670204535126686, + 0.016530683264136314, + 0.01773575134575367, + 0.01292925700545311, + 0.014021075330674648, + 0.015247474424540997, + 0.01669454388320446, + 0.01373021025210619, + 0.01499613095074892, + 0.01623532734811306, + 0.01758282259106636, + 0.015234442427754402, + 0.01637510396540165, + 0.01772587187588215, + 0.01901337131857872, + 0.017032917588949203, + 0.01834850199520588, + 0.019697511568665504, + 0.021141810342669487, + 0.019469235092401505, + 0.02084389142692089, + 0.022254284471273422, + 0.02407064661383629, + 0.022380243986845016, + 0.023947052657604218, + 0.025450291112065315, + 0.027382872998714447, + 0.025761734694242477, + 0.02756396308541298, + 0.02946169301867485 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.9468072852978366e-06, + "first_to_last_ratio": 6.831203575113915, + "imbalance_abs_log_ratio": 1.9215008769619122, + "last_quartile_mean": 4.313745378681233e-07, + "mean": 1.4344362195828353e-06, + "normalized": [ + 2.0208098689060576, + 2.70247744984379, + 2.3511261487968675, + 2.0828624204646062, + 1.7252416990779311, + 2.0561625392735503, + 1.8889131659507596, + 1.6070574509858848, + 1.2980615117448049, + 1.4730921817018607, + 1.3154183683511145, + 1.1251444251921345, + 0.9479215504591184, + 0.9438177843313365, + 0.9908118070600781, + 0.7983125303617123, + 0.663485001791352, + 0.7166985535761707, + 0.6372153189923393, + 0.4803275608465089, + 0.44713051888100325, + 0.4971914883998871, + 0.44243986640593336, + 0.382460121340808, + 0.33911149104781046, + 0.37657627859451315, + 0.3537072871377602, + 0.301474679623081, + 0.272351762260187, + 0.2855781634777493, + 0.2572154244268084, + 0.21980558069648082 + ], + "peak_layer": 2, + "peak_normalized": 2.70247744984379, + "population_cv": 0.7068955562190269, + "rest_mean": 1.588062559676473e-06, + "spike_contrast": 0.3808754128681272, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.048539830771916e-07, + "top_five_layers": [ + 2, + 3, + 4, + 6, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.8987228688492905e-06, + 3.876531536661787e-06, + 3.3725405046425294e-06, + 2.9877332963224035e-06, + 2.4747491806920152e-06, + 2.9494340196833946e-06, + 2.7095254608866526e-06, + 2.3052214146446204e-06, + 1.861986447693198e-06, + 2.1130567802174482e-06, + 1.8868837514673942e-06, + 1.6139479157573078e-06, + 1.3597330053016776e-06, + 1.35384641453129e-06, + 1.421256342837296e-06, + 1.145128408097662e-06, + 9.517269177194976e-07, + 1.0280583637722884e-06, + 9.140447332356416e-07, + 6.889992505421105e-07, + 6.413802111637779e-07, + 7.131894790290971e-07, + 6.346517693600617e-07, + 5.486146505973011e-07, + 4.864338052357198e-07, + 5.40174653451686e-07, + 5.073705438007892e-07, + 4.3244619973847875e-07, + 3.9067123225322575e-07, + 4.0964366121443163e-07, + 3.689591210331855e-07, + 3.1529708621746977e-07 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.710511570418021e-06, + "first_to_last_ratio": 6.354606328287203, + "imbalance_abs_log_ratio": 1.8491799559542976, + "last_quartile_mean": 1.0560074414911469e-06, + "mean": 3.2492914723292188e-06, + "normalized": [ + 1.783276830307924, + 2.779944343220534, + 2.3950635818648984, + 2.1106770285738783, + 1.7960465411480506, + 2.159230004292565, + 1.9196468120338586, + 1.5778990753682762, + 1.2819032996284179, + 1.4557156863533565, + 1.3120099400931993, + 1.1009867702080869, + 0.8992855746074583, + 0.9377834334418341, + 0.8845056521165512, + 0.7577187265725858, + 0.6356736108971007, + 0.6642988512627761, + 0.5934764503981409, + 0.5228260514911628, + 0.46756909304244726, + 0.49649554676318175, + 0.45575067618660486, + 0.41224667168429796, + 0.3602883651000989, + 0.3931940610190251, + 0.3661495872210781, + 0.32797257011813363, + 0.29319922194152587, + 0.31032847914557965, + 0.2896492848736135, + 0.25918817902375785 + ], + "peak_layer": 2, + "peak_normalized": 2.779944343220534, + "population_cv": 0.7087840069849813, + "rest_mean": 3.5871756225518496e-06, + "spike_contrast": 0.39716958717329126, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4247170611270122e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.794386197521817e-06, + 9.032849447976332e-06, + 7.782259672239888e-06, + 6.858204869786277e-06, + 5.835878710058751e-06, + 7.015967639745213e-06, + 6.237492016225588e-06, + 5.1270540097902995e-06, + 4.165277459833305e-06, + 4.730044565803837e-06, + 4.2631027099560015e-06, + 3.577426923584426e-06, + 2.9220409487606958e-06, + 3.047131713174167e-06, + 2.8740166726493044e-06, + 2.462048996676458e-06, + 2.0654888430726714e-06, + 2.1585005924862344e-06, + 1.9283779693068936e-06, + 1.6988142306217924e-06, + 1.519268266747531e-06, + 1.6132587461470393e-06, + 1.4808667856414104e-06, + 1.3395095947998925e-06, + 1.1706819122991874e-06, + 1.2776021094396128e-06, + 1.1897267313543125e-06, + 1.0656784752427484e-06, + 9.526897315481619e-07, + 1.0083476809086278e-06, + 9.411549513060891e-07, + 8.421779398304352e-07 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.005950948863755912, + "first_to_last_ratio": 6.354606465537837, + "imbalance_abs_log_ratio": 1.8491799775529028, + "last_quartile_mean": 0.0009364779543830082, + "mean": 0.0028815042023779824, + "normalized": [ + 1.7832768321055497, + 2.779944541805248, + 2.3950634036601817, + 2.110677044556637, + 1.796046582443378, + 2.1592300209325574, + 1.9196467896009393, + 1.5778990914249873, + 1.281903295419947, + 1.4557156625259682, + 1.3120099546166246, + 1.1009867557766109, + 0.8992854956916723, + 0.9377834188412186, + 0.884505639688408, + 0.7577187722730399, + 0.635673635605656, + 0.6642988751267256, + 0.5934764053600845, + 0.5228260433663814, + 0.46756910562655507, + 0.49649557415115175, + 0.45575067494350857, + 0.4122466780510639, + 0.36028835142322546, + 0.3931940647806284, + 0.3661495503121345, + 0.32797257179832034, + 0.2931992062983405, + 0.3103284950057511, + 0.2896492814770661, + 0.25918818531043897 + ], + "peak_layer": 2, + "peak_normalized": 2.779944541805248, + "population_cv": 0.7087840141850674, + "rest_mean": 0.0031811432455907816, + "spike_contrast": 0.3971695932838214, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0012634533690288664, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.005138519685715437, + 0.008010421879589558, + 0.006901385262608528, + 0.006081924773752689, + 0.005175315774977207, + 0.006221830379217863, + 0.005531470291316509, + 0.004546722862869501, + 0.0036938097327947617, + 0.004194650799036026, + 0.00378056219778955, + 0.0031724979635328054, + 0.0025912949349731207, + 0.0027022268623113632, + 0.002548706717789173, + 0.0021833698265254498, + 0.0018316962523385882, + 0.0019141800003126264, + 0.0017101047560572624, + 0.0015065254410728812, + 0.0013473023427650332, + 0.001430654083378613, + 0.0013132474850863218, + 0.0011878905352205038, + 0.0010381723986938596, + 0.0011329903500154614, + 0.001055061467923224, + 0.0009450543439015746, + 0.0008448547450825572, + 0.0008942128624767065, + 0.000834625621791929, + 0.0007468518451787531 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.7645694729017123e-06, + "first_to_last_ratio": 5.854487285906525, + "imbalance_abs_log_ratio": 1.7672084246254944, + "last_quartile_mean": 4.722137632029444e-07, + "mean": 1.404573582775015e-06, + "normalized": [ + 1.7305940535699378, + 2.5618749599510693, + 2.200447296481188, + 1.980294414184873, + 1.7177222534372074, + 2.006950537243153, + 1.8986342274283854, + 1.6495820380503576, + 1.3071680990137817, + 1.5435943853891647, + 1.37926355112175, + 1.1804343531650965, + 0.9505292781052922, + 0.987344076938019, + 0.9642189006877869, + 0.8086506810691724, + 0.6736023260407531, + 0.710546872437511, + 0.640481569051335, + 0.5322781307138237, + 0.48482706689793736, + 0.5138657129886831, + 0.47272339176998346, + 0.4147938968760285, + 0.36560568881857053, + 0.4082117778611776, + 0.3821008292991752, + 0.34425699387548675, + 0.3079872683634573, + 0.32257229483060335, + 0.30100273359597224, + 0.25784034074326717 + ], + "peak_layer": 2, + "peak_normalized": 2.5618749599510693, + "population_cv": 0.6600852637188594, + "rest_mean": 1.5475375452869802e-06, + "spike_contrast": 0.40875789226367176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.325681852104025e-07, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.4307466901518637e-06, + 3.5983418911200715e-06, + 3.090690142926178e-06, + 2.7814692202809965e-06, + 2.4126672997226706e-06, + 2.8189097065478563e-06, + 2.66677147919836e-06, + 2.316959353265702e-06, + 1.836013780120993e-06, + 2.1680918962374562e-06, + 1.9372771475900663e-06, + 1.658006908655807e-06, + 1.3350883136808989e-06, + 1.3867974075765233e-06, + 1.3543163959184312e-06, + 1.1358093843227834e-06, + 9.461240324526443e-07, + 9.980153663491365e-07, + 8.996034921437968e-07, + 7.476238010895031e-07, + 6.809752903791377e-07, + 7.217622055577522e-07, + 6.639747880399227e-07, + 5.826085498483735e-07, + 5.135200922268268e-07, + 5.733634793614328e-07, + 5.36688730790047e-07, + 4.835342792830488e-07, + 4.325907809743512e-07, + 4.5307652385417896e-07, + 4.227804879519681e-07, + 3.621557311817014e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.024905337018936e-06, + "first_to_last_ratio": 5.991275482114614, + "imbalance_abs_log_ratio": 1.7903043247043764, + "last_quartile_mean": 6.717944032175183e-07, + "mean": 2.0148276771436713e-06, + "normalized": [ + 1.5675942414338733, + 2.5250440978186566, + 2.231455540086097, + 2.0780670863790145, + 1.7996941190454676, + 2.2162616169225418, + 1.9414222145713715, + 1.621600841461054, + 1.3376231697058878, + 1.5231899507700504, + 1.3715764047674524, + 1.1419542658104653, + 0.9487730313859013, + 0.9689325883752917, + 0.9148133638638105, + 0.7812038234732276, + 0.6554220232920718, + 0.683968041755189, + 0.6172110636131051, + 0.5352507957421382, + 0.47670028302923295, + 0.5067173036495194, + 0.46642566869070556, + 0.4216965387703109, + 0.36513142756884537, + 0.4048860931364547, + 0.37578964789761116, + 0.34071121185725145, + 0.3026913075409305, + 0.3181401874842009, + 0.29774178307895716, + 0.26231026702331217 + ], + "peak_layer": 2, + "peak_normalized": 2.5250440978186566, + "population_cv": 0.6756995388559863, + "rest_mean": 2.2210362439417857e-06, + "spike_contrast": 0.40580221006851624, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.013014164338528e-07, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.1584322641720064e-06, + 5.087528734293301e-06, + 4.495998382481048e-06, + 4.186947080597747e-06, + 3.6260735214455053e-06, + 4.465385245566722e-06, + 3.911631210939959e-06, + 3.2672462566551985e-06, + 2.6950801839120686e-06, + 3.0689652703586034e-06, + 2.763490101642674e-06, + 2.3008410607872065e-06, + 1.911614162963815e-06, + 1.952232196344994e-06, + 1.8431912849337095e-06, + 1.5739910850243177e-06, + 1.3205624327383703e-06, + 1.378077740810113e-06, + 1.2435739336069673e-06, + 1.078438117474434e-06, + 9.6046892394952e-07, + 1.0209480478806654e-06, + 9.397673466082779e-07, + 8.496458576701116e-07, + 7.356769060606894e-07, + 8.157757065418991e-07, + 7.57151383368182e-07, + 6.864743795631512e-07, + 6.098708240642736e-07, + 6.409976549548446e-07, + 5.9989838518959e-07, + 5.285099859975162e-07 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.5129092762199434e-06, + "first_to_last_ratio": 5.690023270721292, + "imbalance_abs_log_ratio": 1.7387143378875727, + "last_quartile_mean": 4.4163427048715675e-07, + "mean": 1.3025998217841561e-06, + "normalized": [ + 1.3333242329011745, + 2.2929798702209006, + 2.055055088077371, + 2.0612267847935852, + 1.8004956832661445, + 2.300472589669482, + 1.9555756876630987, + 1.6340628779322937, + 1.3881426615583095, + 1.6201188266700812, + 1.4367832841182007, + 1.1670671159103638, + 1.0033834447630112, + 1.0052901825823841, + 0.9417324589159911, + 0.8084681814074165, + 0.6736539539316041, + 0.710866891998549, + 0.6407147024486859, + 0.5472927533253719, + 0.486548923284392, + 0.5119738979313811, + 0.4796067027050235, + 0.43283812540794353, + 0.36960358799802223, + 0.40990700009728237, + 0.38098457936452723, + 0.3475616176385816, + 0.31215171064803027, + 0.32339061526715857, + 0.3036943199348575, + 0.2650316475687802 + ], + "peak_layer": 6, + "peak_normalized": 2.300472589669482, + "population_cv": 0.6517350453796433, + "rest_mean": 1.4337971336957438e-06, + "spike_contrast": 0.41437824326663764, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.941343374615827e-07, + "top_five_layers": [ + 6, + 2, + 4, + 3, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.7367879081575666e-06, + 2.9868351703044027e-06, + 2.676914391486207e-06, + 2.6849536425288534e-06, + 2.345325356145622e-06, + 2.9965951853228034e-06, + 2.547332542235381e-06, + 2.128530013578711e-06, + 1.8081943835568381e-06, + 2.110366494889604e-06, + 1.8715536498348229e-06, + 1.520221417194989e-06, + 1.307007096329471e-06, + 1.3094908126731752e-06, + 1.226700533152325e-06, + 1.0531105090194615e-06, + 8.775015203354997e-07, + 9.259750868295669e-07, + 8.345948572241468e-07, + 7.129034429453895e-07, + 6.33778540759522e-07, + 6.668971082035569e-07, + 6.247356054700504e-07, + 5.638148650177754e-07, + 4.814455678570084e-07, + 5.33944785274798e-07, + 4.962704451827449e-07, + 4.527337011950294e-07, + 4.0660876265974366e-07, + 4.212485578136693e-07, + 3.955921670240059e-07, + 3.4523017689025437e-07 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 1.2106409826628806e-05, + "first_to_last_ratio": 6.182837050272253, + "imbalance_abs_log_ratio": 1.8217772357527195, + "last_quartile_mean": 1.9580671022367824e-06, + "mean": 5.891290232540314e-06, + "normalized": [ + 1.6628561901458256, + 2.679541050672962, + 2.4593584605705066, + 2.112226792311699, + 1.796452079584457, + 2.148898726796378, + 1.979566597291414, + 1.6008399202436439, + 1.3322729283852512, + 1.4727285184978303, + 1.3085955832877125, + 1.1187775970056968, + 0.8873455238855094, + 0.9252291834138445, + 0.8986949676582121, + 0.75315156809204, + 0.6225658481990487, + 0.659474986853632, + 0.5815564660361224, + 0.5264648905960624, + 0.46580075393312903, + 0.4965536609025122, + 0.450087000441461, + 0.40202926580138304, + 0.37131699924705175, + 0.4000060060879296, + 0.37557443972643606, + 0.33475575489567644, + 0.30250455585342495, + 0.3159959000194193, + 0.2955172411886233, + 0.2632605423751041 + ], + "peak_layer": 2, + "peak_normalized": 2.679541050672962, + "population_cv": 0.7042782314383109, + "rest_mean": 6.505339919621528e-06, + "spike_contrast": 0.3958935204190824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5754219223017572e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.796368431125302e-06, + 1.5785954019520432e-05, + 1.4488794477074407e-05, + 1.2443741070455872e-05, + 1.0583420589682646e-05, + 1.265978607989382e-05, + 1.1662201359285973e-05, + 9.431012585991994e-06, + 7.848806490073912e-06, + 8.676271136209834e-06, + 7.709316378168296e-06, + 6.591043529624585e-06, + 5.227610017755069e-06, + 5.450793651107233e-06, + 5.294472884997958e-06, + 4.437034476723056e-06, + 3.6677161006082315e-06, + 3.885158548655454e-06, + 3.426117928029271e-06, + 3.1015574677439872e-06, + 2.744167431956157e-06, + 2.925341732407105e-06, + 2.651593149494147e-06, + 2.3684710868110415e-06, + 2.187536210840335e-06, + 2.356551476623281e-06, + 2.2126180283521535e-06, + 1.972143309103558e-06, + 1.782142135198228e-06, + 1.8616235593071906e-06, + 1.7409778365617967e-06, + 1.5509442619077163e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "51c96475782d47da4b4460ab98a6f5efa352f41865c65bc22f8b46e27bda20aa", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.013703026808798313, + 0.014947476796805859, + 0.015775443986058235, + 0.016744175925850868, + 0.012103146873414516, + 0.013081531040370464, + 0.01416850183159113, + 0.015494133345782757, + 0.012773395515978336, + 0.013903867453336716, + 0.015176740474998951, + 0.016410861164331436, + 0.014334342442452908, + 0.015468464232981205, + 0.01670125499367714, + 0.017849057912826538, + 0.01606663130223751, + 0.0173187293112278, + 0.018571525812149048, + 0.019770435988903046, + 0.018272221088409424, + 0.01950262486934662, + 0.02077464945614338, + 0.0220524650067091, + 0.020764723420143127, + 0.02203267626464367, + 0.023422541096806526, + 0.024811791256070137, + 0.023658527061343193, + 0.025185640901327133, + 0.026624467223882675, + 0.02854793891310692 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.74612942677777e-06, + "first_to_last_ratio": 4.290754879791884, + "imbalance_abs_log_ratio": 1.4564626801157643, + "last_quartile_mean": 1.8053069084089657e-06, + "mean": 4.49623530585086e-06, + "normalized": [ + 1.8981071323025052, + 1.739149695276559, + 1.5735966428132337, + 1.5026215379163184, + 1.9831005875602767, + 1.854738075795215, + 1.6911251838970416, + 1.5399882637631246, + 1.7056874629197014, + 1.4591941238713075, + 1.3137324489958155, + 1.17819115097449, + 1.2205893686523535, + 1.0594273921988615, + 0.9634337939297237, + 0.867129494870504, + 0.8508597774565135, + 0.7919049093759021, + 0.7029364663408488, + 0.6435982792516399, + 0.6460999667655339, + 0.576190114496073, + 0.5231361846468904, + 0.5033401852027535, + 0.47554278072756984, + 0.4897885434342915, + 0.44031530870935176, + 0.391536447771622, + 0.3980832380767055, + 0.38067347531577295, + 0.33586151098891653, + 0.3003204557025833 + ], + "peak_layer": 5, + "peak_normalized": 1.9831005875602767, + "population_cv": 0.5381157672273565, + "rest_mean": 4.875199793888088e-06, + "spike_contrast": 0.5025080353673121, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4498270704498283e-06, + "top_five_layers": [ + 5, + 1, + 6, + 2, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.534336302545853e-06, + 7.81962626206223e-06, + 7.0752607825852465e-06, + 6.756140010111267e-06, + 8.9164868768421e-06, + 8.339338819496334e-06, + 7.603696758451406e-06, + 6.924149602127727e-06, + 7.66917219152674e-06, + 6.560880137840286e-06, + 5.906850219616899e-06, + 5.297424650052562e-06, + 5.4880570132809225e-06, + 4.7634348447900265e-06, + 4.331825039116666e-06, + 3.898818249581382e-06, + 3.8256657717283815e-06, + 3.5605908124125563e-06, + 3.1605677577317692e-06, + 2.8937693059560843e-06, + 2.905017481680261e-06, + 2.590686335679493e-06, + 2.352143383177463e-06, + 2.2631359115621308e-06, + 2.1381522401497932e-06, + 2.202204541390529e-06, + 1.979761236725608e-06, + 1.7604399999981979e-06, + 1.7898759097079164e-06, + 1.711597519715724e-06, + 1.510112383584783e-06, + 1.3503114359991741e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.130438562948257e-05, + "first_to_last_ratio": 4.214157668538487, + "imbalance_abs_log_ratio": 1.4384497300922356, + "last_quartile_mean": 7.428385004004667e-06, + "mean": 1.8293917648293245e-05, + "normalized": [ + 1.8783239189466776, + 1.7239775883511843, + 1.5628409896597186, + 1.5021774100270706, + 1.9687179771495062, + 1.8492004689267585, + 1.6770781946618867, + 1.5272107503967014, + 1.6975023935986964, + 1.4550858154880342, + 1.3108870533979449, + 1.1834221941571295, + 1.2280669813964384, + 1.0632172174009653, + 0.9681770292934567, + 0.8701237722001045, + 0.8538823529559815, + 0.7967879122792568, + 0.7086904702827618, + 0.651045975234203, + 0.652290955492651, + 0.5835747270147889, + 0.5308413464878547, + 0.5084153950901334, + 0.48184390105002417, + 0.49559938674797777, + 0.443562473293985, + 0.39641716095408386, + 0.40208621599683964, + 0.38482821036485854, + 0.3408788461690432, + 0.30324491553328303 + ], + "peak_layer": 5, + "peak_normalized": 1.9687179771495062, + "population_cv": 0.5313999208641894, + "rest_mean": 1.9813690734601415e-05, + "spike_contrast": 0.5090996481848562, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0087142982229125e-05, + "top_five_layers": [ + 5, + 1, + 6, + 2, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.4361903090029955e-05, + 3.153830402879976e-05, + 2.859048436221201e-05, + 2.7480709832161665e-05, + 3.601556454668753e-05, + 3.3829121093731374e-05, + 3.068033038289286e-05, + 2.7938667699345388e-05, + 3.1053968996275216e-05, + 2.6619220079737715e-05, + 2.3981259801075794e-05, + 2.1649428163073026e-05, + 2.246615622425452e-05, + 1.9450408217380755e-05, + 1.7711750842863694e-05, + 1.5917972632450983e-05, + 1.5620853446307592e-05, + 1.4576372450392228e-05, + 1.2964725101483054e-05, + 1.1910181456187274e-05, + 1.1932957022509072e-05, + 1.0675867997633759e-05, + 9.711167876957916e-06, + 9.300909368903376e-06, + 8.814812645141501e-06, + 9.06645436771214e-06, + 8.114495358313434e-06, + 7.252022896864219e-06, + 7.355732122960035e-06, + 7.040015589154791e-06, + 6.236009539861698e-06, + 5.547537512029521e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.027761041885241866, + "first_to_last_ratio": 4.214157540695516, + "imbalance_abs_log_ratio": 1.438449699755693, + "last_quartile_mean": 0.006587566225789487, + "mean": 0.016223229031311348, + "normalized": [ + 1.8783239576926372, + 1.7239774602868518, + 1.5628410093355816, + 1.502177362926827, + 1.968717976965925, + 1.8492004888075664, + 1.6770782262083783, + 1.5272107169113356, + 1.697502508427555, + 1.455085816293378, + 1.3108869922362147, + 1.183422230798549, + 1.2280668728519175, + 1.0632172396078363, + 0.9681770638198094, + 0.8701237808458149, + 0.8538823828074513, + 0.7967879426055737, + 0.7086904796790885, + 0.6510459681632486, + 0.6522909479780994, + 0.5835746869352977, + 0.5308413214129329, + 0.5084153812334933, + 0.4818439286563117, + 0.4955993855362714, + 0.44356247849658154, + 0.3964171704578387, + 0.40208622884148165, + 0.38482821497145886, + 0.3408788545627423, + 0.30324492364595074 + ], + "peak_layer": 5, + "peak_normalized": 1.968717976965925, + "population_cv": 0.5313999193824042, + "rest_mean": 0.017570979030871833, + "spike_contrast": 0.5090996362791106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00894537903368473, + "top_five_layers": [ + 5, + 1, + 6, + 2, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03047247976064682, + 0.027968481183052063, + 0.025354327633976936, + 0.024370167404413223, + 0.03193896263837814, + 0.030000003054738045, + 0.027207624167203903, + 0.024776289239525795, + 0.027538971975445747, + 0.0236061904579401, + 0.021266819909214973, + 0.019198929890990257, + 0.01992321014404297, + 0.017248816788196564, + 0.01570695824921131, + 0.014116217382252216, + 0.013852729462087154, + 0.012926473282277584, + 0.011497247964143753, + 0.010562067851424217, + 0.010582265444099903, + 0.009467465803027153, + 0.008611960336565971, + 0.008248139172792435, + 0.00781706441193819, + 0.008040222339332104, + 0.007196015678346157, + 0.006431166548281908, + 0.006523136980831623, + 0.006243156269192696, + 0.0055301557295024395, + 0.004919611848890781 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.045660991711884e-06, + "first_to_last_ratio": 3.917022593501096, + "imbalance_abs_log_ratio": 1.3653318227296314, + "last_quartile_mean": 5.222489640743788e-07, + "mean": 1.2342234425588572e-06, + "normalized": [ + 1.8213398189861276, + 1.6645319684866091, + 1.4617715403691671, + 1.4547475368447418, + 1.9368035492846856, + 1.8317059236170603, + 1.6013392411126162, + 1.4873432240414948, + 1.6809680722553373, + 1.3943888205708208, + 1.3087026096396304, + 1.1611861005014494, + 1.226147655357962, + 1.097886062602167, + 1.0319446198761002, + 0.9091144134925789, + 0.8857120604478002, + 0.8417980143622228, + 0.7482542320108061, + 0.6799549053136131, + 0.6827139372140408, + 0.6176009501656174, + 0.5476403828473109, + 0.5412865865177412, + 0.5057577756670864, + 0.5227785606025747, + 0.47425131333003206, + 0.40558303186112465, + 0.41965194834280006, + 0.3928760379302379, + 0.35229520413941656, + 0.31192390220902655 + ], + "peak_layer": 5, + "peak_normalized": 1.9368035492846856, + "population_cv": 0.5025309389629137, + "rest_mean": 1.3304471759022785e-06, + "spike_contrast": 0.5371241304787221, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.146152825043828e-07, + "top_five_layers": [ + 5, + 6, + 1, + 9, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.247940301458584e-06, + 2.054404376394814e-06, + 1.804152702788997e-06, + 1.7954835129785351e-06, + 2.390448344158358e-06, + 2.2607343908020994e-06, + 1.976410430870601e-06, + 1.8357138742430834e-06, + 2.074690200970508e-06, + 1.7209873703905032e-06, + 1.6152314401551848e-06, + 1.433163106412394e-06, + 1.513340180281375e-06, + 1.3550367157222354e-06, + 1.2736502412735717e-06, + 1.1220503211006871e-06, + 1.0931665883617825e-06, + 1.038966843225353e-06, + 9.23512914141611e-07, + 8.392162840209494e-07, + 8.42621545871225e-07, + 7.622575708410295e-07, + 6.759105986020586e-07, + 6.680685942228592e-07, + 6.242181029847416e-07, + 6.452255547628738e-07, + 5.853320885762514e-07, + 5.005800858270959e-07, + 5.179442723601824e-07, + 4.848968160331424e-07, + 4.348109996499261e-07, + 3.8498379240081704e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.4131825966833276e-05, + "first_to_last_ratio": 3.655390206056544, + "imbalance_abs_log_ratio": 1.2962028470263438, + "last_quartile_mean": 6.60170996980014e-06, + "mean": 1.4587583734737564e-05, + "normalized": [ + 1.850614794041918, + 1.6756697305772237, + 1.532740691958691, + 1.4525582288852792, + 1.9097065779239775, + 1.773563531018016, + 1.5841340955031835, + 1.4551854143356047, + 1.616552010268487, + 1.4126736159014799, + 1.2481506316585642, + 1.139663047863614, + 1.206304825795999, + 1.058901504186884, + 0.9570224096844311, + 0.8837934651949423, + 0.8892675475561976, + 0.8327723366593228, + 0.742233508291339, + 0.6863736151537033, + 0.7069606538289486, + 0.6275682542143274, + 0.5826463632089418, + 0.5544888806084907, + 0.5307528105903948, + 0.5485698264270499, + 0.49089718904506907, + 0.4444636283252074, + 0.45082728024375657, + 0.4306907622876734, + 0.381786289322936, + 0.34246647943834635 + ], + "peak_layer": 5, + "peak_normalized": 1.9097065779239775, + "population_cv": 0.4863857904662779, + "rest_mean": 1.5666839654268524e-05, + "spike_contrast": 0.5591173435469335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.759601769270375e-06, + "top_five_layers": [ + 5, + 1, + 6, + 2, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.699599826883059e-05, + 2.4443972506560385e-05, + 2.2358983187587e-05, + 2.11893147934461e-05, + 2.7858004614245147e-05, + 2.587200651760213e-05, + 2.3108688765205443e-05, + 2.122763908118941e-05, + 2.358158781134989e-05, + 2.0607494661817327e-05, + 1.820750185288489e-05, + 1.6624930140096694e-05, + 1.7597072655917145e-05, + 1.5446814359165728e-05, + 1.3960644537291955e-05, + 1.2892411177745089e-05, + 1.2972264812560752e-05, + 1.2148136192990933e-05, + 1.0827393452927936e-05, + 1.0012532584369183e-05, + 1.0312847734894603e-05, + 9.154704457614571e-06, + 8.499402611050755e-06, + 8.088652975857258e-06, + 7.74240106693469e-06, + 8.002308277355041e-06, + 7.16100385034224e-06, + 6.483650395239238e-06, + 6.576480700459797e-06, + 6.282737558649387e-06, + 5.569339464273071e-06, + 4.995758445147658e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.8469200995241408e-05, + "first_to_last_ratio": 3.207575199084929, + "imbalance_abs_log_ratio": 1.1655152619982017, + "last_quartile_mean": 5.7579946997066145e-06, + "mean": 1.1597108013461366e-05, + "normalized": [ + 1.81806399507725, + 1.625366158769584, + 1.5063686258414184, + 1.3954385973171766, + 1.8443701608420582, + 1.6859765108020661, + 1.4840450012318553, + 1.3809272845448963, + 1.548769019794943, + 1.3994132960456658, + 1.2002346351153552, + 1.120141499141413, + 1.2021529711241468, + 1.0647775737601772, + 0.9335838124381102, + 0.9011530250739159, + 0.9289007857250168, + 0.8471652887398277, + 0.7610194462198521, + 0.7208419881303229, + 0.7713258850383539, + 0.6593132095004427, + 0.6413626788985748, + 0.5872674621057953, + 0.57359062275249, + 0.6016506688370158, + 0.5329045048635649, + 0.4939332120091125, + 0.4968495752468759, + 0.4799066396346222, + 0.4191450071900165, + 0.37404085818808425 + ], + "peak_layer": 5, + "peak_normalized": 1.8443701608420582, + "population_cv": 0.44401994718402354, + "rest_mean": 1.2356134498582429e-05, + "spike_contrast": 0.6068536235719905, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.498364993807627e-06, + "top_five_layers": [ + 5, + 1, + 6, + 2, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.108428452629596e-05, + 1.8849546904675663e-05, + 1.74695196619723e-05, + 1.6183052139240317e-05, + 2.138935997209046e-05, + 1.9552451703930274e-05, + 1.721063017612323e-05, + 1.601476287760306e-05, + 1.796124161046464e-05, + 1.6229147149715573e-05, + 1.3919250704930164e-05, + 1.2990401955903508e-05, + 1.3941497854830232e-05, + 1.2348340533208102e-05, + 1.082687231246382e-05, + 1.0450768968439661e-05, + 1.0772562745842151e-05, + 9.824667358770967e-06, + 8.825624718156178e-06, + 8.35968239698559e-06, + 8.945149602368474e-06, + 7.646126505278517e-06, + 7.437952262989711e-06, + 6.810604190832237e-06, + 6.651992407569196e-06, + 6.977407792874146e-06, + 6.18015110376291e-06, + 5.72819681110559e-06, + 5.76201819058042e-06, + 5.565529136219993e-06, + 4.860869921685662e-06, + 4.337792233854998e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.355659252221812e-05, + "first_to_last_ratio": 4.688745822260651, + "imbalance_abs_log_ratio": 1.5451651314001054, + "last_quartile_mean": 1.355513711587264e-05, + "mean": 3.660362690993679e-05, + "normalized": [ + 1.8604312730573458, + 1.729027777106884, + 1.5601949774918833, + 1.5188228334740854, + 1.995241408516474, + 1.8645405891130502, + 1.7681697430427887, + 1.5943462503396746, + 1.722236164777989, + 1.4743310838751806, + 1.323156933502311, + 1.1917678488813146, + 1.2713083492940296, + 1.0938173223246854, + 1.0218191154232952, + 0.8916141451419608, + 0.8287888488337875, + 0.7802753941234802, + 0.720014469658079, + 0.6521922684818504, + 0.6212316142834627, + 0.5670057570442619, + 0.503851683647319, + 0.4832362271528006, + 0.44920645672067455, + 0.4521315093539983, + 0.4042296913438992, + 0.3615519057236543, + 0.36249030689593087, + 0.352153631754142, + 0.30928834976812813, + 0.2715260698515788 + ], + "peak_layer": 5, + "peak_normalized": 1.995241408516474, + "population_cv": 0.5545162509282403, + "rest_mean": 3.9824025205032536e-05, + "spike_contrast": 0.4824594203498991, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.921347611641977e-05, + "top_five_layers": [ + 5, + 6, + 1, + 7, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.809853221056983e-05, + 6.328868767013773e-05, + 5.710879486287013e-05, + 5.559442433877848e-05, + 7.30330721125938e-05, + 6.824894808232784e-05, + 6.472142558777705e-05, + 5.835885531269014e-05, + 6.304009002633393e-05, + 5.396586493588984e-05, + 4.843234273721464e-05, + 4.362302570370957e-05, + 4.653449650504626e-05, + 4.003768117399886e-05, + 3.740228567039594e-05, + 3.2636311516398564e-05, + 3.033667780982796e-05, + 2.8560909413499758e-05, + 2.635514101712033e-05, + 2.3872602469054982e-05, + 2.2739330233889632e-05, + 2.075446718663443e-05, + 1.8442799046169966e-05, + 1.7688198568066582e-05, + 1.644258554733824e-05, + 1.6549653082620353e-05, + 1.4796272807870992e-05, + 1.3234111065685283e-05, + 1.3268459952087142e-05, + 1.2890100151707884e-05, + 1.1321075362502597e-05, + 9.938838957168628e-06 + ] + } + } + } + } + } + }, + "step": 100 + }, + { + "modes": { + "learned": { + "bits_per_byte": 3.059021014366605, + "forward": { + "activation_sha256": { + "attention_branch_output": "d7913db2408d5062b288c4b95e9ffb2f0d06e58bee52e81d9300885e31436501", + "mlp_branch_output": "490a3ed9260ad0b92a1839e225a59c197ccf4984373e834d9df6ed56c379de39", + "post_attention_state": "40fd0d817b1a8fb04957cc5ff2887bc65937f1776b53a68242e4c7341723dda7", + "post_mlp_state": "79ad652a3aa7b0850b826f149977b0495d569b726bb8d66c66c52638215ffd53", + "pre_attention_input": "0db2d20deb5c148ef675d303417844641f9632a980cb7b1bc2b39e8bf6c514f2", + "pre_mlp_input": "f2daf0e86334d46ef8de0e95de81ea9f5fd3989918e22ce51c09039c50c57cf2" + }, + "logits_sha256": "1a0f96c15dd52249e0f831e43c96af3ac5c3a1538f81e995c2942a1eb3edcccc", + "loss_nats": 2.120351791381836, + "mixer_summary_sha256": "c3dc8918354fb21100a845676f3328608d15f24ff7f23ed2830e4f5211e0fe8d" + }, + "loss_nats": 2.120351791381836, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6572662591934204, + "group": 1, + "latest_source_mass": 0.6088608503341675, + "layer": 1, + "max_source_mass": 0.6088608503341675, + "mixer_index": 1, + "normalized_entropy": 0.9482347726819877, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3911391496658325, + "median": 0.3669866621494293, + "p05": 0.30033501982688904, + "p95": 0.5365557670593262 + }, + { + "label": "current_group_partial", + "mean": 0.6088608503341675, + "median": 0.6330133676528931, + "p05": 0.4634442627429962, + "p95": 0.6996649503707886 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6751130819320679, + "group": 1, + "latest_source_mass": 0.5585282444953918, + "layer": 2, + "max_source_mass": 0.5585282444953918, + "mixer_index": 2, + "normalized_entropy": 0.9739822953426588, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.44147175550460815, + "median": 0.4214625358581543, + "p05": 0.3529680669307709, + "p95": 0.5934914946556091 + }, + { + "label": "current_group_partial", + "mean": 0.5585282444953918, + "median": 0.5785374641418457, + "p05": 0.40650850534439087, + "p95": 0.6470319032669067 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6885842084884644, + "group": 1, + "latest_source_mass": 0.5103577375411987, + "layer": 2, + "max_source_mass": 0.5103577375411987, + "mixer_index": 3, + "normalized_entropy": 0.9934170228207596, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4896422028541565, + "median": 0.48023492097854614, + "p05": 0.42685163021087646, + "p95": 0.5744246244430542 + }, + { + "label": "current_group_partial", + "mean": 0.5103577375411987, + "median": 0.5197650790214539, + "p05": 0.4255753755569458, + "p95": 0.5731483697891235 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.668085515499115, + "group": 1, + "latest_source_mass": 0.5667530298233032, + "layer": 3, + "max_source_mass": 0.5667530298233032, + "mixer_index": 4, + "normalized_entropy": 0.96384366010032, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4332469701766968, + "median": 0.4057430326938629, + "p05": 0.3270769417285919, + "p95": 0.6122807264328003 + }, + { + "label": "current_group_partial", + "mean": 0.5667530298233032, + "median": 0.5942569375038147, + "p05": 0.3877193331718445, + "p95": 0.6729230880737305 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6886032223701477, + "group": 1, + "latest_source_mass": 0.4881485104560852, + "layer": 3, + "max_source_mass": 0.5118515491485596, + "mixer_index": 5, + "normalized_entropy": 0.9934444540535723, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5118515491485596, + "median": 0.5089530944824219, + "p05": 0.44159775972366333, + "p95": 0.5947571992874146 + }, + { + "label": "current_group_partial", + "mean": 0.4881485104560852, + "median": 0.4910469055175781, + "p05": 0.40524277091026306, + "p95": 0.5584021806716919 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6789880990982056, + "group": 1, + "latest_source_mass": 0.5300825834274292, + "layer": 4, + "max_source_mass": 0.5300825834274292, + "mixer_index": 6, + "normalized_entropy": 0.9795727633916053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4699174761772156, + "median": 0.45400309562683105, + "p05": 0.3678591251373291, + "p95": 0.6229325532913208 + }, + { + "label": "current_group_partial", + "mean": 0.5300825834274292, + "median": 0.545996904373169, + "p05": 0.3770674467086792, + "p95": 0.6321409344673157 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6772009134292603, + "group": 1, + "latest_source_mass": 0.5604768991470337, + "layer": 4, + "max_source_mass": 0.5604768991470337, + "mixer_index": 7, + "normalized_entropy": 0.97699439948987, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4395230710506439, + "median": 0.4313446283340454, + "p05": 0.34554681181907654, + "p95": 0.5607337951660156 + }, + { + "label": "current_group_partial", + "mean": 0.5604768991470337, + "median": 0.5686553716659546, + "p05": 0.4392662048339844, + "p95": 0.6544531583786011 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6781555414199829, + "group": 2, + "latest_source_mass": 0.5458660125732422, + "layer": 5, + "max_source_mass": 0.5458660125732422, + "mixer_index": 8, + "normalized_entropy": 0.9783716365579794, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4541339576244354, + "median": 0.4442256689071655, + "p05": 0.3537072241306305, + "p95": 0.5944923758506775 + }, + { + "label": "completed_group_1", + "mean": 0.5458660125732422, + "median": 0.5557742714881897, + "p05": 0.40550756454467773, + "p95": 0.6462928056716919 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0917184352874756, + "group": 2, + "latest_source_mass": 0.33584368228912354, + "layer": 5, + "max_source_mass": 0.355538010597229, + "mixer_index": 9, + "normalized_entropy": 0.9937249442303328, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30861833691596985, + "median": 0.3010689616203308, + "p05": 0.2663846015930176, + "p95": 0.37415313720703125 + }, + { + "label": "completed_group_1", + "mean": 0.355538010597229, + "median": 0.3584616184234619, + "p05": 0.2822456955909729, + "p95": 0.4176298975944519 + }, + { + "label": "current_group_partial", + "mean": 0.33584368228912354, + "median": 0.3367723822593689, + "p05": 0.30035722255706787, + "p95": 0.37120699882507324 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0860013961791992, + "group": 2, + "latest_source_mass": 0.37406760454177856, + "layer": 6, + "max_source_mass": 0.37406760454177856, + "mixer_index": 10, + "normalized_entropy": 0.9885210709738198, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30104559659957886, + "median": 0.28779512643814087, + "p05": 0.24668964743614197, + "p95": 0.39145994186401367 + }, + { + "label": "completed_group_1", + "mean": 0.3248867988586426, + "median": 0.3244113028049469, + "p05": 0.2869458496570587, + "p95": 0.3649960160255432 + }, + { + "label": "current_group_partial", + "mean": 0.37406760454177856, + "median": 0.3894866406917572, + "p05": 0.25732842087745667, + "p95": 0.4404592216014862 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0827364921569824, + "group": 2, + "latest_source_mass": 0.378738671541214, + "layer": 6, + "max_source_mass": 0.378738671541214, + "mixer_index": 11, + "normalized_entropy": 0.9855492272616264, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2735002934932709, + "median": 0.2623218297958374, + "p05": 0.22696587443351746, + "p95": 0.3791309595108032 + }, + { + "label": "completed_group_1", + "mean": 0.34776100516319275, + "median": 0.34822437167167664, + "p05": 0.2800303101539612, + "p95": 0.40881502628326416 + }, + { + "label": "current_group_partial", + "mean": 0.378738671541214, + "median": 0.3773481547832489, + "p05": 0.3319735825061798, + "p95": 0.42474985122680664 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0724949836730957, + "group": 2, + "latest_source_mass": 0.40881654620170593, + "layer": 7, + "max_source_mass": 0.40881654620170593, + "mixer_index": 12, + "normalized_entropy": 0.9762270044997612, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2886848449707031, + "median": 0.2789040505886078, + "p05": 0.2284647673368454, + "p95": 0.38948193192481995 + }, + { + "label": "completed_group_1", + "mean": 0.30249863862991333, + "median": 0.29331475496292114, + "p05": 0.26091599464416504, + "p95": 0.38118377327919006 + }, + { + "label": "current_group_partial", + "mean": 0.40881654620170593, + "median": 0.43531113862991333, + "p05": 0.24153971672058105, + "p95": 0.48550277948379517 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0897927284240723, + "group": 2, + "latest_source_mass": 0.3727659583091736, + "layer": 7, + "max_source_mass": 0.3727659583091736, + "mixer_index": 13, + "normalized_entropy": 0.9919720903042785, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2973601222038269, + "median": 0.28749629855155945, + "p05": 0.25842589139938354, + "p95": 0.36914706230163574 + }, + { + "label": "completed_group_1", + "mean": 0.3298739790916443, + "median": 0.32824352383613586, + "p05": 0.2834911346435547, + "p95": 0.37481051683425903 + }, + { + "label": "current_group_partial", + "mean": 0.3727659583091736, + "median": 0.369523286819458, + "p05": 0.3233802914619446, + "p95": 0.4292686879634857 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0697213411331177, + "group": 2, + "latest_source_mass": 0.4076794981956482, + "layer": 8, + "max_source_mass": 0.4076794981956482, + "mixer_index": 14, + "normalized_entropy": 0.9737023262592323, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3028595447540283, + "median": 0.29215747117996216, + "p05": 0.25508639216423035, + "p95": 0.3769429922103882 + }, + { + "label": "completed_group_1", + "mean": 0.28946101665496826, + "median": 0.268873929977417, + "p05": 0.2331261932849884, + "p95": 0.3979876637458801 + }, + { + "label": "current_group_partial", + "mean": 0.4076794981956482, + "median": 0.4429859519004822, + "p05": 0.22969894111156464, + "p95": 0.4997333288192749 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0826778411865234, + "group": 2, + "latest_source_mass": 0.3901388645172119, + "layer": 8, + "max_source_mass": 0.3901388645172119, + "mixer_index": 15, + "normalized_entropy": 0.9854958408476349, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2862996757030487, + "median": 0.2773204445838928, + "p05": 0.22970792651176453, + "p95": 0.39824816584587097 + }, + { + "label": "completed_group_1", + "mean": 0.3235614597797394, + "median": 0.32365044951438904, + "p05": 0.26518940925598145, + "p95": 0.38037896156311035 + }, + { + "label": "current_group_partial", + "mean": 0.3901388645172119, + "median": 0.38918495178222656, + "p05": 0.3267495334148407, + "p95": 0.44896650314331055 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0579445362091064, + "group": 3, + "latest_source_mass": 0.4269215166568756, + "layer": 9, + "max_source_mass": 0.4269215166568756, + "mixer_index": 16, + "normalized_entropy": 0.9629826164530652, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2899969816207886, + "median": 0.2869234085083008, + "p05": 0.22596141695976257, + "p95": 0.37520572543144226 + }, + { + "label": "completed_group_1", + "mean": 0.2830814719200134, + "median": 0.2585422396659851, + "p05": 0.2167314738035202, + "p95": 0.408857524394989 + }, + { + "label": "completed_group_2", + "mean": 0.4269215166568756, + "median": 0.4538877308368683, + "p05": 0.23425813019275665, + "p95": 0.5443560481071472 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3809051513671875, + "group": 3, + "latest_source_mass": 0.2620403468608856, + "layer": 9, + "max_source_mass": 0.26517200469970703, + "mixer_index": 17, + "normalized_entropy": 0.9961125069077325, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23795948922634125, + "median": 0.23573458194732666, + "p05": 0.20756788551807404, + "p95": 0.2841742932796478 + }, + { + "label": "completed_group_1", + "mean": 0.2348281741142273, + "median": 0.23269692063331604, + "p05": 0.20418217778205872, + "p95": 0.2688288688659668 + }, + { + "label": "completed_group_2", + "mean": 0.26517200469970703, + "median": 0.2671319544315338, + "p05": 0.22843149304389954, + "p95": 0.29650676250457764 + }, + { + "label": "current_group_partial", + "mean": 0.2620403468608856, + "median": 0.26025551557540894, + "p05": 0.2228235900402069, + "p95": 0.30686914920806885 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3605632781982422, + "group": 3, + "latest_source_mass": 0.27637380361557007, + "layer": 10, + "max_source_mass": 0.27637380361557007, + "mixer_index": 18, + "normalized_entropy": 0.9814389471361176, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24186556041240692, + "median": 0.23976491391658783, + "p05": 0.19832007586956024, + "p95": 0.3010247051715851 + }, + { + "label": "completed_group_1", + "mean": 0.22099661827087402, + "median": 0.19567278027534485, + "p05": 0.1509825587272644, + "p95": 0.35698726773262024 + }, + { + "label": "completed_group_2", + "mean": 0.2607640027999878, + "median": 0.2613059878349304, + "p05": 0.2048056572675705, + "p95": 0.31461119651794434 + }, + { + "label": "current_group_partial", + "mean": 0.27637380361557007, + "median": 0.2970566749572754, + "p05": 0.1543806940317154, + "p95": 0.359146386384964 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3829561471939087, + "group": 3, + "latest_source_mass": 0.26782411336898804, + "layer": 10, + "max_source_mass": 0.26782411336898804, + "mixer_index": 19, + "normalized_entropy": 0.9975919876617797, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23731297254562378, + "median": 0.23403030633926392, + "p05": 0.21617025136947632, + "p95": 0.2765912413597107 + }, + { + "label": "completed_group_1", + "mean": 0.23829799890518188, + "median": 0.23762314021587372, + "p05": 0.21581488847732544, + "p95": 0.26157575845718384 + }, + { + "label": "completed_group_2", + "mean": 0.2565649151802063, + "median": 0.256875216960907, + "p05": 0.235240638256073, + "p95": 0.2770921587944031 + }, + { + "label": "current_group_partial", + "mean": 0.26782411336898804, + "median": 0.2663458585739136, + "p05": 0.2370651215314865, + "p95": 0.3028164207935333 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.35977041721344, + "group": 3, + "latest_source_mass": 0.29567578434944153, + "layer": 11, + "max_source_mass": 0.29567578434944153, + "mixer_index": 20, + "normalized_entropy": 0.9808670188306733, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23858636617660522, + "median": 0.23665651679039001, + "p05": 0.19799774885177612, + "p95": 0.2946481704711914 + }, + { + "label": "completed_group_1", + "mean": 0.21978360414505005, + "median": 0.19543592631816864, + "p05": 0.158006951212883, + "p95": 0.33895981311798096 + }, + { + "label": "completed_group_2", + "mean": 0.2459542453289032, + "median": 0.24462920427322388, + "p05": 0.21809649467468262, + "p95": 0.27666720747947693 + }, + { + "label": "current_group_partial", + "mean": 0.29567578434944153, + "median": 0.3254140615463257, + "p05": 0.15821409225463867, + "p95": 0.38422760367393494 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3790044784545898, + "group": 3, + "latest_source_mass": 0.27211782336235046, + "layer": 11, + "max_source_mass": 0.27211782336235046, + "mixer_index": 21, + "normalized_entropy": 0.9947414612150541, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24452988803386688, + "median": 0.24109914898872375, + "p05": 0.21188999712467194, + "p95": 0.29188552498817444 + }, + { + "label": "completed_group_1", + "mean": 0.2358427196741104, + "median": 0.2335224598646164, + "p05": 0.20158399641513824, + "p95": 0.27844953536987305 + }, + { + "label": "completed_group_2", + "mean": 0.24750956892967224, + "median": 0.24642077088356018, + "p05": 0.21784070134162903, + "p95": 0.28041350841522217 + }, + { + "label": "current_group_partial", + "mean": 0.27211782336235046, + "median": 0.276747465133667, + "p05": 0.20197471976280212, + "p95": 0.3271600306034088 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3556556701660156, + "group": 3, + "latest_source_mass": 0.3048465847969055, + "layer": 12, + "max_source_mass": 0.3048465847969055, + "mixer_index": 22, + "normalized_entropy": 0.9778988562507576, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2410905957221985, + "median": 0.23850542306900024, + "p05": 0.20414574444293976, + "p95": 0.28753191232681274 + }, + { + "label": "completed_group_1", + "mean": 0.22088873386383057, + "median": 0.19732099771499634, + "p05": 0.15404488146305084, + "p95": 0.34053435921669006 + }, + { + "label": "completed_group_2", + "mean": 0.23317405581474304, + "median": 0.231553316116333, + "p05": 0.20150601863861084, + "p95": 0.26888900995254517 + }, + { + "label": "current_group_partial", + "mean": 0.3048465847969055, + "median": 0.3246956467628479, + "p05": 0.15267115831375122, + "p95": 0.41261592507362366 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3804312944412231, + "group": 3, + "latest_source_mass": 0.2690383195877075, + "layer": 12, + "max_source_mass": 0.2690383195877075, + "mixer_index": 23, + "normalized_entropy": 0.9957706913891425, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.26568183302879333, + "median": 0.2650548219680786, + "p05": 0.22915047407150269, + "p95": 0.3039885461330414 + }, + { + "label": "completed_group_1", + "mean": 0.22685569524765015, + "median": 0.2285763919353485, + "p05": 0.19875401258468628, + "p95": 0.2494552731513977 + }, + { + "label": "completed_group_2", + "mean": 0.2384241372346878, + "median": 0.23920457065105438, + "p05": 0.2169211506843567, + "p95": 0.25894689559936523 + }, + { + "label": "current_group_partial", + "mean": 0.2690383195877075, + "median": 0.27138766646385193, + "p05": 0.2234886884689331, + "p95": 0.312222421169281 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3533532619476318, + "group": 4, + "latest_source_mass": 0.3041262626647949, + "layer": 13, + "max_source_mass": 0.3041262626647949, + "mixer_index": 24, + "normalized_entropy": 0.9762380197913754, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24537263810634613, + "median": 0.24389782547950745, + "p05": 0.2054842710494995, + "p95": 0.2901798486709595 + }, + { + "label": "completed_group_1", + "mean": 0.2171444594860077, + "median": 0.1941797286272049, + "p05": 0.1490350365638733, + "p95": 0.338962197303772 + }, + { + "label": "completed_group_2", + "mean": 0.23335666954517365, + "median": 0.23076492547988892, + "p05": 0.20317009091377258, + "p95": 0.27066874504089355 + }, + { + "label": "completed_group_3", + "mean": 0.3041262626647949, + "median": 0.3265917897224426, + "p05": 0.1414138674736023, + "p95": 0.42294132709503174 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6053518056869507, + "group": 4, + "latest_source_mass": 0.22539189457893372, + "layer": 13, + "max_source_mass": 0.22539189457893372, + "mixer_index": 25, + "normalized_entropy": 0.9974611591316562, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20177139341831207, + "median": 0.20180779695510864, + "p05": 0.18717515468597412, + "p95": 0.21589727699756622 + }, + { + "label": "completed_group_1", + "mean": 0.18913303315639496, + "median": 0.19024527072906494, + "p05": 0.17409206926822662, + "p95": 0.20153141021728516 + }, + { + "label": "completed_group_2", + "mean": 0.18395330011844635, + "median": 0.18139007687568665, + "p05": 0.16835211217403412, + "p95": 0.20520012080669403 + }, + { + "label": "completed_group_3", + "mean": 0.1997503787279129, + "median": 0.19939276576042175, + "p05": 0.1866363286972046, + "p95": 0.2140733301639557 + }, + { + "label": "current_group_partial", + "mean": 0.22539189457893372, + "median": 0.2293616533279419, + "p05": 0.18638378381729126, + "p95": 0.24480541050434113 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.585533618927002, + "group": 4, + "latest_source_mass": 0.25353068113327026, + "layer": 14, + "max_source_mass": 0.25353068113327026, + "mixer_index": 26, + "normalized_entropy": 0.9851474273580733, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19244730472564697, + "median": 0.19150400161743164, + "p05": 0.1739657074213028, + "p95": 0.21505704522132874 + }, + { + "label": "completed_group_1", + "mean": 0.1780712604522705, + "median": 0.16121169924736023, + "p05": 0.1377859115600586, + "p95": 0.2661179304122925 + }, + { + "label": "completed_group_2", + "mean": 0.17078758776187897, + "median": 0.16340723633766174, + "p05": 0.14458762109279633, + "p95": 0.22640380263328552 + }, + { + "label": "completed_group_3", + "mean": 0.20516318082809448, + "median": 0.20959466695785522, + "p05": 0.15043368935585022, + "p95": 0.2540265917778015 + }, + { + "label": "current_group_partial", + "mean": 0.25353068113327026, + "median": 0.2702726125717163, + "p05": 0.14368262887001038, + "p95": 0.29619652032852173 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6022908687591553, + "group": 4, + "latest_source_mass": 0.23394522070884705, + "layer": 14, + "max_source_mass": 0.23394522070884705, + "mixer_index": 27, + "normalized_entropy": 0.9955592920859333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1985350400209427, + "median": 0.1984509825706482, + "p05": 0.1769370585680008, + "p95": 0.22112208604812622 + }, + { + "label": "completed_group_1", + "mean": 0.18978449702262878, + "median": 0.1900787055492401, + "p05": 0.1711708903312683, + "p95": 0.20756162703037262 + }, + { + "label": "completed_group_2", + "mean": 0.17792725563049316, + "median": 0.17577432096004486, + "p05": 0.1577838957309723, + "p95": 0.20609596371650696 + }, + { + "label": "completed_group_3", + "mean": 0.1998080015182495, + "median": 0.200008824467659, + "p05": 0.18536703288555145, + "p95": 0.2129323035478592 + }, + { + "label": "current_group_partial", + "mean": 0.23394522070884705, + "median": 0.23855753242969513, + "p05": 0.18709585070610046, + "p95": 0.2655036449432373 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5833771228790283, + "group": 4, + "latest_source_mass": 0.2590453028678894, + "layer": 15, + "max_source_mass": 0.2590453028678894, + "mixer_index": 28, + "normalized_entropy": 0.9838075210272276, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1926397681236267, + "median": 0.19182440638542175, + "p05": 0.1704544872045517, + "p95": 0.21726183593273163 + }, + { + "label": "completed_group_1", + "mean": 0.1801614761352539, + "median": 0.16366446018218994, + "p05": 0.13534875214099884, + "p95": 0.27078741788864136 + }, + { + "label": "completed_group_2", + "mean": 0.17439624667167664, + "median": 0.16901585459709167, + "p05": 0.14859820902347565, + "p95": 0.22471660375595093 + }, + { + "label": "completed_group_3", + "mean": 0.19375719130039215, + "median": 0.19451642036437988, + "p05": 0.16403654217720032, + "p95": 0.22201430797576904 + }, + { + "label": "current_group_partial", + "mean": 0.2590453028678894, + "median": 0.27778831124305725, + "p05": 0.1263412982225418, + "p95": 0.3228909373283386 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.601109504699707, + "group": 4, + "latest_source_mass": 0.23593464493751526, + "layer": 15, + "max_source_mass": 0.23593464493751526, + "mixer_index": 29, + "normalized_entropy": 0.994825269325365, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19747966527938843, + "median": 0.19566109776496887, + "p05": 0.17110133171081543, + "p95": 0.22994962334632874 + }, + { + "label": "completed_group_1", + "mean": 0.19028550386428833, + "median": 0.1902298927307129, + "p05": 0.16050578653812408, + "p95": 0.22037073969841003 + }, + { + "label": "completed_group_2", + "mean": 0.18151576817035675, + "median": 0.18003535270690918, + "p05": 0.15826183557510376, + "p95": 0.21004454791545868 + }, + { + "label": "completed_group_3", + "mean": 0.19478440284729004, + "median": 0.1937907189130783, + "p05": 0.17653080821037292, + "p95": 0.2156463861465454 + }, + { + "label": "current_group_partial", + "mean": 0.23593464493751526, + "median": 0.23639161884784698, + "p05": 0.1930353045463562, + "p95": 0.27262991666793823 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5755603313446045, + "group": 4, + "latest_source_mass": 0.27402210235595703, + "layer": 16, + "max_source_mass": 0.27402210235595703, + "mixer_index": 30, + "normalized_entropy": 0.9789506753707202, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1969345510005951, + "median": 0.19588357210159302, + "p05": 0.17255066335201263, + "p95": 0.22505135834217072 + }, + { + "label": "completed_group_1", + "mean": 0.17712508141994476, + "median": 0.1627851128578186, + "p05": 0.13340646028518677, + "p95": 0.26109084486961365 + }, + { + "label": "completed_group_2", + "mean": 0.16844677925109863, + "median": 0.16122135519981384, + "p05": 0.14004012942314148, + "p95": 0.22884467244148254 + }, + { + "label": "completed_group_3", + "mean": 0.18347150087356567, + "median": 0.18296413123607635, + "p05": 0.16371525824069977, + "p95": 0.2048267126083374 + }, + { + "label": "current_group_partial", + "mean": 0.27402210235595703, + "median": 0.29705047607421875, + "p05": 0.12304861843585968, + "p95": 0.34848910570144653 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5980476140975952, + "group": 4, + "latest_source_mass": 0.24492081999778748, + "layer": 16, + "max_source_mass": 0.24492081999778748, + "mixer_index": 31, + "normalized_entropy": 0.9929228097284731, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19043892621994019, + "median": 0.18689709901809692, + "p05": 0.16917146742343903, + "p95": 0.22622565925121307 + }, + { + "label": "completed_group_1", + "mean": 0.1923028826713562, + "median": 0.19136476516723633, + "p05": 0.1641806662082672, + "p95": 0.22264797985553741 + }, + { + "label": "completed_group_2", + "mean": 0.1810823678970337, + "median": 0.1799810826778412, + "p05": 0.1545257568359375, + "p95": 0.21300536394119263 + }, + { + "label": "completed_group_3", + "mean": 0.19125500321388245, + "median": 0.19091501832008362, + "p05": 0.16802990436553955, + "p95": 0.21442335844039917 + }, + { + "label": "current_group_partial", + "mean": 0.24492081999778748, + "median": 0.24817338585853577, + "p05": 0.19009609520435333, + "p95": 0.29222598671913147 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5842485427856445, + "group": 5, + "latest_source_mass": 0.26211416721343994, + "layer": 17, + "max_source_mass": 0.26211416721343994, + "mixer_index": 32, + "normalized_entropy": 0.9843489646578789, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1964084655046463, + "median": 0.1943213939666748, + "p05": 0.1771012842655182, + "p95": 0.22198832035064697 + }, + { + "label": "completed_group_1", + "mean": 0.17743000388145447, + "median": 0.16406384110450745, + "p05": 0.13566289842128754, + "p95": 0.25637227296829224 + }, + { + "label": "completed_group_2", + "mean": 0.17621049284934998, + "median": 0.1722719371318817, + "p05": 0.15086707472801208, + "p95": 0.22157853841781616 + }, + { + "label": "completed_group_3", + "mean": 0.1878368854522705, + "median": 0.18780654668807983, + "p05": 0.16647562384605408, + "p95": 0.20967811346054077 + }, + { + "label": "completed_group_4", + "mean": 0.26211416721343994, + "median": 0.27968263626098633, + "p05": 0.1302976906299591, + "p95": 0.32829993963241577 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7856926918029785, + "group": 5, + "latest_source_mass": 0.18537181615829468, + "layer": 17, + "max_source_mass": 0.18537181615829468, + "mixer_index": 33, + "normalized_entropy": 0.9966140670501437, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15997162461280823, + "median": 0.15724682807922363, + "p05": 0.13894931972026825, + "p95": 0.18793770670890808 + }, + { + "label": "completed_group_1", + "mean": 0.16431674361228943, + "median": 0.1631746143102646, + "p05": 0.1443376988172531, + "p95": 0.18849408626556396 + }, + { + "label": "completed_group_2", + "mean": 0.15234553813934326, + "median": 0.15064480900764465, + "p05": 0.13384775817394257, + "p95": 0.17680883407592773 + }, + { + "label": "completed_group_3", + "mean": 0.15752777457237244, + "median": 0.15624287724494934, + "p05": 0.1438755989074707, + "p95": 0.17557287216186523 + }, + { + "label": "completed_group_4", + "mean": 0.18046650290489197, + "median": 0.18079496920108795, + "p05": 0.1579463630914688, + "p95": 0.20150543749332428 + }, + { + "label": "current_group_partial", + "mean": 0.18537181615829468, + "median": 0.18655891716480255, + "p05": 0.15393848717212677, + "p95": 0.2133883237838745 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7751184701919556, + "group": 5, + "latest_source_mass": 0.2034376710653305, + "layer": 18, + "max_source_mass": 0.2034376710653305, + "mixer_index": 34, + "normalized_entropy": 0.9907124816015239, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16064734756946564, + "median": 0.1604275405406952, + "p05": 0.135296031832695, + "p95": 0.1862303614616394 + }, + { + "label": "completed_group_1", + "mean": 0.15155573189258575, + "median": 0.14294931292533875, + "p05": 0.12762469053268433, + "p95": 0.20150375366210938 + }, + { + "label": "completed_group_2", + "mean": 0.14907032251358032, + "median": 0.14577648043632507, + "p05": 0.13263380527496338, + "p95": 0.1819881796836853 + }, + { + "label": "completed_group_3", + "mean": 0.14832918345928192, + "median": 0.1450541615486145, + "p05": 0.12743958830833435, + "p95": 0.18020303547382355 + }, + { + "label": "completed_group_4", + "mean": 0.18695974349975586, + "median": 0.19112101197242737, + "p05": 0.14380104839801788, + "p95": 0.21154803037643433 + }, + { + "label": "current_group_partial", + "mean": 0.2034376710653305, + "median": 0.2150307595729828, + "p05": 0.12007137387990952, + "p95": 0.24437281489372253 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7829411029815674, + "group": 5, + "latest_source_mass": 0.1976606249809265, + "layer": 18, + "max_source_mass": 0.1976606249809265, + "mixer_index": 35, + "normalized_entropy": 0.9950783760890145, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14932186901569366, + "median": 0.14249257743358612, + "p05": 0.12489590048789978, + "p95": 0.19243758916854858 + }, + { + "label": "completed_group_1", + "mean": 0.16724151372909546, + "median": 0.1661989390850067, + "p05": 0.1489294171333313, + "p95": 0.186150923371315 + }, + { + "label": "completed_group_2", + "mean": 0.15592648088932037, + "median": 0.15542590618133545, + "p05": 0.13526445627212524, + "p95": 0.1783086359500885 + }, + { + "label": "completed_group_3", + "mean": 0.1555134654045105, + "median": 0.154983788728714, + "p05": 0.14075525104999542, + "p95": 0.17211037874221802 + }, + { + "label": "completed_group_4", + "mean": 0.17433607578277588, + "median": 0.17505210638046265, + "p05": 0.15327315032482147, + "p95": 0.19426627457141876 + }, + { + "label": "current_group_partial", + "mean": 0.1976606249809265, + "median": 0.20035341382026672, + "p05": 0.15607678890228271, + "p95": 0.22393837571144104 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7661566734313965, + "group": 5, + "latest_source_mass": 0.22923919558525085, + "layer": 19, + "max_source_mass": 0.22923919558525085, + "mixer_index": 36, + "normalized_entropy": 0.9857108075964633, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16592846810817719, + "median": 0.16519835591316223, + "p05": 0.13959640264511108, + "p95": 0.1943950057029724 + }, + { + "label": "completed_group_1", + "mean": 0.143815815448761, + "median": 0.13836468756198883, + "p05": 0.12640488147735596, + "p95": 0.17898298799991608 + }, + { + "label": "completed_group_2", + "mean": 0.1375015527009964, + "median": 0.13198094069957733, + "p05": 0.11646819114685059, + "p95": 0.1853182464838028 + }, + { + "label": "completed_group_3", + "mean": 0.144493967294693, + "median": 0.14179405570030212, + "p05": 0.1222185343503952, + "p95": 0.17581477761268616 + }, + { + "label": "completed_group_4", + "mean": 0.17902101576328278, + "median": 0.18112671375274658, + "p05": 0.15200620889663696, + "p95": 0.1975291222333908 + }, + { + "label": "current_group_partial", + "mean": 0.22923919558525085, + "median": 0.24496859312057495, + "p05": 0.13256870210170746, + "p95": 0.270551860332489 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7807259559631348, + "group": 5, + "latest_source_mass": 0.20281299948692322, + "layer": 19, + "max_source_mass": 0.20281299948692322, + "mixer_index": 37, + "normalized_entropy": 0.9938420789986538, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14958029985427856, + "median": 0.14521853625774384, + "p05": 0.12654969096183777, + "p95": 0.20142483711242676 + }, + { + "label": "completed_group_1", + "mean": 0.1695217788219452, + "median": 0.16831448674201965, + "p05": 0.1474662870168686, + "p95": 0.19261527061462402 + }, + { + "label": "completed_group_2", + "mean": 0.15695258975028992, + "median": 0.15581071376800537, + "p05": 0.13313542306423187, + "p95": 0.18386399745941162 + }, + { + "label": "completed_group_3", + "mean": 0.1499582827091217, + "median": 0.14832203090190887, + "p05": 0.1333591639995575, + "p95": 0.17367161810398102 + }, + { + "label": "completed_group_4", + "mean": 0.1711740493774414, + "median": 0.17146730422973633, + "p05": 0.15243321657180786, + "p95": 0.18792594969272614 + }, + { + "label": "current_group_partial", + "mean": 0.20281299948692322, + "median": 0.2034434974193573, + "p05": 0.15488654375076294, + "p95": 0.2422361671924591 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.781292200088501, + "group": 5, + "latest_source_mass": 0.20378397405147552, + "layer": 20, + "max_source_mass": 0.20378397405147552, + "mixer_index": 38, + "normalized_entropy": 0.994158105862243, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1682473123073578, + "median": 0.17088356614112854, + "p05": 0.1362982541322708, + "p95": 0.19248411059379578 + }, + { + "label": "completed_group_1", + "mean": 0.15929323434829712, + "median": 0.15772488713264465, + "p05": 0.13811591267585754, + "p95": 0.1835828274488449 + }, + { + "label": "completed_group_2", + "mean": 0.14874720573425293, + "median": 0.14682769775390625, + "p05": 0.1326565444469452, + "p95": 0.17428536713123322 + }, + { + "label": "completed_group_3", + "mean": 0.15058940649032593, + "median": 0.14999014139175415, + "p05": 0.13366414606571198, + "p95": 0.16969719529151917 + }, + { + "label": "completed_group_4", + "mean": 0.1693388670682907, + "median": 0.16931521892547607, + "p05": 0.15269556641578674, + "p95": 0.18495050072669983 + }, + { + "label": "current_group_partial", + "mean": 0.20378397405147552, + "median": 0.21180889010429382, + "p05": 0.14455918967723846, + "p95": 0.23620733618736267 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7818934917449951, + "group": 5, + "latest_source_mass": 0.1928463578224182, + "layer": 20, + "max_source_mass": 0.1928463578224182, + "mixer_index": 39, + "normalized_entropy": 0.9944936931253889, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14812925457954407, + "median": 0.14193585515022278, + "p05": 0.11988933384418488, + "p95": 0.20610599219799042 + }, + { + "label": "completed_group_1", + "mean": 0.18386004865169525, + "median": 0.18534164130687714, + "p05": 0.1614244282245636, + "p95": 0.20129990577697754 + }, + { + "label": "completed_group_2", + "mean": 0.1664430797100067, + "median": 0.16720223426818848, + "p05": 0.14779648184776306, + "p95": 0.1832200437784195 + }, + { + "label": "completed_group_3", + "mean": 0.15062816441059113, + "median": 0.14718246459960938, + "p05": 0.13040047883987427, + "p95": 0.18431158363819122 + }, + { + "label": "completed_group_4", + "mean": 0.15809309482574463, + "median": 0.15818026661872864, + "p05": 0.14130482077598572, + "p95": 0.17517155408859253 + }, + { + "label": "current_group_partial", + "mean": 0.1928463578224182, + "median": 0.1947423219680786, + "p05": 0.1518069952726364, + "p95": 0.22260980308055878 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.77958083152771, + "group": 6, + "latest_source_mass": 0.19859939813613892, + "layer": 21, + "max_source_mass": 0.19859939813613892, + "mixer_index": 40, + "normalized_entropy": 0.9932029728825198, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19125157594680786, + "median": 0.19011858105659485, + "p05": 0.15831239521503448, + "p95": 0.2248726487159729 + }, + { + "label": "completed_group_1", + "mean": 0.15093469619750977, + "median": 0.15087208151817322, + "p05": 0.12483120709657669, + "p95": 0.17756037414073944 + }, + { + "label": "completed_group_2", + "mean": 0.14643779397010803, + "median": 0.14498162269592285, + "p05": 0.12900538742542267, + "p95": 0.16938214004039764 + }, + { + "label": "completed_group_3", + "mean": 0.1490234136581421, + "median": 0.1482788324356079, + "p05": 0.12938444316387177, + "p95": 0.17225822806358337 + }, + { + "label": "completed_group_4", + "mean": 0.16375310719013214, + "median": 0.16389217972755432, + "p05": 0.14641568064689636, + "p95": 0.1809726506471634 + }, + { + "label": "completed_group_5", + "mean": 0.19859939813613892, + "median": 0.20098775625228882, + "p05": 0.15739423036575317, + "p95": 0.23065634071826935 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9289655685424805, + "group": 6, + "latest_source_mass": 0.19567109644412994, + "layer": 21, + "max_source_mass": 0.19567109644412994, + "mixer_index": 41, + "normalized_entropy": 0.9912922081623045, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12320417165756226, + "median": 0.121359683573246, + "p05": 0.09729336947202682, + "p95": 0.16812030971050262 + }, + { + "label": "completed_group_1", + "mean": 0.15015369653701782, + "median": 0.1501747965812683, + "p05": 0.13342028856277466, + "p95": 0.16470867395401 + }, + { + "label": "completed_group_2", + "mean": 0.13329030573368073, + "median": 0.13304202258586884, + "p05": 0.11676464229822159, + "p95": 0.14988435804843903 + }, + { + "label": "completed_group_3", + "mean": 0.12274879962205887, + "median": 0.12164658308029175, + "p05": 0.10655395686626434, + "p95": 0.1420663595199585 + }, + { + "label": "completed_group_4", + "mean": 0.13076716661453247, + "median": 0.13035811483860016, + "p05": 0.11734984070062637, + "p95": 0.14464935660362244 + }, + { + "label": "completed_group_5", + "mean": 0.1441647708415985, + "median": 0.14637865126132965, + "p05": 0.11776107549667358, + "p95": 0.16295388340950012 + }, + { + "label": "current_group_partial", + "mean": 0.19567109644412994, + "median": 0.19740021228790283, + "p05": 0.16954219341278076, + "p95": 0.21879082918167114 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9340521097183228, + "group": 6, + "latest_source_mass": 0.18921294808387756, + "layer": 22, + "max_source_mass": 0.18921294808387756, + "mixer_index": 42, + "normalized_entropy": 0.9939061732409653, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14550700783729553, + "median": 0.14537246525287628, + "p05": 0.13202586770057678, + "p95": 0.16015800833702087 + }, + { + "label": "completed_group_1", + "mean": 0.14042077958583832, + "median": 0.13958397507667542, + "p05": 0.1146586537361145, + "p95": 0.1715676635503769 + }, + { + "label": "completed_group_2", + "mean": 0.13418063521385193, + "median": 0.1339511275291443, + "p05": 0.1221425011754036, + "p95": 0.14647553861141205 + }, + { + "label": "completed_group_3", + "mean": 0.1260203719139099, + "median": 0.1256953775882721, + "p05": 0.11505919694900513, + "p95": 0.13822400569915771 + }, + { + "label": "completed_group_4", + "mean": 0.12906625866889954, + "median": 0.12889152765274048, + "p05": 0.11804929375648499, + "p95": 0.14088976383209229 + }, + { + "label": "completed_group_5", + "mean": 0.1355919986963272, + "median": 0.13481314480304718, + "p05": 0.12486162781715393, + "p95": 0.1482546031475067 + }, + { + "label": "current_group_partial", + "mean": 0.18921294808387756, + "median": 0.1889541745185852, + "p05": 0.1605755239725113, + "p95": 0.21719029545783997 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9230798482894897, + "group": 6, + "latest_source_mass": 0.20240548253059387, + "layer": 22, + "max_source_mass": 0.20240548253059387, + "mixer_index": 43, + "normalized_entropy": 0.9882675462806405, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12067099660634995, + "median": 0.1175689548254013, + "p05": 0.09146378189325333, + "p95": 0.17015382647514343 + }, + { + "label": "completed_group_1", + "mean": 0.15888366103172302, + "median": 0.15990646183490753, + "p05": 0.13809531927108765, + "p95": 0.17797793447971344 + }, + { + "label": "completed_group_2", + "mean": 0.14002521336078644, + "median": 0.14027830958366394, + "p05": 0.12228793650865555, + "p95": 0.15711908042430878 + }, + { + "label": "completed_group_3", + "mean": 0.12021137773990631, + "median": 0.11844314634799957, + "p05": 0.1043848991394043, + "p95": 0.14366739988327026 + }, + { + "label": "completed_group_4", + "mean": 0.12209220230579376, + "median": 0.12163396179676056, + "p05": 0.10983633995056152, + "p95": 0.13694660365581512 + }, + { + "label": "completed_group_5", + "mean": 0.13571105897426605, + "median": 0.13670915365219116, + "p05": 0.11317556351423264, + "p95": 0.15392079949378967 + }, + { + "label": "current_group_partial", + "mean": 0.20240548253059387, + "median": 0.20184719562530518, + "p05": 0.1681356430053711, + "p95": 0.24128922820091248 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9308109283447266, + "group": 6, + "latest_source_mass": 0.1935279220342636, + "layer": 23, + "max_source_mass": 0.1935279220342636, + "mixer_index": 44, + "normalized_entropy": 0.9922405355057545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1450628638267517, + "median": 0.14652788639068604, + "p05": 0.12564095854759216, + "p95": 0.16459055244922638 + }, + { + "label": "completed_group_1", + "mean": 0.14527733623981476, + "median": 0.14751458168029785, + "p05": 0.11515176296234131, + "p95": 0.17206212878227234 + }, + { + "label": "completed_group_2", + "mean": 0.13644321262836456, + "median": 0.13537836074829102, + "p05": 0.1257452517747879, + "p95": 0.15082380175590515 + }, + { + "label": "completed_group_3", + "mean": 0.12497065216302872, + "median": 0.12392216920852661, + "p05": 0.11321288347244263, + "p95": 0.14013291895389557 + }, + { + "label": "completed_group_4", + "mean": 0.12634633481502533, + "median": 0.12607431411743164, + "p05": 0.11311027407646179, + "p95": 0.14066654443740845 + }, + { + "label": "completed_group_5", + "mean": 0.1283716857433319, + "median": 0.1277504414319992, + "p05": 0.11627928167581558, + "p95": 0.14233072102069855 + }, + { + "label": "current_group_partial", + "mean": 0.1935279220342636, + "median": 0.19237324595451355, + "p05": 0.1571470946073532, + "p95": 0.23567645251750946 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9273775815963745, + "group": 6, + "latest_source_mass": 0.18937146663665771, + "layer": 23, + "max_source_mass": 0.18937146663665771, + "mixer_index": 45, + "normalized_entropy": 0.9904761443029958, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1260491907596588, + "median": 0.12174050509929657, + "p05": 0.10315698385238647, + "p95": 0.16765159368515015 + }, + { + "label": "completed_group_1", + "mean": 0.16657932102680206, + "median": 0.16951650381088257, + "p05": 0.13692717254161835, + "p95": 0.18870727717876434 + }, + { + "label": "completed_group_2", + "mean": 0.14038510620594025, + "median": 0.13948792219161987, + "p05": 0.12602533400058746, + "p95": 0.15653200447559357 + }, + { + "label": "completed_group_3", + "mean": 0.1219596415758133, + "median": 0.12005066871643066, + "p05": 0.10611658543348312, + "p95": 0.1454348862171173 + }, + { + "label": "completed_group_4", + "mean": 0.12248437106609344, + "median": 0.12224026769399643, + "p05": 0.10896626859903336, + "p95": 0.13705895841121674 + }, + { + "label": "completed_group_5", + "mean": 0.13317090272903442, + "median": 0.13336879014968872, + "p05": 0.11660954356193542, + "p95": 0.14904874563217163 + }, + { + "label": "current_group_partial", + "mean": 0.18937146663665771, + "median": 0.18636475503444672, + "p05": 0.15230347216129303, + "p95": 0.23292618989944458 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.918284296989441, + "group": 6, + "latest_source_mass": 0.20540057122707367, + "layer": 24, + "max_source_mass": 0.20540057122707367, + "mixer_index": 46, + "normalized_entropy": 0.9858031204167963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13994047045707703, + "median": 0.14368271827697754, + "p05": 0.10786622762680054, + "p95": 0.16540095210075378 + }, + { + "label": "completed_group_1", + "mean": 0.15779414772987366, + "median": 0.1629778891801834, + "p05": 0.10690005868673325, + "p95": 0.19786833226680756 + }, + { + "label": "completed_group_2", + "mean": 0.13759920001029968, + "median": 0.1359691321849823, + "p05": 0.12253671139478683, + "p95": 0.15824976563453674 + }, + { + "label": "completed_group_3", + "mean": 0.11949826776981354, + "median": 0.117921382188797, + "p05": 0.10491804778575897, + "p95": 0.13834650814533234 + }, + { + "label": "completed_group_4", + "mean": 0.11786683648824692, + "median": 0.11655203253030777, + "p05": 0.1013876423239708, + "p95": 0.13959234952926636 + }, + { + "label": "completed_group_5", + "mean": 0.12190047651529312, + "median": 0.12099424004554749, + "p05": 0.10532263666391373, + "p95": 0.1426703929901123 + }, + { + "label": "current_group_partial", + "mean": 0.20540057122707367, + "median": 0.20053882896900177, + "p05": 0.15279710292816162, + "p95": 0.27538907527923584 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9060933589935303, + "group": 6, + "latest_source_mass": 0.19080331921577454, + "layer": 24, + "max_source_mass": 0.19080331921577454, + "mixer_index": 47, + "normalized_entropy": 0.9795382175887654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11515207588672638, + "median": 0.108605295419693, + "p05": 0.07161381095647812, + "p95": 0.19197940826416016 + }, + { + "label": "completed_group_1", + "mean": 0.18610318005084991, + "median": 0.1929539144039154, + "p05": 0.13255266845226288, + "p95": 0.22670131921768188 + }, + { + "label": "completed_group_2", + "mean": 0.15217435359954834, + "median": 0.15105080604553223, + "p05": 0.12881332635879517, + "p95": 0.17896413803100586 + }, + { + "label": "completed_group_3", + "mean": 0.11993157863616943, + "median": 0.11775963753461838, + "p05": 0.09839600324630737, + "p95": 0.1517515480518341 + }, + { + "label": "completed_group_4", + "mean": 0.11246243864297867, + "median": 0.11158700287342072, + "p05": 0.09335656464099884, + "p95": 0.1348860114812851 + }, + { + "label": "completed_group_5", + "mean": 0.12337306886911392, + "median": 0.12389068305492401, + "p05": 0.098457470536232, + "p95": 0.14636507630348206 + }, + { + "label": "current_group_partial", + "mean": 0.19080331921577454, + "median": 0.18048369884490967, + "p05": 0.1278817355632782, + "p95": 0.28530141711235046 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9199097156524658, + "group": 7, + "latest_source_mass": 0.15970462560653687, + "layer": 25, + "max_source_mass": 0.16625350713729858, + "mixer_index": 48, + "normalized_entropy": 0.9866384203733816, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13630829751491547, + "median": 0.14164748787879944, + "p05": 0.0576973557472229, + "p95": 0.20787456631660461 + }, + { + "label": "completed_group_1", + "mean": 0.16625350713729858, + "median": 0.17280495166778564, + "p05": 0.10803423821926117, + "p95": 0.20908519625663757 + }, + { + "label": "completed_group_2", + "mean": 0.14684012532234192, + "median": 0.1455259621143341, + "p05": 0.12492340058088303, + "p95": 0.1712140440940857 + }, + { + "label": "completed_group_3", + "mean": 0.12322510778903961, + "median": 0.12119361758232117, + "p05": 0.0957479178905487, + "p95": 0.1575886607170105 + }, + { + "label": "completed_group_4", + "mean": 0.12947362661361694, + "median": 0.12904725968837738, + "p05": 0.10726519674062729, + "p95": 0.1540917158126831 + }, + { + "label": "completed_group_5", + "mean": 0.1381947249174118, + "median": 0.139154314994812, + "p05": 0.11157024651765823, + "p95": 0.1611405611038208 + }, + { + "label": "completed_group_6", + "mean": 0.15970462560653687, + "median": 0.1496606469154358, + "p05": 0.10531163960695267, + "p95": 0.24789606034755707 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.0490760803222656, + "group": 7, + "latest_source_mass": 0.09463626891374588, + "layer": 25, + "max_source_mass": 0.1722080111503601, + "mixer_index": 49, + "normalized_entropy": 0.9853972998283761, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10742150247097015, + "median": 0.10384121537208557, + "p05": 0.07562363147735596, + "p95": 0.16456669569015503 + }, + { + "label": "completed_group_1", + "mean": 0.1722080111503601, + "median": 0.17752903699874878, + "p05": 0.12821395695209503, + "p95": 0.20726709067821503 + }, + { + "label": "completed_group_2", + "mean": 0.14153704047203064, + "median": 0.14094984531402588, + "p05": 0.1209082081913948, + "p95": 0.16446956992149353 + }, + { + "label": "completed_group_3", + "mean": 0.1159135028719902, + "median": 0.11460001766681671, + "p05": 0.09666494280099869, + "p95": 0.14028386771678925 + }, + { + "label": "completed_group_4", + "mean": 0.10868977010250092, + "median": 0.10807925462722778, + "p05": 0.09490883350372314, + "p95": 0.124228335916996 + }, + { + "label": "completed_group_5", + "mean": 0.1118093952536583, + "median": 0.11115369200706482, + "p05": 0.0934501588344574, + "p95": 0.13121479749679565 + }, + { + "label": "completed_group_6", + "mean": 0.1477845162153244, + "median": 0.143342986702919, + "p05": 0.10309553891420364, + "p95": 0.20393821597099304 + }, + { + "label": "current_group_partial", + "mean": 0.09463626891374588, + "median": 0.0896594375371933, + "p05": 0.07582583278417587, + "p95": 0.13509176671504974 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.051286220550537, + "group": 7, + "latest_source_mass": 0.10080308467149734, + "layer": 26, + "max_source_mass": 0.16089829802513123, + "mixer_index": 50, + "normalized_entropy": 0.9864601526107082, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1131708100438118, + "median": 0.11772071570158005, + "p05": 0.05498850345611572, + "p95": 0.16554684937000275 + }, + { + "label": "completed_group_1", + "mean": 0.16089829802513123, + "median": 0.16788098216056824, + "p05": 0.10815531760454178, + "p95": 0.1984335482120514 + }, + { + "label": "completed_group_2", + "mean": 0.14448881149291992, + "median": 0.14400863647460938, + "p05": 0.12385094165802002, + "p95": 0.1663750410079956 + }, + { + "label": "completed_group_3", + "mean": 0.11956015229225159, + "median": 0.11957795917987823, + "p05": 0.09828256070613861, + "p95": 0.14187976717948914 + }, + { + "label": "completed_group_4", + "mean": 0.10795272141695023, + "median": 0.10684113204479218, + "p05": 0.09203589707612991, + "p95": 0.12730176746845245 + }, + { + "label": "completed_group_5", + "mean": 0.11352235078811646, + "median": 0.1128544956445694, + "p05": 0.09500996023416519, + "p95": 0.1338450163602829 + }, + { + "label": "completed_group_6", + "mean": 0.13960374891757965, + "median": 0.13686895370483398, + "p05": 0.09854506701231003, + "p95": 0.19470667839050293 + }, + { + "label": "current_group_partial", + "mean": 0.10080308467149734, + "median": 0.0929388776421547, + "p05": 0.07505455613136292, + "p95": 0.15377722680568695 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0247201919555664, + "group": 7, + "latest_source_mass": 0.08503472805023193, + "layer": 26, + "max_source_mass": 0.1894242763519287, + "mixer_index": 51, + "normalized_entropy": 0.9736845933740154, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.101108118891716, + "median": 0.09478609263896942, + "p05": 0.06526951491832733, + "p95": 0.17756468057632446 + }, + { + "label": "completed_group_1", + "mean": 0.1894242763519287, + "median": 0.19696936011314392, + "p05": 0.12617433071136475, + "p95": 0.23825061321258545 + }, + { + "label": "completed_group_2", + "mean": 0.1531166434288025, + "median": 0.15302017331123352, + "p05": 0.11950686573982239, + "p95": 0.18814897537231445 + }, + { + "label": "completed_group_3", + "mean": 0.11718504130840302, + "median": 0.11499293893575668, + "p05": 0.09171593189239502, + "p95": 0.15106697380542755 + }, + { + "label": "completed_group_4", + "mean": 0.10284576565027237, + "median": 0.10174860805273056, + "p05": 0.08644768595695496, + "p95": 0.1233588457107544 + }, + { + "label": "completed_group_5", + "mean": 0.10429200530052185, + "median": 0.10377363860607147, + "p05": 0.08251910656690598, + "p95": 0.1272168904542923 + }, + { + "label": "completed_group_6", + "mean": 0.14699342846870422, + "median": 0.14241871237754822, + "p05": 0.09243378043174744, + "p95": 0.21625562012195587 + }, + { + "label": "current_group_partial", + "mean": 0.08503472805023193, + "median": 0.07344722747802734, + "p05": 0.06163846701383591, + "p95": 0.15428426861763 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.047537326812744, + "group": 7, + "latest_source_mass": 0.09834913909435272, + "layer": 27, + "max_source_mass": 0.16533835232257843, + "mixer_index": 52, + "normalized_entropy": 0.9846573158092636, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111423671245575, + "median": 0.11424723267555237, + "p05": 0.05927120894193649, + "p95": 0.1600918024778366 + }, + { + "label": "completed_group_1", + "mean": 0.16533835232257843, + "median": 0.17344561219215393, + "p05": 0.10133285820484161, + "p95": 0.21326853334903717 + }, + { + "label": "completed_group_2", + "mean": 0.14553974568843842, + "median": 0.14541476964950562, + "p05": 0.11970965564250946, + "p95": 0.17045748233795166 + }, + { + "label": "completed_group_3", + "mean": 0.12043843418359756, + "median": 0.12007981538772583, + "p05": 0.09830569475889206, + "p95": 0.14429020881652832 + }, + { + "label": "completed_group_4", + "mean": 0.10876711457967758, + "median": 0.10606391727924347, + "p05": 0.08962530642747879, + "p95": 0.13553042709827423 + }, + { + "label": "completed_group_5", + "mean": 0.11067016422748566, + "median": 0.10974282026290894, + "p05": 0.09159430861473083, + "p95": 0.13197509944438934 + }, + { + "label": "completed_group_6", + "mean": 0.13975468277931213, + "median": 0.13449960947036743, + "p05": 0.10173755139112473, + "p95": 0.2017771154642105 + }, + { + "label": "current_group_partial", + "mean": 0.09834913909435272, + "median": 0.09059631079435349, + "p05": 0.07406806200742722, + "p95": 0.15292926132678986 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0282247066497803, + "group": 7, + "latest_source_mass": 0.08871228992938995, + "layer": 27, + "max_source_mass": 0.18681631982326508, + "mixer_index": 53, + "normalized_entropy": 0.9753699086973703, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.100978784263134, + "median": 0.0939556136727333, + "p05": 0.06883562356233597, + "p95": 0.17509761452674866 + }, + { + "label": "completed_group_1", + "mean": 0.18681631982326508, + "median": 0.19304953515529633, + "p05": 0.12605154514312744, + "p95": 0.2314973771572113 + }, + { + "label": "completed_group_2", + "mean": 0.15267829596996307, + "median": 0.15211889147758484, + "p05": 0.11771412938833237, + "p95": 0.18910512328147888 + }, + { + "label": "completed_group_3", + "mean": 0.11643697321414948, + "median": 0.11495602130889893, + "p05": 0.08927997201681137, + "p95": 0.14835557341575623 + }, + { + "label": "completed_group_4", + "mean": 0.10204558074474335, + "median": 0.10098521411418915, + "p05": 0.08531612157821655, + "p95": 0.12325739860534668 + }, + { + "label": "completed_group_5", + "mean": 0.10493150353431702, + "median": 0.10501007735729218, + "p05": 0.08253021538257599, + "p95": 0.12569722533226013 + }, + { + "label": "completed_group_6", + "mean": 0.14740024507045746, + "median": 0.14431840181350708, + "p05": 0.09808903187513351, + "p95": 0.2124406099319458 + }, + { + "label": "current_group_partial", + "mean": 0.08871228992938995, + "median": 0.07655761390924454, + "p05": 0.06286359578371048, + "p95": 0.16396106779575348 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.053830623626709, + "group": 7, + "latest_source_mass": 0.11169404536485672, + "layer": 28, + "max_source_mass": 0.1550440490245819, + "mixer_index": 54, + "normalized_entropy": 0.9876837518440468, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11781683564186096, + "median": 0.12262599170207977, + "p05": 0.05666963383555412, + "p95": 0.16679181158542633 + }, + { + "label": "completed_group_1", + "mean": 0.1550440490245819, + "median": 0.1601896733045578, + "p05": 0.09854641556739807, + "p95": 0.19885312020778656 + }, + { + "label": "completed_group_2", + "mean": 0.1398259699344635, + "median": 0.1393846720457077, + "p05": 0.12018831819295883, + "p95": 0.16097646951675415 + }, + { + "label": "completed_group_3", + "mean": 0.11913558095693588, + "median": 0.1185217946767807, + "p05": 0.09514695405960083, + "p95": 0.14609722793102264 + }, + { + "label": "completed_group_4", + "mean": 0.10805961489677429, + "median": 0.10712219774723053, + "p05": 0.08963555097579956, + "p95": 0.12859195470809937 + }, + { + "label": "completed_group_5", + "mean": 0.11164885014295578, + "median": 0.11060681939125061, + "p05": 0.09239380061626434, + "p95": 0.13463559746742249 + }, + { + "label": "completed_group_6", + "mean": 0.13677509129047394, + "median": 0.13695205748081207, + "p05": 0.09874161332845688, + "p95": 0.17497648298740387 + }, + { + "label": "current_group_partial", + "mean": 0.11169404536485672, + "median": 0.10361628234386444, + "p05": 0.07534734159708023, + "p95": 0.17376387119293213 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.017721176147461, + "group": 7, + "latest_source_mass": 0.08305738866329193, + "layer": 28, + "max_source_mass": 0.19308976829051971, + "mixer_index": 55, + "normalized_entropy": 0.9703187782415296, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09720280766487122, + "median": 0.08876170963048935, + "p05": 0.0643138587474823, + "p95": 0.18079209327697754 + }, + { + "label": "completed_group_1", + "mean": 0.19308976829051971, + "median": 0.19938363134860992, + "p05": 0.1292668581008911, + "p95": 0.24086137115955353 + }, + { + "label": "completed_group_2", + "mean": 0.16317343711853027, + "median": 0.1630440354347229, + "p05": 0.12323655933141708, + "p95": 0.20307078957557678 + }, + { + "label": "completed_group_3", + "mean": 0.11856385320425034, + "median": 0.11627572774887085, + "p05": 0.09080541133880615, + "p95": 0.15461181104183197 + }, + { + "label": "completed_group_4", + "mean": 0.10023905336856842, + "median": 0.09881964325904846, + "p05": 0.08244071900844574, + "p95": 0.1235516369342804 + }, + { + "label": "completed_group_5", + "mean": 0.10130266845226288, + "median": 0.10165216773748398, + "p05": 0.07800924777984619, + "p95": 0.12226968258619308 + }, + { + "label": "completed_group_6", + "mean": 0.14337103068828583, + "median": 0.1400463879108429, + "p05": 0.08998478949069977, + "p95": 0.21036171913146973 + }, + { + "label": "current_group_partial", + "mean": 0.08305738866329193, + "median": 0.07284967601299286, + "p05": 0.05788487195968628, + "p95": 0.15074607729911804 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0608584880828857, + "group": 8, + "latest_source_mass": 0.1310214400291443, + "layer": 29, + "max_source_mass": 0.1425117403268814, + "mixer_index": 56, + "normalized_entropy": 0.9910634402437022, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12434183061122894, + "median": 0.12991920113563538, + "p05": 0.07406175881624222, + "p95": 0.15820357203483582 + }, + { + "label": "completed_group_1", + "mean": 0.14232872426509857, + "median": 0.14891478419303894, + "p05": 0.08827720582485199, + "p95": 0.17772579193115234 + }, + { + "label": "completed_group_2", + "mean": 0.13082580268383026, + "median": 0.13057959079742432, + "p05": 0.11437083035707474, + "p95": 0.1484394371509552 + }, + { + "label": "completed_group_3", + "mean": 0.11684279143810272, + "median": 0.11654500663280487, + "p05": 0.09846262633800507, + "p95": 0.13625562191009521 + }, + { + "label": "completed_group_4", + "mean": 0.10329097509384155, + "median": 0.10113433003425598, + "p05": 0.08762617409229279, + "p95": 0.12809985876083374 + }, + { + "label": "completed_group_5", + "mean": 0.10883669555187225, + "median": 0.10607670247554779, + "p05": 0.08987759053707123, + "p95": 0.13684110343456268 + }, + { + "label": "completed_group_6", + "mean": 0.1425117403268814, + "median": 0.14483797550201416, + "p05": 0.10676668584346771, + "p95": 0.16993491351604462 + }, + { + "label": "completed_group_7", + "mean": 0.1310214400291443, + "median": 0.12577015161514282, + "p05": 0.09179449081420898, + "p95": 0.18496999144554138 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.142296314239502, + "group": 8, + "latest_source_mass": 0.12039357423782349, + "layer": 29, + "max_source_mass": 0.16583400964736938, + "mixer_index": 57, + "normalized_entropy": 0.9750010701394441, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08636108040809631, + "median": 0.08012399077415466, + "p05": 0.05614824593067169, + "p95": 0.1551167219877243 + }, + { + "label": "completed_group_1", + "mean": 0.16583400964736938, + "median": 0.1709868609905243, + "p05": 0.10593608021736145, + "p95": 0.2091270387172699 + }, + { + "label": "completed_group_2", + "mean": 0.14278221130371094, + "median": 0.14347383379936218, + "p05": 0.10352978855371475, + "p95": 0.17850933969020844 + }, + { + "label": "completed_group_3", + "mean": 0.1055997759103775, + "median": 0.1052464097738266, + "p05": 0.07856260240077972, + "p95": 0.1339757740497589 + }, + { + "label": "completed_group_4", + "mean": 0.08843953162431717, + "median": 0.08753003180027008, + "p05": 0.07198000699281693, + "p95": 0.1094239354133606 + }, + { + "label": "completed_group_5", + "mean": 0.08854378759860992, + "median": 0.08859115839004517, + "p05": 0.06814242154359818, + "p95": 0.10749459266662598 + }, + { + "label": "completed_group_6", + "mean": 0.12657995522022247, + "median": 0.12664274871349335, + "p05": 0.08023036271333694, + "p95": 0.1752617508172989 + }, + { + "label": "completed_group_7", + "mean": 0.07546606659889221, + "median": 0.06708197295665741, + "p05": 0.05449982360005379, + "p95": 0.12993836402893066 + }, + { + "label": "current_group_partial", + "mean": 0.12039357423782349, + "median": 0.11719904839992523, + "p05": 0.0764557495713234, + "p95": 0.17428058385849 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.191580057144165, + "group": 8, + "latest_source_mass": 0.10807353258132935, + "layer": 30, + "max_source_mass": 0.12397187203168869, + "mixer_index": 58, + "normalized_entropy": 0.9974310681528523, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1060912236571312, + "median": 0.10528025031089783, + "p05": 0.09126897901296616, + "p95": 0.12398397922515869 + }, + { + "label": "completed_group_1", + "mean": 0.12397187203168869, + "median": 0.12518228590488434, + "p05": 0.10349762439727783, + "p95": 0.14397969841957092 + }, + { + "label": "completed_group_2", + "mean": 0.11601400375366211, + "median": 0.11583627760410309, + "p05": 0.10074462741613388, + "p95": 0.13201698660850525 + }, + { + "label": "completed_group_3", + "mean": 0.1091354638338089, + "median": 0.10811793804168701, + "p05": 0.09470823407173157, + "p95": 0.12769103050231934 + }, + { + "label": "completed_group_4", + "mean": 0.10782144963741302, + "median": 0.10718400776386261, + "p05": 0.09581813216209412, + "p95": 0.1218264177441597 + }, + { + "label": "completed_group_5", + "mean": 0.10360676795244217, + "median": 0.1029055118560791, + "p05": 0.09420947730541229, + "p95": 0.11479298770427704 + }, + { + "label": "completed_group_6", + "mean": 0.12169313430786133, + "median": 0.12091503292322159, + "p05": 0.10930042713880539, + "p95": 0.13614466786384583 + }, + { + "label": "completed_group_7", + "mean": 0.10359255224466324, + "median": 0.1012670248746872, + "p05": 0.09212177246809006, + "p95": 0.12151782214641571 + }, + { + "label": "current_group_partial", + "mean": 0.10807353258132935, + "median": 0.10744988918304443, + "p05": 0.09496128559112549, + "p95": 0.12403780221939087 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1432788372039795, + "group": 8, + "latest_source_mass": 0.08614903688430786, + "layer": 30, + "max_source_mass": 0.16873395442962646, + "mixer_index": 59, + "normalized_entropy": 0.9754482356111087, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08927534520626068, + "median": 0.08175399154424667, + "p05": 0.05808858573436737, + "p95": 0.16859911382198334 + }, + { + "label": "completed_group_1", + "mean": 0.16873395442962646, + "median": 0.17203307151794434, + "p05": 0.12107796967029572, + "p95": 0.2058679312467575 + }, + { + "label": "completed_group_2", + "mean": 0.14930182695388794, + "median": 0.14880773425102234, + "p05": 0.10963106155395508, + "p95": 0.18977254629135132 + }, + { + "label": "completed_group_3", + "mean": 0.112123504281044, + "median": 0.11111646890640259, + "p05": 0.08233093470335007, + "p95": 0.14486809074878693 + }, + { + "label": "completed_group_4", + "mean": 0.09766747057437897, + "median": 0.09729938209056854, + "p05": 0.08042105287313461, + "p95": 0.11693428456783295 + }, + { + "label": "completed_group_5", + "mean": 0.0965762585401535, + "median": 0.0965472012758255, + "p05": 0.07654033601284027, + "p95": 0.1171490028500557 + }, + { + "label": "completed_group_6", + "mean": 0.1252625733613968, + "median": 0.12540382146835327, + "p05": 0.08324124664068222, + "p95": 0.1696549952030182 + }, + { + "label": "completed_group_7", + "mean": 0.07491002976894379, + "median": 0.06726562976837158, + "p05": 0.055054910480976105, + "p95": 0.1270829290151596 + }, + { + "label": "current_group_partial", + "mean": 0.08614903688430786, + "median": 0.07982201874256134, + "p05": 0.06172287464141846, + "p95": 0.13157080113887787 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1724743843078613, + "group": 8, + "latest_source_mass": 0.10534654557704926, + "layer": 31, + "max_source_mass": 0.12893787026405334, + "mixer_index": 60, + "normalized_entropy": 0.9887357017195011, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1105542704463005, + "median": 0.12003511190414429, + "p05": 0.041383370757102966, + "p95": 0.15075469017028809 + }, + { + "label": "completed_group_1", + "mean": 0.12893787026405334, + "median": 0.1339881867170334, + "p05": 0.07801777124404907, + "p95": 0.16926369071006775 + }, + { + "label": "completed_group_2", + "mean": 0.11523330211639404, + "median": 0.1141275092959404, + "p05": 0.09732259064912796, + "p95": 0.1368464231491089 + }, + { + "label": "completed_group_3", + "mean": 0.10665617883205414, + "median": 0.10615617036819458, + "p05": 0.08223703503608704, + "p95": 0.1332816481590271 + }, + { + "label": "completed_group_4", + "mean": 0.10366716235876083, + "median": 0.10414719581604004, + "p05": 0.08117569237947464, + "p95": 0.12477125227451324 + }, + { + "label": "completed_group_5", + "mean": 0.10828493535518646, + "median": 0.10714839398860931, + "p05": 0.08299056440591812, + "p95": 0.136210098862648 + }, + { + "label": "completed_group_6", + "mean": 0.11819782108068466, + "median": 0.11870056390762329, + "p05": 0.08487735688686371, + "p95": 0.14779172837734222 + }, + { + "label": "completed_group_7", + "mean": 0.10312190651893616, + "median": 0.09999410808086395, + "p05": 0.07080516219139099, + "p95": 0.1448727399110794 + }, + { + "label": "current_group_partial", + "mean": 0.10534654557704926, + "median": 0.09821351617574692, + "p05": 0.06854747235774994, + "p95": 0.16421541571617126 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1452555656433105, + "group": 8, + "latest_source_mass": 0.07998920977115631, + "layer": 31, + "max_source_mass": 0.1701684445142746, + "mixer_index": 61, + "normalized_entropy": 0.9763478834940427, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09202821552753448, + "median": 0.08689712733030319, + "p05": 0.055717833340168, + "p95": 0.16549156606197357 + }, + { + "label": "completed_group_1", + "mean": 0.1701684445142746, + "median": 0.17422428727149963, + "p05": 0.11712116748094559, + "p95": 0.21046262979507446 + }, + { + "label": "completed_group_2", + "mean": 0.14628148078918457, + "median": 0.14571070671081543, + "p05": 0.11122585833072662, + "p95": 0.18218384683132172 + }, + { + "label": "completed_group_3", + "mean": 0.11057832837104797, + "median": 0.10866793990135193, + "p05": 0.08514636754989624, + "p95": 0.14279140532016754 + }, + { + "label": "completed_group_4", + "mean": 0.09616327285766602, + "median": 0.09543465077877045, + "p05": 0.08178231120109558, + "p95": 0.1130019798874855 + }, + { + "label": "completed_group_5", + "mean": 0.09830239415168762, + "median": 0.09682399034500122, + "p05": 0.07916976511478424, + "p95": 0.12123610824346542 + }, + { + "label": "completed_group_6", + "mean": 0.12624147534370422, + "median": 0.1266379952430725, + "p05": 0.08612610399723053, + "p95": 0.166628897190094 + }, + { + "label": "completed_group_7", + "mean": 0.08024715632200241, + "median": 0.0715465098619461, + "p05": 0.05839171260595322, + "p95": 0.13830669224262238 + }, + { + "label": "current_group_partial", + "mean": 0.07998920977115631, + "median": 0.07393841445446014, + "p05": 0.0605103000998497, + "p95": 0.11983495950698853 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1762189865112305, + "group": 8, + "latest_source_mass": 0.09661206603050232, + "layer": 32, + "max_source_mass": 0.13803134858608246, + "mixer_index": 62, + "normalized_entropy": 0.990439943626311, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09908370673656464, + "median": 0.10031558573246002, + "p05": 0.06232558190822601, + "p95": 0.14320872724056244 + }, + { + "label": "completed_group_1", + "mean": 0.13803134858608246, + "median": 0.14098677039146423, + "p05": 0.10051311552524567, + "p95": 0.1691289246082306 + }, + { + "label": "completed_group_2", + "mean": 0.13144394755363464, + "median": 0.13024751842021942, + "p05": 0.10748467594385147, + "p95": 0.15961234271526337 + }, + { + "label": "completed_group_3", + "mean": 0.11672185361385345, + "median": 0.11463914811611176, + "p05": 0.09377485513687134, + "p95": 0.14699220657348633 + }, + { + "label": "completed_group_4", + "mean": 0.1074555367231369, + "median": 0.1059759333729744, + "p05": 0.08800189942121506, + "p95": 0.13215802609920502 + }, + { + "label": "completed_group_5", + "mean": 0.10124591737985611, + "median": 0.10008268058300018, + "p05": 0.08680262416601181, + "p95": 0.11920121312141418 + }, + { + "label": "completed_group_6", + "mean": 0.12084682285785675, + "median": 0.11794044077396393, + "p05": 0.09386572986841202, + "p95": 0.15664330124855042 + }, + { + "label": "completed_group_7", + "mean": 0.08855880051851273, + "median": 0.0855959877371788, + "p05": 0.0724000632762909, + "p95": 0.11697207391262054 + }, + { + "label": "current_group_partial", + "mean": 0.09661206603050232, + "median": 0.09512065351009369, + "p05": 0.07854700088500977, + "p95": 0.12120820581912994 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1389729976654053, + "group": 8, + "latest_source_mass": 0.07340531051158905, + "layer": 32, + "max_source_mass": 0.17082780599594116, + "mixer_index": 63, + "normalized_entropy": 0.9734885635853232, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09089215844869614, + "median": 0.08436901867389679, + "p05": 0.05205666273832321, + "p95": 0.17794299125671387 + }, + { + "label": "completed_group_1", + "mean": 0.17082780599594116, + "median": 0.17341303825378418, + "p05": 0.1192600280046463, + "p95": 0.21146632730960846 + }, + { + "label": "completed_group_2", + "mean": 0.14892268180847168, + "median": 0.14870190620422363, + "p05": 0.11514535546302795, + "p95": 0.18527255952358246 + }, + { + "label": "completed_group_3", + "mean": 0.11308471858501434, + "median": 0.11074972152709961, + "p05": 0.08665508031845093, + "p95": 0.14719238877296448 + }, + { + "label": "completed_group_4", + "mean": 0.10078474879264832, + "median": 0.10026010870933533, + "p05": 0.08329866826534271, + "p95": 0.1204671636223793 + }, + { + "label": "completed_group_5", + "mean": 0.09951519966125488, + "median": 0.09887547045946121, + "p05": 0.0807466059923172, + "p95": 0.12053526192903519 + }, + { + "label": "completed_group_6", + "mean": 0.12574951350688934, + "median": 0.12600116431713104, + "p05": 0.08480837196111679, + "p95": 0.17120841145515442 + }, + { + "label": "completed_group_7", + "mean": 0.07681785523891449, + "median": 0.06763558089733124, + "p05": 0.05441128462553024, + "p95": 0.13893920183181763 + }, + { + "label": "current_group_partial", + "mean": 0.07340531051158905, + "median": 0.06719464063644409, + "p05": 0.054995350539684296, + "p95": 0.1120883971452713 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1635239124298096, + "group": null, + "latest_source_mass": 0.11434720456600189, + "layer": null, + "max_source_mass": 0.14857198297977448, + "mixer_index": null, + "normalized_entropy": 0.9846621664193896, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1242082417011261, + "median": 0.12387055158615112, + "p05": 0.08946356177330017, + "p95": 0.16199731826782227 + }, + { + "label": "completed_group_1", + "mean": 0.0880277156829834, + "median": 0.08818831294775009, + "p05": 0.06631279736757278, + "p95": 0.10954886674880981 + }, + { + "label": "completed_group_2", + "mean": 0.0938715785741806, + "median": 0.0940513014793396, + "p05": 0.07511411607265472, + "p95": 0.11214712262153625 + }, + { + "label": "completed_group_3", + "mean": 0.09339353442192078, + "median": 0.09318573027849197, + "p05": 0.07264634221792221, + "p95": 0.11525531113147736 + }, + { + "label": "completed_group_4", + "mean": 0.09749531745910645, + "median": 0.0970197319984436, + "p05": 0.07785478234291077, + "p95": 0.12001418322324753 + }, + { + "label": "completed_group_5", + "mean": 0.10569863021373749, + "median": 0.10433581471443176, + "p05": 0.08659540861845016, + "p95": 0.1311836838722229 + }, + { + "label": "completed_group_6", + "mean": 0.13438579440116882, + "median": 0.13242986798286438, + "p05": 0.08274075388908386, + "p95": 0.1898222714662552 + }, + { + "label": "completed_group_7", + "mean": 0.14857198297977448, + "median": 0.14630873501300812, + "p05": 0.11486123502254486, + "p95": 0.19245916604995728 + }, + { + "label": "completed_group_8", + "mean": 0.11434720456600189, + "median": 0.1131601482629776, + "p05": 0.05654018372297287, + "p95": 0.17325904965400696 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "c897bc93974dc10587c9be423f78a7546e14b4b7a8b97f6d5a3e52fd21bd21e8", + "optimizer_state_before": "c897bc93974dc10587c9be423f78a7546e14b4b7a8b97f6d5a3e52fd21bd21e8", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "d7913db2408d5062b288c4b95e9ffb2f0d06e58bee52e81d9300885e31436501", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.01139580737799406, + 0.014748631045222282, + 0.012339585460722446, + 0.0120800556614995, + 0.01439172774553299, + 0.0171960536390543, + 0.01820177212357521, + 0.01519495714455843, + 0.016362585127353668, + 0.015944689512252808, + 0.017555685713887215, + 0.01772458106279373, + 0.025544000789523125, + 0.019797103479504585, + 0.02002105489373207, + 0.017730148509144783, + 0.022795414552092552, + 0.02134518511593342, + 0.021650495007634163, + 0.03255617991089821, + 0.033973678946495056, + 0.031387828290462494, + 0.019825322553515434, + 0.013387269340455532, + 0.015395586378872395, + 0.011897368356585503, + 0.014087310060858727, + 0.014466723427176476, + 0.006911881268024445, + 0.005224450025707483, + 0.00758614856749773, + 0.005234629847109318 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.39030686973274e-05, + "first_to_last_ratio": 5.095393144356493, + "imbalance_abs_log_ratio": 1.628336826476301, + "last_quartile_mean": 6.653670823197899e-06, + "mean": 1.8118416377888025e-05, + "normalized": [ + 2.2929757071570536, + 2.1131249790843154, + 2.066690915233284, + 1.9927541789217667, + 1.8585972056365079, + 1.693285810246388, + 1.5304369916151304, + 1.4216847468656184, + 1.289196531005685, + 1.185594524249146, + 1.0961124019425812, + 1.0141508318397905, + 1.0609145912379738, + 0.974788958343087, + 0.8967359470227997, + 0.8249692378890231, + 0.8460887274678097, + 0.7831360031144634, + 0.7104172027135063, + 0.6493842722397042, + 0.8100416294668759, + 0.7277529238767537, + 0.6447870573653334, + 0.5785188547546978, + 0.45250252031967675, + 0.4317242716558631, + 0.41742992665867057, + 0.40386477203384563, + 0.31920966761853126, + 0.30772885346331863, + 0.30340232726708727, + 0.30199743169371174 + ], + "peak_layer": 1, + "peak_normalized": 2.2929757071570536, + "population_cv": 0.5811777617331019, + "rest_mean": 1.931718248602852e-05, + "spike_contrast": 0.6028352945545675, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1645079393929336e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.1545088606653735e-05, + 3.8286478229565546e-05, + 3.744516652659513e-05, + 3.610554995248094e-05, + 3.367483805050142e-05, + 3.067965735681355e-05, + 2.7729094654205255e-05, + 2.575867620180361e-05, + 2.3358199541689828e-05, + 2.1481095245690085e-05, + 1.9859820895362645e-05, + 1.8374807041254826e-05, + 1.9222092305426486e-05, + 1.7661632227827795e-05, + 1.6247435269178823e-05, + 1.4947136151022278e-05, + 1.5329787856899202e-05, + 1.418918418494286e-05, + 1.2871634680777788e-05, + 1.1765814633690752e-05, + 1.4676671526103746e-05, + 1.3185730495024472e-05, + 1.1682520380418282e-05, + 1.048184549290454e-05, + 8.19862907519564e-06, + 7.822160114301369e-06, + 7.563169219793053e-06, + 7.3173901000700425e-06, + 5.7835736697597895e-06, + 5.575559498538496e-06, + 5.497169695445336e-06, + 5.4717152124794666e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013720810875383904, + "first_to_last_ratio": 5.154319968637833, + "imbalance_abs_log_ratio": 1.6398351919170495, + "last_quartile_mean": 2.6620021571943653e-05, + "mean": 7.278344395444947e-05, + "normalized": [ + 2.2311752925780666, + 2.1140511136094218, + 2.0818052914231253, + 2.0170825228677507, + 1.9039940200491434, + 1.731017362744226, + 1.558296121653412, + 1.4438221733118084, + 1.2971888744180369, + 1.1848962105350367, + 1.0932162849170117, + 1.0098532310596642, + 1.045979580290602, + 0.9573868419617885, + 0.8796552407811828, + 0.8149140782597045, + 0.8407378071284205, + 0.7792422299357804, + 0.710697365168306, + 0.6467959271863449, + 0.8010942116833898, + 0.7191139042344474, + 0.6388296904045196, + 0.5732121153785235, + 0.44847063516438496, + 0.42886656601890155, + 0.4146320104757948, + 0.40203589246501603, + 0.31840897516456645, + 0.30737981813304965, + 0.30352323323450536, + 0.3026253777640678 + ], + "peak_layer": 1, + "peak_normalized": 2.2311752925780666, + "population_cv": 0.5853894325437503, + "rest_mean": 7.768764482063018e-05, + "spike_contrast": 0.5959861363280557, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.630075927707367e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001623926218599081, + 0.00015386792074423283, + 0.00015152095875237137, + 0.00014681021275464445, + 0.00013857924204785377, + 0.0001259894052054733, + 0.00011341815843479708, + 0.00010508635023143142, + 9.441387373954058e-05, + 8.62408269313164e-05, + 7.956804620334879e-05, + 7.350059604505077e-05, + 7.61299961595796e-05, + 6.96819115546532e-05, + 6.402433791663498e-05, + 5.931225314270705e-05, + 6.119179306551814e-05, + 5.67159331694711e-05, + 5.17270018463023e-05, + 4.7076035116333514e-05, + 5.8306395658291876e-05, + 5.2339586545713246e-05, + 4.649622496799566e-05, + 4.1720351873664185e-05, + 3.2641237339703366e-05, + 3.1214385671773925e-05, + 3.0178345696185715e-05, + 2.926155684690457e-05, + 2.3174901798483916e-05, + 2.237216176581569e-05, + 2.2091466234996915e-05, + 2.2026117221685126e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12167752534151077, + "first_to_last_ratio": 5.154320181520886, + "imbalance_abs_log_ratio": 1.6398352332189183, + "last_quartile_mean": 0.02360690082423389, + "mean": 0.06454508687602356, + "normalized": [ + 2.231175139109246, + 2.114051177053498, + 2.081805426225287, + 2.0170825984642944, + 1.903994030265683, + 1.7310172806323025, + 1.5582960978629543, + 1.4438222279772381, + 1.29718883510768, + 1.184896287276894, + 1.093216349808963, + 1.0098533014799294, + 1.045979536169102, + 0.9573867748507232, + 0.879655201241054, + 0.8149140774782775, + 0.8407378162482237, + 0.7792421838807176, + 0.7106973963780332, + 0.6467959263501122, + 0.8010942473612533, + 0.7191138457790023, + 0.638829720198085, + 0.5732121198325099, + 0.448470614548316, + 0.42886653521122853, + 0.4146320149591386, + 0.4020358814661456, + 0.31840895515555, + 0.30737981361343175, + 0.30352322421536726, + 0.3026253637997583 + ], + "peak_layer": 1, + "peak_normalized": 2.231175139109246, + "population_cv": 0.5853894401227376, + "rest_mean": 0.06889418133155063, + "spike_contrast": 0.595986134425167, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.04105997681617737, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.14401139318943024, + 0.1364516168832779, + 0.13437031209468842, + 0.13019277155399323, + 0.12289346009492874, + 0.11172866076231003, + 0.1005803570151329, + 0.09319163113832474, + 0.083727166056633, + 0.0764792338013649, + 0.07056174427270889, + 0.06518106907606125, + 0.06751284003257751, + 0.06179461255669594, + 0.05677742138504982, + 0.05259869992733002, + 0.054265495389699936, + 0.05029625445604324, + 0.045872025191783905, + 0.041747499257326126, + 0.051706697791814804, + 0.046415265649557114, + 0.04123331978917122, + 0.03699802607297897, + 0.02894657477736473, + 0.027681227773427963, + 0.0267624594271183, + 0.025949440896511078, + 0.020551733672618866, + 0.019839856773614883, + 0.01959093287587166, + 0.019532980397343636 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.918741178196797e-06, + "first_to_last_ratio": 4.256889559589457, + "imbalance_abs_log_ratio": 1.448538743260589, + "last_quartile_mean": 1.6253043639835596e-06, + "mean": 3.7710106575161717e-06, + "normalized": [ + 3.111036134503483, + 2.3043433850128423, + 2.0000543541988076, + 1.856903042177676, + 1.4761204412583784, + 1.3975532001518793, + 1.3153833234811834, + 1.2163502182025636, + 1.121279589978469, + 1.1184172583382423, + 1.051317552929728, + 0.9856458779932212, + 1.025317558169754, + 0.9223028019770371, + 0.9122247681464483, + 0.8823925941638188, + 0.8042436205537018, + 0.7521991935038654, + 0.7015280489244919, + 0.6885237102932811, + 0.8268157151112371, + 0.7604455810889451, + 0.6890904244634548, + 0.6325142134972163, + 0.49413520276931655, + 0.48659857961175057, + 0.4811786179015675, + 0.4807183548291107, + 0.37702217293214657, + 0.3708307648499783, + 0.37628259255004715, + 0.3812311064363565 + ], + "peak_layer": 1, + "peak_normalized": 3.111036134503483, + "population_cv": 0.6047224613054547, + "rest_mean": 3.994058795763488e-06, + "spike_contrast": 0.6425921205023355, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.566550710980664e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1731750419130549e-05, + 8.689703463460319e-06, + 7.542226285295328e-06, + 7.002401162026217e-06, + 5.566465915762819e-06, + 5.2701880122185685e-06, + 4.9603245315665845e-06, + 4.586869636113988e-06, + 4.22835728386417e-06, + 4.217563400743529e-06, + 3.964529696531827e-06, + 3.7168811104493216e-06, + 3.8664834391966e-06, + 3.478013695712434e-06, + 3.440009322730475e-06, + 3.3275118767051026e-06, + 3.0328112643474014e-06, + 2.836551175278146e-06, + 2.6454697490407852e-06, + 2.59643024946854e-06, + 3.11793087348633e-06, + 2.8676483907474903e-06, + 2.5985673346440308e-06, + 2.385217840128462e-06, + 1.8633891158970073e-06, + 1.8349684296481428e-06, + 1.814529696275713e-06, + 1.812794039324217e-06, + 1.4217546322470298e-06, + 1.3984067663841415e-06, + 1.4189656667440431e-06, + 1.4376265653481823e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00011491361965454416, + "first_to_last_ratio": 4.815696874018557, + "imbalance_abs_log_ratio": 1.5718807645842947, + "last_quartile_mean": 2.3862303351052105e-05, + "mean": 6.221126920991082e-05, + "normalized": [ + 2.275296751999321, + 2.0925224842292702, + 2.024985525325661, + 1.949051152845169, + 1.8210682543299608, + 1.673027934661814, + 1.5224711978555032, + 1.4187861804383168, + 1.274895563153476, + 1.1755832824885524, + 1.0931627851870758, + 1.0178889780531333, + 1.046098089023552, + 0.9645004914703617, + 0.8923564167503102, + 0.8324219825464898, + 0.8548630177301194, + 0.7953291565277032, + 0.7278927794580229, + 0.6660752428029298, + 0.8185859571667472, + 0.7386897130300317, + 0.6593898256040199, + 0.5965066478095707, + 0.4749516886939756, + 0.4558898522635054, + 0.4413896066740623, + 0.4292309588085209, + 0.3268633002656605, + 0.31649448274376557, + 0.312303845893163, + 0.31142685417023475 + ], + "peak_layer": 1, + "peak_normalized": 2.275296751999321, + "population_cv": 0.5648737407715131, + "rest_mean": 6.615563917741249e-05, + "spike_contrast": 0.6184154804352692, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.091167138540186e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014154909877106547, + 0.0001301784795941785, + 0.00012597691966220737, + 0.00012125294597353786, + 0.00011329096741974354, + 0.00010408119123894721, + 9.471486555412412e-05, + 8.826448902254924e-05, + 7.931287109386176e-05, + 7.313452806556597e-05, + 6.800704431952909e-05, + 6.332416523946449e-05, + 6.507908983621746e-05, + 6.000279972795397e-05, + 5.5514625273644924e-05, + 5.1786028052447364e-05, + 5.318211333360523e-05, + 4.947843626723625e-05, + 4.528313365881331e-05, + 4.1437386244069785e-05, + 5.092527135275304e-05, + 4.595482459990308e-05, + 4.102147795492783e-05, + 3.710943565238267e-05, + 2.9547347367042676e-05, + 2.836148632923141e-05, + 2.745940764725674e-05, + 2.6703002731665038e-05, + 2.033458076766692e-05, + 1.9689523469423875e-05, + 1.942881863215007e-05, + 1.9374259863980114e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.519367267785128e-05, + "first_to_last_ratio": 4.049851770423372, + "imbalance_abs_log_ratio": 1.3986802805531915, + "last_quartile_mean": 2.3505470835516462e-05, + "mean": 5.353606127300736e-05, + "normalized": [ + 2.290218656747677, + 2.0391521977718905, + 1.9205616721349126, + 1.8371747485589947, + 1.7099486540024496, + 1.5877756040230135, + 1.4632374899841143, + 1.3769106131996964, + 1.2365773178603239, + 1.1501221443012748, + 1.081965026408811, + 1.0124849839355623, + 1.0387551076173185, + 0.9627440254142965, + 0.895438640452568, + 0.8412318940948519, + 0.8632752234551885, + 0.8102372145057791, + 0.7438108799024353, + 0.6860735938593548, + 0.8437350402761642, + 0.7647523383937855, + 0.6911829721367653, + 0.640164753988397, + 0.5390194638264668, + 0.5259797411671606, + 0.5155398289698411, + 0.5072164058311368, + 0.36377694228816476, + 0.35556193943755593, + 0.35248648635161195, + 0.35288839910243647 + ], + "peak_layer": 1, + "peak_normalized": 2.290218656747677, + "population_cv": 0.522006447212346, + "rest_mean": 5.6552214421875156e-05, + "spike_contrast": 0.6586627004779664, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.724883426912129e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012260928633622825, + 0.00010916817700490355, + 0.00010281930735800415, + 9.835509990807623e-05, + 9.154391591437161e-05, + 8.500325202476233e-05, + 7.833597192075104e-05, + 7.37143709557131e-05, + 6.62014790577814e-05, + 6.157300958875567e-05, + 5.7924145949073136e-05, + 5.420445813797414e-05, + 5.5610857089050114e-05, + 5.154152313480154e-05, + 4.793825792148709e-05, + 4.5036242227070034e-05, + 4.621635525836609e-05, + 4.3376909161452204e-05, + 3.98207048419863e-05, + 3.672967795864679e-05, + 4.5170250814408064e-05, + 4.0941828046925366e-05, + 3.700321394717321e-05, + 3.4271899494342506e-05, + 2.8856979042757303e-05, + 2.8158883651485667e-05, + 2.759997187240515e-05, + 2.715436858125031e-05, + 1.9475184672046453e-05, + 1.903538577607833e-05, + 1.8870738131226972e-05, + 1.8892254956881516e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002514238494768506, + "first_to_last_ratio": 5.858443137507712, + "imbalance_abs_log_ratio": 1.7678838920975262, + "last_quartile_mean": 4.2916495658573695e-05, + "mean": 0.00013052156418780214, + "normalized": [ + 2.175700208718937, + 2.1468533781910737, + 2.156626189980414, + 2.081820074147806, + 1.9605560874678556, + 1.78239871858387, + 1.6139249690844666, + 1.4925302040410395, + 1.335030130570538, + 1.2140953753546508, + 1.1065838540738395, + 1.0144108621297776, + 1.060298570529704, + 0.9686444554003223, + 0.8827093572640272, + 0.8050838137717637, + 0.8356205616435275, + 0.769998030276327, + 0.6982893436799542, + 0.6331263204330165, + 0.7830016139470163, + 0.6976170559290904, + 0.6139283242518799, + 0.5406908798379542, + 0.3991410841103365, + 0.37768194897807716, + 0.3609359356216191, + 0.34776185509470936, + 0.29820014430703956, + 0.284663078565391, + 0.28009631595127243, + 0.2819812580627016 + ], + "peak_layer": 1, + "peak_normalized": 2.175700208718937, + "population_cv": 0.6126337690901321, + "rest_mean": 0.0001400236357801111, + "spike_contrast": 0.5656929071155059, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.92103775893338e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002839757944457233, + 0.0002802106610033661, + 0.00028148622368462384, + 0.0002717224124353379, + 0.00025589484721422195, + 0.00023264146875590086, + 0.0002106520114466548, + 0.00019480737682897598, + 0.00017425022087991238, + 0.0001584656274644658, + 0.00014443305553868413, + 0.0001324024924542755, + 0.00013839182793162763, + 0.0001264289894606918, + 0.0001152126060333103, + 0.0001050807986757718, + 0.00010906650277320296, + 0.00010050134733319283, + 9.114181739278138e-05, + 8.263663767138496e-05, + 0.00010219859541393816, + 9.105406934395432e-05, + 8.013088518055156e-05, + 7.057181937852874e-05, + 5.209651862969622e-05, + 4.929563874611631e-05, + 4.7109922888921574e-05, + 4.5390421291813254e-05, + 3.8921549275983125e-05, + 3.715467028087005e-05, + 3.6558609281200916e-05, + 3.680463487398811e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "490a3ed9260ad0b92a1839e225a59c197ccf4984373e834d9df6ed56c379de39", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.01652366667985916, + 0.013563322834670544, + 0.012220539152622223, + 0.0097816726192832, + 0.008202088065445423, + 0.008006753399968147, + 0.008192610926926136, + 0.008106541819870472, + 0.008609342388808727, + 0.008666405454277992, + 0.009109876118600368, + 0.008975221775472164, + 0.009615846909582615, + 0.009856111370027065, + 0.009463186375796795, + 0.009926749393343925, + 0.010939708910882473, + 0.010758070275187492, + 0.011536303907632828, + 0.012654256075620651, + 0.014695252291858196, + 0.016267452389001846, + 0.01879013702273369, + 0.02227928675711155, + 0.021389417350292206, + 0.023454682901501656, + 0.024781694635748863, + 0.026336129754781723, + 0.026170482859015465, + 0.026235800236463547, + 0.030007388442754745, + 0.031163375824689865 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.201577646905207e-05, + "first_to_last_ratio": 4.932110786508481, + "imbalance_abs_log_ratio": 1.595767047845531, + "last_quartile_mean": 6.491293049748492e-06, + "mean": 1.705986865374598e-05, + "normalized": [ + 2.1062843954634993, + 2.172473088763457, + 2.1330173481310855, + 2.026075301568175, + 1.8981950970169568, + 1.7049265180158335, + 1.5444094898875882, + 1.4279941247542707, + 1.304143449910112, + 1.1896224712687855, + 1.096253324873095, + 1.0266908387336267, + 1.0572033947314374, + 0.9622174038057444, + 0.8910292233541588, + 0.8167935724505643, + 0.8366571216551983, + 0.7772132951252987, + 0.7011662644209955, + 0.6475304539316058, + 0.7729030247398233, + 0.689598266398322, + 0.6131100793867855, + 0.5604863427749205, + 0.46076833070284307, + 0.44397840145700934, + 0.4298672372830245, + 0.41973114466892847, + 0.32654557794018146, + 0.3218478363715071, + 0.32040670799723164, + 0.320860872417935 + ], + "peak_layer": 2, + "peak_normalized": 2.172473088763457, + "population_cv": 0.5805644210561929, + "rest_mean": 1.8262358110034796e-05, + "spike_contrast": 0.5785904277049709, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0566425589786377e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5932935134042054e-05, + 3.706210554810241e-05, + 3.638899579527788e-05, + 3.456457852735184e-05, + 3.238295903429389e-05, + 2.90858224616386e-05, + 2.6347423045081086e-05, + 2.4361392206628807e-05, + 2.224851596110966e-05, + 2.029480310739018e-05, + 1.870193773356732e-05, + 1.7515210856799968e-05, + 1.8035751054412685e-05, + 1.6415302525274456e-05, + 1.520084151707124e-05, + 1.3934391063230578e-05, + 1.4273260603658855e-05, + 1.3259156730782706e-05, + 1.1961804375459906e-05, + 1.1046784493373707e-05, + 1.3185624084144365e-05, + 1.1764455848606303e-05, + 1.0459577424626332e-05, + 9.56182338995859e-06, + 7.860647201596294e-06, + 7.574213213956682e-06, + 7.333478606597055e-06, + 7.1605581979383714e-06, + 5.570824669121066e-06, + 5.490681814990239e-06, + 5.466096354211913e-06, + 5.473844339576317e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001296065474889474, + "first_to_last_ratio": 4.981819427000543, + "imbalance_abs_log_ratio": 1.6057951711009326, + "last_quartile_mean": 2.6015906314569293e-05, + "mean": 6.850327099527931e-05, + "normalized": [ + 2.0631317176336217, + 2.1730824382529383, + 2.146215362336739, + 2.044146942206416, + 1.9438216945498896, + 1.7429561173297294, + 1.573898233305266, + 1.4485558265602336, + 1.3070218414216235, + 1.1814367373147445, + 1.0902159864460161, + 1.0200822906031923, + 1.0387457708358308, + 0.9441162107193557, + 0.8721012120223935, + 0.8075112656119094, + 0.832890665117298, + 0.7730404866597239, + 0.7008773204267549, + 0.6442617083188491, + 0.7641479384870525, + 0.683465457573547, + 0.6090232197249015, + 0.557044614214288, + 0.45807193966692283, + 0.441967695148959, + 0.42799701004285134, + 0.41848476044449356, + 0.3263378031557906, + 0.3220971579939778, + 0.3213355025096009, + 0.32191707336508907 + ], + "peak_layer": 2, + "peak_normalized": 2.1730824382529383, + "population_cv": 0.5858238895510721, + "rest_mean": 7.339553822357966e-05, + "spike_contrast": 0.573400358946299, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.208502796245739e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014133127115201205, + 0.00014886325516272336, + 0.0001470227725803852, + 0.00014003075193613768, + 0.00013315814430825412, + 0.0001193981952383183, + 0.00010781717719510198, + 9.92308123386465e-05, + 8.953527139965445e-05, + 8.093228098005056e-05, + 7.46833611628972e-05, + 6.987897359067574e-05, + 7.115748303476721e-05, + 6.467504863394424e-05, + 5.974178566248156e-05, + 5.53171630599536e-05, + 5.7055734941968694e-05, + 5.295580194797367e-05, + 4.80123890156392e-05, + 4.413403439684771e-05, + 5.234663331066258e-05, + 4.681961945607327e-05, + 4.172008266323246e-05, + 3.815937816398218e-05, + 3.137942621833645e-05, + 3.0276232791948132e-05, + 2.9319195164134726e-05, + 2.8667574952123687e-05, + 2.235520696558524e-05, + 2.2064708900870755e-05, + 2.2012533008819446e-05, + 2.2052372514735907e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11493638530373573, + "first_to_last_ratio": 4.98181949999669, + "imbalance_abs_log_ratio": 1.60579518575344, + "last_quartile_mean": 0.02307116612792015, + "mean": 0.060749385971575975, + "normalized": [ + 2.0631316304082516, + 2.1730824619815525, + 2.1462154581134283, + 2.0441469912308308, + 1.9438217938629945, + 1.7429561579567567, + 1.5738982612782981, + 1.4485559076637762, + 1.3070217763738827, + 1.1814367092114706, + 1.0902160293917031, + 1.0200822388374744, + 1.0387456719603603, + 0.9441161484281956, + 0.8721011745721409, + 0.8075112852333144, + 0.8328906721337106, + 0.7730404424285328, + 0.7008773139661862, + 0.6442617114020169, + 0.7641479159706749, + 0.6834654683906575, + 0.6090232364111592, + 0.5570445786770215, + 0.45807193597544066, + 0.4419676958416594, + 0.42799701081524794, + 0.41848476148852237, + 0.3263378087680775, + 0.32209715911319503, + 0.32133550584852344, + 0.3219170862649435 + ], + "peak_layer": 2, + "peak_normalized": 2.1730824619815525, + "population_cv": 0.5858239032428009, + "rest_mean": 0.06508789756507785, + "spike_contrast": 0.5734003518757106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03732142336666584, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.1253339797258377, + 0.13201342523097992, + 0.1303812712430954, + 0.12418067455291748, + 0.11808598041534424, + 0.10588351637125015, + 0.09561335295438766, + 0.0879988819360733, + 0.07940077036619186, + 0.07177155464887619, + 0.06622995436191559, + 0.061969369649887085, + 0.06310316175222397, + 0.05735447630286217, + 0.05297961086034775, + 0.04905581474304199, + 0.050597596913576126, + 0.04696173220872879, + 0.04257786646485329, + 0.03913850337266922, + 0.04642151668667793, + 0.04152010753750801, + 0.03699778765439987, + 0.0338401161134243, + 0.027827588841319084, + 0.02684926614165306, + 0.026000555604696274, + 0.02542269229888916, + 0.01982482150197029, + 0.019567204639315605, + 0.01952093467116356, + 0.019556265324354172 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.753675563686556e-06, + "first_to_last_ratio": 4.253918229238526, + "imbalance_abs_log_ratio": 1.447840494505505, + "last_quartile_mean": 1.587636432986983e-06, + "mean": 3.648644131715173e-06, + "normalized": [ + 2.903986991371279, + 2.3738972852772044, + 2.0636484550173555, + 1.9730309898065614, + 1.5024757487770555, + 1.4408542771106638, + 1.3294926268411962, + 1.2206909715315049, + 1.1367895670998007, + 1.1343245437154552, + 1.0524555559248938, + 0.9965717010048886, + 1.0284766664266607, + 0.9159044401569835, + 0.9130676317207874, + 0.8768166573288402, + 0.7991841246832521, + 0.7460017873265686, + 0.6928987175840244, + 0.6772842439447505, + 0.7872818930048806, + 0.7106755394489379, + 0.6442079001086913, + 0.598937492646923, + 0.49155975663697377, + 0.488564194861926, + 0.4847420251719816, + 0.4867047087469123, + 0.3737366741036465, + 0.3778045297753075, + 0.3854675643472102, + 0.3924647384968829 + ], + "peak_layer": 1, + "peak_normalized": 2.903986991371279, + "population_cv": 0.6010395319776336, + "rest_mean": 3.88747321688209e-06, + "spike_contrast": 0.6068124306476403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.3589670718138223e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.0595615094644018e-05, + 8.661506399221253e-06, + 7.529518825322157e-06, + 7.19888794264989e-06, + 5.481999323819764e-06, + 5.257164502836531e-06, + 4.850845471082721e-06, + 4.453866949916119e-06, + 4.14774058299372e-06, + 4.138746589887887e-06, + 3.840035788016394e-06, + 3.6361354887048947e-06, + 3.752545353563619e-06, + 3.3418093607906485e-06, + 3.3314588563371217e-06, + 3.1991919513529865e-06, + 2.915938466685475e-06, + 2.7218950435781153e-06, + 2.5281408397859195e-06, + 2.4711691821721615e-06, + 2.87251145891787e-06, + 2.5930021365638822e-06, + 2.350485374336131e-06, + 2.185309767810395e-06, + 1.793526621440833e-06, + 1.7825968825491145e-06, + 1.7686511455394793e-06, + 1.775812279447564e-06, + 1.3636321227750159e-06, + 1.378474280500086e-06, + 1.4064339666219894e-06, + 1.4319641650217818e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00010877794466068735, + "first_to_last_ratio": 4.650144040991601, + "imbalance_abs_log_ratio": 1.5368981956768348, + "last_quartile_mean": 2.3392381763187586e-05, + "mean": 5.88290772043365e-05, + "normalized": [ + 2.086094665047828, + 2.1387631343390847, + 2.081886589830107, + 1.9779132190019841, + 1.858425229920235, + 1.685830327312371, + 1.5378399999356094, + 1.4256522001239635, + 1.285691895424893, + 1.177016349317481, + 1.0935933359436882, + 1.0283102880092312, + 1.0433458951143084, + 0.9547749651290882, + 0.8884378576765547, + 0.8276442560837406, + 0.8483650440474175, + 0.7907127191539125, + 0.7197037997675502, + 0.663207322715193, + 0.7855863227954794, + 0.7052947535916103, + 0.632643492598849, + 0.5822024313323265, + 0.48579336759194897, + 0.46994818464757, + 0.45555568062012447, + 0.4458186318974359, + 0.33465168362726294, + 0.3304765071871079, + 0.32922282877056575, + 0.32959702144547803 + ], + "peak_layer": 2, + "peak_normalized": 2.1387631343390847, + "population_cv": 0.5615221836591825, + "rest_mean": 6.276949157095428e-05, + "spike_contrast": 0.598233929968243, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.755083962460048e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012272302410565317, + 0.00012582146155182272, + 0.0001224754669237882, + 0.00011635880946414545, + 0.00010932944132946432, + 9.917584247887135e-05, + 9.04697080841288e-05, + 8.386980334762484e-05, + 7.563606777694076e-05, + 6.924278568476439e-05, + 6.433508679037914e-05, + 6.049454532330856e-05, + 6.137907621450722e-05, + 5.616853013634682e-05, + 5.226597932050936e-05, + 4.8689547838876024e-05, + 4.990853267372586e-05, + 4.651689960155636e-05, + 4.2339510400779545e-05, + 3.90158747904934e-05, + 4.621531843440607e-05, + 4.1491839510854334e-05, + 3.7217832868918777e-05, + 3.425043178140186e-05, + 2.8578775527421385e-05, + 2.764661803666968e-05, + 2.6799920306075364e-05, + 2.622709871502593e-05, + 1.9687249732669443e-05, + 1.944162795552984e-05, + 1.936787521117367e-05, + 1.938988862093538e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.041007069754414e-05, + "first_to_last_ratio": 3.896952313368097, + "imbalance_abs_log_ratio": 1.360194789527301, + "last_quartile_mean": 2.320019939361373e-05, + "mean": 5.0960711291736516e-05, + "normalized": [ + 2.073418805084712, + 2.070705493300103, + 1.9695194092323627, + 1.868517934306777, + 1.7463516565856263, + 1.6026793418463672, + 1.4766395559839594, + 1.3850734214754388, + 1.2521428459550328, + 1.1598174296073966, + 1.0827997233727307, + 1.021319953269383, + 1.0310088635522379, + 0.9522405115352638, + 0.8934796928616261, + 0.8382858426322578, + 0.8561291086272991, + 0.804471244488103, + 0.7365257997466831, + 0.683193532115906, + 0.8124716878605815, + 0.7378124954845751, + 0.6735135453221345, + 0.6298294254760799, + 0.554636180125419, + 0.5428431825046466, + 0.5332516810286245, + 0.5270828016374611, + 0.3733719900070218, + 0.36998938084721245, + 0.3703962562554523, + 0.37048120787152483 + ], + "peak_layer": 1, + "peak_normalized": 2.073418805084712, + "population_cv": 0.5133586984393086, + "rest_mean": 5.396500879821057e-05, + "spike_contrast": 0.6437042359553637, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4737504756776615e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010566289711277932, + 0.00010552462481427938, + 0.0001003681099973619, + 9.522100299363956e-05, + 8.89953225851059e-05, + 8.167367923306301e-05, + 7.525060209445655e-05, + 7.058432674966753e-05, + 6.381009006872773e-05, + 5.9105121181346476e-05, + 5.5180244089569896e-05, + 5.2047191275050864e-05, + 5.2540945034706965e-05, + 4.8526853788644075e-05, + 4.5532360672950745e-05, + 4.271964280633256e-05, + 4.362894833320752e-05, + 4.09964268328622e-05, + 3.753387863980606e-05, + 3.4816028346540406e-05, + 4.140413511777297e-05, + 3.759944956982508e-05, + 3.432272933423519e-05, + 3.209655551472679e-05, + 2.826465424732305e-05, + 2.766367470030673e-05, + 2.7174884962732904e-05, + 2.6860514481086284e-05, + 1.902730218716897e-05, + 1.885492201836314e-05, + 1.887565667857416e-05, + 1.88799858733546e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00023664305626880378, + "first_to_last_ratio": 5.672568947923063, + "imbalance_abs_log_ratio": 1.73564209236627, + "last_quartile_mean": 4.171708769717952e-05, + "mean": 0.00012206438873363368, + "normalized": [ + 2.020436105172798, + 2.2182139141989996, + 2.231345216253622, + 2.0960079716438154, + 2.010914874012047, + 1.7992008227464464, + 1.6294774204031262, + 1.5037956431282506, + 1.3510265318569439, + 1.199041021272032, + 1.1081855437866828, + 1.0311083485666759, + 1.0583798698536107, + 0.9525451374397906, + 0.8640397467231776, + 0.7956174467156442, + 0.8229808232176812, + 0.7579481058232506, + 0.6814030270894333, + 0.6327437253152498, + 0.744247194141427, + 0.6591213123640995, + 0.579305210409028, + 0.5188112493408454, + 0.40414005715569307, + 0.38553326632489926, + 0.3713389234486457, + 0.36290767571642146, + 0.30471206207452944, + 0.2988035841468308, + 0.30136351925091265, + 0.3053046504073908 + ], + "peak_layer": 3, + "peak_normalized": 2.231345216253622, + "population_cv": 0.6172237569075844, + "rest_mean": 0.00013153285544831306, + "spike_contrast": 0.5392923937718314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.093466847436502e-05, + "top_five_layers": [ + 3, + 2, + 4, + 1, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002466232981532812, + 0.0002707649255171418, + 0.00027236778987571597, + 0.0002558479318395257, + 0.00024546109489165246, + 0.00021961834863759577, + 0.00019890116527676582, + 0.00018355989595875144, + 0.0001649122277740389, + 0.00014636020932812244, + 0.00013526999100577086, + 0.00012586161028593779, + 0.00012919049186166376, + 0.00011627183994278312, + 0.00010546848352532834, + 9.711655729915947e-05, + 0.0001004566511255689, + 9.251847222913057e-05, + 8.31750439829193e-05, + 7.723547605564818e-05, + 9.084607881959528e-05, + 8.045524009503424e-05, + 7.071253639878705e-05, + 6.33283780189231e-05, + 4.9331109039485455e-05, + 4.705988249043003e-05, + 4.532725870376453e-05, + 4.429810360306874e-05, + 3.7194491596892476e-05, + 3.647327685030177e-05, + 3.67857537639793e-05, + 3.726682552951388e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "40fd0d817b1a8fb04957cc5ff2887bc65937f1776b53a68242e4c7341723dda7", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.01139580737799406, + 0.02950318343937397, + 0.04104945436120033, + 0.05214058980345726, + 0.01439172774553299, + 0.024067245423793793, + 0.030465533956885338, + 0.036618731915950775, + 0.016362585127353668, + 0.02618422359228134, + 0.033149950206279755, + 0.04017872363328934, + 0.025544000789523125, + 0.030907167121767998, + 0.03762345016002655, + 0.04704070836305618, + 0.022795414552092552, + 0.03890290856361389, + 0.04891306161880493, + 0.060658931732177734, + 0.033973678946495056, + 0.03364448994398117, + 0.04609670862555504, + 0.05855504423379898, + 0.015395586378872395, + 0.03421308845281601, + 0.058928847312927246, + 0.08527950197458267, + 0.006911881268024445, + 0.028406694531440735, + 0.05527278408408165, + 0.0854114517569542 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.390311508155719e-05, + "first_to_last_ratio": 5.095387404603287, + "imbalance_abs_log_ratio": 1.6283357000163252, + "last_quartile_mean": 6.6536874214762065e-06, + "mean": 1.811846108523696e-05, + "normalized": [ + 2.2929525806332673, + 2.1130976782069264, + 2.0666904338039136, + 1.9927924313254255, + 1.8585936234857137, + 1.6932727973648873, + 1.5304627311689392, + 1.421671801794904, + 1.2892190508392531, + 1.1855900928778078, + 1.0961219453815518, + 1.014146923900456, + 1.060927032566774, + 0.9747790234797166, + 0.8967222893794864, + 0.8249674532609624, + 0.8460940689165917, + 0.7831291514058804, + 0.7104177588251133, + 0.649386033089901, + 0.8100343599886985, + 0.7277646813689918, + 0.6447855165487525, + 0.5785206900712379, + 0.45250607210141436, + 0.4317189898164091, + 0.4174322598568066, + 0.40385800282682455, + 0.31921174120413165, + 0.30773080478647463, + 0.30340519281425327, + 0.30199678690853393 + ], + "peak_layer": 1, + "peak_normalized": 2.2929525806332673, + "population_cv": 0.5811766764858837, + "rest_mean": 1.9317224558579727e-05, + "spike_contrast": 0.6028370325080594, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1645138329186011e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.154477210249752e-05, + 3.8286078051896766e-05, + 3.7445250200107694e-05, + 3.610633211792447e-05, + 3.3674856240395457e-05, + 3.067949728574604e-05, + 2.7729629437089898e-05, + 2.5758505216799676e-05, + 2.3358665202977136e-05, + 2.1481067960849032e-05, + 1.9860042812069878e-05, + 1.8374781575403176e-05, + 1.9222365153837018e-05, + 1.766149580362253e-05, + 1.624722790438682e-05, + 1.4947140698495787e-05, + 1.5329922462115064e-05, + 1.4189095054462086e-05, + 1.287167651753407e-05, + 1.1765875569835771e-05, + 1.467657602916006e-05, + 1.3185976058593951e-05, + 1.1682521289912984e-05, + 1.0481904610060155e-05, + 8.198713658202905e-06, + 7.82208371674642e-06, + 7.563230155938072e-06, + 7.317285508179339e-06, + 5.7836255109577905e-06, + 5.575608611252392e-06, + 5.497235179063864e-06, + 5.47171703146887e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013720781316806097, + "first_to_last_ratio": 5.154303273542655, + "imbalance_abs_log_ratio": 1.6398319528627574, + "last_quartile_mean": 2.6620050448400434e-05, + "mean": 7.278342172867269e-05, + "normalized": [ + 2.231162578299261, + 2.1140725523598243, + 2.081809525961795, + 2.017090336462668, + 1.9039602127395285, + 1.731013692718718, + 1.5582801029140219, + 1.4438270127683217, + 1.297193569130097, + 1.1848923737412382, + 1.0932281149833414, + 1.009860837045549, + 1.045980299568947, + 0.957387934055637, + 0.8796484117264644, + 0.8149115280262805, + 0.8407436620287138, + 0.7792370696611136, + 0.7106987318162484, + 0.6467969744190775, + 0.8010932067219945, + 0.7191204217652744, + 0.6388312850244199, + 0.5732129901902088, + 0.448470472211554, + 0.42886454768570625, + 0.41463001278762157, + 0.40203646508675156, + 0.3184117215284888, + 0.30738136152216167, + 0.30352664983164235, + 0.30262534521733025 + ], + "peak_layer": 1, + "peak_normalized": 2.231162578299261, + "population_cv": 0.5853882312557181, + "rest_mean": 7.768760001959486e-05, + "spike_contrast": 0.5959877631181125, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.630085895769298e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001623916468815878, + 0.00015386943414341658, + 0.00015152122068684548, + 0.00014681073662359267, + 0.0001385767391184345, + 0.0001259890996152535, + 0.00011341695790179074, + 0.00010508667037356645, + 9.441418660571799e-05, + 8.624052134109661e-05, + 7.956888293847442e-05, + 7.35011271899566e-05, + 7.613002526341006e-05, + 6.968196976231411e-05, + 6.402382132364437e-05, + 5.931204941589385e-05, + 6.119220051914454e-05, + 5.6715540267759934e-05, + 5.1727085519814864e-05, + 4.7076096961973235e-05, + 5.83063047088217e-05, + 5.234004493104294e-05, + 4.649632683140226e-05, + 4.1720402805367485e-05, + 3.2641215511830524e-05, + 3.121422923868522e-05, + 3.0178191082086414e-05, + 2.9261589588713832e-05, + 2.317509461136069e-05, + 2.2372267267201096e-05, + 2.2091708160587586e-05, + 2.202610812673811e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12167725898325443, + "first_to_last_ratio": 5.154303204824563, + "imbalance_abs_log_ratio": 1.639831939530579, + "last_quartile_mean": 0.02360692690126598, + "mean": 0.06454506580485031, + "normalized": [ + 2.231162477353291, + 2.1140724141359577, + 2.0818095688115155, + 2.0170904137525367, + 1.9039602530341841, + 1.7310136901744795, + 1.5582800997709945, + 1.443826970315116, + 1.2971936450083206, + 1.1848924031019537, + 1.0932281344871677, + 1.009860787951356, + 1.045980339365098, + 0.9573879531378027, + 0.8796484470446824, + 0.8149115154229624, + 0.8407436891757111, + 0.7792370706711085, + 0.7106987249961478, + 0.6467969455265957, + 0.8010931814132939, + 0.7191204293109883, + 0.6388312562189619, + 0.5732129995548365, + 0.44847050123207355, + 0.4288645397214359, + 0.41463001482239054, + 0.40203647444238044, + 0.3184117140434955, + 0.30738135686247064, + 0.3035266419790624, + 0.30262534716162937 + ], + "peak_layer": 1, + "peak_normalized": 2.231162477353291, + "population_cv": 0.5853882259295132, + "rest_mean": 0.06889414014639678, + "spike_contrast": 0.5959877614155383, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.04106006436049938, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.144010528922081, + 0.13645294308662415, + 0.13437053561210632, + 0.13019323348999023, + 0.12289123982191086, + 0.11172839254140854, + 0.10057929158210754, + 0.09319190680980682, + 0.08372744917869568, + 0.07647895812988281, + 0.07056248188018799, + 0.06518153101205826, + 0.0675128698348999, + 0.061794668436050415, + 0.05677696689963341, + 0.05259851738810539, + 0.05426585674285889, + 0.050295908004045486, + 0.045872095972299576, + 0.041747551411390305, + 0.05170661211013794, + 0.046415675431489944, + 0.041233405470848083, + 0.036998070776462555, + 0.028946558013558388, + 0.02768108993768692, + 0.02676232159137726, + 0.025949470698833466, + 0.020551905035972595, + 0.019839949905872345, + 0.01959114708006382, + 0.01953297294676304 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.918673363998096e-06, + "first_to_last_ratio": 4.2569175865719195, + "imbalance_abs_log_ratio": 1.4485453271500246, + "last_quartile_mean": 1.6252777328418233e-06, + "mean": 3.7709897107163215e-06, + "normalized": [ + 3.1110512448069554, + 2.304346296539455, + 1.999936311003599, + 1.856948328142694, + 1.4760735306186028, + 1.3975733840426199, + 1.315376520925139, + 1.216376148656068, + 1.121245902760062, + 1.1184741190475735, + 1.0513651171862035, + 0.9856602767147932, + 1.025270072900951, + 0.9223172106183702, + 0.9122023405577108, + 0.882390621923502, + 0.8042742561465184, + 0.7522122351984912, + 0.7014935374901973, + 0.6885154154574921, + 0.8268174136591959, + 0.7604637334111901, + 0.6891177674106737, + 0.632538167109275, + 0.4941282098312776, + 0.4865824100430069, + 0.48117691929198236, + 0.4807033886549141, + 0.37702330245768634, + 0.3708218207847471, + 0.3762822105801532, + 0.38124178602889996 + ], + "peak_layer": 1, + "peak_normalized": 3.1110512448069554, + "population_cv": 0.6047199362907585, + "rest_mean": 3.994027649780576e-06, + "spike_contrast": 0.64260567648057, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.566584839769348e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1731742233678233e-05, + 8.689666174177546e-06, + 7.541739250882529e-06, + 7.002533038757974e-06, + 5.566258096223464e-06, + 5.27023485119571e-06, + 4.960271326126531e-06, + 4.586941940942779e-06, + 4.228206762491027e-06, + 4.2177543946309015e-06, + 3.964687039115233e-06, + 3.716914761753287e-06, + 3.866282895614859e-06, + 3.4780487112584524e-06, + 3.439905640334473e-06, + 3.327485956106102e-06, + 3.032909944522544e-06, + 2.836584599208436e-06, + 2.645324912009528e-06, + 2.5963845473597758e-06, + 3.1179199595499085e-06, + 2.8677009140665177e-06, + 2.5986560103774536e-06, + 2.385294919804437e-06, + 1.8633523950484232e-06, + 1.834897261687729e-06, + 1.8145132116842433e-06, + 1.8127275325241499e-06, + 1.4217509942682227e-06, + 1.3983652706883731e-06, + 1.41895634442335e-06, + 1.4376588524100953e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001149136905951309, + "first_to_last_ratio": 4.815694891169908, + "imbalance_abs_log_ratio": 1.5718803528372292, + "last_quartile_mean": 2.3862327907409053e-05, + "mean": 6.221135890882579e-05, + "normalized": [ + 2.275293471382189, + 2.0925239114508343, + 2.0249889212120458, + 1.9490506817315036, + 1.821047383587751, + 1.6730348788569944, + 1.5224659618516898, + 1.418792087744366, + 1.2749012101051704, + 1.1755812366175094, + 1.093166238104123, + 1.0178928903694313, + 1.046098218092314, + 0.9644986329917807, + 0.8923499840739568, + 0.8324245833781815, + 0.8548661709826579, + 0.7953276004450885, + 0.7278943614497286, + 0.6660761537152818, + 0.818586121883297, + 0.7386933261746851, + 0.6593902783338745, + 0.5965063725188312, + 0.4749508284557732, + 0.4558878499549265, + 0.44138695277868795, + 0.4292311586137051, + 0.3268647879844354, + 0.31649601465159427, + 0.3123056762318561, + 0.311426054275736 + ], + "peak_layer": 1, + "peak_normalized": 2.275293471382189, + "population_cv": 0.5648730691909721, + "rest_mean": 6.615572743208354e-05, + "spike_contrast": 0.6184161292041498, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.091176888323389e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014154909877106547, + 0.00013017875608056784, + 0.00012597731256391853, + 0.00012125309149269015, + 0.00011328983237035573, + 0.00010408177331555635, + 9.471467637922615e-05, + 8.8264983787667e-05, + 7.931333675514907e-05, + 7.313450623769313e-05, + 6.80073571857065e-05, + 6.332449993351474e-05, + 6.507919169962406e-05, + 6.0002770624123514e-05, + 5.55143051315099e-05, + 5.1786264521069825e-05, + 5.318238618201576e-05, + 4.94784108013846e-05, + 4.528329736785963e-05, + 4.143750265939161e-05, + 5.0925355026265606e-05, + 4.5955115638207644e-05, + 4.10215652664192e-05, + 3.710947203217074e-05, + 2.9547336453106254e-05, + 2.8361402655718848e-05, + 2.7459282136987895e-05, + 2.6703053663368337e-05, + 2.033470263995696e-05, + 1.9689647160703316e-05, + 1.9428960513323545e-05, + 1.9374238036107272e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.519206923869206e-05, + "first_to_last_ratio": 4.049909152070383, + "imbalance_abs_log_ratio": 1.3986944492792963, + "last_quartile_mean": 2.350474187551299e-05, + "mean": 5.3535959864348115e-05, + "normalized": [ + 2.2900191331294404, + 2.039391045052068, + 1.9210026615791371, + 1.8368270426601154, + 1.71008739314826, + 1.5872578127019539, + 1.4631636096399043, + 1.3770182781394782, + 1.236995810064802, + 1.1502352236926436, + 1.0819737353690493, + 1.0125428958379479, + 1.0387169824256042, + 0.9627940963486861, + 0.895516173191146, + 0.8412775217154583, + 0.8632420662731606, + 0.8101070545557904, + 0.7437235410096641, + 0.6860674864525296, + 0.8435748401878083, + 0.7646895025830382, + 0.6913289553009525, + 0.640080208740369, + 0.5389256211609706, + 0.5259226029001551, + 0.515575733832938, + 0.5073004063094942, + 0.36380392953064605, + 0.3554700596975052, + 0.3524837223636654, + 0.3528848544056177 + ], + "peak_layer": 1, + "peak_normalized": 2.2900191331294404, + "population_cv": 0.5220149329094871, + "rest_mean": 5.65526137911042e-05, + "spike_contrast": 0.6586084384613202, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.724602865986526e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012259837239980698, + 0.00010918075713561848, + 0.0001028427213896066, + 9.833629883360118e-05, + 9.155117004411295e-05, + 8.497537055518478e-05, + 7.833186828065664e-05, + 7.371999527094886e-05, + 6.622375803999603e-05, + 6.157894677016884e-05, + 5.792450247099623e-05, + 5.4207455832511187e-05, + 5.560871068155393e-05, + 5.154410609975457e-05, + 4.794231790583581e-05, + 4.503859963733703e-05, + 4.621449261321686e-05, + 4.336985875852406e-05, + 3.981595364166424e-05, + 3.6729281418956816e-05, + 4.516158878686838e-05, + 4.093838651897386e-05, + 3.7010959204053506e-05, + 3.4267308365087956e-05, + 2.8851900424342602e-05, + 2.81557713606162e-05, + 2.7601841793511994e-05, + 2.7158814191352576e-05, + 1.9476592569844797e-05, + 1.903043084894307e-05, + 1.8870554413297214e-05, + 1.8892029402195476e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002514511506888084, + "first_to_last_ratio": 5.859513217612488, + "imbalance_abs_log_ratio": 1.7680665314717174, + "last_quartile_mean": 4.291331742933835e-05, + "mean": 0.0001305278289009948, + "normalized": [ + 2.175519083700028, + 2.147448905449854, + 2.155906169303229, + 2.0822753527857363, + 1.9604833952769096, + 1.7827704840526475, + 1.6143096143512923, + 1.49263047981926, + 1.3352686262026217, + 1.2139066669363794, + 1.1066600661067776, + 1.0143193649474604, + 1.0602634006626006, + 0.9683991870554761, + 0.8824502085414744, + 0.8051197014148838, + 0.8356431104994041, + 0.7697672013591956, + 0.6982515926862864, + 0.6332597050821357, + 0.7831169912368967, + 0.6976137861149029, + 0.6139728290209445, + 0.5405034987503755, + 0.3991384270450618, + 0.3775843888731109, + 0.3608789237031527, + 0.34760168279002573, + 0.2982009104896796, + 0.2846383790650737, + 0.2801016579127231, + 0.2819962087644011 + ], + "peak_layer": 1, + "peak_normalized": 2.175519083700028, + "population_cv": 0.6126834994698729, + "rest_mean": 0.00014003051816231432, + "spike_contrast": 0.5656860228000465, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.921330688986927e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002839657827280462, + 0.00028030184330418706, + 0.00028140575159341097, + 0.00027179488097317517, + 0.0002558976411819458, + 0.00023270116071216762, + 0.00021071232913527638, + 0.00019482981588225812, + 0.00017428971477784216, + 0.0001584486017236486, + 0.00014444993576034904, + 0.0001323969045188278, + 0.00013839387975167483, + 0.00012640304339583963, + 0.00011518430983414873, + 0.00010509052663110197, + 0.0001090746809495613, + 0.0001004760415526107, + 9.11412644200027e-05, + 8.265801443485543e-05, + 0.00010221856064163148, + 9.105801291298121e-05, + 8.014054037630558e-05, + 7.055074820527807e-05, + 5.209867231315002e-05, + 4.92852705065161e-05, + 4.7104742407100275e-05, + 4.5371692976914346e-05, + 3.892351742251776e-05, + 3.715322964126244e-05, + 3.6561061278916895e-05, + 3.6808352888328955e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "79ad652a3aa7b0850b826f149977b0495d569b726bb8d66c66c52638215ffd53", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.021054552868008614, + 0.038058556616306305, + 0.04883526638150215, + 0.05791458487510681, + 0.016990283504128456, + 0.027475843206048012, + 0.03445403650403023, + 0.040619369596242905, + 0.019055280834436417, + 0.029903952032327652, + 0.037492990493774414, + 0.04475866258144379, + 0.0285621490329504, + 0.03507128730416298, + 0.04228900745511055, + 0.05219251662492752, + 0.027010982856154442, + 0.04389137029647827, + 0.054182786494493484, + 0.06662839651107788, + 0.03837457299232483, + 0.041143495589494705, + 0.05716610327363014, + 0.07329611480236053, + 0.028997479006648064, + 0.053451184183359146, + 0.08026835322380066, + 0.10854664444923401, + 0.027061885222792625, + 0.05247335880994797, + 0.08323372155427933, + 0.11460508406162262 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.201575941602641e-05, + "first_to_last_ratio": 4.932102112877248, + "imbalance_abs_log_ratio": 1.5957652892396856, + "last_quartile_mean": 6.491301007827133e-06, + "mean": 1.705989417644105e-05, + "normalized": [ + 2.106292333194075, + 2.172491163348758, + 2.1330241796302363, + 2.0259918761437463, + 1.8982135819397121, + 1.704913518211797, + 1.5444411923191312, + 1.4279770610625449, + 1.3041442710446836, + 1.1896258094559513, + 1.0962581888548657, + 1.0266942074295144, + 1.0572238841977288, + 0.9622016766881734, + 0.891040738475761, + 0.8167872858482652, + 0.8366683449379246, + 0.7772035491548255, + 0.7011655353024898, + 0.6475278325168645, + 0.7728997359529347, + 0.6896078437753965, + 0.6131124674701751, + 0.5604884364037679, + 0.4607666284385362, + 0.44397715080639827, + 0.4298684867451416, + 0.41972950379888196, + 0.3265460490205608, + 0.3218480479207054, + 0.3204072149176842, + 0.3208622049927692 + ], + "peak_layer": 2, + "peak_normalized": 2.172491163348758, + "population_cv": 0.5805628153884638, + "rest_mean": 1.8262379702112714e-05, + "spike_contrast": 0.5785923034220807, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0566472337814047e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.593312430894002e-05, + 3.706246934598312e-05, + 3.638916678028181e-05, + 3.456320700934157e-05, + 3.23833228321746e-05, + 2.9085644200677052e-05, + 2.6348003302700818e-05, + 2.436113754811231e-05, + 2.2248563254834153e-05, + 2.029489041888155e-05, + 1.8702048691920936e-05, + 1.751529453031253e-05, + 1.803612758521922e-05, + 1.641505878069438e-05, + 1.5201060705294367e-05, + 1.393430466123391e-05, + 1.427347342541907e-05, + 1.3259010302135721e-05, + 1.1961809832428116e-05, + 1.1046756299037952e-05, + 1.3185587704356294e-05, + 1.1764636838051956e-05, + 1.0459633813297842e-05, + 9.561873412167188e-06, + 7.86062992119696e-06, + 7.5742032095149625e-06, + 7.333510893658968e-06, + 7.160540917539038e-06, + 5.570841040025698e-06, + 5.490693638421362e-06, + 5.466113179863896e-06, + 5.473875262396177e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.000129606451082509, + "first_to_last_ratio": 4.9818074922726145, + "imbalance_abs_log_ratio": 1.6057927754415957, + "last_quartile_mean": 2.601594928819395e-05, + "mean": 6.850338621688934e-05, + "normalized": [ + 2.0631371693799077, + 2.173061151786652, + 2.146221311616304, + 2.04412417319785, + 1.9437850741564366, + 1.7429527608527258, + 1.5739287245233284, + 1.448561249882341, + 1.3070356812110293, + 1.181442503713818, + 1.0902177639657034, + 1.0200948073975256, + 1.038760061861336, + 0.9441053821903842, + 0.8720979926456799, + 0.8075042780959905, + 0.832896699126015, + 0.7730365841967101, + 0.7008771505867426, + 0.6442638641806842, + 0.7641491492103867, + 0.6834677068149604, + 0.6090231512754765, + 0.5570467574542299, + 0.45807302792658366, + 0.4419659692950432, + 0.4279961308389998, + 0.4184841096663127, + 0.3263401485679923, + 0.32209706763737483, + 0.3213341654300619, + 0.32191823131541414 + ], + "peak_layer": 2, + "peak_normalized": 2.173061151786652, + "population_cv": 0.5858213540607353, + "rest_mean": 7.33956448702547e-05, + "spike_contrast": 0.5734017265344913, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2085189488716426e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014133188233245164, + 0.0001488620473537594, + 0.00014702342741657048, + 0.0001400294277118519, + 0.00013315585965756327, + 0.00011939816613448784, + 0.0001078194472938776, + 9.923135075950995e-05, + 8.953637006925419e-05, + 8.09328121249564e-05, + 7.468360854545608e-05, + 6.987994856899604e-05, + 7.115858170436695e-05, + 6.467441562563181e-05, + 5.974166560918093e-05, + 5.531677743420005e-05, + 5.705624425900169e-05, + 5.295562368701212e-05, + 4.8012458137236536e-05, + 4.413425631355494e-05, + 5.2346804295666516e-05, + 4.681985228671692e-05, + 4.172014814685099e-05, + 3.8159589166752994e-05, + 3.1379553547594696e-05, + 3.02761654893402e-05, + 2.9319184250198305e-05, + 2.8667578590102494e-05, + 2.2355405235430226e-05, + 2.2064739823690616e-05, + 2.201247843913734e-05, + 2.2052488930057734e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11493630055338144, + "first_to_last_ratio": 4.9818077321878835, + "imbalance_abs_log_ratio": 1.6057928235998724, + "last_quartile_mean": 0.02307120361365378, + "mean": 0.06074948905734345, + "normalized": [ + 2.063137205162244, + 2.1730611136953453, + 2.1462213824598626, + 2.0441242673588023, + 1.9437851361343719, + 1.7429527097582802, + 1.5739287045014374, + 1.4485612988475673, + 1.3070356248971926, + 1.1814425536696513, + 1.0902177360905216, + 1.020094857247625, + 1.0387600983655463, + 0.9441054093552858, + 0.8720980390057368, + 0.8075042733286111, + 0.8328966787842622, + 0.7730365551257176, + 0.7008771058026076, + 0.6442638069074199, + 0.7641491334974229, + 0.6834676813377316, + 0.6090231227925368, + 0.557046760858952, + 0.4580729983384807, + 0.4419659647123029, + 0.42799613124198876, + 0.418484082023093, + 0.32634013714750504, + 0.3220970724629912, + 0.32133413272526656, + 0.3219182263636403 + ], + "peak_layer": 2, + "peak_normalized": 2.1730611136953453, + "population_cv": 0.5858213710563731, + "rest_mean": 0.06508799324984904, + "spike_contrast": 0.5734017067410487, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0373215664178133, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.12533453106880188, + 0.13201235234737396, + 0.13038185238838196, + 0.12417950481176376, + 0.11808395385742188, + 0.10588348656892776, + 0.09561536461114883, + 0.0879993587732315, + 0.07940174639225006, + 0.0717720314860344, + 0.0662301704287529, + 0.061970241367816925, + 0.06310414522886276, + 0.0573539212346077, + 0.05297951027750969, + 0.049055472016334534, + 0.05059804767370224, + 0.046961575746536255, + 0.04257792606949806, + 0.03913869708776474, + 0.04642166942358017, + 0.041520312428474426, + 0.03699784353375435, + 0.03384030610322952, + 0.027827700600028038, + 0.026849206537008286, + 0.026000546291470528, + 0.02542269416153431, + 0.01982499659061432, + 0.019567232578992844, + 0.01952088437974453, + 0.01955636776983738 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.7535999619394715e-06, + "first_to_last_ratio": 4.253913636706542, + "imbalance_abs_log_ratio": 1.4478394149044844, + "last_quartile_mean": 1.5876203747211548e-06, + "mean": 3.648633160935333e-06, + "normalized": [ + 2.9040847125163616, + 2.373904173886903, + 2.063563551858565, + 1.9729206379098345, + 1.502439510808684, + 1.4407906834302517, + 1.3294895201906802, + 1.2207633158069082, + 1.136833740861507, + 1.1342758569990883, + 1.0524618363430551, + 0.9966191922064839, + 1.0284603781292727, + 0.9159325573360332, + 0.9130731814372068, + 0.8768378643711425, + 0.7992107068730414, + 0.7459864568856505, + 0.6929026705299385, + 0.6772775559667719, + 0.7873543673999568, + 0.7106391017928749, + 0.6441870912399603, + 0.5989718855930185, + 0.49154718207331655, + 0.4885345051390287, + 0.4847150036087574, + 0.4867019034315765, + 0.3737492954398724, + 0.37780977871978977, + 0.38548196584593536, + 0.3924798153685328 + ], + "peak_layer": 1, + "peak_normalized": 2.9040847125163616, + "population_cv": 0.6010372604548386, + "rest_mean": 3.887456521759439e-06, + "spike_contrast": 0.6068201661629119, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.3589870124851585e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.059593978425255e-05, + 8.661505489726551e-06, + 7.529186405008659e-06, + 7.198463663371513e-06, + 5.481850621436024e-06, + 5.256916665530298e-06, + 4.85081955048372e-06, + 4.454117515706457e-06, + 4.14788928537746e-06, + 4.138556505495217e-06, + 3.840047156700166e-06, + 3.636297833509161e-06, + 3.752474640350556e-06, + 3.341901901876554e-06, + 3.3314690881525166e-06, + 3.1992597087082686e-06, + 2.916026687671547e-06, + 2.7218309242016403e-06, + 2.528147660996183e-06, + 2.4711373498575995e-06, + 2.872767254302744e-06, + 2.592861392258783e-06, + 2.350402382944594e-06, + 2.185428684242652e-06, + 1.7934753486770205e-06, + 1.782483195711393e-06, + 1.7685472357698018e-06, + 1.775796704350796e-06, + 1.3636740732181352e-06, + 1.3784892871626653e-06, + 1.406482283528021e-06, + 1.4320148693514057e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001087780183297582, + "first_to_last_ratio": 4.650142263541028, + "imbalance_abs_log_ratio": 1.536897813441167, + "last_quartile_mean": 2.339240654691821e-05, + "mean": 5.882917196231574e-05, + "normalized": [ + 2.086098725675131, + 2.138759442007871, + 2.081889667800162, + 1.9778974178151685, + 1.8584148157319034, + 1.6858251383046516, + 1.537848282997998, + 1.4256580666230054, + 1.285692298109234, + 1.1770210084696262, + 1.0935923165353414, + 1.0283135788544562, + 1.0433544799560879, + 0.9547703352579274, + 0.888436859519986, + 0.8276396454668482, + 0.848368006338972, + 0.7907124968033461, + 0.7197062272212832, + 0.6632080478180333, + 0.7855866652595299, + 0.7052955964220388, + 0.6326444524509259, + 0.5822042763475935, + 0.48579283246827537, + 0.46994724216827105, + 0.45555494684240355, + 0.44581791380348257, + 0.3346526596662172, + 0.330475418320936, + 0.3292227313590583, + 0.32959840758423514 + ], + "peak_layer": 2, + "peak_normalized": 2.138759442007871, + "population_cv": 0.5615212363220359, + "rest_mean": 6.27695851478536e-05, + "spike_contrast": 0.5982346493442672, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.755094076041132e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012272346066311002, + 0.0001258214469999075, + 0.00012247584527358413, + 0.0001163580673164688, + 0.00010932900477200747, + 9.917569695971906e-05, + 9.047034109244123e-05, + 8.387028356082737e-05, + 7.563621329609305e-05, + 6.924317131051794e-05, + 6.433513044612482e-05, + 6.0494836361613125e-05, + 6.13796801189892e-05, + 5.616834823740646e-05, + 5.226600478636101e-05, + 4.868935502599925e-05, + 4.9908787332242355e-05, + 4.651696144719608e-05, + 4.233972140355036e-05, + 3.9015980291878805e-05, + 4.6215413021855056e-05, + 4.149195592617616e-05, + 3.7217949284240603e-05, + 3.425059549044818e-05, + 2.8578790079336613e-05, + 2.7646607122733258e-05, + 2.6799920306075364e-05, + 2.622709871502593e-05, + 1.9687338863150217e-05, + 1.9441595213720575e-05, + 1.9367900677025318e-05, + 1.93900013982784e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.041136763698887e-05, + "first_to_last_ratio": 3.896756618520285, + "imbalance_abs_log_ratio": 1.3601445708574575, + "last_quartile_mean": 2.320169733138755e-05, + "mean": 5.096003008020489e-05, + "normalized": [ + 2.073194376167996, + 2.071105252365612, + 1.96987669569892, + 1.868253501375736, + 1.7461955294439868, + 1.6029130762574872, + 1.4764811084477982, + 1.3852794036995026, + 1.2521995618218273, + 1.1598580624238308, + 1.0825747595090334, + 1.0211644142159193, + 1.0309911631402593, + 0.9519908151976145, + 0.8935121251146112, + 0.8382787729230907, + 0.8561012891122731, + 0.8043820538962967, + 0.7366169572228874, + 0.6831815336454671, + 0.8123345595077685, + 0.737879683501474, + 0.6735151955051606, + 0.6297835892231947, + 0.5547719514435682, + 0.5428008237338267, + 0.5332896849902665, + 0.5271935397804924, + 0.37339953996344866, + 0.3699162629784263, + 0.37044439781611194, + 0.3705203198761118 + ], + "peak_layer": 1, + "peak_normalized": 2.073194376167996, + "population_cv": 0.5133726204518009, + "rest_mean": 5.3964246574280505e-05, + "spike_contrast": 0.64370881124749, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.473726101219654e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010565004777163267, + 0.00010554358595982194, + 0.00010038497566711158, + 9.520625462755561e-05, + 8.898617670638487e-05, + 8.16844985820353e-05, + 7.524152169935405e-05, + 7.059388008201495e-05, + 6.38121273368597e-05, + 5.910640174988657e-05, + 5.516804230865091e-05, + 5.2038569265278056e-05, + 5.253934068605304e-05, + 4.8513480578549206e-05, + 4.553340477286838e-05, + 4.271871148375794e-05, + 4.362694744486362e-05, + 4.099133366253227e-05, + 3.753802229766734e-05, + 3.481495150481351e-05, + 4.1396593587705866e-05, + 3.760237086680718e-05, + 3.432235462241806e-05, + 3.20937906508334e-05, + 2.8271195333218202e-05, + 2.76611463050358e-05, + 2.717645838856697e-05, + 2.6865798645303585e-05, + 1.902845178847201e-05, + 1.8850943888537586e-05, + 1.887785765575245e-05, + 1.8881726646213792e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00023663779211346991, + "first_to_last_ratio": 5.672677986629943, + "imbalance_abs_log_ratio": 1.7356613142849313, + "last_quartile_mean": 4.171535783825675e-05, + "mean": 0.00012206177893858694, + "normalized": [ + 2.0204564142034975, + 2.2187758849519086, + 2.231495928529079, + 2.0959130632005167, + 2.010595686112895, + 1.799588002892838, + 1.6290705588593046, + 1.5034830183184902, + 1.3508380843403627, + 1.1990338730444932, + 1.108139972391173, + 1.0309048348839545, + 1.0584610348587842, + 0.9524995764002239, + 0.8641330297603085, + 0.795578425477979, + 0.8231670526464523, + 0.757893675020928, + 0.6814420357365967, + 0.6326769608985996, + 0.7444868783224939, + 0.6591411274404338, + 0.5794466495741255, + 0.5187294118731414, + 0.40422067568857084, + 0.3855350120263851, + 0.37131717783399165, + 0.36287239745994854, + 0.304694852796421, + 0.298784221840333, + 0.30136301825835976, + 0.30526146435741114 + ], + "peak_layer": 3, + "peak_normalized": 2.231495928529079, + "population_cv": 0.6172354521995556, + "rest_mean": 0.00013152823615840775, + "spike_contrast": 0.5393739931714252, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.094290995155462e-05, + "top_five_layers": [ + 3, + 2, + 4, + 1, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00024662050418555737, + 0.00027082773158326745, + 0.0002723803627304733, + 0.00025583087699487805, + 0.0002454168861731887, + 0.00021966091298963875, + 0.00019884725043084472, + 0.000183517811819911, + 0.0001648856996325776, + 0.00014635620755143464, + 0.00013526153634302318, + 0.00012583407806232572, + 0.0001291976368520409, + 0.00011626379273366183, + 0.00010547761485213414, + 9.710971789900213e-05, + 0.00010047723480965942, + 9.250985021935776e-05, + 8.317802712554112e-05, + 7.722567534074187e-05, + 9.087339276447892e-05, + 8.04559385869652e-05, + 7.072828884702176e-05, + 6.331703480100259e-05, + 4.933989475830458e-05, + 4.705908941105008e-05, + 4.5323635276872665e-05, + 4.42928503616713e-05, + 3.719159576576203e-05, + 3.6470133636612445e-05, + 3.678490611491725e-05, + 3.726075738086365e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "0db2d20deb5c148ef675d303417844641f9632a980cb7b1bc2b39e8bf6c514f2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04405451565980911, + 0.0240375604480505, + 0.030349314212799072, + 0.034919846802949905, + 0.038470037281513214, + 0.02659580484032631, + 0.028048591688275337, + 0.029997555539011955, + 0.031361404806375504, + 0.0246393084526062, + 0.025546060875058174, + 0.02669469080865383, + 0.027990806847810745, + 0.02195541374385357, + 0.023044798523187637, + 0.02394026517868042, + 0.025360919535160065, + 0.02148214913904667, + 0.02322523109614849, + 0.023834573104977608, + 0.02502058818936348, + 0.01980534754693508, + 0.02057415433228016, + 0.02201046608388424, + 0.02379545010626316, + 0.022529911249876022, + 0.023956971243023872, + 0.02642984129488468, + 0.030454812571406364, + 0.02776513062417507, + 0.030121317133307457, + 0.03041756898164749 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.9001171494892333e-05, + "first_to_last_ratio": 4.396109463475654, + "imbalance_abs_log_ratio": 1.480719936931006, + "last_quartile_mean": 4.322269873568985e-06, + "mean": 1.3701130082921509e-05, + "normalized": [ + 1.0077312064371016, + 2.0216860298126127, + 1.9115786719811696, + 1.5699942654414412, + 1.2912410675604775, + 1.3448969617811173, + 1.0829684927533307, + 0.864562725693079, + 1.472694590311165, + 1.8785554526189956, + 1.4195158427110435, + 1.1323106177566131, + 1.5536879058091357, + 1.3271158818759365, + 1.4866965999456043, + 1.0117258821924624, + 0.6982931262632973, + 0.8871860339483304, + 1.0814660916038894, + 0.9542868684601444, + 1.3260318801629343, + 0.8711143311473454, + 0.8200899788684414, + 0.46082446220796464, + 0.6310998230199332, + 0.5173662703583513, + 0.4802648347536266, + 0.33073499649835436, + 0.17451899850392552, + 0.1289952247275187, + 0.17263606827795114, + 0.08812881651670629 + ], + "peak_layer": 2, + "peak_normalized": 2.0216860298126127, + "population_cv": 0.5264799713353324, + "rest_mean": 1.4153185942531207e-05, + "spike_contrast": 0.7955825979217899, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1260028441029134e-05, + "top_five_layers": [ + 2, + 3, + 10, + 4, + 13 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.3807056348014157e-05, + 2.7699383281287737e-05, + 2.619078804855235e-05, + 2.1510695660253987e-05, + 1.7691461835056543e-05, + 1.8426608221489005e-05, + 1.4837892194918822e-05, + 1.184548636956606e-05, + 2.017758015426807e-05, + 2.5738332624314353e-05, + 1.9448971215751953e-05, + 1.551393506815657e-05, + 2.128728010575287e-05, + 1.81829873326933e-05, + 2.0369423509691842e-05, + 1.3861787920177449e-05, + 9.56740495894337e-06, + 1.2155451258877292e-05, + 1.4817307601333596e-05, + 1.3074808521196246e-05, + 1.816813528421335e-05, + 1.1935250768146943e-05, + 1.1236159480176866e-05, + 6.31381590210367e-06, + 8.646780770504847e-06, + 7.0885025706957094e-06, + 6.580170975212241e-06, + 4.5314432099985424e-06, + 2.391107500443468e-06, + 1.7673803540674271e-06, + 2.365309228480328e-06, + 1.2074643791493145e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.191661304910667e-05, + "first_to_last_ratio": 3.797990683695372, + "imbalance_abs_log_ratio": 1.3344721594369466, + "last_quartile_mean": 1.6302465752460193e-05, + "mean": 4.225009619318598e-05, + "normalized": [ + 0.9174753286219919, + 2.2626595758448156, + 1.7722563272438279, + 1.463236175471811, + 1.3866667550506941, + 1.6673315743394923, + 1.2737488857960582, + 0.9804544105398723, + 1.1727460803814453, + 1.3858997244842928, + 1.1458588519796304, + 1.012187071256809, + 1.1964358555891934, + 1.2230862069043256, + 1.1426917941369261, + 1.0392305817059992, + 0.812301299394157, + 0.8125846736189017, + 1.1589483129789653, + 1.1474138368245612, + 1.578094611394048, + 0.9268792886892305, + 0.8930796579738131, + 0.5418827272180105, + 0.8135175439777245, + 0.634708408163733, + 0.5701529871597378, + 0.4148669257029605, + 0.21665436196405244, + 0.1431611577547963, + 0.20865076223123533, + 0.0851382456068877 + ], + "peak_layer": 2, + "peak_normalized": 2.2626595758448156, + "population_cv": 0.48660988012988077, + "rest_mean": 4.2635896172079675e-05, + "spike_contrast": 0.9420882381607695, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.0166776307160036e-05, + "top_five_layers": [ + 2, + 3, + 6, + 21, + 4 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.8763420889154077e-05, + 9.559758473187685e-05, + 7.487800030503422e-05, + 6.182186916703358e-05, + 5.858680378878489e-05, + 7.044491940177977e-05, + 5.3816012950846925e-05, + 4.142429315834306e-05, + 4.954863470629789e-05, + 5.855439667357132e-05, + 4.8412646719953045e-05, + 4.276500112609938e-05, + 5.054952998762019e-05, + 5.1675509894266725e-05, + 4.8278838221449405e-05, + 4.3907592043979093e-05, + 3.43198080372531e-05, + 3.4331780625507236e-05, + 4.89656777062919e-05, + 4.8478344979230314e-05, + 6.667464913334697e-05, + 3.916073910659179e-05, + 3.773270145757124e-05, + 2.2894597350386903e-05, + 3.437119448790327e-05, + 2.6816491299541667e-05, + 2.4089018552331254e-05, + 1.7528167518321425e-05, + 9.15366763365455e-06, + 6.048572686268017e-06, + 8.81551477505127e-06, + 3.597099066610099e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.054908270947635174, + "first_to_last_ratio": 3.797990637672001, + "imbalance_abs_log_ratio": 1.3344721473191257, + "last_quartile_mean": 0.01445718965260312, + "mean": 0.03746780757501256, + "normalized": [ + 0.9174753670275252, + 2.2626595668392633, + 1.7722562063201177, + 1.4632360819755696, + 1.3866667949681875, + 1.6673315955377577, + 1.2737488958767915, + 0.9804544532014017, + 1.1727461605745655, + 1.3858997200707546, + 1.1458588680193664, + 1.0121870939065154, + 1.1964358991827213, + 1.2230861591318958, + 1.1426918379859348, + 1.0392305853812045, + 0.8123012456223383, + 0.8125847103705666, + 1.158948357357931, + 1.1474137979378307, + 1.5780945861023632, + 0.926879317506979, + 0.8930796013491593, + 0.5418826865229676, + 0.8135175788378592, + 0.6347083934759172, + 0.5701530007014057, + 0.41486691525311564, + 0.21665436405077207, + 0.1431611567511669, + 0.2086507602497586, + 0.08513824191029651 + ], + "peak_layer": 2, + "peak_normalized": 2.2626595668392633, + "population_cv": 0.4866098746823269, + "rest_mean": 0.03780993893397627, + "spike_contrast": 0.9420882244430143, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.035620298236608505, + "top_five_layers": [ + 2, + 3, + 6, + 21, + 4 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.034375790506601334, + 0.08477689325809479, + 0.06640255451202393, + 0.05482424795627594, + 0.051955364644527435, + 0.062471259385347366, + 0.04772457852959633, + 0.03673547878861427, + 0.0439402274787426, + 0.0519266240298748, + 0.042932819575071335, + 0.03792443126440048, + 0.04482783004641533, + 0.04582635685801506, + 0.04281415790319443, + 0.03893769159913063, + 0.030435146763920784, + 0.0304457675665617, + 0.043423254042863846, + 0.04299107939004898, + 0.059127744287252426, + 0.03472813591361046, + 0.033461734652519226, + 0.020303156226873398, + 0.030480720102787018, + 0.023781131953001022, + 0.021362382918596268, + 0.01554415374994278, + 0.008117564022541046, + 0.005363934673368931, + 0.007817686535418034, + 0.003189943265169859 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.201552186103072e-05, + "first_to_last_ratio": 5.505409601247022, + "imbalance_abs_log_ratio": 1.705731172719516, + "last_quartile_mean": 2.182493716418321e-06, + "mean": 8.921801814665287e-06, + "normalized": [ + 1.8522267980560887, + 1.3991189499642396, + 1.5189623710595221, + 1.7095356122655745, + 1.0880389595429378, + 0.9409894011627431, + 1.2244978988385578, + 1.0407064664287273, + 0.8019872963111272, + 1.6142561275113876, + 1.5038986961003484, + 1.2795093808294922, + 1.518204136149992, + 1.3587318804660677, + 1.8279910088209055, + 1.9335497857688762, + 1.3708485517952869, + 1.171301878733052, + 0.9049971990240303, + 0.9405561531974967, + 0.9612715223781346, + 0.8560463973337525, + 0.6765022731234531, + 0.5492730955683951, + 0.30272551699640965, + 0.3421729481134854, + 0.33897188277967555, + 0.26236357588022213, + 0.18272919759715678, + 0.18185216354703881, + 0.19745928436567797, + 0.14872359029014504 + ], + "peak_layer": 16, + "peak_normalized": 1.9335497857688762, + "population_cv": 0.5460082571286404, + "rest_mean": 9.468404658508479e-06, + "spike_contrast": 0.6305335136418331, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.970146457912051e-06, + "top_five_layers": [ + 16, + 1, + 15, + 4, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.6525200408068486e-05, + 1.2482661986723542e-05, + 1.355188123852713e-05, + 1.5252137927745935e-05, + 9.707267963676713e-06, + 8.395320946874563e-06, + 1.0924727575911675e-05, + 9.284976840717718e-06, + 7.155171715567121e-06, + 1.4402073247765657e-05, + 1.3417486115940847e-05, + 1.1415529115765821e-05, + 1.3545116416935343e-05, + 1.212233655678574e-05, + 1.6308973499690183e-05, + 1.7250747987418436e-05, + 1.223043909703847e-05, + 1.0450123227201402e-05, + 8.074205652519595e-06, + 8.391455594392028e-06, + 8.576274012739304e-06, + 7.637476301169954e-06, + 6.035619207978016e-06, + 4.900505700788926e-06, + 2.700857066884055e-06, + 3.052799229408265e-06, + 3.024239958904218e-06, + 2.3407558273902396e-06, + 1.630273686714645e-06, + 1.6224489627347793e-06, + 1.7616926015762147e-06, + 1.3268823977341526e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.77098144529009e-05, + "first_to_last_ratio": 5.786351228183689, + "imbalance_abs_log_ratio": 1.7555019078197227, + "last_quartile_mean": 8.245233061643376e-06, + "mean": 3.073857808999492e-05, + "normalized": [ + 0.8362577566227221, + 2.373745168443029, + 1.8117198501099894, + 1.5751424117641866, + 1.5278011706530124, + 1.815487592878854, + 1.4046395949024106, + 1.0721280996591929, + 1.2352562716436093, + 1.4523847721238452, + 1.2122549920407257, + 1.0201321842789222, + 1.2203169082313816, + 1.2783429873250631, + 1.0525616380439955, + 1.010813722620164, + 0.8760394275203864, + 0.9273309105161657, + 1.2093309822135678, + 1.0693992526347946, + 1.428708411485824, + 0.9918157658703739, + 0.9182234708569954, + 0.534268291669333, + 0.5596528890045244, + 0.41161184317293326, + 0.4078398397243235, + 0.27957479810482855, + 0.14962079640520384, + 0.11736499492778929, + 0.14511937346068207, + 0.07511383109117112 + ], + "peak_layer": 2, + "peak_normalized": 2.373745168443029, + "population_cv": 0.5427210092082145, + "rest_mean": 3.138446525688913e-05, + "spike_contrast": 0.8682890457336828, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.7250787388766186e-05, + "top_five_layers": [ + 2, + 6, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.5705374355311505e-05, + 7.296555122593418e-05, + 5.568969208979979e-05, + 4.8417638026876375e-05, + 4.696243559010327e-05, + 5.5805507145123556e-05, + 4.317662387620658e-05, + 3.295569331385195e-05, + 3.797002136707306e-05, + 4.464424273464829e-05, + 3.726299473783001e-05, + 3.1357412808574736e-05, + 3.7510806578211486e-05, + 3.929444574168883e-05, + 3.235424810554832e-05, + 3.107097654719837e-05, + 2.692820635274984e-05, + 2.8504833608167246e-05, + 3.7173114833422005e-05, + 3.287181243649684e-05, + 4.391646507428959e-05, + 3.0487006370094605e-05, + 2.8224883863003924e-05, + 1.6422647604485974e-05, + 1.720293403195683e-05, + 1.265236278413795e-05, + 1.253641676157713e-05, + 8.593731763539836e-06, + 4.599130534188589e-06, + 3.6076330616197083e-06, + 4.460763193492312e-06, + 2.3088923626346514e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.6240980534785194e-05, + "first_to_last_ratio": 8.81501945263024, + "imbalance_abs_log_ratio": 2.176457022612032, + "last_quartile_mean": 4.111276297180666e-06, + "mean": 2.2519588263492096e-05, + "normalized": [ + 0.7284164778364034, + 2.389714365593298, + 1.8344675527063727, + 1.6124517982477564, + 1.660867848829399, + 1.9609031976200701, + 1.5230683202508775, + 1.1645840254645428, + 1.3073819892255008, + 1.4901651727836733, + 1.2625908178604672, + 1.0494073392406849, + 1.21297826715993, + 1.378958269273871, + 0.9917742879119321, + 0.9095282929708826, + 0.9198153022966056, + 1.0487426528518113, + 1.2584126391221928, + 0.9987518391352217, + 1.3085415757850667, + 1.0724225800006402, + 0.9485461675021898, + 0.5069936019893233, + 0.33739074000009295, + 0.273271867252871, + 0.28263222104028385, + 0.19007385384653275, + 0.10699675859843288, + 0.10147279993418688, + 0.10196626637254216, + 0.0667111112963436 + ], + "peak_layer": 2, + "peak_normalized": 2.389714365593298, + "population_cv": 0.5911439983506431, + "rest_mean": 2.320860870794503e-05, + "spike_contrast": 0.8099958985051443, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.879887786344625e-05, + "top_five_layers": [ + 2, + 6, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.640363916521892e-05, + 5.38153835805133e-05, + 4.13114539696835e-05, + 3.63117505912669e-05, + 3.74020601157099e-05, + 4.4158732634969056e-05, + 3.4298871469218284e-05, + 2.6225952751701698e-05, + 2.944170410046354e-05, + 3.355790613568388e-05, + 2.8433025363483466e-05, + 2.3632221200386994e-05, + 2.731577114900574e-05, + 3.105357245658524e-05, + 2.233434861409478e-05, + 2.048220267170109e-05, + 2.0713861886179075e-05, + 2.3617252736585215e-05, + 2.833893449860625e-05, + 2.2491480194730684e-05, + 2.9467817512340844e-05, + 2.4150514946086332e-05, + 2.136086914106272e-05, + 1.1417287169024348e-05, + 7.597900548717007e-06, + 6.153969934530323e-06, + 6.3647612478234805e-06, + 4.280384928279091e-06, + 2.409522949164966e-06, + 2.2851256744615966e-06, + 2.296238335475209e-06, + 1.5023067589936545e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011762118538172217, + "first_to_last_ratio": 4.228719768518167, + "imbalance_abs_log_ratio": 1.4418992920522087, + "last_quartile_mean": 2.78148451116067e-05, + "mean": 8.002406778473414e-05, + "normalized": [ + 0.9210338573301938, + 2.3087354790458674, + 1.72875802725484, + 1.5227237833085348, + 1.3615568761930628, + 1.6698981403826305, + 1.262630340241389, + 0.9832444969618955, + 1.1432437822572334, + 1.444183076942276, + 1.2101593686516263, + 1.0400906407977921, + 1.2308343305675216, + 1.1478679901804696, + 1.125011533923313, + 1.162732392396995, + 0.8650388354268836, + 0.807876464469789, + 1.2254554690773087, + 1.1623670673328852, + 1.5536447585331754, + 0.8757401866950771, + 0.9111182549928211, + 0.5554068863324947, + 0.8132826025949527, + 0.4896112785691883, + 0.5227427522902547, + 0.38035098946634355, + 0.18821446379390167, + 0.12866317105200828, + 0.18038891087911704, + 0.07739379205815819 + ], + "peak_layer": 2, + "peak_normalized": 2.3087354790458674, + "population_cv": 0.5019050745008682, + "rest_mean": 8.088597830161841e-05, + "spike_contrast": 0.9318024282590781, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.536975099355913e-05, + "top_five_layers": [ + 2, + 3, + 6, + 21, + 4 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 7.370487583102658e-05, + 0.00018475440447218716, + 0.00013834224955644459, + 0.000121854551252909, + 0.00010895731975324452, + 0.00013363204197958112, + 0.00010104081593453884, + 7.868322427384555e-05, + 9.14870179258287e-05, + 0.00011556940444279462, + 9.684187534730881e-05, + 8.323228394147009e-05, + 9.849636990111321e-05, + 9.185706585412845e-05, + 9.002799924928695e-05, + 9.304657578468323e-05, + 6.922392640262842e-05, + 6.464956095442176e-05, + 9.806593152461573e-05, + 9.301734098698944e-05, + 0.00012432897347025573, + 7.008029206190258e-05, + 7.29113889974542e-05, + 4.444591831997968e-05, + 6.508218211820349e-05, + 3.9180686144391075e-05, + 4.183200144325383e-05, + 3.043723336304538e-05, + 1.5061687008710578e-05, + 1.0296150321664754e-05, + 1.4435454431804828e-05, + 6.19336606177967e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "f2daf0e86334d46ef8de0e95de81ea9f5fd3989918e22ce51c09039c50c57cf2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.018189694732427597, + 0.02713710628449917, + 0.03182194381952286, + 0.03608781099319458, + 0.026185201480984688, + 0.02762730047106743, + 0.028690988197922707, + 0.03034786321222782, + 0.02331089787185192, + 0.024178674444556236, + 0.024928271770477295, + 0.0256717037409544, + 0.020905915647745132, + 0.021578654646873474, + 0.022400982677936554, + 0.023724498227238655, + 0.020535636693239212, + 0.021988606080412865, + 0.023164646700024605, + 0.024238524958491325, + 0.019457563757896423, + 0.019637931138277054, + 0.02042427472770214, + 0.021937908604741096, + 0.021518733352422714, + 0.02267260104417801, + 0.024102909490466118, + 0.025153208523988724, + 0.023361310362815857, + 0.025513608008623123, + 0.027568820863962173, + 0.028944427147507668 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.237074980053876e-05, + "first_to_last_ratio": 1.549599178512509, + "imbalance_abs_log_ratio": 0.43799630298177145, + "last_quartile_mean": 7.983193313521042e-06, + "mean": 1.1284510890163801e-05, + "normalized": [ + 2.2240740776654793, + 1.123463076357621, + 0.8892756502185444, + 0.8391508499567661, + 1.020497607750464, + 0.9770305811455078, + 0.8809641928112601, + 0.815618216100643, + 1.047627354359627, + 1.038392580788716, + 0.9776631042995007, + 0.8263946439643135, + 1.1834865254731135, + 1.1158848074227978, + 0.9831938132537951, + 0.9253651743484881, + 1.2047165912881532, + 1.0585100087162846, + 1.0183273795435872, + 0.8939052877072882, + 1.565309272455062, + 1.3951478097472831, + 1.2557330331978191, + 1.0806924055185287, + 0.9345587629892653, + 0.8411301445051553, + 0.7796712675898905, + 0.7463030115246023, + 0.6524170468838495, + 0.5916843062269816, + 0.5583702111657051, + 0.5554412050239066 + ], + "peak_layer": 1, + "peak_normalized": 2.2240740776654793, + "population_cv": 0.31201770965801834, + "rest_mean": 1.0769836350206148e-05, + "spike_contrast": 1.3058465280826606, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4063753405935131e-05, + "top_five_layers": [ + 1, + 21, + 22, + 23, + 17 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.5097588149947114e-05, + 1.2677731319854502e-05, + 1.0035040759248659e-05, + 9.469406904827338e-06, + 1.1515816368046217e-05, + 1.102531223295955e-05, + 9.941250027623028e-06, + 9.203852641803678e-06, + 1.1821962289104704e-05, + 1.1717752386175562e-05, + 1.1032449947379064e-05, + 9.325459359388333e-06, + 1.3355066585063469e-05, + 1.2592214261530899e-05, + 1.1094861292804126e-05, + 1.0442293387313839e-05, + 1.3594637493952177e-05, + 1.1944767720706295e-05, + 1.1491326404211577e-05, + 1.00872839539079e-05, + 1.7663749531493522e-05, + 1.5743560652481392e-05, + 1.4170333088259213e-05, + 1.2195085218991153e-05, + 1.0546038538450375e-05, + 9.491742275713477e-06, + 8.798208909865934e-06, + 8.421664460911416e-06, + 7.362207270489307e-06, + 6.6768679971573874e-06, + 6.30093472864246e-06, + 6.267882326937979e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.889938600172172e-05, + "first_to_last_ratio": 1.5413227470241377, + "imbalance_abs_log_ratio": 0.43264097439819243, + "last_quartile_mean": 3.172559809172526e-05, + "mean": 4.484832186335552e-05, + "normalized": [ + 2.1812015108286227, + 1.1117106638221061, + 0.8712690674441437, + 0.8155092170381977, + 1.0253782370170077, + 0.9956135898204023, + 0.8876686512602217, + 0.8342739405890812, + 1.055082370640812, + 1.0480930529822718, + 0.9841224192619671, + 0.8329693296087033, + 1.1966925672041921, + 1.1146712863155361, + 0.9871271696154558, + 0.9208744603675149, + 1.199191469514801, + 1.0575028324354707, + 1.0341642237425372, + 0.898469999535679, + 1.5531345456552004, + 1.3827320560469147, + 1.2641281302304002, + 1.089237902148047, + 0.9325460360008964, + 0.8434023238170155, + 0.7808660077834011, + 0.7497730633285921, + 0.6533877584038497, + 0.584941351024475, + 0.5602379828066166, + 0.5540267837098669 + ], + "peak_layer": 1, + "peak_normalized": 2.1812015108286227, + "population_cv": 0.30617930626208134, + "rest_mean": 4.2818887676629754e-05, + "spike_contrast": 1.303332933193729, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5807266471674664e-05, + "top_five_layers": [ + 1, + 21, + 22, + 23, + 17 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.782322740647942e-05, + 4.985835767001845e-05, + 3.907495556632057e-05, + 3.6574219848262146e-05, + 4.598649320541881e-05, + 4.465159872779623e-05, + 3.981044937972911e-05, + 3.741578620974906e-05, + 4.73186737508513e-05, + 4.700521458289586e-05, + 4.413623901200481e-05, + 3.73572765965946e-05, + 5.366965342545882e-05, + 4.999113662051968e-05, + 4.42709970229771e-05, + 4.1299674194306135e-05, + 5.3781725000590086e-05, + 4.742722740047611e-05, + 4.638052996597253e-05, + 4.029487172374502e-05, + 6.965547800064087e-05, + 6.201321230037138e-05, + 5.6694025261094794e-05, + 4.885049202130176e-05, + 4.1823124774964526e-05, + 3.782517887884751e-05, + 3.502053004922345e-05, + 3.362606366863474e-05, + 2.9303344490472227e-05, + 2.623363798193168e-05, + 2.512573337298818e-05, + 2.4847171516739763e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04336446523666382, + "first_to_last_ratio": 1.541322758743648, + "imbalance_abs_log_ratio": 0.4326409820017331, + "last_quartile_mean": 0.02813457790762186, + "mean": 0.03977194067556411, + "normalized": [ + 2.1812015617016365, + 1.1117106345862457, + 0.8712690698977081, + 0.8155092439280437, + 1.025378223837616, + 0.9956136045129053, + 0.8876686330275743, + 0.834273974654176, + 1.0550823347362368, + 1.0480930495251253, + 0.9841223425346584, + 0.8329693905077741, + 1.1966926583705826, + 1.1146713324813935, + 0.9871271572544008, + 0.9208743646706732, + 1.1991914877687524, + 1.0575028591124085, + 1.034164215905152, + 0.8984699490823292, + 1.553134589516788, + 1.3827319963584066, + 1.2641280815855902, + 1.089237936269389, + 0.9325460279316578, + 0.8434023170272189, + 0.7808660625001403, + 0.7497730677902867, + 0.653387781511157, + 0.5849413243159465, + 0.5602379632013756, + 0.5540267638966512 + ], + "peak_layer": 1, + "peak_normalized": 2.1812015617016365, + "population_cv": 0.30617931423170164, + "rest_mean": 0.03797221818455943, + "spike_contrast": 1.3033329232031423, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.049490442126989366, + "top_five_layers": [ + 1, + 21, + 22, + 23, + 17 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.08675061911344528, + 0.0442148894071579, + 0.03465206176042557, + 0.032434385269880295, + 0.040781281888484955, + 0.03959748521447182, + 0.03530430421233177, + 0.03318069502711296, + 0.041962672024965286, + 0.04168469458818436, + 0.039140455424785614, + 0.03312880918383598, + 0.047594789415597916, + 0.04433264210820198, + 0.03925996273756027, + 0.0366249606013298, + 0.04769417271018028, + 0.04205894097685814, + 0.04113071784377098, + 0.035733893513679504, + 0.061771176755428314, + 0.05499393492937088, + 0.050276827067136765, + 0.04332110658288002, + 0.037089165300130844, + 0.033543746918439865, + 0.031056558713316917, + 0.029819929972290993, + 0.025986500084400177, + 0.02326425164937973, + 0.022281751036643982, + 0.022034719586372375 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.2504997307114536e-06, + "first_to_last_ratio": 1.534048564025481, + "imbalance_abs_log_ratio": 0.42791036086994516, + "last_quartile_mean": 2.1189027563650598e-06, + "mean": 2.8327239434133844e-06, + "normalized": [ + 2.434101831970979, + 1.2717248047474439, + 1.00501292981449, + 0.978793620303735, + 0.982603804852325, + 0.9590124302490006, + 0.7831434651115216, + 0.7654630979136043, + 0.9086838647658697, + 0.9473256649897754, + 0.8950270312608294, + 0.7521042950228368, + 1.0341349274760525, + 1.0071135921377148, + 0.8968409003212485, + 0.9087337907131031, + 1.1168468910837492, + 0.9975784586172981, + 0.9974908072758529, + 0.913717074488086, + 1.5359343623090123, + 1.463151481657458, + 1.3013629981753376, + 1.160026574630551, + 0.9743714014341565, + 0.8914931250247432, + 0.814663192509027, + 0.7848098840673302, + 0.7014159737591589, + 0.636528977982725, + 0.5932070215396675, + 0.5875817237953173 + ], + "peak_layer": 1, + "peak_normalized": 2.434101831970979, + "population_cv": 0.3367092305215161, + "rest_mean": 2.6821859827813804e-06, + "spike_contrast": 1.3592006498541729, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6456289308262057e-06, + "top_five_layers": [ + 1, + 21, + 22, + 23, + 2 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.895138540130574e-06, + 3.6024453038407955e-06, + 2.8469241897255415e-06, + 2.772652123894659e-06, + 2.7834453248942737e-06, + 2.7166174731974024e-06, + 2.2184292447491316e-06, + 2.168345645259251e-06, + 2.574050540715689e-06, + 2.6835120934265433e-06, + 2.535364501454751e-06, + 2.1305038444552338e-06, + 2.9294187697814777e-06, + 2.8528747861855663e-06, + 2.540502691772417e-06, + 2.5741919671418145e-06, + 3.163718929499737e-06, + 2.8258643851586385e-06, + 2.8256160931050545e-06, + 2.588308234408032e-06, + 4.350878043624107e-06, + 4.144704234931851e-06, + 3.686402124003507e-06, + 3.2860350529517746e-06, + 2.7601251986197894e-06, + 2.525353920646012e-06, + 2.307715931237908e-06, + 2.2231497496250086e-06, + 1.9869178231601836e-06, + 1.803110876608116e-06, + 1.6803917333163554e-06, + 1.664456817707105e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.123088797314267e-05, + "first_to_last_ratio": 1.5481267051258016, + "imbalance_abs_log_ratio": 0.4370456226743978, + "last_quartile_mean": 2.663276063685771e-05, + "mean": 3.721600086237231e-05, + "normalized": [ + 2.2576717646411373, + 1.1550152331301582, + 0.8940234958179116, + 0.8176409477374584, + 1.026845887840395, + 0.9928503960506618, + 0.8891512866739572, + 0.8298463487568712, + 1.0382073376210066, + 1.010525228672297, + 0.9673644078560969, + 0.8327734670628495, + 1.1597014181578864, + 1.092703899157089, + 0.9631469488595324, + 0.9043154291215759, + 1.184106843777168, + 1.0436930451114852, + 1.0042890707665433, + 0.902293602113743, + 1.5231038792179041, + 1.386951430682882, + 1.2813911026130895, + 1.1173746266426017, + 0.9589627259110588, + 0.8812873444364125, + 0.807412211801708, + 0.7778540745678935, + 0.6383882364209196, + 0.5722421467697438, + 0.546122523093635, + 0.5427436389163276 + ], + "peak_layer": 1, + "peak_normalized": 2.2576717646411373, + "population_cv": 0.31425595490925345, + "rest_mean": 3.5468525244073976e-05, + "spike_contrast": 1.3153174224230797, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.665236920118332e-05, + "top_five_layers": [ + 1, + 21, + 22, + 23, + 17 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.402151433983818e-05, + 4.298504791222513e-05, + 3.3271979191340506e-05, + 3.0429326216108166e-05, + 3.82150974473916e-05, + 3.694992119562812e-05, + 3.309065505163744e-05, + 3.0883562430972233e-05, + 3.863792517222464e-05, + 3.760770778171718e-05, + 3.6001434637000784e-05, + 3.099249806837179e-05, + 4.31594489782583e-05, + 4.0666069253347814e-05, + 3.584447767934762e-05, + 3.365500379004516e-05, + 4.406772131915204e-05, + 3.8842081266921014e-05, + 3.7375622923718765e-05, + 3.357975947437808e-05, + 5.668383528245613e-05, + 5.161678564036265e-05, + 4.768825237988494e-05, + 4.1584215068724006e-05, + 3.5688757634488866e-05, + 3.2797990570543334e-05, + 3.00486535707023e-05, + 2.894861790991854e-05, + 2.3758257157169282e-05, + 2.129656422766857e-05, + 2.0324496290413663e-05, + 2.0198747733957134e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.4926876878671465e-05, + "first_to_last_ratio": 1.476240806784429, + "imbalance_abs_log_ratio": 0.3894988610989889, + "last_quartile_mean": 2.3659335738557274e-05, + "mean": 3.1687308080563525e-05, + "normalized": [ + 2.2762986475001763, + 1.1662144265070453, + 0.8845459641933642, + 0.8067880948230711, + 1.022807332270146, + 0.9684203821336351, + 0.8774829326589954, + 0.815326475621621, + 1.0196651328217694, + 0.9647280193706825, + 0.9452574962733946, + 0.828508179687011, + 1.129511920443437, + 1.0667478383346118, + 0.9293458708194444, + 0.8844500989246517, + 1.1690667339751835, + 1.0196011843730592, + 0.9801960813926914, + 0.9044117736621221, + 1.5006890378974727, + 1.3936090252586848, + 1.313619848705842, + 1.1595056946364453, + 1.0169445109383177, + 0.9550874175993482, + 0.867249516139161, + 0.8392884874733272, + 0.6350726181832244, + 0.5697204034365697, + 0.5445274700537234, + 0.5453113838917707 + ], + "peak_layer": 1, + "peak_normalized": 2.2762986475001763, + "population_cv": 0.3151213209355983, + "rest_mean": 3.0062608116336548e-05, + "spike_contrast": 1.3458808274656038, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.046068788738921e-05, + "top_five_layers": [ + 1, + 21, + 22, + 23, + 17 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.212977652670816e-05, + 3.6954195820726454e-05, + 2.8028880478814244e-05, + 2.5564942916389555e-05, + 3.2410011044703424e-05, + 3.068663500016555e-05, + 2.7805072022601962e-05, + 2.583550121926237e-05, + 3.231044320273213e-05, + 3.0569633963750675e-05, + 2.9952665499877185e-05, + 2.62531939370092e-05, + 3.579119220376015e-05, + 3.380236739758402e-05, + 2.9448468922055326e-05, + 2.8025842766510323e-05, + 3.704457776620984e-05, + 3.230841684853658e-05, + 3.1059775210451335e-05, + 2.8658374503720552e-05, + 4.755279587698169e-05, + 4.415971852722578e-05, + 4.162507684668526e-05, + 3.674161416711286e-05, + 3.222423401894048e-05, + 3.0264149245340377e-05, + 2.7480802600621246e-05, + 2.6594792871037498e-05, + 2.012374170590192e-05, + 1.8052905943477526e-05, + 1.7254609701922163e-05, + 1.7279449821216986e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.852452538121725e-05, + "first_to_last_ratio": 1.5689675763636528, + "imbalance_abs_log_ratio": 0.4504178083771724, + "last_quartile_mean": 5.6422150919388514e-05, + "mean": 8.211212457354122e-05, + "normalized": [ + 2.1219723286224816, + 1.09827042835026, + 0.8643788556282443, + 0.8279612862679215, + 1.017320737753908, + 0.9856182851753849, + 0.8733076450463839, + 0.8359162514841522, + 1.0662813223328673, + 1.0751002353169425, + 1.0056898741580604, + 0.8412786645166292, + 1.2255342764451151, + 1.1374637954311468, + 1.0329797245885708, + 0.9362220937013769, + 1.2341718933129842, + 1.085018444301826, + 1.0615915480189244, + 0.9105849975032602, + 1.5381215664946966, + 1.374354285922899, + 1.2590874380574026, + 1.0946904946303944, + 0.8961975645628097, + 0.7984172853485704, + 0.7473454782550812, + 0.7156455510623411, + 0.6436881832738256, + 0.5784219801847321, + 0.5573798012661504, + 0.5599876829846564 + ], + "peak_layer": 1, + "peak_normalized": 2.1219723286224816, + "population_cv": 0.303114029752154, + "rest_mean": 7.85768893870732e-05, + "spike_contrast": 1.2879409629202945, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010120239458046853, + "top_five_layers": [ + 1, + 21, + 22, + 23, + 17 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00017423965618945658, + 9.018131822813302e-05, + 7.09759842720814e-05, + 6.7985660280101e-05, + 8.353436714969575e-05, + 8.093121141428128e-05, + 7.17091461410746e-05, + 6.863885937491432e-05, + 8.755462476983666e-05, + 8.827876445138827e-05, + 8.257933222921565e-05, + 6.907917850185186e-05, + 0.000100631223176606, + 9.339956886833534e-05, + 8.482015982735902e-05, + 7.687518518650904e-05, + 0.00010134047624887899, + 8.909316966310143e-05, + 8.716953743714839e-05, + 7.477006874978542e-05, + 0.0001262984296772629, + 0.00011285115033388138, + 0.00010338634456275031, + 8.98873622645624e-05, + 7.358868606388569e-05, + 6.555973959621042e-05, + 6.136612500995398e-05, + 5.8763176639331505e-05, + 5.28546042914968e-05, + 4.749545769300312e-05, + 4.576763967634179e-05, + 4.598177838488482e-05 + ] + } + } + } + } + } + }, + "step": 500 + }, + { + "modes": { + "learned": { + "bits_per_byte": 2.064953725142933, + "forward": { + "activation_sha256": { + "attention_branch_output": "3bc7ac091ccda73a60054802f3ec6d7bc68d1af6b8f7940e83f1843e7ab472be", + "mlp_branch_output": "b5dfb9be1f653e88219f0cf5f3837c21e4150bbb11f0520e3acfd9e5ac781830", + "post_attention_state": "e3e18e8967d646eb5fe3ca615db63633333413502a9ff39741999ecb78762644", + "post_mlp_state": "1266324f6216c67f6f92c122a2698de3ea217c2cc1431a3715e69a04aa62551d", + "pre_attention_input": "9ec6f7109f4e916c83054b81e32dd86c0125957a9aa2bade5062f6ae3f8e89ac", + "pre_mlp_input": "f4a14b64dd8b186b4b509748e2f608d960603b5d908ec49d21001a2020554f02" + }, + "logits_sha256": "e65e488a65420b358dedf0b8a5957c28a5e46147110f919cae1df8502eac1276", + "loss_nats": 1.43131685256958, + "mixer_summary_sha256": "34a25dd049f4b5bf7ff812bb03925bb3c0f35a3d7d3518411476b9b87af79ff5" + }, + "loss_nats": 1.43131685256958, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.4580664336681366, + "group": 1, + "latest_source_mass": 0.18571940064430237, + "layer": 1, + "max_source_mass": 0.81428062915802, + "mixer_index": 1, + "normalized_entropy": 0.660850172250714, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.81428062915802, + "median": 0.8348954319953918, + "p05": 0.6508002281188965, + "p95": 0.9081302285194397 + }, + { + "label": "current_group_partial", + "mean": 0.18571940064430237, + "median": 0.16510456800460815, + "p05": 0.0918697863817215, + "p95": 0.3491998016834259 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6350213885307312, + "group": 1, + "latest_source_mass": 0.48109376430511475, + "layer": 2, + "max_source_mass": 0.5189062356948853, + "mixer_index": 2, + "normalized_entropy": 0.9161422080917097, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5189062356948853, + "median": 0.49609383940696716, + "p05": 0.2741084098815918, + "p95": 0.8060141801834106 + }, + { + "label": "current_group_partial", + "mean": 0.48109376430511475, + "median": 0.5039061903953552, + "p05": 0.19398583471775055, + "p95": 0.7258915901184082 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4277062714099884, + "group": 1, + "latest_source_mass": 0.15978552401065826, + "layer": 2, + "max_source_mass": 0.8402144908905029, + "mixer_index": 3, + "normalized_entropy": 0.6170497167202993, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8402144908905029, + "median": 0.8553081154823303, + "p05": 0.7220970392227173, + "p95": 0.9125142097473145 + }, + { + "label": "current_group_partial", + "mean": 0.15978552401065826, + "median": 0.14469186961650848, + "p05": 0.08748584985733032, + "p95": 0.2779030203819275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6868079900741577, + "group": 1, + "latest_source_mass": 0.5064847469329834, + "layer": 3, + "max_source_mass": 0.5064847469329834, + "mixer_index": 4, + "normalized_entropy": 0.9908544813229038, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4935153126716614, + "median": 0.488329142332077, + "p05": 0.41239702701568604, + "p95": 0.5894505381584167 + }, + { + "label": "current_group_partial", + "mean": 0.5064847469329834, + "median": 0.5116708278656006, + "p05": 0.41054946184158325, + "p95": 0.587602972984314 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.38686758279800415, + "group": 1, + "latest_source_mass": 0.13613657653331757, + "layer": 3, + "max_source_mass": 0.863863468170166, + "mixer_index": 5, + "normalized_entropy": 0.5581319431833811, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.863863468170166, + "median": 0.8749346137046814, + "p05": 0.7766229510307312, + "p95": 0.9342007040977478 + }, + { + "label": "current_group_partial", + "mean": 0.13613657653331757, + "median": 0.1250653862953186, + "p05": 0.0657993033528328, + "p95": 0.22337706387043 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6772255897521973, + "group": 1, + "latest_source_mass": 0.43206995725631714, + "layer": 4, + "max_source_mass": 0.5679300427436829, + "mixer_index": 6, + "normalized_entropy": 0.9770299998985986, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5679300427436829, + "median": 0.5676445364952087, + "p05": 0.476777583360672, + "p95": 0.6623597741127014 + }, + { + "label": "current_group_partial", + "mean": 0.43206995725631714, + "median": 0.43235546350479126, + "p05": 0.3376401960849762, + "p95": 0.5232223868370056 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5690979361534119, + "group": 1, + "latest_source_mass": 0.28186240792274475, + "layer": 4, + "max_source_mass": 0.7181376218795776, + "mixer_index": 7, + "normalized_entropy": 0.8210347702686712, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7181376218795776, + "median": 0.7203682661056519, + "p05": 0.5602526664733887, + "p95": 0.9003764390945435 + }, + { + "label": "current_group_partial", + "mean": 0.28186240792274475, + "median": 0.2796317934989929, + "p05": 0.09962356090545654, + "p95": 0.43974730372428894 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6592018008232117, + "group": 2, + "latest_source_mass": 0.4827212691307068, + "layer": 5, + "max_source_mass": 0.517278790473938, + "mixer_index": 8, + "normalized_entropy": 0.9510271689927217, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.517278790473938, + "median": 0.5070376396179199, + "p05": 0.33305567502975464, + "p95": 0.7351903915405273 + }, + { + "label": "completed_group_1", + "mean": 0.4827212691307068, + "median": 0.4929623603820801, + "p05": 0.26480966806411743, + "p95": 0.6669443845748901 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.038456916809082, + "group": 2, + "latest_source_mass": 0.3147233724594116, + "layer": 5, + "max_source_mass": 0.4540913999080658, + "mixer_index": 9, + "normalized_entropy": 0.9452442208415888, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4540913999080658, + "median": 0.4537813663482666, + "p05": 0.2934587895870209, + "p95": 0.613494873046875 + }, + { + "label": "completed_group_1", + "mean": 0.23118522763252258, + "median": 0.23724967241287231, + "p05": 0.17357462644577026, + "p95": 0.2779788076877594 + }, + { + "label": "current_group_partial", + "mean": 0.3147233724594116, + "median": 0.31569617986679077, + "p05": 0.190863698720932, + "p95": 0.4443512260913849 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0903832912445068, + "group": 2, + "latest_source_mass": 0.38192373514175415, + "layer": 6, + "max_source_mass": 0.38192373514175415, + "mixer_index": 10, + "normalized_entropy": 0.9925096437492255, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3152569830417633, + "median": 0.3122430443763733, + "p05": 0.2715557813644409, + "p95": 0.3724803924560547 + }, + { + "label": "completed_group_1", + "mean": 0.30281925201416016, + "median": 0.303215354681015, + "p05": 0.2769315242767334, + "p95": 0.3273058235645294 + }, + { + "label": "current_group_partial", + "mean": 0.38192373514175415, + "median": 0.3863401412963867, + "p05": 0.32811740040779114, + "p95": 0.4194748103618622 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0411981344223022, + "group": 2, + "latest_source_mass": 0.31767070293426514, + "layer": 6, + "max_source_mass": 0.4020236134529114, + "mixer_index": 11, + "normalized_entropy": 0.9477393846418622, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4020236134529114, + "median": 0.386812299489975, + "p05": 0.2018219530582428, + "p95": 0.707051694393158 + }, + { + "label": "completed_group_1", + "mean": 0.2803056538105011, + "median": 0.29224973917007446, + "p05": 0.12760642170906067, + "p95": 0.363227903842926 + }, + { + "label": "current_group_partial", + "mean": 0.31767070293426514, + "median": 0.32260167598724365, + "p05": 0.1550494283437729, + "p95": 0.4480850100517273 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0905810594558716, + "group": 2, + "latest_source_mass": 0.384937047958374, + "layer": 7, + "max_source_mass": 0.384937047958374, + "mixer_index": 12, + "normalized_entropy": 0.9926896601329894, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3278552293777466, + "median": 0.32747673988342285, + "p05": 0.305451899766922, + "p95": 0.3515099585056305 + }, + { + "label": "completed_group_1", + "mean": 0.2872077226638794, + "median": 0.28588157892227173, + "p05": 0.26596081256866455, + "p95": 0.30896520614624023 + }, + { + "label": "current_group_partial", + "mean": 0.384937047958374, + "median": 0.38384974002838135, + "p05": 0.3644270896911621, + "p95": 0.4090334475040436 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0150185823440552, + "group": 2, + "latest_source_mass": 0.29576438665390015, + "layer": 7, + "max_source_mass": 0.48287349939346313, + "mixer_index": 13, + "normalized_entropy": 0.9239097294047216, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48287349939346313, + "median": 0.4650489091873169, + "p05": 0.3346910774707794, + "p95": 0.7498793601989746 + }, + { + "label": "completed_group_1", + "mean": 0.2213621288537979, + "median": 0.22629523277282715, + "p05": 0.10256442427635193, + "p95": 0.29524046182632446 + }, + { + "label": "current_group_partial", + "mean": 0.29576438665390015, + "median": 0.3073716163635254, + "p05": 0.14070476591587067, + "p95": 0.3917995095252991 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0816482305526733, + "group": 2, + "latest_source_mass": 0.40547415614128113, + "layer": 8, + "max_source_mass": 0.40547415614128113, + "mixer_index": 14, + "normalized_entropy": 0.9845586488605524, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3306047320365906, + "median": 0.33274394273757935, + "p05": 0.2933180034160614, + "p95": 0.3644672632217407 + }, + { + "label": "completed_group_1", + "mean": 0.2639211118221283, + "median": 0.2629454731941223, + "p05": 0.23683591187000275, + "p95": 0.2942429184913635 + }, + { + "label": "current_group_partial", + "mean": 0.40547415614128113, + "median": 0.40589621663093567, + "p05": 0.37255728244781494, + "p95": 0.43844079971313477 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0114538669586182, + "group": 2, + "latest_source_mass": 0.3466050922870636, + "layer": 8, + "max_source_mass": 0.4296638071537018, + "mixer_index": 15, + "normalized_entropy": 0.9206649856291366, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4296638071537018, + "median": 0.39453762769699097, + "p05": 0.24606716632843018, + "p95": 0.7861940860748291 + }, + { + "label": "completed_group_1", + "mean": 0.22373110055923462, + "median": 0.22628825902938843, + "p05": 0.07940524816513062, + "p95": 0.3281305432319641 + }, + { + "label": "current_group_partial", + "mean": 0.3466050922870636, + "median": 0.36169061064720154, + "p05": 0.12893560528755188, + "p95": 0.47660863399505615 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0918333530426025, + "group": 3, + "latest_source_mass": 0.32777342200279236, + "layer": 9, + "max_source_mass": 0.36806726455688477, + "mixer_index": 16, + "normalized_entropy": 0.9938295468788851, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.36806726455688477, + "median": 0.3683367669582367, + "p05": 0.31737297773361206, + "p95": 0.4160827398300171 + }, + { + "label": "completed_group_1", + "mean": 0.3041592836380005, + "median": 0.3026740550994873, + "p05": 0.27596747875213623, + "p95": 0.3395172357559204 + }, + { + "label": "completed_group_2", + "mean": 0.32777342200279236, + "median": 0.3294421434402466, + "p05": 0.2683165669441223, + "p95": 0.3802773654460907 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3290088176727295, + "group": 3, + "latest_source_mass": 0.2603103518486023, + "layer": 9, + "max_source_mass": 0.35130923986434937, + "mixer_index": 17, + "normalized_entropy": 0.9586772152770757, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35130923986434937, + "median": 0.3600649833679199, + "p05": 0.20772139728069305, + "p95": 0.5109238028526306 + }, + { + "label": "completed_group_1", + "mean": 0.17179939150810242, + "median": 0.16660934686660767, + "p05": 0.11614341288805008, + "p95": 0.23857468366622925 + }, + { + "label": "completed_group_2", + "mean": 0.21658103168010712, + "median": 0.21380692720413208, + "p05": 0.1328548789024353, + "p95": 0.30206120014190674 + }, + { + "label": "current_group_partial", + "mean": 0.2603103518486023, + "median": 0.25967299938201904, + "p05": 0.21334272623062134, + "p95": 0.3113977313041687 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.379033088684082, + "group": 3, + "latest_source_mass": 0.24449527263641357, + "layer": 10, + "max_source_mass": 0.29040855169296265, + "mixer_index": 18, + "normalized_entropy": 0.9947620991331576, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29040855169296265, + "median": 0.29490548372268677, + "p05": 0.2500562369823456, + "p95": 0.32532158493995667 + }, + { + "label": "completed_group_1", + "mean": 0.23445811867713928, + "median": 0.2324211150407791, + "p05": 0.21483668684959412, + "p95": 0.26111143827438354 + }, + { + "label": "completed_group_2", + "mean": 0.2306380271911621, + "median": 0.23122423887252808, + "p05": 0.19837205111980438, + "p95": 0.26181501150131226 + }, + { + "label": "current_group_partial", + "mean": 0.24449527263641357, + "median": 0.2435547560453415, + "p05": 0.21777315437793732, + "p95": 0.2725067138671875 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3394957780838013, + "group": 3, + "latest_source_mass": 0.26184359192848206, + "layer": 10, + "max_source_mass": 0.3262045383453369, + "mixer_index": 19, + "normalized_entropy": 0.9662419581666017, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3262045383453369, + "median": 0.33913201093673706, + "p05": 0.15982496738433838, + "p95": 0.49248045682907104 + }, + { + "label": "completed_group_1", + "mean": 0.18669261038303375, + "median": 0.1907009780406952, + "p05": 0.1366541087627411, + "p95": 0.22178497910499573 + }, + { + "label": "completed_group_2", + "mean": 0.22525924444198608, + "median": 0.22064685821533203, + "p05": 0.14132791757583618, + "p95": 0.31671440601348877 + }, + { + "label": "current_group_partial", + "mean": 0.26184359192848206, + "median": 0.2529178857803345, + "p05": 0.2070307433605194, + "p95": 0.33866065740585327 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3677098751068115, + "group": 3, + "latest_source_mass": 0.2845887541770935, + "layer": 11, + "max_source_mass": 0.29889947175979614, + "mixer_index": 20, + "normalized_entropy": 0.9865941270957302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29889947175979614, + "median": 0.2991558313369751, + "p05": 0.25637567043304443, + "p95": 0.345245897769928 + }, + { + "label": "completed_group_1", + "mean": 0.22409284114837646, + "median": 0.2220568209886551, + "p05": 0.19557906687259674, + "p95": 0.2607859671115875 + }, + { + "label": "completed_group_2", + "mean": 0.1924189329147339, + "median": 0.1933085322380066, + "p05": 0.16921265423297882, + "p95": 0.21061491966247559 + }, + { + "label": "current_group_partial", + "mean": 0.2845887541770935, + "median": 0.286131888628006, + "p05": 0.25166112184524536, + "p95": 0.31440335512161255 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2910308837890625, + "group": 3, + "latest_source_mass": 0.2587660849094391, + "layer": 11, + "max_source_mass": 0.3990757167339325, + "mixer_index": 21, + "normalized_entropy": 0.9312819268384881, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3990757167339325, + "median": 0.4163215160369873, + "p05": 0.22181591391563416, + "p95": 0.5799376368522644 + }, + { + "label": "completed_group_1", + "mean": 0.18164023756980896, + "median": 0.17469573020935059, + "p05": 0.11628367006778717, + "p95": 0.24950891733169556 + }, + { + "label": "completed_group_2", + "mean": 0.16051796078681946, + "median": 0.15820392966270447, + "p05": 0.09586596488952637, + "p95": 0.21985307335853577 + }, + { + "label": "current_group_partial", + "mean": 0.2587660849094391, + "median": 0.2571001946926117, + "p05": 0.18264421820640564, + "p95": 0.3371993899345398 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3623619079589844, + "group": 3, + "latest_source_mass": 0.29877203702926636, + "layer": 12, + "max_source_mass": 0.3024129569530487, + "mixer_index": 22, + "normalized_entropy": 0.9827363842542266, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3024129569530487, + "median": 0.3032229542732239, + "p05": 0.26408907771110535, + "p95": 0.3383038640022278 + }, + { + "label": "completed_group_1", + "mean": 0.21113446354866028, + "median": 0.20936083793640137, + "p05": 0.19063444435596466, + "p95": 0.236852765083313 + }, + { + "label": "completed_group_2", + "mean": 0.18768057227134705, + "median": 0.1870611608028412, + "p05": 0.16947540640830994, + "p95": 0.20746007561683655 + }, + { + "label": "current_group_partial", + "mean": 0.29877203702926636, + "median": 0.30005669593811035, + "p05": 0.26102375984191895, + "p95": 0.332477867603302 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2281851768493652, + "group": 3, + "latest_source_mass": 0.23528897762298584, + "layer": 12, + "max_source_mass": 0.46696996688842773, + "mixer_index": 23, + "normalized_entropy": 0.8859483319669569, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46696996688842773, + "median": 0.4326247572898865, + "p05": 0.3197334408760071, + "p95": 0.6868066787719727 + }, + { + "label": "completed_group_1", + "mean": 0.16045212745666504, + "median": 0.16051796078681946, + "p05": 0.08773931860923767, + "p95": 0.22867971658706665 + }, + { + "label": "completed_group_2", + "mean": 0.13728895783424377, + "median": 0.14161056280136108, + "p05": 0.08086587488651276, + "p95": 0.1866428703069687 + }, + { + "label": "current_group_partial", + "mean": 0.23528897762298584, + "median": 0.24016538262367249, + "p05": 0.13368576765060425, + "p95": 0.32655879855155945 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3772211074829102, + "group": 4, + "latest_source_mass": 0.24272704124450684, + "layer": 13, + "max_source_mass": 0.28748467564582825, + "mixer_index": 24, + "normalized_entropy": 0.9934550309866003, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28748467564582825, + "median": 0.293403685092926, + "p05": 0.22645097970962524, + "p95": 0.3287804424762726 + }, + { + "label": "completed_group_1", + "mean": 0.24638661742210388, + "median": 0.247311532497406, + "p05": 0.21059969067573547, + "p95": 0.27817288041114807 + }, + { + "label": "completed_group_2", + "mean": 0.22340166568756104, + "median": 0.22116877138614655, + "p05": 0.19812123477458954, + "p95": 0.2556689977645874 + }, + { + "label": "completed_group_3", + "mean": 0.24272704124450684, + "median": 0.24190634489059448, + "p05": 0.20034869015216827, + "p95": 0.2857007384300232 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5519046783447266, + "group": 4, + "latest_source_mass": 0.23496176302433014, + "layer": 13, + "max_source_mass": 0.30400484800338745, + "mixer_index": 25, + "normalized_entropy": 0.9642525917620761, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30400484800338745, + "median": 0.3028053641319275, + "p05": 0.22878815233707428, + "p95": 0.400842547416687 + }, + { + "label": "completed_group_1", + "mean": 0.1470816731452942, + "median": 0.1489969789981842, + "p05": 0.11740177869796753, + "p95": 0.17312948405742645 + }, + { + "label": "completed_group_2", + "mean": 0.13316048681735992, + "median": 0.13380160927772522, + "p05": 0.10824885964393616, + "p95": 0.15745775401592255 + }, + { + "label": "completed_group_3", + "mean": 0.1807912290096283, + "median": 0.17882078886032104, + "p05": 0.14052768051624298, + "p95": 0.22473205626010895 + }, + { + "label": "current_group_partial", + "mean": 0.23496176302433014, + "median": 0.23252201080322266, + "p05": 0.20341888070106506, + "p95": 0.27739572525024414 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.572178602218628, + "group": 4, + "latest_source_mass": 0.28295373916625977, + "layer": 14, + "max_source_mass": 0.28295373916625977, + "mixer_index": 26, + "normalized_entropy": 0.9768494889255332, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21885336935520172, + "median": 0.2175014615058899, + "p05": 0.1686568409204483, + "p95": 0.2588198482990265 + }, + { + "label": "completed_group_1", + "mean": 0.2029750645160675, + "median": 0.20364594459533691, + "p05": 0.16090315580368042, + "p95": 0.2408662736415863 + }, + { + "label": "completed_group_2", + "mean": 0.1456144154071808, + "median": 0.1434675008058548, + "p05": 0.12856581807136536, + "p95": 0.16848507523536682 + }, + { + "label": "completed_group_3", + "mean": 0.14960341155529022, + "median": 0.14882786571979523, + "p05": 0.11882077157497406, + "p95": 0.17920038104057312 + }, + { + "label": "current_group_partial", + "mean": 0.28295373916625977, + "median": 0.28187495470046997, + "p05": 0.24177348613739014, + "p95": 0.3278854489326477 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5391805171966553, + "group": 4, + "latest_source_mass": 0.2634100914001465, + "layer": 14, + "max_source_mass": 0.2898171544075012, + "mixer_index": 27, + "normalized_entropy": 0.9563466259278133, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2898171544075012, + "median": 0.2940465807914734, + "p05": 0.18881390988826752, + "p95": 0.4135587215423584 + }, + { + "label": "completed_group_1", + "mean": 0.15984712541103363, + "median": 0.15493696928024292, + "p05": 0.1193767637014389, + "p95": 0.2155456840991974 + }, + { + "label": "completed_group_2", + "mean": 0.1228363886475563, + "median": 0.12368190288543701, + "p05": 0.08640466630458832, + "p95": 0.15742656588554382 + }, + { + "label": "completed_group_3", + "mean": 0.16408920288085938, + "median": 0.16305197775363922, + "p05": 0.12961797416210175, + "p95": 0.20428991317749023 + }, + { + "label": "current_group_partial", + "mean": 0.2634100914001465, + "median": 0.26288318634033203, + "p05": 0.2051006406545639, + "p95": 0.3223395347595215 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5521924495697021, + "group": 4, + "latest_source_mass": 0.32056212425231934, + "layer": 15, + "max_source_mass": 0.32056212425231934, + "mixer_index": 28, + "normalized_entropy": 0.9644313940773145, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.210292249917984, + "median": 0.20937448740005493, + "p05": 0.16961288452148438, + "p95": 0.2644428610801697 + }, + { + "label": "completed_group_1", + "mean": 0.185291588306427, + "median": 0.1801799237728119, + "p05": 0.1599384844303131, + "p95": 0.23004677891731262 + }, + { + "label": "completed_group_2", + "mean": 0.14173434674739838, + "median": 0.13837604224681854, + "p05": 0.12188456207513809, + "p95": 0.17588482797145844 + }, + { + "label": "completed_group_3", + "mean": 0.14211969077587128, + "median": 0.14116454124450684, + "p05": 0.1147841066122055, + "p95": 0.17095403373241425 + }, + { + "label": "current_group_partial", + "mean": 0.32056212425231934, + "median": 0.3242885172367096, + "p05": 0.25657448172569275, + "p95": 0.3705737292766571 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5468778610229492, + "group": 4, + "latest_source_mass": 0.26227355003356934, + "layer": 15, + "max_source_mass": 0.2756267488002777, + "mixer_index": 29, + "normalized_entropy": 0.9611292545504065, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2756267488002777, + "median": 0.26299941539764404, + "p05": 0.16425450146198273, + "p95": 0.411077082157135 + }, + { + "label": "completed_group_1", + "mean": 0.16966477036476135, + "median": 0.16762341558933258, + "p05": 0.1361422836780548, + "p95": 0.2147567719221115 + }, + { + "label": "completed_group_2", + "mean": 0.133374884724617, + "median": 0.13428664207458496, + "p05": 0.09884629398584366, + "p95": 0.16465862095355988 + }, + { + "label": "completed_group_3", + "mean": 0.1590600460767746, + "median": 0.1579585075378418, + "p05": 0.12251652777194977, + "p95": 0.20119814574718475 + }, + { + "label": "current_group_partial", + "mean": 0.26227355003356934, + "median": 0.2576714754104614, + "p05": 0.1876349151134491, + "p95": 0.35171017050743103 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5309922695159912, + "group": 4, + "latest_source_mass": 0.2753770351409912, + "layer": 16, + "max_source_mass": 0.2753770351409912, + "mixer_index": 30, + "normalized_entropy": 0.95125898159099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21606208384037018, + "median": 0.21523752808570862, + "p05": 0.17893150448799133, + "p95": 0.25598081946372986 + }, + { + "label": "completed_group_1", + "mean": 0.27386677265167236, + "median": 0.2663733959197998, + "p05": 0.20507416129112244, + "p95": 0.368195503950119 + }, + { + "label": "completed_group_2", + "mean": 0.12764990329742432, + "median": 0.12438307702541351, + "p05": 0.10569629073143005, + "p95": 0.15790577232837677 + }, + { + "label": "completed_group_3", + "mean": 0.10704416781663895, + "median": 0.10739792883396149, + "p05": 0.07143788039684296, + "p95": 0.13995352387428284 + }, + { + "label": "current_group_partial", + "mean": 0.2753770351409912, + "median": 0.278347373008728, + "p05": 0.2043362706899643, + "p95": 0.333025723695755 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5832164287567139, + "group": 4, + "latest_source_mass": 0.1685337871313095, + "layer": 16, + "max_source_mass": 0.24636118113994598, + "mixer_index": 31, + "normalized_entropy": 0.9837076761552552, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22763225436210632, + "median": 0.2050790637731552, + "p05": 0.15834280848503113, + "p95": 0.3526703417301178 + }, + { + "label": "completed_group_1", + "mean": 0.24636118113994598, + "median": 0.24476206302642822, + "p05": 0.2120620459318161, + "p95": 0.2850618064403534 + }, + { + "label": "completed_group_2", + "mean": 0.17577087879180908, + "median": 0.17898494005203247, + "p05": 0.13335087895393372, + "p95": 0.20948299765586853 + }, + { + "label": "completed_group_3", + "mean": 0.1817018985748291, + "median": 0.17961755394935608, + "p05": 0.14637041091918945, + "p95": 0.22162409126758575 + }, + { + "label": "current_group_partial", + "mean": 0.1685337871313095, + "median": 0.1639421284198761, + "p05": 0.12563997507095337, + "p95": 0.22547563910484314 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5793015956878662, + "group": 5, + "latest_source_mass": 0.25108590722084045, + "layer": 17, + "max_source_mass": 0.25108590722084045, + "mixer_index": 32, + "normalized_entropy": 0.9812752536066109, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2158121019601822, + "median": 0.21408912539482117, + "p05": 0.1828576922416687, + "p95": 0.25927743315696716 + }, + { + "label": "completed_group_1", + "mean": 0.21387425065040588, + "median": 0.2010079026222229, + "p05": 0.15640759468078613, + "p95": 0.3137179911136627 + }, + { + "label": "completed_group_2", + "mean": 0.1464191973209381, + "median": 0.14469388127326965, + "p05": 0.1298638880252838, + "p95": 0.16936294734477997 + }, + { + "label": "completed_group_3", + "mean": 0.17280855774879456, + "median": 0.17517459392547607, + "p05": 0.13696318864822388, + "p95": 0.198440819978714 + }, + { + "label": "completed_group_4", + "mean": 0.25108590722084045, + "median": 0.2582312226295471, + "p05": 0.1583583801984787, + "p95": 0.3231808543205261 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7719972133636475, + "group": 5, + "latest_source_mass": 0.1918708086013794, + "layer": 17, + "max_source_mass": 0.20328448712825775, + "mixer_index": 33, + "normalized_entropy": 0.9889704749974495, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16962553560733795, + "median": 0.1690855771303177, + "p05": 0.1343761533498764, + "p95": 0.2083701342344284 + }, + { + "label": "completed_group_1", + "mean": 0.20328448712825775, + "median": 0.20764055848121643, + "p05": 0.16696703433990479, + "p95": 0.2276618480682373 + }, + { + "label": "completed_group_2", + "mean": 0.1659279465675354, + "median": 0.16634680330753326, + "p05": 0.1471528857946396, + "p95": 0.18308526277542114 + }, + { + "label": "completed_group_3", + "mean": 0.14008209109306335, + "median": 0.1373065710067749, + "p05": 0.10879024863243103, + "p95": 0.18058663606643677 + }, + { + "label": "completed_group_4", + "mean": 0.12920911610126495, + "median": 0.1269148886203766, + "p05": 0.10906878858804703, + "p95": 0.1541941612958908 + }, + { + "label": "current_group_partial", + "mean": 0.1918708086013794, + "median": 0.19034206867218018, + "p05": 0.14745062589645386, + "p95": 0.23773568868637085 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.689546823501587, + "group": 5, + "latest_source_mass": 0.3272479772567749, + "layer": 18, + "max_source_mass": 0.3272479772567749, + "mixer_index": 34, + "normalized_entropy": 0.9429540362521402, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16601097583770752, + "median": 0.16331082582473755, + "p05": 0.13003051280975342, + "p95": 0.20780879259109497 + }, + { + "label": "completed_group_1", + "mean": 0.1523149609565735, + "median": 0.15154623985290527, + "p05": 0.1387515515089035, + "p95": 0.16826456785202026 + }, + { + "label": "completed_group_2", + "mean": 0.0875597670674324, + "median": 0.08667673170566559, + "p05": 0.0783165767788887, + "p95": 0.09952021390199661 + }, + { + "label": "completed_group_3", + "mean": 0.10126751661300659, + "median": 0.0997447669506073, + "p05": 0.08770889788866043, + "p95": 0.12069930881261826 + }, + { + "label": "completed_group_4", + "mean": 0.1655988097190857, + "median": 0.16505587100982666, + "p05": 0.15017777681350708, + "p95": 0.18333116173744202 + }, + { + "label": "current_group_partial", + "mean": 0.3272479772567749, + "median": 0.32916754484176636, + "p05": 0.2944076955318451, + "p95": 0.355704128742218 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5251271724700928, + "group": 5, + "latest_source_mass": 0.4654960334300995, + "layer": 18, + "max_source_mass": 0.4654960334300995, + "mixer_index": 35, + "normalized_entropy": 0.8511896817976157, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13745036721229553, + "median": 0.12849631905555725, + "p05": 0.08826753497123718, + "p95": 0.21900935471057892 + }, + { + "label": "completed_group_1", + "mean": 0.11702707409858704, + "median": 0.11415569484233856, + "p05": 0.09654439240694046, + "p95": 0.14336207509040833 + }, + { + "label": "completed_group_2", + "mean": 0.07592720538377762, + "median": 0.07502187043428421, + "p05": 0.061439283192157745, + "p95": 0.09369536489248276 + }, + { + "label": "completed_group_3", + "mean": 0.08887732028961182, + "median": 0.08338192105293274, + "p05": 0.0636318027973175, + "p95": 0.1313587874174118 + }, + { + "label": "completed_group_4", + "mean": 0.11522199958562851, + "median": 0.11553025990724564, + "p05": 0.09672605991363525, + "p95": 0.13243958353996277 + }, + { + "label": "current_group_partial", + "mean": 0.4654960334300995, + "median": 0.47634756565093994, + "p05": 0.36450207233428955, + "p95": 0.5321885943412781 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7189464569091797, + "group": 5, + "latest_source_mass": 0.29706352949142456, + "layer": 19, + "max_source_mass": 0.29706352949142456, + "mixer_index": 36, + "normalized_entropy": 0.9593622840736289, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1744518131017685, + "median": 0.1729438602924347, + "p05": 0.14070478081703186, + "p95": 0.20897650718688965 + }, + { + "label": "completed_group_1", + "mean": 0.15982553362846375, + "median": 0.15947404503822327, + "p05": 0.14682501554489136, + "p95": 0.1730521321296692 + }, + { + "label": "completed_group_2", + "mean": 0.09849169850349426, + "median": 0.09798875451087952, + "p05": 0.09000753611326218, + "p95": 0.1088048666715622 + }, + { + "label": "completed_group_3", + "mean": 0.10780535638332367, + "median": 0.10671155154705048, + "p05": 0.09614568203687668, + "p95": 0.12314765155315399 + }, + { + "label": "completed_group_4", + "mean": 0.16236206889152527, + "median": 0.16202759742736816, + "p05": 0.14780421555042267, + "p95": 0.17789581418037415 + }, + { + "label": "current_group_partial", + "mean": 0.29706352949142456, + "median": 0.297796368598938, + "p05": 0.2712000906467438, + "p95": 0.32343345880508423 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5877950191497803, + "group": 5, + "latest_source_mass": 0.4179171919822693, + "layer": 19, + "max_source_mass": 0.4179171919822693, + "mixer_index": 37, + "normalized_entropy": 0.8861652729726335, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14238224923610687, + "median": 0.13146832585334778, + "p05": 0.08928614854812622, + "p95": 0.2557840645313263 + }, + { + "label": "completed_group_1", + "mean": 0.1503014862537384, + "median": 0.14734625816345215, + "p05": 0.12638163566589355, + "p95": 0.1823616921901703 + }, + { + "label": "completed_group_2", + "mean": 0.10275131464004517, + "median": 0.10360147058963776, + "p05": 0.08169187605381012, + "p95": 0.1201353371143341 + }, + { + "label": "completed_group_3", + "mean": 0.10159259289503098, + "median": 0.09856455028057098, + "p05": 0.08053024113178253, + "p95": 0.1288338005542755 + }, + { + "label": "completed_group_4", + "mean": 0.08505518734455109, + "median": 0.0846521332859993, + "p05": 0.06617184728384018, + "p95": 0.10538893193006516 + }, + { + "label": "current_group_partial", + "mean": 0.4179171919822693, + "median": 0.42233192920684814, + "p05": 0.3364676535129547, + "p95": 0.48611512780189514 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7369492053985596, + "group": 5, + "latest_source_mass": 0.2724200189113617, + "layer": 20, + "max_source_mass": 0.2724200189113617, + "mixer_index": 38, + "normalized_entropy": 0.9694098093126812, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17980656027793884, + "median": 0.18241192400455475, + "p05": 0.1317644566297531, + "p95": 0.2213428020477295 + }, + { + "label": "completed_group_1", + "mean": 0.1706826537847519, + "median": 0.17070558667182922, + "p05": 0.15632681548595428, + "p95": 0.1849895715713501 + }, + { + "label": "completed_group_2", + "mean": 0.10907261073589325, + "median": 0.10894737392663956, + "p05": 0.09957940876483917, + "p95": 0.11852658540010452 + }, + { + "label": "completed_group_3", + "mean": 0.1106589287519455, + "median": 0.10897988080978394, + "p05": 0.097733274102211, + "p95": 0.12891732156276703 + }, + { + "label": "completed_group_4", + "mean": 0.15735921263694763, + "median": 0.15671271085739136, + "p05": 0.14351971447467804, + "p95": 0.17215019464492798 + }, + { + "label": "current_group_partial", + "mean": 0.2724200189113617, + "median": 0.27173393964767456, + "p05": 0.24179266393184662, + "p95": 0.3049001395702362 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5987625122070312, + "group": 5, + "latest_source_mass": 0.39448028802871704, + "layer": 20, + "max_source_mass": 0.39448028802871704, + "mixer_index": 39, + "normalized_entropy": 0.8922863473945123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15493065118789673, + "median": 0.1367141604423523, + "p05": 0.08384928107261658, + "p95": 0.33080407977104187 + }, + { + "label": "completed_group_1", + "mean": 0.15333226323127747, + "median": 0.14968673884868622, + "p05": 0.1258050799369812, + "p95": 0.1917010396718979 + }, + { + "label": "completed_group_2", + "mean": 0.10736657679080963, + "median": 0.10799242556095123, + "p05": 0.07847796380519867, + "p95": 0.1328052282333374 + }, + { + "label": "completed_group_3", + "mean": 0.1186048835515976, + "median": 0.1159338653087616, + "p05": 0.08940468728542328, + "p95": 0.15236710011959076 + }, + { + "label": "completed_group_4", + "mean": 0.07128533720970154, + "median": 0.06884299218654633, + "p05": 0.04878491908311844, + "p95": 0.10267870128154755 + }, + { + "label": "current_group_partial", + "mean": 0.39448028802871704, + "median": 0.4018329381942749, + "p05": 0.2799415588378906, + "p95": 0.47351574897766113 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.75795578956604, + "group": 6, + "latest_source_mass": 0.2277819961309433, + "layer": 21, + "max_source_mass": 0.2277819961309433, + "mixer_index": 40, + "normalized_entropy": 0.9811338071640952, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1946164071559906, + "median": 0.19585168361663818, + "p05": 0.1496070772409439, + "p95": 0.23593875765800476 + }, + { + "label": "completed_group_1", + "mean": 0.17554128170013428, + "median": 0.17525508999824524, + "p05": 0.16232231259346008, + "p95": 0.1893068552017212 + }, + { + "label": "completed_group_2", + "mean": 0.11572675406932831, + "median": 0.11579035222530365, + "p05": 0.10456694662570953, + "p95": 0.126620814204216 + }, + { + "label": "completed_group_3", + "mean": 0.1155121773481369, + "median": 0.11377307772636414, + "p05": 0.1026163250207901, + "p95": 0.13427132368087769 + }, + { + "label": "completed_group_4", + "mean": 0.1708213835954666, + "median": 0.17137710750102997, + "p05": 0.15489280223846436, + "p95": 0.18481197953224182 + }, + { + "label": "completed_group_5", + "mean": 0.2277819961309433, + "median": 0.22828495502471924, + "p05": 0.1998254656791687, + "p95": 0.25523197650909424 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5344648361206055, + "group": 6, + "latest_source_mass": 0.5061697959899902, + "layer": 21, + "max_source_mass": 0.5061697959899902, + "mixer_index": 41, + "normalized_entropy": 0.7885589357070504, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0905577763915062, + "median": 0.07774583250284195, + "p05": 0.049066200852394104, + "p95": 0.19859620928764343 + }, + { + "label": "completed_group_1", + "mean": 0.0786280632019043, + "median": 0.07735998928546906, + "p05": 0.06383179128170013, + "p95": 0.09802225232124329 + }, + { + "label": "completed_group_2", + "mean": 0.06272733211517334, + "median": 0.062279097735881805, + "p05": 0.04816657304763794, + "p95": 0.07942435145378113 + }, + { + "label": "completed_group_3", + "mean": 0.07359343767166138, + "median": 0.07242873311042786, + "p05": 0.056279584765434265, + "p95": 0.09239646047353745 + }, + { + "label": "completed_group_4", + "mean": 0.050107162445783615, + "median": 0.048957135528326035, + "p05": 0.038674917072057724, + "p95": 0.06534890085458755 + }, + { + "label": "completed_group_5", + "mean": 0.13821640610694885, + "median": 0.13806892931461334, + "p05": 0.10529474169015884, + "p95": 0.17281517386436462 + }, + { + "label": "current_group_partial", + "mean": 0.5061697959899902, + "median": 0.5132234692573547, + "p05": 0.41420048475265503, + "p95": 0.5726657509803772 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7138500213623047, + "group": 6, + "latest_source_mass": 0.4071047902107239, + "layer": 22, + "max_source_mass": 0.4071047902107239, + "mixer_index": 42, + "normalized_entropy": 0.8807446850484503, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13302071392536163, + "median": 0.13715070486068726, + "p05": 0.08387725800275803, + "p95": 0.173191100358963 + }, + { + "label": "completed_group_1", + "mean": 0.10984362661838531, + "median": 0.11014518141746521, + "p05": 0.08778759092092514, + "p95": 0.12984654307365417 + }, + { + "label": "completed_group_2", + "mean": 0.07243296504020691, + "median": 0.07197447121143341, + "p05": 0.06215493753552437, + "p95": 0.08360114693641663 + }, + { + "label": "completed_group_3", + "mean": 0.07083482295274734, + "median": 0.06908848881721497, + "p05": 0.05891891568899155, + "p95": 0.08817940950393677 + }, + { + "label": "completed_group_4", + "mean": 0.08885612338781357, + "median": 0.08808258175849915, + "p05": 0.07647298276424408, + "p95": 0.10338804125785828 + }, + { + "label": "completed_group_5", + "mean": 0.11790697276592255, + "median": 0.11741673201322556, + "p05": 0.09878336638212204, + "p95": 0.13930705189704895 + }, + { + "label": "current_group_partial", + "mean": 0.4071047902107239, + "median": 0.4047229290008545, + "p05": 0.34799832105636597, + "p95": 0.4778216481208801 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4565383195877075, + "group": 6, + "latest_source_mass": 0.5493720769882202, + "layer": 22, + "max_source_mass": 0.5493720769882202, + "mixer_index": 43, + "normalized_entropy": 0.7485126280341451, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09418222308158875, + "median": 0.08733601868152618, + "p05": 0.04874149709939957, + "p95": 0.1813039481639862 + }, + { + "label": "completed_group_1", + "mean": 0.07457256317138672, + "median": 0.07398833334445953, + "p05": 0.05570545420050621, + "p95": 0.09413403272628784 + }, + { + "label": "completed_group_2", + "mean": 0.057202231138944626, + "median": 0.05683571845293045, + "p05": 0.04444708675146103, + "p95": 0.07107516378164291 + }, + { + "label": "completed_group_3", + "mean": 0.06210128217935562, + "median": 0.06029999628663063, + "p05": 0.04672754555940628, + "p95": 0.08132980763912201 + }, + { + "label": "completed_group_4", + "mean": 0.04410415142774582, + "median": 0.043332912027835846, + "p05": 0.03435574471950531, + "p95": 0.05708703026175499 + }, + { + "label": "completed_group_5", + "mean": 0.11846547573804855, + "median": 0.1162438839673996, + "p05": 0.09147419780492783, + "p95": 0.15436163544654846 + }, + { + "label": "current_group_partial", + "mean": 0.5493720769882202, + "median": 0.5507825016975403, + "p05": 0.4547572731971741, + "p95": 0.6371357440948486 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7785513401031494, + "group": 6, + "latest_source_mass": 0.3567505478858948, + "layer": 23, + "max_source_mass": 0.3567505478858948, + "mixer_index": 44, + "normalized_entropy": 0.9139945854985072, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1471671462059021, + "median": 0.15290190279483795, + "p05": 0.09501753002405167, + "p95": 0.18610240519046783 + }, + { + "label": "completed_group_1", + "mean": 0.12308044731616974, + "median": 0.12249830365180969, + "p05": 0.10776080191135406, + "p95": 0.14060497283935547 + }, + { + "label": "completed_group_2", + "mean": 0.07605893164873123, + "median": 0.07562456279993057, + "p05": 0.06604670733213425, + "p95": 0.08747582882642746 + }, + { + "label": "completed_group_3", + "mean": 0.07563325762748718, + "median": 0.07386856526136398, + "p05": 0.06278492510318756, + "p95": 0.09318976104259491 + }, + { + "label": "completed_group_4", + "mean": 0.1090458557009697, + "median": 0.10803408920764923, + "p05": 0.094943106174469, + "p95": 0.12811031937599182 + }, + { + "label": "completed_group_5", + "mean": 0.11226382851600647, + "median": 0.1111060380935669, + "p05": 0.09618639945983887, + "p95": 0.13308405876159668 + }, + { + "label": "current_group_partial", + "mean": 0.3567505478858948, + "median": 0.35415834188461304, + "p05": 0.2961513102054596, + "p95": 0.4284268319606781 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5135828256607056, + "group": 6, + "latest_source_mass": 0.5200446844100952, + "layer": 23, + "max_source_mass": 0.5200446844100952, + "mixer_index": 45, + "normalized_entropy": 0.77782770514636, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10121430456638336, + "median": 0.08898357301950455, + "p05": 0.06175067648291588, + "p95": 0.20128758251667023 + }, + { + "label": "completed_group_1", + "mean": 0.07933001220226288, + "median": 0.07804793119430542, + "p05": 0.06303977221250534, + "p95": 0.0987483337521553 + }, + { + "label": "completed_group_2", + "mean": 0.061728835105895996, + "median": 0.06086552143096924, + "p05": 0.04682660102844238, + "p95": 0.0792360007762909 + }, + { + "label": "completed_group_3", + "mean": 0.06817515194416046, + "median": 0.0653572827577591, + "p05": 0.050377603620290756, + "p95": 0.0938427746295929 + }, + { + "label": "completed_group_4", + "mean": 0.048567142337560654, + "median": 0.04761301726102829, + "p05": 0.03707665950059891, + "p95": 0.06356129050254822 + }, + { + "label": "completed_group_5", + "mean": 0.12093988060951233, + "median": 0.11864300072193146, + "p05": 0.09093102067708969, + "p95": 0.16081735491752625 + }, + { + "label": "current_group_partial", + "mean": 0.5200446844100952, + "median": 0.5256286263465881, + "p05": 0.41434088349342346, + "p95": 0.606282114982605 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7546210289001465, + "group": 6, + "latest_source_mass": 0.3774413764476776, + "layer": 24, + "max_source_mass": 0.3774413764476776, + "mixer_index": 46, + "normalized_entropy": 0.9016968382388917, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14427465200424194, + "median": 0.14563661813735962, + "p05": 0.09999129176139832, + "p95": 0.19150450825691223 + }, + { + "label": "completed_group_1", + "mean": 0.12224892526865005, + "median": 0.1226624846458435, + "p05": 0.09991119801998138, + "p95": 0.1422707736492157 + }, + { + "label": "completed_group_2", + "mean": 0.08193168044090271, + "median": 0.08116704970598221, + "p05": 0.07012136280536652, + "p95": 0.09678678959608078 + }, + { + "label": "completed_group_3", + "mean": 0.07876674830913544, + "median": 0.0766892209649086, + "p05": 0.06478038430213928, + "p95": 0.09861677885055542 + }, + { + "label": "completed_group_4", + "mean": 0.09007281810045242, + "median": 0.08881355822086334, + "p05": 0.07590551674365997, + "p95": 0.11008355021476746 + }, + { + "label": "completed_group_5", + "mean": 0.10526381433010101, + "median": 0.10412947833538055, + "p05": 0.08725899457931519, + "p95": 0.12721270322799683 + }, + { + "label": "current_group_partial", + "mean": 0.3774413764476776, + "median": 0.37542667984962463, + "p05": 0.3145911991596222, + "p95": 0.44844120740890503 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5284571647644043, + "group": 6, + "latest_source_mass": 0.5105152726173401, + "layer": 24, + "max_source_mass": 0.5105152726173401, + "mixer_index": 47, + "normalized_entropy": 0.7854716033555963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10831617563962936, + "median": 0.09708426892757416, + "p05": 0.04612221196293831, + "p95": 0.24017156660556793 + }, + { + "label": "completed_group_1", + "mean": 0.08754566311836243, + "median": 0.08632134646177292, + "p05": 0.06808170676231384, + "p95": 0.10904698818922043 + }, + { + "label": "completed_group_2", + "mean": 0.06345342099666595, + "median": 0.06274682283401489, + "p05": 0.047288306057453156, + "p95": 0.08281171321868896 + }, + { + "label": "completed_group_3", + "mean": 0.06688661873340607, + "median": 0.06420055776834488, + "p05": 0.048897963017225266, + "p95": 0.09314233064651489 + }, + { + "label": "completed_group_4", + "mean": 0.05242554843425751, + "median": 0.051159247756004333, + "p05": 0.03922025114297867, + "p95": 0.07083941996097565 + }, + { + "label": "completed_group_5", + "mean": 0.11085730046033859, + "median": 0.10786899924278259, + "p05": 0.08087077736854553, + "p95": 0.15321578085422516 + }, + { + "label": "current_group_partial", + "mean": 0.5105152726173401, + "median": 0.5166662931442261, + "p05": 0.39341118931770325, + "p95": 0.6058120131492615 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8309729099273682, + "group": 7, + "latest_source_mass": 0.31088557839393616, + "layer": 25, + "max_source_mass": 0.31088557839393616, + "mixer_index": 48, + "normalized_entropy": 0.9409339433355933, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1468513309955597, + "median": 0.14672473073005676, + "p05": 0.08069700002670288, + "p95": 0.2256799042224884 + }, + { + "label": "completed_group_1", + "mean": 0.13658970594406128, + "median": 0.13762319087982178, + "p05": 0.11331360042095184, + "p95": 0.1581028401851654 + }, + { + "label": "completed_group_2", + "mean": 0.09501184523105621, + "median": 0.09421288967132568, + "p05": 0.07920350134372711, + "p95": 0.11351083964109421 + }, + { + "label": "completed_group_3", + "mean": 0.0928751602768898, + "median": 0.09060071408748627, + "p05": 0.07744356989860535, + "p95": 0.11498282104730606 + }, + { + "label": "completed_group_4", + "mean": 0.10057065635919571, + "median": 0.09926527738571167, + "p05": 0.08480366319417953, + "p95": 0.12013090401887894 + }, + { + "label": "completed_group_5", + "mean": 0.11721574515104294, + "median": 0.1158604770898819, + "p05": 0.09652461111545563, + "p95": 0.14446426928043365 + }, + { + "label": "completed_group_6", + "mean": 0.31088557839393616, + "median": 0.3095095753669739, + "p05": 0.24580135941505432, + "p95": 0.3798484206199646 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.6937754154205322, + "group": 7, + "latest_source_mass": 0.26301223039627075, + "layer": 25, + "max_source_mass": 0.3669250011444092, + "mixer_index": 49, + "normalized_entropy": 0.8145337974022819, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09603364765644073, + "median": 0.08905445039272308, + "p05": 0.058942534029483795, + "p95": 0.15364733338356018 + }, + { + "label": "completed_group_1", + "mean": 0.06767556071281433, + "median": 0.06639593839645386, + "p05": 0.04534030333161354, + "p95": 0.09348243474960327 + }, + { + "label": "completed_group_2", + "mean": 0.04511266574263573, + "median": 0.044085465371608734, + "p05": 0.0323045440018177, + "p95": 0.060699306428432465 + }, + { + "label": "completed_group_3", + "mean": 0.04789547994732857, + "median": 0.04490892216563225, + "p05": 0.03212590143084526, + "p95": 0.07155333459377289 + }, + { + "label": "completed_group_4", + "mean": 0.03816477954387665, + "median": 0.036960676312446594, + "p05": 0.028077732771635056, + "p95": 0.05199173092842102 + }, + { + "label": "completed_group_5", + "mean": 0.07518067210912704, + "median": 0.07277381420135498, + "p05": 0.0526883490383625, + "p95": 0.10617287456989288 + }, + { + "label": "completed_group_6", + "mean": 0.3669250011444092, + "median": 0.36637505888938904, + "p05": 0.23747950792312622, + "p95": 0.49873870611190796 + }, + { + "label": "current_group_partial", + "mean": 0.26301223039627075, + "median": 0.26092374324798584, + "p05": 0.11381205916404724, + "p95": 0.4189145267009735 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9070472717285156, + "group": 7, + "latest_source_mass": 0.2382715940475464, + "layer": 26, + "max_source_mass": 0.24526362121105194, + "mixer_index": 50, + "normalized_entropy": 0.9170958805545191, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10919073224067688, + "median": 0.11166982352733612, + "p05": 0.04006436467170715, + "p95": 0.17950788140296936 + }, + { + "label": "completed_group_1", + "mean": 0.11743451654911041, + "median": 0.11847291141748428, + "p05": 0.078579381108284, + "p95": 0.15376348793506622 + }, + { + "label": "completed_group_2", + "mean": 0.07601121068000793, + "median": 0.07483125478029251, + "p05": 0.06124042719602585, + "p95": 0.09465125948190689 + }, + { + "label": "completed_group_3", + "mean": 0.06814020872116089, + "median": 0.06591488420963287, + "p05": 0.052837856113910675, + "p95": 0.08991338312625885 + }, + { + "label": "completed_group_4", + "mean": 0.0658918246626854, + "median": 0.06464026868343353, + "p05": 0.051906269043684006, + "p95": 0.08394908905029297 + }, + { + "label": "completed_group_5", + "mean": 0.07979628443717957, + "median": 0.07902438938617706, + "p05": 0.06372269243001938, + "p95": 0.0988437831401825 + }, + { + "label": "completed_group_6", + "mean": 0.24526362121105194, + "median": 0.24634721875190735, + "p05": 0.17033326625823975, + "p95": 0.3122789263725281 + }, + { + "label": "current_group_partial", + "mean": 0.2382715940475464, + "median": 0.22752389311790466, + "p05": 0.1175542026758194, + "p95": 0.39684510231018066 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.776512861251831, + "group": 7, + "latest_source_mass": 0.23505288362503052, + "layer": 26, + "max_source_mass": 0.33455103635787964, + "mixer_index": 51, + "normalized_entropy": 0.8543220983344934, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08731269836425781, + "median": 0.07214296609163284, + "p05": 0.03820698708295822, + "p95": 0.2289474606513977 + }, + { + "label": "completed_group_1", + "mean": 0.08430957794189453, + "median": 0.08319784700870514, + "p05": 0.05780494213104248, + "p95": 0.11521004140377045 + }, + { + "label": "completed_group_2", + "mean": 0.056895703077316284, + "median": 0.05547339469194412, + "p05": 0.041310664266347885, + "p95": 0.07652511447668076 + }, + { + "label": "completed_group_3", + "mean": 0.06295455247163773, + "median": 0.0598134770989418, + "p05": 0.04222813621163368, + "p95": 0.09349943697452545 + }, + { + "label": "completed_group_4", + "mean": 0.04594648629426956, + "median": 0.0447731539607048, + "p05": 0.034647077322006226, + "p95": 0.06127878278493881 + }, + { + "label": "completed_group_5", + "mean": 0.09297706186771393, + "median": 0.09176799654960632, + "p05": 0.0599244087934494, + "p95": 0.13207381963729858 + }, + { + "label": "completed_group_6", + "mean": 0.33455103635787964, + "median": 0.3391231298446655, + "p05": 0.2096378207206726, + "p95": 0.4444153904914856 + }, + { + "label": "current_group_partial", + "mean": 0.23505288362503052, + "median": 0.22023122012615204, + "p05": 0.0984557569026947, + "p95": 0.41664981842041016 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9573358297348022, + "group": 7, + "latest_source_mass": 0.2227109968662262, + "layer": 27, + "max_source_mass": 0.22605091333389282, + "mixer_index": 52, + "normalized_entropy": 0.9412795649708946, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11900999397039413, + "median": 0.12137913703918457, + "p05": 0.07443303614854813, + "p95": 0.16705159842967987 + }, + { + "label": "completed_group_1", + "mean": 0.11752338707447052, + "median": 0.1196502298116684, + "p05": 0.0804249495267868, + "p95": 0.14694978296756744 + }, + { + "label": "completed_group_2", + "mean": 0.07979564368724823, + "median": 0.07969067245721817, + "p05": 0.0651809498667717, + "p95": 0.09441538900136948 + }, + { + "label": "completed_group_3", + "mean": 0.07266241312026978, + "median": 0.07125541567802429, + "p05": 0.05800755321979523, + "p95": 0.09137322753667831 + }, + { + "label": "completed_group_4", + "mean": 0.07412993162870407, + "median": 0.07290434837341309, + "p05": 0.05999685451388359, + "p95": 0.09376152604818344 + }, + { + "label": "completed_group_5", + "mean": 0.08811671286821365, + "median": 0.087425097823143, + "p05": 0.07325023412704468, + "p95": 0.10507422685623169 + }, + { + "label": "completed_group_6", + "mean": 0.22605091333389282, + "median": 0.22650857269763947, + "p05": 0.1749759018421173, + "p95": 0.27763867378234863 + }, + { + "label": "current_group_partial", + "mean": 0.2227109968662262, + "median": 0.2145734429359436, + "p05": 0.14350463449954987, + "p95": 0.328477680683136 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7231189012527466, + "group": 7, + "latest_source_mass": 0.2634544372558594, + "layer": 27, + "max_source_mass": 0.3417898416519165, + "mixer_index": 53, + "normalized_entropy": 0.8286450312331257, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09091638028621674, + "median": 0.07839073240756989, + "p05": 0.045465391129255295, + "p95": 0.19545325636863708 + }, + { + "label": "completed_group_1", + "mean": 0.07795695960521698, + "median": 0.07602830231189728, + "p05": 0.053584761917591095, + "p95": 0.1065598800778389 + }, + { + "label": "completed_group_2", + "mean": 0.05235600098967552, + "median": 0.05057134851813316, + "p05": 0.03418106958270073, + "p95": 0.07707518339157104 + }, + { + "label": "completed_group_3", + "mean": 0.053361356258392334, + "median": 0.05035271495580673, + "p05": 0.031062742695212364, + "p95": 0.08295466005802155 + }, + { + "label": "completed_group_4", + "mean": 0.04162060469388962, + "median": 0.040454644709825516, + "p05": 0.029850685968995094, + "p95": 0.05732571333646774 + }, + { + "label": "completed_group_5", + "mean": 0.07854442298412323, + "median": 0.07773076742887497, + "p05": 0.053876958787441254, + "p95": 0.10653360933065414 + }, + { + "label": "completed_group_6", + "mean": 0.3417898416519165, + "median": 0.348370760679245, + "p05": 0.1893925964832306, + "p95": 0.4778548777103424 + }, + { + "label": "current_group_partial", + "mean": 0.2634544372558594, + "median": 0.2390930950641632, + "p05": 0.11348576098680496, + "p95": 0.4878687262535095 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8967883586883545, + "group": 7, + "latest_source_mass": 0.23599036037921906, + "layer": 28, + "max_source_mass": 0.261149525642395, + "mixer_index": 54, + "normalized_entropy": 0.9121623862318685, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1089041605591774, + "median": 0.1106237843632698, + "p05": 0.0463758185505867, + "p95": 0.1698746681213379 + }, + { + "label": "completed_group_1", + "mean": 0.10000172257423401, + "median": 0.09870216995477676, + "p05": 0.07011638581752777, + "p95": 0.13429011404514313 + }, + { + "label": "completed_group_2", + "mean": 0.07706406712532043, + "median": 0.0757090300321579, + "p05": 0.05966329202055931, + "p95": 0.09858658909797668 + }, + { + "label": "completed_group_3", + "mean": 0.07702124863862991, + "median": 0.07400410622358322, + "p05": 0.05783015862107277, + "p95": 0.1036195158958435 + }, + { + "label": "completed_group_4", + "mean": 0.06017591059207916, + "median": 0.05848415568470955, + "p05": 0.04412911832332611, + "p95": 0.08321075141429901 + }, + { + "label": "completed_group_5", + "mean": 0.07969298958778381, + "median": 0.07816708087921143, + "p05": 0.057708315551280975, + "p95": 0.10603620111942291 + }, + { + "label": "completed_group_6", + "mean": 0.261149525642395, + "median": 0.2585980296134949, + "p05": 0.1826809048652649, + "p95": 0.34350520372390747 + }, + { + "label": "current_group_partial", + "mean": 0.23599036037921906, + "median": 0.22840067744255066, + "p05": 0.1214975118637085, + "p95": 0.3747939467430115 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.801592230796814, + "group": 7, + "latest_source_mass": 0.1870974600315094, + "layer": 28, + "max_source_mass": 0.3473600149154663, + "mixer_index": 55, + "normalized_entropy": 0.8663827256915495, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09432460367679596, + "median": 0.07062139362096786, + "p05": 0.0350821278989315, + "p95": 0.24959293007850647 + }, + { + "label": "completed_group_1", + "mean": 0.09541776776313782, + "median": 0.09502051025629044, + "p05": 0.06895694881677628, + "p95": 0.12235064059495926 + }, + { + "label": "completed_group_2", + "mean": 0.06706254929304123, + "median": 0.0657147765159607, + "p05": 0.049946803599596024, + "p95": 0.08819274604320526 + }, + { + "label": "completed_group_3", + "mean": 0.07247316837310791, + "median": 0.06966830790042877, + "p05": 0.05053037777543068, + "p95": 0.09898276627063751 + }, + { + "label": "completed_group_4", + "mean": 0.048089414834976196, + "median": 0.0466206818819046, + "p05": 0.035116635262966156, + "p95": 0.06637651473283768 + }, + { + "label": "completed_group_5", + "mean": 0.08817502856254578, + "median": 0.08746744692325592, + "p05": 0.062397219240665436, + "p95": 0.11551516503095627 + }, + { + "label": "completed_group_6", + "mean": 0.3473600149154663, + "median": 0.354450523853302, + "p05": 0.20967835187911987, + "p95": 0.4590650796890259 + }, + { + "label": "current_group_partial", + "mean": 0.1870974600315094, + "median": 0.15943653881549835, + "p05": 0.072502501308918, + "p95": 0.38723331689834595 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.919801950454712, + "group": 8, + "latest_source_mass": 0.19988907873630524, + "layer": 29, + "max_source_mass": 0.27338963747024536, + "mixer_index": 56, + "normalized_entropy": 0.9232295844699908, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10887958109378815, + "median": 0.10778149962425232, + "p05": 0.06027446314692497, + "p95": 0.16913464665412903 + }, + { + "label": "completed_group_1", + "mean": 0.10445677489042282, + "median": 0.10656317323446274, + "p05": 0.06373343616724014, + "p95": 0.13975882530212402 + }, + { + "label": "completed_group_2", + "mean": 0.0838727131485939, + "median": 0.08285491168498993, + "p05": 0.06476438045501709, + "p95": 0.10624834150075912 + }, + { + "label": "completed_group_3", + "mean": 0.0852852314710617, + "median": 0.0823126882314682, + "p05": 0.06312129646539688, + "p95": 0.1139926165342331 + }, + { + "label": "completed_group_4", + "mean": 0.05865198373794556, + "median": 0.055648073554039, + "p05": 0.04130495712161064, + "p95": 0.08606290817260742 + }, + { + "label": "completed_group_5", + "mean": 0.08557502925395966, + "median": 0.08467335999011993, + "p05": 0.06446348130702972, + "p95": 0.10952071845531464 + }, + { + "label": "completed_group_6", + "mean": 0.27338963747024536, + "median": 0.2741563320159912, + "p05": 0.19311219453811646, + "p95": 0.3497011959552765 + }, + { + "label": "completed_group_7", + "mean": 0.19988907873630524, + "median": 0.1887303590774536, + "p05": 0.10821843147277832, + "p95": 0.3243493437767029 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.9332969188690186, + "group": 8, + "latest_source_mass": 0.25055450201034546, + "layer": 29, + "max_source_mass": 0.2511133551597595, + "mixer_index": 57, + "normalized_entropy": 0.8798813461356915, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07704077661037445, + "median": 0.0657581090927124, + "p05": 0.031858570873737335, + "p95": 0.16932255029678345 + }, + { + "label": "completed_group_1", + "mean": 0.08837630599737167, + "median": 0.08667970448732376, + "p05": 0.05486329644918442, + "p95": 0.12723307311534882 + }, + { + "label": "completed_group_2", + "mean": 0.05746292322874069, + "median": 0.05622619390487671, + "p05": 0.04283035546541214, + "p95": 0.0759863406419754 + }, + { + "label": "completed_group_3", + "mean": 0.06001049652695656, + "median": 0.05710422247648239, + "p05": 0.0414297915995121, + "p95": 0.08556482195854187 + }, + { + "label": "completed_group_4", + "mean": 0.04015304893255234, + "median": 0.03914806246757507, + "p05": 0.02911726012825966, + "p95": 0.05505921691656113 + }, + { + "label": "completed_group_5", + "mean": 0.06584024429321289, + "median": 0.06438547372817993, + "p05": 0.04402649402618408, + "p95": 0.09364330768585205 + }, + { + "label": "completed_group_6", + "mean": 0.2511133551597595, + "median": 0.24885429441928864, + "p05": 0.16403883695602417, + "p95": 0.34440478682518005 + }, + { + "label": "completed_group_7", + "mean": 0.10944834351539612, + "median": 0.08987390995025635, + "p05": 0.04556909203529358, + "p95": 0.22957125306129456 + }, + { + "label": "current_group_partial", + "mean": 0.25055450201034546, + "median": 0.2451469898223877, + "p05": 0.11517379432916641, + "p95": 0.4031696617603302 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0115504264831543, + "group": 8, + "latest_source_mass": 0.16160473227500916, + "layer": 30, + "max_source_mass": 0.2653427720069885, + "mixer_index": 58, + "normalized_entropy": 0.9154960522614556, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08485299348831177, + "median": 0.07469280064105988, + "p05": 0.04144430533051491, + "p95": 0.1810980588197708 + }, + { + "label": "completed_group_1", + "mean": 0.08635596185922623, + "median": 0.08461107313632965, + "p05": 0.05276995152235031, + "p95": 0.12535785138607025 + }, + { + "label": "completed_group_2", + "mean": 0.06204982101917267, + "median": 0.061112865805625916, + "p05": 0.045534878969192505, + "p95": 0.08121293038129807 + }, + { + "label": "completed_group_3", + "mean": 0.06870189309120178, + "median": 0.06677035242319107, + "p05": 0.046723417937755585, + "p95": 0.0936926007270813 + }, + { + "label": "completed_group_4", + "mean": 0.0476498119533062, + "median": 0.042902059853076935, + "p05": 0.030372878536581993, + "p95": 0.08035364747047424 + }, + { + "label": "completed_group_5", + "mean": 0.08640170097351074, + "median": 0.08549860864877701, + "p05": 0.059603940695524216, + "p95": 0.11531224846839905 + }, + { + "label": "completed_group_6", + "mean": 0.2653427720069885, + "median": 0.26641854643821716, + "p05": 0.1776670217514038, + "p95": 0.3456951975822449 + }, + { + "label": "completed_group_7", + "mean": 0.13704031705856323, + "median": 0.13101622462272644, + "p05": 0.0725383460521698, + "p95": 0.22235709428787231 + }, + { + "label": "current_group_partial", + "mean": 0.16160473227500916, + "median": 0.15514495968818665, + "p05": 0.08618976920843124, + "p95": 0.255747526884079 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781036376953125, + "group": 8, + "latest_source_mass": 0.15038734674453735, + "layer": 30, + "max_source_mass": 0.29481440782546997, + "mixer_index": 59, + "normalized_entropy": 0.9002737626817574, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08389372378587723, + "median": 0.06660648435354233, + "p05": 0.04010086879134178, + "p95": 0.22044557332992554 + }, + { + "label": "completed_group_1", + "mean": 0.08841055631637573, + "median": 0.08784788101911545, + "p05": 0.06586075574159622, + "p95": 0.11172220855951309 + }, + { + "label": "completed_group_2", + "mean": 0.060679107904434204, + "median": 0.0598948709666729, + "p05": 0.045160289853811264, + "p95": 0.07931925356388092 + }, + { + "label": "completed_group_3", + "mean": 0.0658775195479393, + "median": 0.06333823502063751, + "p05": 0.047597575932741165, + "p95": 0.09037967771291733 + }, + { + "label": "completed_group_4", + "mean": 0.045975662767887115, + "median": 0.04452216252684593, + "p05": 0.03386770188808441, + "p95": 0.06284140050411224 + }, + { + "label": "completed_group_5", + "mean": 0.08423998951911926, + "median": 0.0840892568230629, + "p05": 0.05936872214078903, + "p95": 0.1089530736207962 + }, + { + "label": "completed_group_6", + "mean": 0.29481440782546997, + "median": 0.3008817434310913, + "p05": 0.17160090804100037, + "p95": 0.396139919757843 + }, + { + "label": "completed_group_7", + "mean": 0.12572166323661804, + "median": 0.10853725671768188, + "p05": 0.06020044535398483, + "p95": 0.24606993794441223 + }, + { + "label": "current_group_partial", + "mean": 0.15038734674453735, + "median": 0.14419245719909668, + "p05": 0.07393782585859299, + "p95": 0.24871884286403656 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0586161613464355, + "group": 8, + "latest_source_mass": 0.1407114714384079, + "layer": 31, + "max_source_mass": 0.22791332006454468, + "mixer_index": 60, + "normalized_entropy": 0.9369165913127441, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1038442850112915, + "median": 0.10749004781246185, + "p05": 0.021369464695453644, + "p95": 0.1852288693189621 + }, + { + "label": "completed_group_1", + "mean": 0.09918913245201111, + "median": 0.09865356981754303, + "p05": 0.057106852531433105, + "p95": 0.1428801417350769 + }, + { + "label": "completed_group_2", + "mean": 0.08148117363452911, + "median": 0.08018780499696732, + "p05": 0.060506708920001984, + "p95": 0.10691888630390167 + }, + { + "label": "completed_group_3", + "mean": 0.0698266476392746, + "median": 0.06695331633090973, + "p05": 0.0502583347260952, + "p95": 0.09614066034555435 + }, + { + "label": "completed_group_4", + "mean": 0.059573933482170105, + "median": 0.05744876712560654, + "p05": 0.04474463313817978, + "p95": 0.08047857880592346 + }, + { + "label": "completed_group_5", + "mean": 0.08160553872585297, + "median": 0.08077611029148102, + "p05": 0.060524459928274155, + "p95": 0.10581386834383011 + }, + { + "label": "completed_group_6", + "mean": 0.22791332006454468, + "median": 0.22841286659240723, + "p05": 0.15760843455791473, + "p95": 0.29978859424591064 + }, + { + "label": "completed_group_7", + "mean": 0.13585451245307922, + "median": 0.12730783224105835, + "p05": 0.05973683297634125, + "p95": 0.2417306751012802 + }, + { + "label": "current_group_partial", + "mean": 0.1407114714384079, + "median": 0.1365126073360443, + "p05": 0.052046023309230804, + "p95": 0.2530304193496704 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.022735118865967, + "group": 8, + "latest_source_mass": 0.14314599335193634, + "layer": 31, + "max_source_mass": 0.24251294136047363, + "mixer_index": 61, + "normalized_entropy": 0.9205864251337508, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09707076847553253, + "median": 0.09347058087587357, + "p05": 0.03571572154760361, + "p95": 0.18511834740638733 + }, + { + "label": "completed_group_1", + "mean": 0.09551472961902618, + "median": 0.0936870127916336, + "p05": 0.07090321183204651, + "p95": 0.12828916311264038 + }, + { + "label": "completed_group_2", + "mean": 0.06701318919658661, + "median": 0.06529247015714645, + "p05": 0.04918193817138672, + "p95": 0.09029978513717651 + }, + { + "label": "completed_group_3", + "mean": 0.06734597682952881, + "median": 0.06535796821117401, + "p05": 0.04815492779016495, + "p95": 0.09132430702447891 + }, + { + "label": "completed_group_4", + "mean": 0.04480960592627525, + "median": 0.04339977353811264, + "p05": 0.034005217254161835, + "p95": 0.06039253994822502 + }, + { + "label": "completed_group_5", + "mean": 0.08329526335000992, + "median": 0.08157584071159363, + "p05": 0.057632118463516235, + "p95": 0.11397058516740799 + }, + { + "label": "completed_group_6", + "mean": 0.24251294136047363, + "median": 0.24409571290016174, + "p05": 0.14804454147815704, + "p95": 0.3317493200302124 + }, + { + "label": "completed_group_7", + "mean": 0.15929152071475983, + "median": 0.14274996519088745, + "p05": 0.07706648111343384, + "p95": 0.29802852869033813 + }, + { + "label": "current_group_partial", + "mean": 0.14314599335193634, + "median": 0.13742807507514954, + "p05": 0.05530726909637451, + "p95": 0.25646957755088806 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9317231178283691, + "group": 8, + "latest_source_mass": 0.09359145164489746, + "layer": 32, + "max_source_mass": 0.3483121693134308, + "mixer_index": 62, + "normalized_entropy": 0.8791650784146389, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07746735960245132, + "median": 0.058669812977313995, + "p05": 0.03224737197160721, + "p95": 0.21910449862480164 + }, + { + "label": "completed_group_1", + "mean": 0.08971680700778961, + "median": 0.08852710574865341, + "p05": 0.051282018423080444, + "p95": 0.13161666691303253 + }, + { + "label": "completed_group_2", + "mean": 0.06909166276454926, + "median": 0.06691395491361618, + "p05": 0.0473506934940815, + "p95": 0.09751966595649719 + }, + { + "label": "completed_group_3", + "mean": 0.08098623156547546, + "median": 0.07793208211660385, + "p05": 0.05465787276625633, + "p95": 0.11334780603647232 + }, + { + "label": "completed_group_4", + "mean": 0.04421798139810562, + "median": 0.03908399119973183, + "p05": 0.025279562920331955, + "p95": 0.07900907099246979 + }, + { + "label": "completed_group_5", + "mean": 0.0916394293308258, + "median": 0.09000804275274277, + "p05": 0.05948498472571373, + "p95": 0.1287805736064911 + }, + { + "label": "completed_group_6", + "mean": 0.3483121693134308, + "median": 0.3540991246700287, + "p05": 0.21832552552223206, + "p95": 0.45785486698150635 + }, + { + "label": "completed_group_7", + "mean": 0.10497690737247467, + "median": 0.09170983731746674, + "p05": 0.0477774515748024, + "p95": 0.20629161596298218 + }, + { + "label": "current_group_partial", + "mean": 0.09359145164489746, + "median": 0.09055774658918381, + "p05": 0.05538146197795868, + "p95": 0.14527763426303864 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.053220272064209, + "group": 8, + "latest_source_mass": 0.10435353964567184, + "layer": 32, + "max_source_mass": 0.26201331615448, + "mixer_index": 63, + "normalized_entropy": 0.934460816269135, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09178322553634644, + "median": 0.07441723346710205, + "p05": 0.05449800193309784, + "p95": 0.19793085753917694 + }, + { + "label": "completed_group_1", + "mean": 0.10647952556610107, + "median": 0.10643037408590317, + "p05": 0.08102327585220337, + "p95": 0.13105911016464233 + }, + { + "label": "completed_group_2", + "mean": 0.07523074746131897, + "median": 0.07466189563274384, + "p05": 0.0580206960439682, + "p95": 0.09448636323213577 + }, + { + "label": "completed_group_3", + "mean": 0.08394691348075867, + "median": 0.08230198919773102, + "p05": 0.06343884021043777, + "p95": 0.10998549312353134 + }, + { + "label": "completed_group_4", + "mean": 0.04770101234316826, + "median": 0.046069372445344925, + "p05": 0.036705970764160156, + "p95": 0.06487860530614853 + }, + { + "label": "completed_group_5", + "mean": 0.09626082330942154, + "median": 0.09582744538784027, + "p05": 0.072615846991539, + "p95": 0.1200246512889862 + }, + { + "label": "completed_group_6", + "mean": 0.26201331615448, + "median": 0.2656516432762146, + "p05": 0.17182716727256775, + "p95": 0.3362464904785156 + }, + { + "label": "completed_group_7", + "mean": 0.13223090767860413, + "median": 0.12176793813705444, + "p05": 0.0719129890203476, + "p95": 0.22232075035572052 + }, + { + "label": "current_group_partial", + "mean": 0.10435353964567184, + "median": 0.09856735169887543, + "p05": 0.06300172954797745, + "p95": 0.16237276792526245 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.843801498413086, + "group": null, + "latest_source_mass": 0.19098761677742004, + "layer": null, + "max_source_mass": 0.3250349760055542, + "mixer_index": null, + "normalized_entropy": 0.8391502249844656, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11151862144470215, + "median": 0.11458685994148254, + "p05": 0.04668144881725311, + "p95": 0.1599162518978119 + }, + { + "label": "completed_group_1", + "mean": 0.061874739825725555, + "median": 0.06061127781867981, + "p05": 0.04380229860544205, + "p95": 0.08225110918283463 + }, + { + "label": "completed_group_2", + "mean": 0.05308101326227188, + "median": 0.05128483101725578, + "p05": 0.03527887910604477, + "p95": 0.07726569473743439 + }, + { + "label": "completed_group_3", + "mean": 0.040848225355148315, + "median": 0.038592323660850525, + "p05": 0.024852607399225235, + "p95": 0.06626670062541962 + }, + { + "label": "completed_group_4", + "mean": 0.042743273079395294, + "median": 0.04188907518982887, + "p05": 0.03039409965276718, + "p95": 0.0579918697476387 + }, + { + "label": "completed_group_5", + "mean": 0.053944505751132965, + "median": 0.05261969938874245, + "p05": 0.037368807941675186, + "p95": 0.07573261857032776 + }, + { + "label": "completed_group_6", + "mean": 0.1199670284986496, + "median": 0.10659539699554443, + "p05": 0.05845983326435089, + "p95": 0.2257011979818344 + }, + { + "label": "completed_group_7", + "mean": 0.3250349760055542, + "median": 0.3179755210876465, + "p05": 0.12162069231271744, + "p95": 0.540380597114563 + }, + { + "label": "completed_group_8", + "mean": 0.19098761677742004, + "median": 0.1842590719461441, + "p05": 0.022588828578591347, + "p95": 0.4012773036956787 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "b1f88fe26521a0d9ae8700286d108b588142750400633243896bf96a8e44a5fb", + "optimizer_state_before": "b1f88fe26521a0d9ae8700286d108b588142750400633243896bf96a8e44a5fb", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "3bc7ac091ccda73a60054802f3ec6d7bc68d1af6b8f7940e83f1843e7ab472be", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.051234737038612366, + 0.11323792487382889, + 0.05108218640089035, + 0.03771067038178444, + 0.08815877884626389, + 0.03706873208284378, + 0.0438041090965271, + 0.030363978818058968, + 0.03348636254668236, + 0.027010800316929817, + 0.032826121896505356, + 0.031702958047389984, + 0.038218315690755844, + 0.03341353312134743, + 0.04180334508419037, + 0.2476748526096344, + 0.05078785493969917, + 0.0355411171913147, + 0.052597854286432266, + 0.06600882858037949, + 0.050489071756601334, + 0.022040290758013725, + 0.028325051069259644, + 0.02171224355697632, + 0.024430278688669205, + 0.015834001824259758, + 0.018936052918434143, + 0.02182060107588768, + 0.030554499477148056, + 0.016542606055736542, + 0.015565099194645882, + 0.013472299091517925 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1110142281722801e-05, + "first_to_last_ratio": 1.1615894542734118, + "imbalance_abs_log_ratio": 0.1497892864258095, + "last_quartile_mean": 9.56460326051456e-06, + "mean": 1.3397138474147141e-05, + "normalized": [ + 0.9679183570279057, + 1.0382751270469845, + 0.9205849321245912, + 0.8794696386057994, + 0.7953559954151083, + 0.7288729389713584, + 0.666621226062364, + 0.6372392908707228, + 0.6772537249699894, + 0.6436456735607675, + 0.6038963391182133, + 0.5706397399299389, + 0.5513013139860934, + 0.5295867329502082, + 0.5131681370281796, + 0.5688577678712857, + 1.1361761926655034, + 1.1401114120766789, + 1.0979237717774961, + 1.045258619130553, + 3.411268167406657, + 2.8827386084395865, + 2.3438877456548695, + 1.9385179883302053, + 1.2367088391016392, + 1.0924371253612384, + 0.9526017067211789, + 0.8037245228882399, + 0.49817350982557956, + 0.40700227563318875, + 0.3703725294044812, + 0.35041005004339426 + ], + "peak_layer": 21, + "peak_normalized": 3.411268167406657, + "population_cv": 0.694220928383246, + "rest_mean": 1.0016533653672215e-05, + "spike_contrast": 3.160015789804438, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.165240450471174e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2967336260771845e-05, + 1.3909915651311167e-05, + 1.2333203812886495e-05, + 1.1782376532210037e-05, + 1.0655494406819344e-05, + 9.764811693457887e-06, + 8.930816875363234e-06, + 8.537183020962402e-06, + 9.07326193555491e-06, + 8.62301021697931e-06, + 8.090482879197225e-06, + 7.644939614692703e-06, + 7.385860044450965e-06, + 7.0949467954051215e-06, + 6.874984592286637e-06, + 7.621066288265865e-06, + 1.522150978416903e-05, + 1.52742304635467e-05, + 1.4709036804561038e-05, + 1.4003474461787846e-05, + 4.5701232011197135e-05, + 3.862044832203537e-05, + 3.140138869639486e-05, + 2.5970593924284913e-05, + 1.6568359569646418e-05, + 1.463553144276375e-05, + 1.2762136975652538e-05, + 1.0767608728201594e-05, + 6.674099495285191e-06, + 5.452665845950833e-06, + 4.961932063451968e-06, + 4.694491963164182e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.520096763371839e-05, + "first_to_last_ratio": 1.1880767713048908, + "imbalance_abs_log_ratio": 0.17233584116303574, + "last_quartile_mean": 3.804549396591028e-05, + "mean": 5.4024936389396316e-05, + "normalized": [ + 0.9708707190563224, + 1.0451943937367467, + 0.9250577208903767, + 0.8842743984856529, + 0.8031024942014594, + 0.7427444322392254, + 0.6768740767497347, + 0.6452305260972615, + 0.6822213900165217, + 0.6476553992317551, + 0.6085575017002121, + 0.5751826010571539, + 0.5529292217889952, + 0.5299244079385589, + 0.5136466137046081, + 0.568975504947234, + 1.1417881140585813, + 1.159000740319458, + 1.1039052119717943, + 1.0463955173718469, + 3.422947145673778, + 2.8780196348647618, + 2.3242594176046136, + 1.9174749293029072, + 1.2126088201065046, + 1.0731537668257296, + 0.9360922740994, + 0.7921003983334755, + 0.4948022653288053, + 0.40618173797049695, + 0.3695278378158856, + 0.34930078651014324 + ], + "peak_layer": 21, + "peak_normalized": 3.422947145673778, + "population_cv": 0.6923039078369757, + "rest_mean": 4.050807749283397e-05, + "spike_contrast": 3.1355715277601766, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012701597443083302, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.2451228839345276e-05, + 5.6466560636181384e-05, + 4.9976184527622536e-05, + 4.777286812895909e-05, + 4.338756116339937e-05, + 4.012672070530243e-05, + 3.6568078940035775e-05, + 3.485853812890127e-05, + 3.685696719912812e-05, + 3.4989541745744646e-05, + 3.28772803186439e-05, + 3.107420343440026e-05, + 2.9871966034988873e-05, + 2.862913243006915e-05, + 2.7749725632020272e-05, + 3.073886546189897e-05, + 6.168503023218364e-05, + 6.261494127102196e-05, + 5.963840885669924e-05, + 5.653145126416348e-05, + 0.00018492450180929154, + 0.00015548482770100236, + 0.00012556796718854457, + 0.00010359146108385175, + 6.551111437147483e-05, + 5.797706398880109e-05, + 5.0572325562825426e-05, + 4.2793173633981496e-05, + 2.6731660909717903e-05, + 2.194394255639054e-05, + 1.9963717932114378e-05, + 1.887095277197659e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04008466936647892, + "first_to_last_ratio": 1.1880767439372768, + "imbalance_abs_log_ratio": 0.17233581812781198, + "last_quartile_mean": 0.03373912465758622, + "mean": 0.04790985374711454, + "normalized": [ + 0.970870766181603, + 1.0451943834429158, + 0.9250577200396625, + 0.8842743365685358, + 0.8031024935917312, + 0.7427444421233665, + 0.6768740172187802, + 0.645230491498922, + 0.682221391904326, + 0.6476553588595259, + 0.6085574586207388, + 0.5751825734843286, + 0.552929202866552, + 0.5299244347807472, + 0.5136466315472489, + 0.5689755646385108, + 1.1417881666708014, + 1.1590007554748112, + 1.1039052466293215, + 1.0463955619068368, + 3.422947136129388, + 2.878019556230048, + 2.324259489013662, + 1.9174748970647042, + 1.212608709608152, + 1.0731538183017837, + 0.9360922641989038, + 0.7921004515426608, + 0.49480228907579926, + 0.40618174778590915, + 0.3695278429438553, + 0.3493008000558679 + ], + "peak_layer": 21, + "peak_normalized": 3.422947136129388, + "population_cv": 0.692303902772671, + "rest_mean": 0.0359229684151985, + "spike_contrast": 3.135571460508958, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11263903453946114, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.046514276415109634, + 0.05007511004805565, + 0.044319380074739456, + 0.04236545413732529, + 0.038476523011922836, + 0.03558477759361267, + 0.032428935170173645, + 0.03091289848089218, + 0.03268512710928917, + 0.031029073521494865, + 0.029155898839235306, + 0.02755691297352314, + 0.02649075724184513, + 0.025388602167367935, + 0.024608734995126724, + 0.02725953608751297, + 0.05470290407538414, + 0.05552755668759346, + 0.0528879389166832, + 0.05013265833258629, + 0.16399289667606354, + 0.13788549602031708, + 0.11135493218898773, + 0.09186594188213348, + 0.05809590592980385, + 0.05141464248299599, + 0.04484804347157478, + 0.037949416786432266, + 0.02370590530335903, + 0.019460108131170273, + 0.01770402491092682, + 0.016734950244426727 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8672176622658299e-06, + "first_to_last_ratio": 0.7390774425516208, + "imbalance_abs_log_ratio": 0.3023525698145062, + "last_quartile_mean": 2.526416793102726e-06, + "mean": 2.8231130428935103e-06, + "normalized": [ + 1.0710459542190351, + 0.7807005916993321, + 0.7446702739163947, + 0.7228305497755413, + 0.5071025265492799, + 0.49702926125096536, + 0.48433377289158164, + 0.4835173786395229, + 0.5122484319159906, + 0.5011695032399489, + 0.4949613551079442, + 0.4928871668407519, + 0.4875154271640947, + 0.48919988211251697, + 0.49217873653037764, + 0.5089309468432903, + 0.8206332361532853, + 0.8408570852012511, + 0.8609730105775593, + 0.8854026612180381, + 3.5996391090811306, + 3.2819833392882214, + 2.8213892706690173, + 2.4595638858289472, + 1.4921873165629496, + 1.3536774320369742, + 1.2002934104350649, + 1.0429420663389846, + 0.6391162919820471, + 0.5173593426294659, + 0.47263883361652936, + 0.44102194968396574 + ], + "peak_layer": 21, + "peak_normalized": 3.5996391090811306, + "population_cv": 0.8312576809575059, + "rest_mean": 1.9181732619068093e-06, + "spike_contrast": 4.019338614155003, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.709787860221695e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0236838028940838e-06, + 2.2040060230210656e-06, + 2.1022883629484568e-06, + 2.0406323528732173e-06, + 1.4316077567855245e-06, + 1.4031697901373263e-06, + 1.3673289913640474e-06, + 1.365024218102917e-06, + 1.4461352293437812e-06, + 1.4148581612971611e-06, + 1.3973318573334836e-06, + 1.3914761893829564e-06, + 1.3763111610387568e-06, + 1.3810665677738143e-06, + 1.3894762105337577e-06, + 1.4367695939654368e-06, + 2.31674039241625e-06, + 2.373834604441072e-06, + 2.4306241357408e-06, + 2.4995918010972673e-06, + 1.0162188118556514e-05, + 9.265409971703775e-06, + 7.965100849105511e-06, + 6.943626885913545e-06, + 4.21261347582913e-06, + 3.8215844142541755e-06, + 3.388563982298365e-06, + 2.944343350463896e-06, + 1.8042975398202543e-06, + 1.4605639080400579e-06, + 1.3343128557607997e-06, + 1.245054818355129e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.5158251421307796e-05, + "first_to_last_ratio": 1.372951179293099, + "imbalance_abs_log_ratio": 0.3169625684647144, + "last_quartile_mean": 2.5607794327697775e-05, + "mean": 3.930288400511017e-05, + "normalized": [ + 1.0457352487354723, + 1.1170219051554786, + 0.9973660844453545, + 0.9523381538668013, + 0.848321809820657, + 0.7912232478279534, + 0.7196571910711226, + 0.6847071570545322, + 0.7251137125737372, + 0.6895283748593514, + 0.6494373653889238, + 0.6137885297016834, + 0.6090137783434721, + 0.5840034448633752, + 0.5660471252539855, + 0.6192854547978126, + 1.2024011313555776, + 1.209946097482875, + 1.1323117727181873, + 1.0603690322871115, + 3.309119658074755, + 2.746842516693067, + 2.1557717289384435, + 1.7582494269549842, + 1.1449656413436453, + 0.9974022764386288, + 0.8715835424185431, + 0.7496749321117195, + 0.4337889933103568, + 0.3592093283156041, + 0.33316446845998277, + 0.32261086933680555 + ], + "peak_layer": 21, + "peak_normalized": 3.309119658074755, + "population_cv": 0.6495478333321385, + "rest_mean": 3.0401582888670317e-05, + "spike_contrast": 2.8738605602817255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.736991003388539e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.11004111811053e-05, + 4.390218236949295e-05, + 3.9199363527586684e-05, + 3.7429635995067656e-05, + 3.3341493690386415e-05, + 3.109735553152859e-05, + 2.828460310411174e-05, + 2.6910965971183032e-05, + 2.849906013580039e-05, + 2.710045373532921e-05, + 2.5524761440465227e-05, + 2.412365938653238e-05, + 2.3935997887747362e-05, + 2.295301965204999e-05, + 2.224728450528346e-05, + 2.433970439597033e-05, + 4.72578321932815e-05, + 4.755437112180516e-05, + 4.4503118260763586e-05, + 4.167556107859127e-05, + 0.00013005794608034194, + 0.00010795883281389251, + 8.472804620396346e-05, + 6.910427327966318e-05, + 4.5000451791565865e-05, + 3.9200785977300256e-05, + 3.425574686843902e-05, + 2.9464386898325756e-05, + 1.7049158486770466e-05, + 1.4117962564341724e-05, + 1.309432445850689e-05, + 1.2679537576332223e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.67776620148652e-05, + "first_to_last_ratio": 1.489713992627352, + "imbalance_abs_log_ratio": 0.39858415027555666, + "last_quartile_mean": 1.7975035575545917e-05, + "mean": 2.834268283891106e-05, + "normalized": [ + 1.106950131948834, + 1.1808729223414434, + 1.058395413505648, + 1.0131391464097315, + 0.8887556018396708, + 0.8336258013743472, + 0.7556781237788452, + 0.7208403934347858, + 0.7653210639837278, + 0.7236752838504213, + 0.6826925954443256, + 0.6471110368851702, + 0.6603765937793802, + 0.6314762042173024, + 0.6140873745590488, + 0.6701983351410581, + 1.245239798504983, + 1.2536001966002472, + 1.1579582641113197, + 1.0597246133471363, + 3.138008050522486, + 2.5781227127810102, + 1.9478107697932208, + 1.59270961868747, + 1.1031600095096015, + 0.9421470006775309, + 0.8542088565799294, + 0.7657556804352058, + 0.41919703377159895, + 0.3415465986591183, + 0.32698829528201995, + 0.3206264782433822 + ], + "peak_layer": 21, + "peak_normalized": 3.138008050522486, + "population_cv": 0.5950468550457515, + "rest_mean": 2.2716333660355303e-05, + "spike_contrast": 2.585142887982815, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.8724968403112145e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.137393650831655e-05, + 3.346910671098158e-05, + 2.99977655231487e-05, + 2.8715081498376094e-05, + 2.518971814424731e-05, + 2.362719169468619e-05, + 2.1417945390567183e-05, + 2.0430550648598e-05, + 2.1691252186428756e-05, + 2.0510899048531428e-05, + 1.9349339709151536e-05, + 1.8340862879995257e-05, + 1.8716844351729378e-05, + 1.7897729776450433e-05, + 1.74048836925067e-05, + 1.899521885206923e-05, + 3.5293436667416245e-05, + 3.5530392779037356e-05, + 3.2819643820403144e-05, + 3.003543861268554e-05, + 8.893956692190841e-05, + 7.307091436814517e-05, + 5.520618287846446e-05, + 4.514166357694194e-05, + 3.126651427010074e-05, + 2.670297362783458e-05, + 2.4210570700233802e-05, + 2.1703570382669568e-05, + 1.1881168575200718e-05, + 9.680346920504235e-06, + 9.267725545214489e-06, + 9.087414582609199e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.124184998654528e-05, + "first_to_last_ratio": 1.1295643250458662, + "imbalance_abs_log_ratio": 0.12183200532280404, + "last_quartile_mean": 8.07761434771237e-05, + "mean": 0.0001131998267283052, + "normalized": [ + 0.9391291478086077, + 0.9994540633197246, + 0.8981563809156281, + 0.8600090937959485, + 0.7637374860603604, + 0.7087527588156981, + 0.6498801446644327, + 0.6290780728741097, + 0.6694944081285747, + 0.6399004336382168, + 0.5999417388211115, + 0.5645228757078199, + 0.523886204579862, + 0.5044184246208804, + 0.48957885496141057, + 0.5347736119904029, + 1.1366324271717236, + 1.1355437314234091, + 1.0782169392550793, + 1.0313830324460944, + 3.519062137269261, + 2.963582030922574, + 2.41128509662923, + 2.041010789775174, + 1.2315559238773703, + 1.0997748890711536, + 0.9743076211416006, + 0.8521146034254664, + 0.47220702964033107, + 0.3868732211836673, + 0.3540802629057541, + 0.3376565631593225 + ], + "peak_layer": 21, + "peak_normalized": 3.519062137269261, + "population_cv": 0.7265013683583667, + "rest_mean": 8.315367476488636e-05, + "spike_contrast": 3.3125300608732915, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002754490473307669, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.0001063092568074353, + 0.0001131380267906934, + 0.00010167114669457078, + 9.735288040246814e-05, + 8.645495108794421e-05, + 8.023068949114531e-05, + 7.356631977017969e-05, + 7.121152884792536e-05, + 7.57866509957239e-05, + 7.24366182112135e-05, + 6.791330088162795e-05, + 6.390389171428978e-05, + 5.9303827583789825e-05, + 5.7100078265648335e-05, + 5.542024155147374e-05, + 6.0536280216183513e-05, + 0.0001286665938096121, + 0.00012854335363954306, + 0.00012205397069919854, + 0.00011675238056341186, + 0.00039835722418501973, + 0.0003354769723955542, + 0.00027295705513097346, + 0.00023104206775315106, + 0.00013941191718913615, + 0.00012449432688299567, + 0.00011029145389329642, + 9.64592254604213e-05, + 5.3453753935173154e-05, + 4.3793981603812426e-05, + 4.008182440884411e-05, + 3.822266444331035e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b5dfb9be1f653e88219f0cf5f3837c21e4150bbb11f0520e3acfd9e5ac781830", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.07594770938158035, + 0.04037529230117798, + 0.04642988368868828, + 0.02321692928671837, + 0.015561277978122234, + 0.01583549566566944, + 0.020819898694753647, + 0.027020027860999107, + 0.0216005090624094, + 0.024715786799788475, + 0.03509491682052612, + 0.05490833893418312, + 0.037195492535829544, + 0.042240556329488754, + 0.04918703809380531, + 0.04376247525215149, + 0.02373618632555008, + 0.01774558797478676, + 0.022557877004146576, + 0.03756217285990715, + 0.03271714597940445, + 0.03932206705212593, + 0.04476166144013405, + 0.049177221953868866, + 0.0453680194914341, + 0.0437605082988739, + 0.052028607577085495, + 0.060354240238666534, + 0.06733348965644836, + 0.07210637629032135, + 0.09804103523492813, + 0.1243291050195694 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0990880355166155e-05, + "first_to_last_ratio": 1.2944234394570906, + "imbalance_abs_log_ratio": 0.2580653755149735, + "last_quartile_mean": 8.49094664090444e-06, + "mean": 1.235348285888449e-05, + "normalized": [ + 1.0282294525621116, + 1.1178781387751966, + 0.9935960877446004, + 0.9494515014869882, + 0.8590253613234318, + 0.7848034670921512, + 0.7091367306470445, + 0.6754706186705651, + 0.7068887402186641, + 0.6749526105270502, + 0.6307964655317274, + 0.6026934930305775, + 0.5785908008702298, + 0.5570004776549429, + 0.5565634542979354, + 0.6239921973919045, + 1.2244474917910158, + 1.2505131815537707, + 1.1714527408188027, + 1.1440936503929904, + 3.1783601472255105, + 2.5998507728769873, + 2.116333755445443, + 1.7672210038875287, + 1.1859194213452948, + 1.0362571799584626, + 0.8733376553978273, + 0.8036621734585419, + 0.442583220280086, + 0.40164066204717863, + 0.38000807949784143, + 0.3752492661975972 + ], + "peak_layer": 21, + "peak_normalized": 3.1783601472255105, + "population_cv": 0.6201763550129065, + "rest_mean": 9.677954056786365e-06, + "spike_contrast": 2.769318621782541, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6801338390214368e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.2702214917226229e-05, + 1.3809688425681088e-05, + 1.2274372238607612e-05, + 1.1729032848961651e-05, + 1.061195507645607e-05, + 9.695056178316008e-06, + 8.760308446653653e-06, + 8.344414709426928e-06, + 8.732537935429718e-06, + 8.338015504705254e-06, + 7.792533324391115e-06, + 7.445363735314459e-06, + 7.147611540858634e-06, + 6.88089585310081e-06, + 6.875497092551086e-06, + 7.70847691455856e-06, + 1.512619110144442e-05, + 1.5448193153133616e-05, + 1.4471521353698336e-05, + 1.4133541299088392e-05, + 3.926381759811193e-05, + 3.2117211958393455e-05, + 2.6144092771573924e-05, + 2.1831334379385225e-05, + 1.4650235243607312e-05, + 1.2801385310012847e-05, + 1.0788761755975429e-05, + 9.928026884153951e-06, + 5.467444225359941e-06, + 4.96166103403084e-06, + 4.6944232963141985e-06, + 4.635635377781e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.471112742976402e-05, + "first_to_last_ratio": 1.3207778481961452, + "imbalance_abs_log_ratio": 0.2782208419780995, + "last_quartile_mean": 3.385211789463938e-05, + "mean": 4.9810070152034314e-05, + "normalized": [ + 1.0308939392414505, + 1.1248976892653257, + 0.9983120541152328, + 0.9550492360145139, + 0.8676012153591989, + 0.7999351969960798, + 0.720536877837289, + 0.6838321262895888, + 0.712031571445422, + 0.6791934724729616, + 0.6349550259349246, + 0.6062750337433573, + 0.5788419659788087, + 0.5569013182098084, + 0.5581310424199538, + 0.6247833800612929, + 1.2301955910359532, + 1.2710297092994107, + 1.176719130778367, + 1.1460218197327094, + 3.189763931827885, + 2.58430753345956, + 2.094796372809775, + 1.7380029619955741, + 1.165603481494619, + 1.0196500369605523, + 0.8607727657791588, + 0.7949214981109073, + 0.4416983921863331, + 0.40097204003750814, + 0.37888754586879986, + 0.37448604323767737 + ], + "peak_layer": 21, + "peak_normalized": 3.189763931827885, + "population_cv": 0.6175106053465529, + "rest_mean": 3.9160909081046055e-05, + "spike_contrast": 2.7403740733718513, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010731553993537091, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.1348899432923645e-05, + 5.6031232816167176e-05, + 4.9725993449101225e-05, + 4.757106944452971e-05, + 4.3215277401031926e-05, + 3.9844828279456124e-05, + 3.5889992432203144e-05, + 3.4061726182699203e-05, + 3.54663425241597e-05, + 3.3830674510682e-05, + 3.162715438520536e-05, + 3.0198601962183602e-05, + 2.883215893234592e-05, + 2.7739293727790937e-05, + 2.780054637696594e-05, + 3.112050399067812e-05, + 6.127612869022414e-05, + 6.331007898552343e-05, + 5.861246245331131e-05, + 5.7083427236648276e-05, + 0.00015888236521277577, + 0.00012872453953605145, + 0.00010434195428388193, + 8.657004946144298e-05, + 5.805879118270241e-05, + 5.078883987152949e-05, + 4.28751518484205e-05, + 3.9595095586264506e-05, + 2.200102790084202e-05, + 1.997244544327259e-05, + 1.887241523945704e-05, + 1.8653176084626466e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.0396502751391381, + "first_to_last_ratio": 1.3207778547199724, + "imbalance_abs_log_ratio": 0.27822084691748217, + "last_quartile_mean": 0.030020396690815687, + "mean": 0.044172068301122636, + "normalized": [ + 1.0308939792489324, + 1.1248976196994809, + 0.9983120766392304, + 0.9550492033649147, + 0.8676012439568811, + 0.7999351931463972, + 0.7205369249138741, + 0.6838321362546834, + 0.7120315669729911, + 0.6791934519566257, + 0.6349550233781368, + 0.6062750468095437, + 0.5788419762424374, + 0.5569013220973693, + 0.5581310235791198, + 0.6247833992671911, + 1.2301955213417186, + 1.2710297742491206, + 1.1767191511915103, + 1.1460218181799973, + 3.189763837856072, + 2.58430760811575, + 2.094796347658092, + 1.7380029451802739, + 1.165603513395908, + 1.0196500665782366, + 0.860772736339134, + 0.7949215093532017, + 0.44169838206421963, + 0.40097203691663513, + 0.37888754792641754, + 0.3744860161259034 + ], + "peak_layer": 21, + "peak_normalized": 3.189763837856072, + "population_cv": 0.6175105993619994, + "rest_mean": 0.03472828582205154, + "spike_contrast": 2.740374062104645, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09516849368810654, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.045536719262599945, + 0.04968905448913574, + 0.04409750923514366, + 0.04218649864196777, + 0.038323741406202316, + 0.035334791988134384, + 0.03182760626077652, + 0.030206279829144478, + 0.03145190700888634, + 0.030001379549503326, + 0.02804727666079998, + 0.026780422776937485, + 0.02556864731013775, + 0.024599483236670494, + 0.024653801694512367, + 0.027597974985837936, + 0.05434028059244156, + 0.05614401400089264, + 0.05197811871767044, + 0.05062215402722359, + 0.1408984661102295, + 0.11415421217679977, + 0.0925314873456955, + 0.07677118480205536, + 0.051487118005752563, + 0.04504005238413811, + 0.03802211210131645, + 0.03511332720518112, + 0.019510731101036072, + 0.017711764201521873, + 0.016736246645450592, + 0.016541821882128716 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8471863825197943e-06, + "first_to_last_ratio": 0.8244526663369597, + "imbalance_abs_log_ratio": 0.19303554756074884, + "last_quartile_mean": 2.240500222683295e-06, + "mean": 2.553070206801067e-06, + "normalized": [ + 1.146978253733447, + 0.8593901687387808, + 0.8257545739301104, + 0.8007460276482833, + 0.5570271643774695, + 0.5465938673635234, + 0.5274890368921664, + 0.5241465220341085, + 0.5489616323415754, + 0.5406808894253674, + 0.5308783967421735, + 0.5296171888458094, + 0.5282959997667881, + 0.528928139981341, + 0.5448236432095829, + 0.5665606991220963, + 0.8991729664453528, + 0.9130923412964861, + 0.9272977733988372, + 0.9661483913891897, + 3.350792350323582, + 2.970835292391852, + 2.5766840237816986, + 2.2685371323868515, + 1.4765630521283415, + 1.319483155407689, + 1.1214078347249905, + 1.0614105324031717, + 0.5682912920566121, + 0.5158443159181672, + 0.4828769301866722, + 0.4746904116078829 + ], + "peak_layer": 21, + "peak_normalized": 3.350792350323582, + "population_cv": 0.7395032169136886, + "rest_mean": 1.8303232780925572e-06, + "spike_contrast": 3.5271930915695604, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.455903621827019e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.928316007455578e-06, + 2.194083435824723e-06, + 2.108209400830674e-06, + 2.0443608264031354e-06, + 1.422129457750998e-06, + 1.3954925179859856e-06, + 1.346716544503579e-06, + 1.3381828694036813e-06, + 1.401537588208157e-06, + 1.3803962701786077e-06, + 1.3553698181567597e-06, + 1.3521498658519704e-06, + 1.34877677737677e-06, + 1.350390675725066e-06, + 1.3909730114392005e-06, + 1.4464692412730074e-06, + 2.2956517113925656e-06, + 2.33118885262229e-06, + 2.3674563180975383e-06, + 2.4666446734045167e-06, + 8.55480811878806e-06, + 7.584751074318774e-06, + 6.578455213457346e-06, + 5.791734565718798e-06, + 3.769769136852119e-06, + 3.3687331324472325e-06, + 2.863032932509668e-06, + 2.709855607463396e-06, + 1.4508875665342202e-06, + 1.31698675431835e-06, + 1.2328187040111516e-06, + 1.211917947330221e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.482982242530852e-05, + "first_to_last_ratio": 1.509442359779984, + "imbalance_abs_log_ratio": 0.41174028446052374, + "last_quartile_mean": 2.307462898443191e-05, + "mean": 3.667055275968778e-05, + "normalized": [ + 1.0984852795741773, + 1.1887269251256418, + 1.0648771915091166, + 1.0174128492633152, + 0.9061095386257815, + 0.8430641277424028, + 0.7598659375227713, + 0.7198868197780367, + 0.7526461911930203, + 0.7186513938933602, + 0.6733570083146694, + 0.6414550341700876, + 0.6322836859770623, + 0.6081147060393997, + 0.6055693001423398, + 0.6699313875166385, + 1.2812295994523142, + 1.3084849615457006, + 1.195994737290222, + 1.1475500305866695, + 3.1043926075673, + 2.476737924383059, + 1.9537955304358658, + 1.5974462445862758, + 1.0791314645326522, + 0.9527353825517174, + 0.812033687878467, + 0.7520888457646006, + 0.38753562965456556, + 0.35900639892934844, + 0.3463472518949916, + 0.34505232655842927 + ], + "peak_layer": 21, + "peak_normalized": 3.1043926075673, + "population_cv": 0.586465117433983, + "rest_mean": 2.9592451870417722e-05, + "spike_contrast": 2.5307905505664654, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.489229756174609e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.028206240036525e-05, + 4.3591273424681276e-05, + 3.904963523382321e-05, + 3.730909156729467e-05, + 3.3227537642233074e-05, + 3.091562757617794e-05, + 2.7864703952218406e-05, + 2.639864760567434e-05, + 2.759995186352171e-05, + 2.6353343855589628e-05, + 2.4692373699508607e-05, + 2.352251067350153e-05, + 2.3186192265711725e-05, + 2.229990241175983e-05, + 2.2206560970516875e-05, + 2.456675429129973e-05, + 4.698339762398973e-05, + 4.798286681761965e-05, + 4.385778811411001e-05, + 4.208129394100979e-05, + 0.0001138397929025814, + 9.082334872800857e-05, + 7.164676208049059e-05, + 5.857923679286614e-05, + 3.957234730478376e-05, + 3.493733311188407e-05, + 2.9777724193991162e-05, + 2.757951369858347e-05, + 1.4211145753506571e-05, + 1.316496309300419e-05, + 1.2700745173788164e-05, + 1.2653259545913897e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6579402856441448e-05, + "first_to_last_ratio": 1.6134108475690156, + "imbalance_abs_log_ratio": 0.47835047692011057, + "last_quartile_mean": 1.6474044969072565e-05, + "mean": 2.676485377151039e-05, + "normalized": [ + 1.150774081299089, + 1.2446775351438402, + 1.1194934771549703, + 1.0710868905687188, + 0.9358496415246254, + 0.8795825855253837, + 0.7921399944267447, + 0.7509646183368756, + 0.7889671267541278, + 0.7481474630705149, + 0.7042600007590066, + 0.6732934483973314, + 0.6778913404746827, + 0.6508461877769323, + 0.6465004342221662, + 0.7150440537203975, + 1.3113655238548305, + 1.3302347281579627, + 1.2033980414753724, + 1.1364499627034836, + 2.9708738613390233, + 2.354543738257344, + 1.7687393668749452, + 1.4507929617425857, + 1.0149536514834854, + 0.914200457095794, + 0.823519211238322, + 0.7833869818285496, + 0.3624534078136484, + 0.3469854230526441, + 0.34042254954577716, + 0.3381612543808245 + ], + "peak_layer": 21, + "peak_normalized": 2.9708738613390233, + "population_cv": 0.5457414223482011, + "rest_mean": 2.2244662937333083e-05, + "spike_contrast": 2.300501671804746, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.1173884276067835e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.0800300010014325e-05, + 3.3313612220808864e-05, + 2.9963079214212485e-05, + 2.866748400265351e-05, + 2.5047878807527013e-05, + 2.3541899281553924e-05, + 2.1201511117396876e-05, + 2.0099458197364584e-05, + 2.1116589778102934e-05, + 2.00240574486088e-05, + 1.8849415937438607e-05, + 1.8020600691670552e-05, + 1.8143662600778043e-05, + 1.7419803043594584e-05, + 1.7303489585174248e-05, + 1.9138049538014457e-05, + 3.5098506486974657e-05, + 3.560353798093274e-05, + 3.220877260901034e-05, + 3.0416917070397176e-05, + 7.951500447234139e-05, + 6.301901885308325e-05, + 4.7340050514321774e-05, + 3.883026147377677e-05, + 2.7165086066816002e-05, + 2.4468441552016884e-05, + 2.2041371266823262e-05, + 2.0967238015145995e-05, + 9.70101245911792e-06, + 9.28701410884969e-06, + 9.111359759117477e-06, + 9.050836524693295e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.01899757081992e-05, + "first_to_last_ratio": 1.2429080864500646, + "imbalance_abs_log_ratio": 0.2174538648629477, + "last_quartile_mean": 7.256367280206177e-05, + "mean": 0.00010441220069878909, + "normalized": [ + 0.9923583084473725, + 1.071937739818941, + 0.9702274874563575, + 0.9286563808868895, + 0.8280637355418319, + 0.7639625829780191, + 0.6921006886798987, + 0.6629947539328518, + 0.7029534904330149, + 0.6708874789375917, + 0.6281382091404577, + 0.5914397520967082, + 0.550684307713307, + 0.5301298645217033, + 0.5304323668326573, + 0.5859521330031094, + 1.2281373620329439, + 1.245755247142365, + 1.1474546090559081, + 1.1339348253988921, + 3.2366950076393746, + 2.635045576071926, + 2.215617097734964, + 1.8966560409115039, + 1.197288676699708, + 1.059685246423848, + 0.9248320715514479, + 0.8492170162384304, + 0.4187477261886761, + 0.38321178810426515, + 0.36578292338610474, + 0.3610195049989306 + ], + "peak_layer": 21, + "peak_normalized": 3.2366950076393746, + "population_cv": 0.6454384585452078, + "rest_mean": 8.050837156285222e-05, + "spike_contrast": 2.90023103809226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00023349287803284824, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0001036143148667179, + 0.00011192337842658162, + 0.00010130358714377508, + 9.696305642137304e-05, + 8.645995694678277e-05, + 7.976701454026625e-05, + 7.226375601021573e-05, + 6.922474130988121e-05, + 7.339692092500627e-05, + 7.004883809713647e-05, + 6.558529275935143e-05, + 6.175352609716356e-05, + 5.749816045863554e-05, + 5.535202581086196e-05, + 5.538361074286513e-05, + 6.118055171100423e-05, + 0.00012823252473026514, + 0.00013007204688619822, + 0.00011980826093349606, + 0.00011839663056889549, + 0.00033795044873841107, + 0.00027513090753927827, + 0.00023133745708037168, + 0.00019803403120022267, + 0.0001250115456059575, + 0.00011064406862715259, + 9.656375186750665e-05, + 8.866861753631383e-05, + 4.372237162897363e-05, + 4.0011986129684374e-05, + 3.8192200008779764e-05, + 3.7694841012125835e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e3e18e8967d646eb5fe3ca615db63633333413502a9ff39741999ecb78762644", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.051234737038612366, + 0.1796528398990631, + 0.21105031669139862, + 0.23972436785697937, + 0.08815877884626389, + 0.1068187803030014, + 0.12298524379730225, + 0.13432784378528595, + 0.03348636254668236, + 0.051035381853580475, + 0.07822930067777634, + 0.11166217923164368, + 0.038218315690755844, + 0.07459890842437744, + 0.12532564997673035, + 0.36271119117736816, + 0.05078785493969917, + 0.0794382095336914, + 0.10824274271726608, + 0.14269457757472992, + 0.050489071756601334, + 0.06721649318933487, + 0.09199884533882141, + 0.11444488167762756, + 0.024430278688669205, + 0.05695623904466629, + 0.09002252668142319, + 0.130380317568779, + 0.030554499477148056, + 0.07779927551746368, + 0.13734298944473267, + 0.22044414281845093 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1110131481473218e-05, + "first_to_last_ratio": 1.1615905687058017, + "imbalance_abs_log_ratio": 0.1497902458283344, + "last_quartile_mean": 9.56458478640343e-06, + "mean": 1.3397171542806063e-05, + "normalized": [ + 0.9679237070104568, + 1.0382438480099463, + 0.9205891769744879, + 0.8794638018815956, + 0.7953494837793593, + 0.7288626539878529, + 0.6666400825149497, + 0.6372419269512295, + 0.6772452645765212, + 0.6436489048118149, + 0.6038970887751886, + 0.5706326967752795, + 0.5512966606689397, + 0.5295937419205439, + 0.5131637814975486, + 0.568852935444849, + 1.1361945010844465, + 1.1401170159003935, + 1.0979049046155025, + 1.0452751153521422, + 3.411273596228361, + 2.8827377384957176, + 2.3438854902907558, + 1.9385604528250762, + 1.2367237086804788, + 1.092419222161065, + 0.9525920914681678, + 0.8037120165341671, + 0.4981781184561606, + 0.4069912576741014, + 0.37037687645015066, + 0.35041213820275013 + ], + "peak_layer": 21, + "peak_normalized": 3.411273596228361, + "population_cv": 0.6942235633030951, + "rest_mean": 1.0016528786191312e-05, + "spike_contrast": 3.160041078518313, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.165264242852572e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2967439943167847e-05, + 1.3909530935052317e-05, + 1.2333291124377865e-05, + 1.1782327419496141e-05, + 1.0655433470674325e-05, + 9.764698006620165e-06, + 8.93109154276317e-06, + 8.537239409633912e-06, + 9.073170986084733e-06, + 8.623074791103136e-06, + 8.090512892522383e-06, + 7.644864126632456e-06, + 7.385815933957929e-06, + 7.095058208506089e-06, + 6.874943210277706e-06, + 7.621020358783426e-06, + 1.522179263702128e-05, + 1.527434324088972e-05, + 1.4708820344822016e-05, + 1.4003730029799044e-05, + 4.5701417548116297e-05, + 3.8620531995547935e-05, + 3.140143599011935e-05, + 2.5971226932597347e-05, + 1.6568599676247686e-05, + 1.4635327715950552e-05, + 1.2762039659719449e-05, + 1.076746775652282e-05, + 6.674177711829543e-06, + 5.452531695482321e-06, + 4.9620025492913555e-06, + 4.6945315261837095e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.520103902905248e-05, + "first_to_last_ratio": 1.1880781863577057, + "imbalance_abs_log_ratio": 0.17233703220725644, + "last_quartile_mean": 3.8045508745199186e-05, + "mean": 5.402507235885423e-05, + "normalized": [ + 0.9708715751809724, + 1.045181729739689, + 0.9250668403006588, + 0.8842698161027324, + 0.80310215643346, + 0.7427420915406581, + 0.6768833494137477, + 0.6452249292071274, + 0.6822190669639766, + 0.6476610418037001, + 0.6085572495277357, + 0.5751764397367072, + 0.5529313317984119, + 0.5299238822976378, + 0.5136471727814265, + 0.5689716487649296, + 1.141806519457199, + 1.159003749171192, + 1.1038866764217172, + 1.0463951059974672, + 3.4229188679498836, + 2.87801535441326, + 2.3242772711715762, + 1.9175102373045263, + 1.212626239200128, + 1.0731375308479167, + 0.9360884367053204, + 0.7920928156744456, + 0.4948063061071149, + 0.406173577795137, + 0.36952801888275133, + 0.3493029713067937 + ], + "peak_layer": 21, + "peak_normalized": 3.4229188679498836, + "population_cv": 0.6923033346437428, + "rest_mean": 4.050810336290548e-05, + "spike_contrast": 3.135587558841143, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012701670493697748, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.2451407100306824e-05, + 5.646601857733913e-05, + 4.997680298401974e-05, + 4.777274079970084e-05, + 4.3387652112869546e-05, + 4.012669523945078e-05, + 3.656867193058133e-05, + 3.4858323488151655e-05, + 3.685693445731886e-05, + 3.498993464745581e-05, + 3.287734944024123e-05, + 3.1073948775883764e-05, + 2.987215520988684e-05, + 2.8629176085814834e-05, + 2.7749825676437467e-05, + 3.073873449466191e-05, + 6.168617983348668e-05, + 6.261526141315699e-05, + 5.9637557569658384e-05, + 5.653157131746411e-05, + 0.00018492343951947987, + 0.00015548498777206987, + 0.00012556924775708467, + 0.00010359362931922078, + 6.551222031703219e-05, + 5.7976332755060866e-05, + 5.057224552729167e-05, + 4.279287168174051e-05, + 2.673194649105426e-05, + 2.1943556930636987e-05, + 1.9963777958764695e-05, + 1.8871118300012313e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.040084732696413994, + "first_to_last_ratio": 1.188078170049551, + "imbalance_abs_log_ratio": 0.1723370184807569, + "last_quartile_mean": 0.03373913746327162, + "mean": 0.04790997417876497, + "normalized": [ + 0.9708715136947069, + 1.0451817255998777, + 0.9250668248545446, + 0.8842697810802376, + 0.8031021854602314, + 0.7427421085449684, + 0.6768833571145957, + 0.6452249040198467, + 0.6822190549510612, + 0.6476610399121447, + 0.6085572507382631, + 0.5751764622807733, + 0.5529313119865983, + 0.5299238802663323, + 0.5136471676571275, + 0.5689716073289056, + 1.1418065239509545, + 1.1590038293019562, + 1.1038866872567357, + 1.046395186500125, + 3.422918937317944, + 2.878015431973323, + 2.3242771288324753, + 1.9175101992153833, + 1.2126262668177996, + 1.073137513398658, + 0.9360884337720428, + 0.7920928619984707, + 0.4948062938209932, + 0.40617357320739117, + 0.3695280026437154, + 0.349302954501817 + ], + "peak_layer": 21, + "peak_normalized": 3.422918937317944, + "population_cv": 0.6923033419361495, + "rest_mean": 0.035922990973900865, + "spike_contrast": 3.135587556360974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11263968348503113, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04651442915201187, + 0.05007462948560715, + 0.04431992769241333, + 0.04236534237861633, + 0.0384766049683094, + 0.03558475524187088, + 0.03242946416139603, + 0.03091270849108696, + 0.03268509730696678, + 0.03102942369878292, + 0.02915596216917038, + 0.027556689456105232, + 0.02649092487990856, + 0.02538863942027092, + 0.024608822539448738, + 0.02725941501557827, + 0.05470392107963562, + 0.05552784353494644, + 0.05288718268275261, + 0.050132766366004944, + 0.16399195790290833, + 0.13788564503192902, + 0.11135605722665787, + 0.09186786413192749, + 0.05809689313173294, + 0.051413990557193756, + 0.04484797269105911, + 0.03794914856553078, + 0.023706156760454178, + 0.019459765404462814, + 0.017704077064990997, + 0.016735095530748367 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8672166390842904e-06, + "first_to_last_ratio": 0.7390827205467719, + "imbalance_abs_log_ratio": 0.3023454285110359, + "last_quartile_mean": 2.5263973668643303e-06, + "mean": 2.8231194875161236e-06, + "normalized": [ + 1.0710441535522213, + 0.7806782718477623, + 0.7446651107672629, + 0.7227944286323706, + 0.5070907376718641, + 0.49705683909512793, + 0.484348171178412, + 0.483537617930567, + 0.5122490344314135, + 0.5011923600503864, + 0.49494085537056437, + 0.4929156803489962, + 0.48753662380682133, + 0.48921410821124456, + 0.49216500849421285, + 0.5089367920233998, + 0.820710291882377, + 0.8409358666404328, + 0.8609799850722372, + 0.8853668132738947, + 3.599560016732762, + 3.2818994953739233, + 2.821405059000161, + 2.4596114274457115, + 1.4921895479857374, + 1.353669348387604, + 1.2002458902354434, + 1.0429447595206092, + 0.6391234507721919, + 0.5173406844488531, + 0.47262998257826977, + 0.441021587237165 + ], + "peak_layer": 21, + "peak_normalized": 3.599560016732762, + "population_cv": 0.8312440892614361, + "rest_mean": 1.918187822243285e-06, + "spike_contrast": 4.0192886174061115, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.709750479989453e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0236856218834873e-06, + 2.2039480427338276e-06, + 2.1022785858804127e-06, + 2.0405350369401276e-06, + 1.431577743460366e-06, + 1.4032508488526219e-06, + 1.3673727607965702e-06, + 1.3650844721269095e-06, + 1.446140231564641e-06, + 1.4149259186524432e-06, + 1.3972771739645395e-06, + 1.3915598628955195e-06, + 1.3763741435468546e-06, + 1.3811098824589862e-06, + 1.3894406265535508e-06, + 1.4367893754752004e-06, + 2.3169632186181843e-06, + 2.374062432863866e-06, + 2.430649374218774e-06, + 2.499496304153581e-06, + 1.0161988029722124e-05, + 9.265194421459455e-06, + 7.965163604239933e-06, + 6.943776952539338e-06, + 4.212629391986411e-06, + 3.821570317086298e-06, + 3.3884375625348184e-06, + 2.944357675005449e-06, + 1.8043218688035267e-06, + 1.4605145679524867e-06, + 1.3342909142011194e-06, + 1.2450566373445326e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.515827052069653e-05, + "first_to_last_ratio": 1.3729523213289132, + "imbalance_abs_log_ratio": 0.31696340027531067, + "last_quartile_mean": 2.5607786938053323e-05, + "mean": 3.930294792553468e-05, + "normalized": [ + 1.0457356769392694, + 1.1170113876105092, + 0.9973655731270212, + 0.9523397521545555, + 0.848322559088178, + 0.7912242750828141, + 0.7196584735613146, + 0.6847053492610473, + 0.7251119779087116, + 0.6895313261932131, + 0.6494379753000808, + 0.613786235590645, + 0.6090154259037742, + 0.5840032818494388, + 0.5660492129431545, + 0.6192857434966186, + 1.2024027857653905, + 1.2099521826215391, + 1.1323058584311179, + 1.0603698069411953, + 3.3091113142819246, + 2.746844343609951, + 2.155778589894086, + 1.758260822046451, + 1.144969795788572, + 0.9973957484994478, + 0.8715780521673296, + 0.7496724170015708, + 0.433790833285331, + 0.35920735567744455, + 0.33316471339849535, + 0.32261115457980727 + ], + "peak_layer": 21, + "peak_normalized": 3.3091113142819246, + "population_cv": 0.6495478539134593, + "rest_mean": 3.040160919664669e-05, + "spike_contrast": 2.873866856730952, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.737017706152982e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.1100494854617864e-05, + 4.390184039948508e-05, + 3.919940718333237e-05, + 3.74297596863471e-05, + 3.334157736389898e-05, + 3.109744648099877e-05, + 2.8284699510550126e-05, + 2.691093868634198e-05, + 2.849903830792755e-05, + 2.7100613806396723e-05, + 2.5524826924083754e-05, + 2.4123608454829082e-05, + 2.3936101570143364e-05, + 2.295305057486985e-05, + 2.224740273959469e-05, + 2.433975532767363e-05, + 4.725797407445498e-05, + 4.755468762596138e-05, + 4.4502958189696074e-05, + 4.1675659304019064e-05, + 0.0001300578296650201, + 0.0001079590801964514, + 8.472845365758985e-05, + 6.910483352839947e-05, + 4.5000688260188326e-05, + 3.920059316442348e-05, + 3.425558679737151e-05, + 2.9464335966622457e-05, + 1.704925853118766e-05, + 1.4117907994659618e-05, + 1.309435538132675e-05, + 1.2679569408646785e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6777070615935372e-05, + "first_to_last_ratio": 1.4895983067726413, + "imbalance_abs_log_ratio": 0.39850649084066486, + "last_quartile_mean": 1.7976034541788977e-05, + "mean": 2.8343781877993024e-05, + "normalized": [ + 1.1067878423232929, + 1.1808830950582898, + 1.0583921093539743, + 1.0130981930448324, + 0.8884743193198895, + 0.8335235896508596, + 0.7557776874483318, + 0.7208607029428713, + 0.7654002308457403, + 0.723786998353269, + 0.6828269487786236, + 0.6470347967107349, + 0.6602591516703425, + 0.6315002997298088, + 0.6140621512563034, + 0.6701035513604336, + 1.2453501569114247, + 1.2538344754875952, + 1.1579706088704604, + 1.0596025963058344, + 3.1377161786254733, + 2.5783790501879347, + 1.9479449698525808, + 1.5927151172754295, + 1.103279856014175, + 0.9422718711690712, + 0.854272511709325, + 0.7657265014248548, + 0.4191617190143686, + 0.34148416420892125, + 0.32691118354419935, + 0.32060737155075375 + ], + "peak_layer": 21, + "peak_normalized": 3.1377161786254733, + "population_cv": 0.5950537042380278, + "rest_mean": 2.2716979367908573e-05, + "spike_contrast": 2.5852255478742316, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.872851543244906e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.137055318802595e-05, + 3.347069286974147e-05, + 2.9998835088917986e-05, + 2.87150342046516e-05, + 2.518272231100127e-05, + 2.3625210815225728e-05, + 2.1421597921289504e-05, + 2.0431918528629467e-05, + 2.169433719245717e-05, + 2.051486080745235e-05, + 1.9353898096596822e-05, + 1.833941314544063e-05, + 1.8714241377892904e-05, + 1.7899106751428917e-05, + 1.7404843674739823e-05, + 1.8993268895428628e-05, + 3.529793320922181e-05, + 3.5538410884328187e-05, + 3.282126635895111e-05, + 3.0033144867047668e-05, + 8.89347429620102e-05, + 7.308101339731365e-05, + 5.521212733583525e-05, + 4.514356987783685e-05, + 3.127112358924933e-05, + 2.67075483861845e-05, + 2.421331373625435e-05, + 2.1703584934584796e-05, + 1.1880628335347865e-05, + 9.678952665126417e-06, + 9.265899279853329e-06, + 9.08722540771123e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.124297230300726e-05, + "first_to_last_ratio": 1.129611466147561, + "imbalance_abs_log_ratio": 0.12187373833170514, + "last_quartile_mean": 8.077376605797326e-05, + "mean": 0.0001132043312281894, + "normalized": [ + 0.9389217132455487, + 0.9994043961548372, + 0.8982931506377284, + 0.859853269169116, + 0.7637743898798451, + 0.7087638918644622, + 0.6499465810809194, + 0.6290624894037778, + 0.6694605698054724, + 0.6398149406563152, + 0.5998145159738899, + 0.5645121104930153, + 0.5238242883776524, + 0.5044864713561084, + 0.48955416809270236, + 0.5348544623331092, + 1.1363983661263592, + 1.1357195169737937, + 1.078307851992808, + 1.031529476521427, + 3.518987411841691, + 2.9635435485495347, + 2.411411019127874, + 2.041586444021295, + 1.2317185051916162, + 1.0997550375371743, + 0.974217691447948, + 0.8518825440407063, + 0.4721291734182047, + 0.38687023180546226, + 0.35404975204423306, + 0.3375520208353729 + ], + "peak_layer": 21, + "peak_normalized": 3.518987411841691, + "population_cv": 0.7265349466170719, + "rest_mean": 8.315383510543379e-05, + "spike_contrast": 3.312859953383778, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00027547701029106975, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00010629000462358817, + 0.00011313690629322082, + 0.00010169067536480725, + 9.733911429066211e-05, + 8.646256901556626e-05, + 8.023514237720519e-05, + 7.357676804531366e-05, + 7.121259841369465e-05, + 7.578583608847111e-05, + 7.242982246680185e-05, + 6.790160114178434e-05, + 6.390521593857557e-05, + 5.929917824687436e-05, + 5.711005360353738e-05, + 5.541965219890699e-05, + 6.054784171283245e-05, + 0.00012864521704614162, + 0.00012856836838182062, + 0.00012206911924295127, + 0.00011677360453177243, + 0.00039836461655795574, + 0.0003354859654791653, + 0.0002729821717366576, + 0.00023111642803996801, + 0.00013943586964160204, + 0.00012449703353922814, + 0.00011028566223103553, + 9.643679368309677e-05, + 5.344706733012572e-05, + 4.3795385863631964e-05, + 4.007996540167369e-05, + 3.8212350773392245e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "1266324f6216c67f6f92c122a2698de3ea217c2cc1431a3715e69a04aa62551d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0955304205417633, + 0.19925470650196075, + 0.2359117865562439, + 0.25103330612182617, + 0.09304249286651611, + 0.11085781455039978, + 0.12910984456539154, + 0.14378803968429565, + 0.04010729491710663, + 0.06472750008106232, + 0.09987020492553711, + 0.14899125695228577, + 0.05702618882060051, + 0.10170809924602509, + 0.15923865139484406, + 0.38537684082984924, + 0.05919196084141731, + 0.08540526777505875, + 0.1145314872264862, + 0.15640178322792053, + 0.06262174993753433, + 0.08427268266677856, + 0.11030438542366028, + 0.13170360028743744, + 0.05311359465122223, + 0.08614693582057953, + 0.12623678147792816, + 0.17503507435321808, + 0.07505694776773453, + 0.13522638380527496, + 0.2187158316373825, + 0.3292153775691986 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0990825558110373e-05, + "first_to_last_ratio": 1.2944181557270227, + "imbalance_abs_log_ratio": 0.25806129358885044, + "last_quartile_mean": 8.490938967042894e-06, + "mean": 1.2353476122939355e-05, + "normalized": [ + 1.0282384798178228, + 1.117858133996928, + 0.9935843345501945, + 0.9494663755935344, + 0.8590058043818248, + 0.7847999930233409, + 0.709132552716486, + 0.6754740791306071, + 0.7068824996302968, + 0.674955776214346, + 0.6307861342112064, + 0.6026941529611411, + 0.5785889076796064, + 0.556995885467898, + 0.5565612914175161, + 0.6240017404560392, + 1.2244356436045847, + 1.2505073846311183, + 1.1714347530624865, + 1.1440952313231665, + 3.1783392045284105, + 2.5998686819467225, + 2.1163742238669494, + 1.767263048891068, + 1.1859101289408378, + 1.03626731593003, + 0.8733325362846178, + 0.8036687223427385, + 0.4425832407384721, + 0.40163477037506473, + 0.38000515774441007, + 0.3752538145405344 + ], + "peak_layer": 21, + "peak_normalized": 3.1783392045284105, + "population_cv": 0.6201793460523679, + "rest_mean": 9.677916666448624e-06, + "spike_contrast": 2.7693457292213166, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.68014971879893e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.2702319509116933e-05, + 1.3809433767164592e-05, + 1.2274220352992415e-05, + 1.1729210200428497e-05, + 1.0611707693897188e-05, + 9.695007975096814e-06, + 8.760252057982143e-06, + 8.344452908204403e-06, + 8.732456080906559e-06, + 8.338050065503921e-06, + 7.792401447659358e-06, + 7.445367828040617e-06, + 7.147584256017581e-06, + 6.880835371703142e-06, + 6.875466624478577e-06, + 7.708590601396281e-06, + 1.5126036487345118e-05, + 1.544811311759986e-05, + 1.4471291251538787e-05, + 1.4133553122519515e-05, + 3.926353747374378e-05, + 3.211741568520665e-05, + 2.614457844174467e-05, + 2.1831841877428815e-05, + 1.4650112461822573e-05, + 1.2801503544324078e-05, + 1.0788692634378094e-05, + 9.928102372214198e-06, + 5.467441496875836e-06, + 4.961585545970593e-06, + 4.694384642789373e-06, + 4.635689037968405e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.4711176087730564e-05, + "first_to_last_ratio": 1.3207760209544162, + "imbalance_abs_log_ratio": 0.27821945851834845, + "last_quartile_mean": 3.385220156815194e-05, + "mean": 4.981010016535947e-05, + "normalized": [ + 1.0309026668033479, + 1.1248865671658381, + 0.9983068512481783, + 0.9550523854303026, + 0.8675947035501784, + 0.799947715571839, + 0.7205411910772573, + 0.68382974224492, + 0.7120277827066004, + 0.6791934284063869, + 0.6349499689734955, + 0.6062730250974693, + 0.5788392434931497, + 0.5568995584252106, + 0.5581244614543389, + 0.6247854138149606, + 1.2301897371875206, + 1.271033179578376, + 1.1767202476646232, + 1.146019522378179, + 3.189751492495938, + 2.584287570955602, + 2.0948183363362958, + 1.7380232415485222, + 1.165593138274326, + 1.019662423146035, + 0.8607718088943986, + 0.7949347500773989, + 0.4416990390008851, + 0.4009654076948961, + 0.3788881940118565, + 0.3744872052916737 + ], + "peak_layer": 21, + "peak_normalized": 3.189751492495938, + "population_cv": 0.6175097196118516, + "rest_mean": 3.9160933603718013e-05, + "spike_contrast": 2.740373880872814, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010731559959822334, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.134936509421095e-05, + 5.603071258519776e-05, + 4.972576425643638e-05, + 4.7571254981448874e-05, + 4.3214979086769745e-05, + 3.9845475839683786e-05, + 3.5890228900825605e-05, + 3.406162795727141e-05, + 3.5466175177134573e-05, + 3.383069270057604e-05, + 3.1626921554561704e-05, + 3.0198520107660443e-05, + 2.883204069803469e-05, + 2.77392227872042e-05, + 2.7800235329777934e-05, + 3.112062404397875e-05, + 6.127587403170764e-05, + 6.331028998829424e-05, + 5.8612553402781487e-05, + 5.708334720111452e-05, + 0.00015888184134382755, + 0.00012872362276539207, + 0.00010434311116114259, + 8.657111175125465e-05, + 5.805831096949987e-05, + 5.078948743175715e-05, + 4.287513002054766e-05, + 3.959577952628024e-05, + 2.2001073375577107e-05, + 1.997212712012697e-05, + 1.8872458895202726e-05, + 1.86532452062238e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03965031821280718, + "first_to_last_ratio": 1.3207760013357848, + "imbalance_abs_log_ratio": 0.2782194436644813, + "last_quartile_mean": 0.0300204714294523, + "mean": 0.04417209519306198, + "normalized": [ + 1.0309027129185282, + 1.124886561555575, + 0.9983068303969317, + 0.9550523327081637, + 0.8675947279179665, + 0.7999476938642114, + 0.7205412090579838, + 0.6838297380461792, + 0.7120277600555036, + 0.6791934601432961, + 0.6349499561794323, + 0.6062730331610577, + 0.5788392624406322, + 0.5568995493471454, + 0.5581244851058754, + 0.6247854224706602, + 1.2301896279139695, + 1.2710332172386067, + 1.1767202901940028, + 1.1460195181019854, + 3.189751438287289, + 2.5842874809086247, + 2.0948183490366055, + 1.7380233083860965, + 1.1655931894927514, + 1.019662433533796, + 0.8607717906221712, + 0.7949347721458104, + 0.4416990408526936, + 0.4009654254500078, + 0.3788882027861667, + 0.37448717968028017 + ], + "peak_layer": 21, + "peak_normalized": 3.189751438287289, + "population_cv": 0.6175097084858858, + "rest_mean": 0.034728307759872186, + "spike_contrast": 2.7403738756960694, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09516854733228683, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.045537132769823074, + 0.04968859627842903, + 0.044097304344177246, + 0.042186662554740906, + 0.038323476910591125, + 0.03533536568284035, + 0.031827814877033234, + 0.030206192284822464, + 0.0314517579972744, + 0.03000139817595482, + 0.028047069907188416, + 0.026780350133776665, + 0.025568543002009392, + 0.02459941990673542, + 0.02465352788567543, + 0.027598081156611443, + 0.05434005334973335, + 0.05614420026540756, + 0.05197820067405701, + 0.050622083246707916, + 0.14089800417423248, + 0.11415339261293411, + 0.09253251552581787, + 0.07677213102579117, + 0.05148669332265854, + 0.04504062607884407, + 0.03802209347486496, + 0.03511393442749977, + 0.019510772079229355, + 0.01771148294210434, + 0.016736285760998726, + 0.01654188334941864 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8471929479346727e-06, + "first_to_last_ratio": 0.824461579019105, + "imbalance_abs_log_ratio": 0.19302473719608554, + "last_quartile_mean": 2.2404839654655007e-06, + "mean": 2.553080481249026e-06, + "normalized": [ + 1.1469614368871983, + 0.8593837713333818, + 0.8257379810924192, + 0.8007786067207996, + 0.5570026135445557, + 0.5466338814455962, + 0.5274997830623889, + 0.5241248198100039, + 0.5489562615578096, + 0.5407197695419355, + 0.5308505223860043, + 0.5295945294910184, + 0.5283032694056181, + 0.5289080215685197, + 0.5448229646529533, + 0.566591415289843, + 0.8991703275174108, + 0.9131167201550765, + 0.9272471968396779, + 0.9661518060876617, + 3.3508739801606664, + 2.970736593720056, + 2.5767839088445323, + 2.268561489055477, + 1.4764458758023058, + 1.3194996647146926, + 1.1214564007083334, + 1.0614159683161588, + 0.5682809452629535, + 0.5158177934133008, + 0.48288389278490795, + 0.4746877888267432 + ], + "peak_layer": 21, + "peak_normalized": 3.3508739801606664, + "population_cv": 0.7395096569005253, + "rest_mean": 1.8303315898635816e-06, + "spike_contrast": 3.5271884780241365, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.455924494730425e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.9282848572620424e-06, + 2.1940759324934334e-06, + 2.108175522153033e-06, + 2.0444522306206636e-06, + 1.4220725006452994e-06, + 1.3956002931081457e-06, + 1.3467493999996805e-06, + 1.3381328471950837e-06, + 1.4015295164426789e-06, + 1.380501089442987e-06, + 1.3553041071645566e-06, + 1.3520974562197807e-06, + 1.3488007652995293e-06, + 1.3503447462426266e-06, + 1.390976876791683e-06, + 1.446553483219759e-06, + 2.2956542125029955e-06, + 2.3312604753300548e-06, + 2.367336719544255e-06, + 2.466663318045903e-06, + 8.555050953873433e-06, + 7.584529612358892e-06, + 6.578736702067545e-06, + 5.791820058220765e-06, + 3.7694851471314905e-06, + 3.368788838997716e-06, + 2.8631684472202323e-06, + 2.7098803911940195e-06, + 1.4508669892165926e-06, + 1.3169243402444408e-06, + 1.2328414413786959e-06, + 1.2119161283408175e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.4829904052458005e-05, + "first_to_last_ratio": 1.509444580974737, + "imbalance_abs_log_ratio": 0.4117417559927783, + "last_quartile_mean": 2.3074649107002188e-05, + "mean": 3.6670617419076734e-05, + "normalized": [ + 1.0984878069834736, + 1.188723837036249, + 1.0648721392453866, + 1.0174200831414708, + 0.9061105203098224, + 0.8430649229703882, + 0.7598681195362218, + 0.7198856496460319, + 0.7526438720220634, + 0.718650325146632, + 0.6733539856914239, + 0.6414557384537676, + 0.6322823726903553, + 0.6081123440925361, + 0.6055674387182303, + 0.669931198330675, + 1.2812282331891511, + 1.3084793805341686, + 1.1959962991108561, + 1.1475497928959553, + 3.1043863400982548, + 2.476737525556709, + 1.953803593412093, + 1.5974619795855607, + 1.0791244029968152, + 0.9527382661611272, + 0.8120339425793232, + 0.7520941665087985, + 0.38753474791935355, + 0.35900363296340954, + 0.3463461699660851, + 0.3450511725076112 + ], + "peak_layer": 21, + "peak_normalized": 3.1043863400982548, + "population_cv": 0.5864654402354135, + "rest_mean": 2.9592490372360096e-05, + "spike_contrast": 2.5307942159980388, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.489250347134658e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.028222610941157e-05, + 4.3591237044893205e-05, + 3.904951881850138e-05, + 3.730942262336612e-05, + 3.322763222968206e-05, + 3.09157112496905e-05, + 2.7864833100466058e-05, + 2.639865124365315e-05, + 2.759991548373364e-05, + 2.6353351131547242e-05, + 2.4692306396900676e-05, + 2.352257797610946e-05, + 2.318618498975411e-05, + 2.229985511803534e-05, + 2.220653186668642e-05, + 2.45667906710878e-05, + 4.6983430365798995e-05, + 4.798274676431902e-05, + 4.385792271932587e-05, + 4.208135942462832e-05, + 0.00011383976379875094, + 9.082349424716085e-05, + 7.164718408603221e-05, + 5.857991709490307e-05, + 3.957215812988579e-05, + 3.49375004589092e-05, + 2.9777786039630882e-05, + 2.7579757443163544e-05, + 1.4211138477548957e-05, + 1.3164884876459837e-05, + 1.270072789338883e-05, + 1.2653239537030458e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6578757797324215e-05, + "first_to_last_ratio": 1.6134215502902036, + "imbalance_abs_log_ratio": 0.47835711049748203, + "last_quartile_mean": 1.6473535879413248e-05, + "mean": 2.6764625772557338e-05, + "normalized": [ + 1.1507208152122448, + 1.2447207601116925, + 1.1194216627013838, + 1.0711038984456216, + 0.9358933619627398, + 0.8793303259328489, + 0.7922968034824025, + 0.7509560638170416, + 0.7890091881742389, + 0.7482141869411878, + 0.7043635941867609, + 0.6733903895614985, + 0.6779168243147607, + 0.6506778841853507, + 0.6465418257154243, + 0.7151576615308665, + 1.3113688112983555, + 1.3303637709400895, + 1.2034267786084025, + 1.1362866793208775, + 2.97087008129982, + 2.355025940458153, + 1.7683296688974177, + 1.4506403077495817, + 1.0150838143878007, + 0.9142558185771078, + 0.8233714760378564, + 0.7833275277830687, + 0.36254973990814826, + 0.34689941606582414, + 0.3403766524569371, + 0.33810826993449583 + ], + "peak_layer": 21, + "peak_normalized": 2.97087008129982, + "population_cv": 0.5457582329285832, + "rest_mean": 2.2244427614000245e-05, + "spike_contrast": 2.300517537100295, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.117369582876563e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.0798611987847835e-05, + 3.3314485335722566e-05, + 2.996090188389644e-05, + 2.866769500542432e-05, + 2.5048835595953278e-05, + 2.3534947104053572e-05, + 2.1205527445999905e-05, + 2.0099058019695804e-05, + 2.1117535652592778e-05, + 2.002567271119915e-05, + 1.88520280062221e-05, + 1.802304177545011e-05, + 1.814419010770507e-05, + 1.7415150068700314e-05, + 1.730445001157932e-05, + 1.9140927179250866e-05, + 3.5098295484203845e-05, + 3.5606688470579684e-05, + 3.22092673741281e-05, + 3.041228774236515e-05, + 7.951422594487667e-05, + 6.303138798102736e-05, + 4.732868183054961e-05, + 3.882584496750496e-05, + 2.7168338419869542e-05, + 2.4469714844599366e-05, + 2.2037229427951388e-05, + 2.0965468138456345e-05, + 9.703508112579584e-06, + 9.284633051720448e-06, + 9.11005372472573e-06, + 9.04934131540358e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.019189837999875e-05, + "first_to_last_ratio": 1.2430263788421747, + "imbalance_abs_log_ratio": 0.21754903421986438, + "last_quartile_mean": 7.255831405927893e-05, + "mean": 0.00010441271081162995, + "normalized": [ + 0.9923145762328835, + 1.0717706255029942, + 0.9701026808081262, + 0.9288730924893803, + 0.8283807269361432, + 0.7640642833999716, + 0.6920059508859676, + 0.6629032941440701, + 0.7029452478860042, + 0.6708830166494891, + 0.6281695645345438, + 0.5913612548049529, + 0.5507136373932213, + 0.5301664721346755, + 0.5303584531978167, + 0.5861270357139448, + 1.228189060760272, + 1.2457938984618147, + 1.1471984869923635, + 1.1338859416919997, + 3.2374008483056524, + 2.634554108633911, + 2.2155512224100757, + 1.8969393106516845, + 1.1972085435140079, + 1.059610663427442, + 0.9247236535108656, + 0.8491302911049953, + 0.4186534528150795, + 0.3832452808390877, + 0.36580012539024503, + 0.36097519877631845 + ], + "peak_layer": 21, + "peak_normalized": 3.2374008483056524, + "population_cv": 0.6454817160990234, + "rest_mean": 8.050740507314913e-05, + "spike_contrast": 2.9003712340158865, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00023350136179942638, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00010361025488236919, + 0.00011190647637704387, + 0.00010129105066880584, + 9.698615758679807e-05, + 8.649347728351131e-05, + 7.97780230641365e-05, + 7.225421722978354e-05, + 6.921552994754165e-05, + 7.339641888393089e-05, + 7.004871440585703e-05, + 6.558888708241284e-05, + 6.174563168315217e-05, + 5.7501503761159256e-05, + 5.535611853701994e-05, + 5.5376163800247014e-05, + 6.119911267887801e-05, + 0.00012823854922316968, + 0.00013007671805098653, + 0.00011978210386587307, + 0.00011839210492325947, + 0.00033802579855546355, + 0.00027508093626238406, + 0.00023133170907385647, + 0.000198064575670287, + 0.0001250037894351408, + 0.00011063682177336887, + 9.65529034147039e-05, + 8.865999552654102e-05, + 4.371274189907126e-05, + 4.001567867817357e-05, + 3.819418270722963e-05, + 3.7690399040002376e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "9ec6f7109f4e916c83054b81e32dd86c0125957a9aa2bade5062f6ae3f8e89ac", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0558863990008831, + 0.061753273010253906, + 0.11003358662128448, + 0.1138310506939888, + 0.13082429766654968, + 0.10403942316770554, + 0.1062953919172287, + 0.10766621679067612, + 0.11238158494234085, + 0.08696430176496506, + 0.08355452120304108, + 0.08609789609909058, + 0.100697360932827, + 0.07805952429771423, + 0.08197034895420074, + 0.10499614477157593, + 0.13756641745567322, + 0.09882861375808716, + 0.10110001266002655, + 0.10269922018051147, + 0.10988812148571014, + 0.0575832724571228, + 0.06622003018856049, + 0.06460635364055634, + 0.07165247201919556, + 0.055589381605386734, + 0.05917651578783989, + 0.06219335272908211, + 0.06750614196062088, + 0.06024058535695076, + 0.06803743541240692, + 0.06952574104070663 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 6.596624018584407e-06, + "first_to_last_ratio": 1.8980506308307645, + "imbalance_abs_log_ratio": 0.640827375717486, + "last_quartile_mean": 3.4754731572661512e-06, + "mean": 8.708262916101717e-06, + "normalized": [ + 1.0256731976862243, + 0.9753411624231779, + 0.9035749226639593, + 0.5582866185427084, + 0.48593023814787983, + 0.8667266687937877, + 0.7602289215288303, + 0.4843437880540975, + 0.4388532976888435, + 0.471538921660211, + 0.6133103789102933, + 0.5130639656537879, + 0.3167954672221293, + 0.31280913276495154, + 0.23475435209498255, + 1.0312032138821217, + 1.0496881251460148, + 1.5459688796414424, + 2.660629290297128, + 2.4820591361597244, + 4.429437174760716, + 2.8054906698151227, + 2.505482199716851, + 1.336005290721962, + 0.7761527446294757, + 0.5408170248947934, + 0.6194156007606676, + 0.405832393835082, + 0.32631761431893097, + 0.23366629172124823, + 0.18561467402758638, + 0.10498864183526842 + ], + "peak_layer": 21, + "peak_normalized": 4.429437174760716, + "population_cv": 0.9558477179995892, + "rest_mean": 6.498116083211816e-06, + "spike_contrast": 3.176775476055201, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0643055813707177e-05, + "top_five_layers": [ + 21, + 22, + 19, + 23, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.931831871450413e-06, + 8.493527275277302e-06, + 7.868567990954034e-06, + 4.861706656811293e-06, + 4.231608272675658e-06, + 7.547683708253317e-06, + 6.620273325097514e-06, + 4.217793048155727e-06, + 3.821649897872703e-06, + 4.106284904992208e-06, + 5.340868028724799e-06, + 4.467895905690966e-06, + 2.7587382191995857e-06, + 2.7240241706749657e-06, + 2.044302618742222e-06, + 8.979988706414588e-06, + 9.140960173681378e-06, + 1.3462703464028891e-05, + 2.3169459382188506e-05, + 2.161442353099119e-05, + 3.85727034881711e-05, + 2.44309503614204e-05, + 2.1818397726747207e-05, + 1.1634285328909755e-05, + 6.758942163287429e-06, + 4.709576842287788e-06, + 5.394033905758988e-06, + 3.534095185386832e-06, + 2.841659579644329e-06, + 2.0348275029391516e-06, + 1.616381382518739e-06, + 9.142686963059532e-07 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.6033237190858927e-05, + "first_to_last_ratio": 1.8862787219924644, + "imbalance_abs_log_ratio": 0.6346059580177699, + "last_quartile_mean": 1.3801373512478676e-05, + "mean": 3.5272740866787444e-05, + "normalized": [ + 0.9568267063421432, + 0.9174711042795557, + 0.911235657888503, + 0.5484831467374834, + 0.4781383342683516, + 0.8432205386406871, + 0.7712046240634284, + 0.47786316065782963, + 0.4398762585196919, + 0.51209123046888, + 0.6108901597314282, + 0.4846724182865656, + 0.3294043935332117, + 0.3091278217388424, + 0.22447937051100367, + 0.999498418571711, + 1.0065120452069123, + 1.5624889800277921, + 2.795100908402575, + 2.4945789402126657, + 4.378703171587316, + 2.953659490750943, + 2.5415441033638264, + 1.3227217970755467, + 0.7766017604446148, + 0.5134703473853324, + 0.605157823112537, + 0.39031143703269466, + 0.3253712384885135, + 0.23092588991327564, + 0.1825610852772617, + 0.10580763747887693 + ], + "peak_layer": 21, + "peak_normalized": 4.378703171587316, + "population_cv": 0.9707222592180655, + "rest_mean": 2.616292804473401e-05, + "spike_contrast": 3.2284509578382985, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.4465730105876e-05, + "top_five_layers": [ + 21, + 22, + 19, + 23, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.3749900467228144e-05, + 3.236172051401809e-05, + 3.2141779229277745e-05, + 1.9346503904671408e-05, + 1.6865249563124962e-05, + 2.9742699553025886e-05, + 2.7202500859857537e-05, + 1.685554343566764e-05, + 1.5515641280217096e-05, + 1.8062861272483133e-05, + 2.154777030227706e-05, + 1.709572461550124e-05, + 1.161899581347825e-05, + 1.0903785550908651e-05, + 7.9180026659742e-06, + 3.525504871504381e-05, + 3.5502438549883664e-05, + 5.511326889973134e-05, + 9.859087003860623e-05, + 8.79906365298666e-05, + 0.0001544488623039797, + 0.00010418366582598537, + 8.964722655946389e-05, + 4.665602318709716e-05, + 2.739287265285384e-05, + 1.811150650610216e-05, + 2.1345575078157708e-05, + 1.3767354175797664e-05, + 1.1476735380711034e-05, + 8.145389074343257e-06, + 6.439429853344336e-06, + 3.732125378519413e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.023086535395123065, + "first_to_last_ratio": 1.8862787345100174, + "imbalance_abs_log_ratio": 0.6346059646538798, + "last_quartile_mean": 0.012239196134032682, + "mean": 0.03128021959128091, + "normalized": [ + 0.9568267121752291, + 0.9174711066988445, + 0.9112356957360986, + 0.5484831570712991, + 0.4781383082120972, + 0.8432205065508228, + 0.7712046480612316, + 0.4778631710159881, + 0.43987625474046216, + 0.512091268376117, + 0.6108901444890714, + 0.4846724370071731, + 0.3294043980010242, + 0.30912781978929554, + 0.22447935000902958, + 0.9994983743937245, + 1.0065119463517638, + 1.5624889224293914, + 2.7951009409274206, + 2.4945790708322866, + 4.378703107551428, + 2.9536595207599716, + 2.541544076160719, + 1.322721846992968, + 0.7766017554969081, + 0.5134703187656182, + 0.6051578466461017, + 0.390311454495132, + 0.3253712457748438, + 0.23092588507713302, + 0.18256107720838338, + 0.10580763220242217 + ], + "peak_layer": 21, + "peak_normalized": 4.378703107551428, + "population_cv": 0.9707222603656619, + "rest_mean": 0.02320154643341623, + "spike_contrast": 3.2284509508326362, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.07490505464375019, + "top_five_layers": [ + 21, + 22, + 19, + 23, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.0299297496676445, + 0.028698697686195374, + 0.028503652662038803, + 0.017156673595309258, + 0.014956271275877953, + 0.02637612260878086, + 0.024123450741171837, + 0.014947664923965931, + 0.013759425841271877, + 0.016018327325582504, + 0.01910877786576748, + 0.01516066025942564, + 0.010303841903805733, + 0.009669586084783077, + 0.007021763361990452, + 0.031264528632164, + 0.03148391470313072, + 0.04887499660253525, + 0.08743137121200562, + 0.07803098112344742, + 0.1369667947292328, + 0.09239111840724945, + 0.07950005680322647, + 0.04137502983212471, + 0.024292273446917534, + 0.016061464324593544, + 0.01892947033047676, + 0.012209028005599976, + 0.010177684016525745, + 0.007223412394523621, + 0.005710550583899021, + 0.003309685969725251 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.8957251032579734e-06, + "first_to_last_ratio": 1.3390904119295601, + "imbalance_abs_log_ratio": 0.2919905864082896, + "last_quartile_mean": 2.1624567523303995e-06, + "mean": 2.6915775031000067e-06, + "normalized": [ + 4.8336725838512145, + 1.7374668605804229, + 0.3213508492275624, + 0.4363891970704281, + 0.32589544632371475, + 0.307241753778847, + 0.3145233096085374, + 0.3302345766920988, + 0.3490931446328018, + 0.3693094318109904, + 0.3998151480537758, + 0.5074504002356933, + 0.29839576596304346, + 0.36719305482228454, + 0.37294658781774515, + 0.5123164707134974, + 0.3979121151789476, + 0.6707129654139522, + 0.9550170379823126, + 1.0953232949941911, + 2.212763417999681, + 3.415237241942596, + 2.685461662904724, + 2.356948728016963, + 1.0448937474521496, + 1.0864393774586762, + 1.3084278349629268, + 0.9447949362821033, + 0.7371038712092777, + 0.5351342382555423, + 0.3756737262173936, + 0.39486122254590533 + ], + "peak_layer": 1, + "peak_normalized": 4.8336725838512145, + "population_cv": 1.0373807348560817, + "rest_mean": 2.022141824521963e-06, + "spike_contrast": 3.1187378110396953, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.3065301674214425e-06, + "top_five_layers": [ + 1, + 22, + 23, + 24, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.301020438404521e-05, + 4.676526714320062e-06, + 8.649407163829892e-07, + 1.1745753454306396e-06, + 8.771728516876465e-07, + 8.26964992484136e-07, + 8.465638643428974e-07, + 8.888519573702069e-07, + 9.396112545800861e-07, + 9.940249583451077e-07, + 1.0761334579001414e-06, + 1.3658420812134864e-06, + 8.031553306864225e-07, + 9.883285656542284e-07, + 1.0038146456281538e-06, + 1.3789394870400429e-06, + 1.071011297426594e-06, + 1.8052759287456865e-06, + 2.570502374510397e-06, + 2.9481475394277368e-06, + 5.955824235570617e-06, + 9.192375728162006e-06, + 7.228128197311889e-06, + 6.343910172290634e-06, + 2.812412503772066e-06, + 2.9242357868497493e-06, + 3.5217349250160623e-06, + 2.5429887955397135e-06, + 1.9839721971948165e-06, + 1.4403552768271766e-06, + 1.0111549499924877e-06, + 1.062799583451124e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.7218609059455048e-05, + "first_to_last_ratio": 2.059812082463918, + "imbalance_abs_log_ratio": 0.7226147565356063, + "last_quartile_mean": 8.35931064102624e-06, + "mean": 2.3338234882430697e-05, + "normalized": [ + 0.7455840782742903, + 0.932126430786308, + 0.9325399041631259, + 0.5978598612170376, + 0.5053742064366669, + 0.8701260784473666, + 0.8098658216052845, + 0.5088068538375754, + 0.46804816039298675, + 0.5744347433377897, + 0.6330397816872262, + 0.500448729191671, + 0.3576625767404614, + 0.35542276690703867, + 0.26113068942082235, + 1.0318778766826857, + 1.0488894418321313, + 1.6253919807257495, + 2.886863167031188, + 2.3788037608248445, + 3.8936434825321258, + 3.128031704756627, + 2.6914081834944965, + 1.3971722912336362, + 0.7333705189828876, + 0.5031325260356582, + 0.6161181986326979, + 0.3294770981758386, + 0.28015866797271904, + 0.16234546962292012, + 0.14387918744395148, + 0.09696576157419173 + ], + "peak_layer": 21, + "peak_normalized": 3.8936434825321258, + "population_cv": 0.9430038169349005, + "rest_mean": 1.7422747650589036e-05, + "spike_contrast": 3.172970592414376, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.528186593437567e-05, + "top_five_layers": [ + 21, + 22, + 19, + 23, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7400616343365982e-05, + 2.1754185581812635e-05, + 2.1763835320598446e-05, + 1.3952993867860641e-05, + 1.1794541933340952e-05, + 2.030720679613296e-05, + 1.890083876787685e-05, + 1.1874653864651918e-05, + 1.0923417903541122e-05, + 1.3406292964646127e-05, + 1.4774031114939135e-05, + 1.167958998848917e-05, + 8.347213224624284e-06, + 8.294940016639885e-06, + 6.094329364714213e-06, + 2.4082208256004378e-05, + 2.447922815917991e-05, + 3.793377982219681e-05, + 6.737429066561162e-05, + 5.551708090933971e-05, + 9.08707661437802e-05, + 7.300273864530027e-05, + 6.28127163508907e-05, + 3.260753510403447e-05, + 1.7115573427872732e-05, + 1.1742225069610868e-05, + 1.4379111235029995e-05, + 7.689413905609399e-06, + 6.538408797496231e-06, + 3.7888567021582276e-06, + 3.357886271260213e-06, + 2.26300971917226e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.1091838814536459e-05, + "first_to_last_ratio": 2.147117923315226, + "imbalance_abs_log_ratio": 0.764126442081038, + "last_quartile_mean": 5.165919716887402e-06, + "mean": 1.5115809230081823e-05, + "normalized": [ + 0.5373176711890457, + 0.9499619969417363, + 0.9455847427341147, + 0.6485396392348263, + 0.5327547371672011, + 0.8976617814793185, + 0.8429317236262398, + 0.5155725553175506, + 0.4928615554351285, + 0.6510671951764587, + 0.6749615278217853, + 0.4894485606904807, + 0.38754415734688374, + 0.4151562078149955, + 0.30831642778913926, + 1.0584316197438957, + 1.0910562730904934, + 1.7032600694319715, + 2.987128042597488, + 2.249255714777125, + 3.2136369586789724, + 3.297496124030594, + 2.9011704426376874, + 1.4748356308081663, + 0.7355649883790728, + 0.5038469292009693, + 0.6237518052151605, + 0.2779007685642119, + 0.24681129799141374, + 0.13171952134177142, + 0.12082312918165286, + 0.09363020456444852 + ], + "peak_layer": 22, + "peak_normalized": 3.297496124030594, + "population_cv": 0.9134577450694712, + "rest_mean": 1.1408122843561244e-05, + "spike_contrast": 3.0800260655612144, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.513731571729295e-05, + "top_five_layers": [ + 22, + 21, + 19, + 23, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.121991413645446e-06, + 1.4359444321598858e-05, + 1.4293278582044877e-05, + 9.803201464819722e-06, + 8.053018973441795e-06, + 1.3568884241976775e-05, + 1.2741595128318295e-05, + 7.793296390445903e-06, + 7.450001248798799e-06, + 9.841407518251799e-06, + 1.0202589692198671e-05, + 7.398411071335431e-06, + 5.858043550688308e-06, + 6.275422038015677e-06, + 4.660452304960927e-06, + 1.5999050447135232e-05, + 1.6492198483319953e-05, + 2.57461542787496e-05, + 4.515285763773136e-05, + 3.399932029424235e-05, + 4.857672320213169e-05, + 4.984432234778069e-05, + 4.3853538954863325e-05, + 2.2293334041023627e-05, + 1.1118660040665418e-05, + 7.616054062964395e-06, + 9.428513294551522e-06, + 4.200695002509747e-06, + 3.730752496267087e-06, + 1.991047156479908e-06, + 1.8263393712913967e-06, + 1.4152963103697402e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 5.0552482662169496e-05, + "first_to_last_ratio": 1.9896078568885895, + "imbalance_abs_log_ratio": 0.6879375624782461, + "last_quartile_mean": 2.5408264491488808e-05, + "mean": 6.979706753895698e-05, + "normalized": [ + 0.8424618714536765, + 0.8875294097392207, + 0.9051459937655827, + 0.5457016616112889, + 0.4773159928049252, + 0.8683162175637178, + 0.7704381752382723, + 0.49731492759757023, + 0.4420323183113381, + 0.5165818577921957, + 0.5794772315022629, + 0.5076225155564954, + 0.3309561455182541, + 0.3052740112561091, + 0.21389119365177953, + 1.0441274181744666, + 1.0008017526930533, + 1.5667387755322346, + 2.8174410811482873, + 2.521932013314648, + 4.568512624917261, + 3.0003142184563067, + 2.5420749618287632, + 1.335753275629179, + 0.7306479606172737, + 0.4947636486838804, + 0.6079571307037338, + 0.3517508043350849, + 0.29228869585353145, + 0.17765058107663878, + 0.15820247401055815, + 0.09898305966240892 + ], + "peak_layer": 21, + "peak_normalized": 4.568512624917261, + "population_cv": 1.0020196982520382, + "rest_mean": 5.1243189571121344e-05, + "spike_contrast": 3.317280012973041, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001699880085652694, + "top_five_layers": [ + 21, + 22, + 19, + 23, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.8801368140848354e-05, + 6.194695015437901e-05, + 6.317653605947271e-05, + 3.8088375731604174e-05, + 3.331525658722967e-05, + 6.060592568246648e-05, + 5.377432535169646e-05, + 3.471112358965911e-05, + 3.08525595755782e-05, + 3.6055898817721754e-05, + 4.0445811464451253e-05, + 3.543056300259195e-05, + 2.3099768441170454e-05, + 2.130723078153096e-05, + 1.4928978089301381e-05, + 7.287703192560002e-05, + 6.985302752582356e-05, + 0.00010935377213172615, + 0.000196649125427939, + 0.00017602345906198025, + 0.0003188687842339277, + 0.00020941313414368778, + 0.00017742937779985368, + 9.323166159447283e-05, + 5.0997085054405034e-05, + 3.4533051803009585e-05, + 4.243362491251901e-05, + 2.455117464705836e-05, + 2.040089384536259e-05, + 1.239948960574111e-05, + 1.1042068763345014e-05, + 6.908727300469764e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "f4a14b64dd8b186b4b509748e2f608d960603b5d908ec49d21001a2020554f02", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04748110845685005, + 0.06085919216275215, + 0.06251973658800125, + 0.08963073790073395, + 0.08407715708017349, + 0.1004372090101242, + 0.09022688120603561, + 0.09703134745359421, + 0.0712415874004364, + 0.07720176130533218, + 0.07461326569318771, + 0.0729745551943779, + 0.06531906127929688, + 0.06964895129203796, + 0.08001933246850967, + 0.11749708652496338, + 0.09965827316045761, + 0.0854005292057991, + 0.08426972478628159, + 0.0886540412902832, + 0.04304825887084007, + 0.0459144227206707, + 0.05584317073225975, + 0.06311307847499847, + 0.05157631263136864, + 0.05562245845794678, + 0.059894196689128876, + 0.06387592852115631, + 0.050557736307382584, + 0.06106090545654297, + 0.06874485313892365, + 0.07015988230705261 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.6830450085290067e-06, + "first_to_last_ratio": 0.27205073217266473, + "imbalance_abs_log_ratio": 1.3017667147372842, + "last_quartile_mean": 1.3538081589103967e-05, + "mean": 9.997345500778465e-06, + "normalized": [ + 0.8238008730306102, + 0.43129250962679994, + 0.49934591673609946, + 0.29782217912043774, + 0.18368099072454586, + 0.1933449363361496, + 0.23685570791400323, + 0.28107523221499575, + 0.3402086077396193, + 0.32325728859883013, + 0.37302383835188735, + 0.43266448277670955, + 0.41282186281653577, + 0.3837163089578277, + 0.40821318475885754, + 0.2906458388933851, + 0.44392610708725316, + 0.5119919320955622, + 0.5900289206272344, + 0.8323966973801565, + 3.728319067589544, + 3.5968610962814633, + 2.95823520549195, + 2.5931302340463516, + 2.1674970568044167, + 1.890620302598499, + 1.7758337928353976, + 1.5730683335489977, + 1.0522330833730802, + 0.8211197895016011, + 0.7910939465119015, + 0.7618746756292967 + ], + "peak_layer": 21, + "peak_normalized": 3.728319067589544, + "population_cv": 0.9878337467468244, + "rest_mean": 6.278317178603848e-06, + "spike_contrast": 4.791108442726131, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.0080098440521397e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.235821951529942e-06, + 4.311780230636941e-06, + 4.992133654013742e-06, + 2.977431222461746e-06, + 1.8363223261985695e-06, + 1.932936129378504e-06, + 2.3679283458477585e-06, + 2.8100062081648502e-06, + 3.401182993911789e-06, + 3.23171479976736e-06, + 3.7292481920303544e-06, + 4.325496320234379e-06, + 4.127122792851878e-06, + 3.836144514934858e-06, + 4.081048246007413e-06, + 2.905686869780766e-06, + 4.43808266936685e-06, + 5.118560238770442e-06, + 5.898722974961856e-06, + 8.321757377416361e-06, + 3.727329385583289e-05, + 3.595906309783459e-05, + 2.9574499421869405e-05, + 2.59244188782759e-05, + 2.16692169487942e-05, + 1.8901184375863522e-05, + 1.775362397893332e-05, + 1.572650762682315e-05, + 1.0519537681830116e-05, + 8.209018233173992e-06, + 7.908839506853838e-06, + 7.616724360559601e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.4919307147920335e-05, + "first_to_last_ratio": 0.2771847156933817, + "imbalance_abs_log_ratio": 1.2830711513778186, + "last_quartile_mean": 5.382442213885952e-05, + "mean": 4.0030962580317464e-05, + "normalized": [ + 0.8279328105716447, + 0.4385340769193414, + 0.5113323447021809, + 0.301967722777521, + 0.17936700215896317, + 0.19494385438915096, + 0.241089047282958, + 0.28638665602796354, + 0.34043590168215715, + 0.32758048161579845, + 0.37911369328600636, + 0.4350549514201117, + 0.4161353264561751, + 0.3865564890531755, + 0.42474039805833697, + 0.30722739751123224, + 0.43933917507773224, + 0.529530389422566, + 0.5935744529609681, + 0.8334989751506177, + 3.7379165275353725, + 3.5870637252031274, + 2.9579615328299957, + 2.566158910032566, + 2.155837666405289, + 1.8800235241322139, + 1.7708849666906623, + 1.5532705423788409, + 1.044510485838323, + 0.8147842371714528, + 0.7815953628912844, + 0.7556513723662709 + ], + "peak_layer": 21, + "peak_normalized": 3.7379165275353725, + "population_cv": 0.9835546199867091, + "rest_mean": 2.519735838817454e-05, + "spike_contrast": 4.767659504905444, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012013242521788925, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.314294735901058e-05, + 1.7554941223352216e-05, + 2.0469125956878997e-05, + 1.208805861097062e-05, + 7.180233751569176e-06, + 7.803790140314959e-06, + 9.651026630308479e-06, + 1.1464333510957658e-05, + 1.3627976841235068e-05, + 1.31133620016044e-05, + 1.5176286069618072e-05, + 1.7415668480680324e-05, + 1.665829768171534e-05, + 1.5474228348466568e-05, + 1.7002766981022432e-05, + 1.2298608453420456e-05, + 1.758717007760424e-05, + 2.1197611204115674e-05, + 2.3761356715112925e-05, + 3.336576628498733e-05, + 0.0001496323966421187, + 0.00014359361375682056, + 0.00011841004743473604, + 0.0001027258113026619, + 8.630025695310906e-05, + 7.525915134465322e-05, + 7.089022983564064e-05, + 6.217891495907679e-05, + 4.181276017334312e-05, + 3.2616597309242934e-05, + 3.1288014724850655e-05, + 3.0249451810959727e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.013230590906459838, + "first_to_last_ratio": 0.27718471002181044, + "imbalance_abs_log_ratio": 1.2830711718391608, + "last_quartile_mean": 0.04773203725926578, + "mean": 0.035499858771800064, + "normalized": [ + 0.8279327972123818, + 0.43853407753129525, + 0.5113323373845421, + 0.3019677236670519, + 0.17936700088464805, + 0.19494385266486466, + 0.24108904976283277, + 0.28638663500812633, + 0.3404359064207765, + 0.3275804556161215, + 0.37911367693924214, + 0.43505493113041177, + 0.4161352869870187, + 0.3865564955173268, + 0.4247403745275919, + 0.30722738227286805, + 0.43933918942416506, + 0.529530400420262, + 0.5935744293554726, + 0.8334989802196542, + 3.7379165882450245, + 3.58706376795423, + 2.95796152343449, + 2.566158906335581, + 2.1558376400844, + 1.8800236046449208, + 1.7708849671242992, + 1.5532705865178382, + 1.044510493614123, + 0.814784201623246, + 0.7815953533826546, + 0.7556513840925386 + ], + "peak_layer": 21, + "peak_normalized": 3.7379165882450245, + "population_cv": 0.9835546333023998, + "rest_mean": 0.022345269840486622, + "spike_contrast": 4.767659543223157, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10653463900089263, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.029391497373580933, + 0.015567897818982601, + 0.018152225762605667, + 0.010719811543822289, + 0.006367503199726343, + 0.006920479238033295, + 0.008558627218008041, + 0.010166685096919537, + 0.012085426598787308, + 0.011629059910774231, + 0.01345848198980093, + 0.01544438861310482, + 0.014772743918001652, + 0.013722700998187065, + 0.015078223310410976, + 0.010906528681516647, + 0.015596479177474976, + 0.018798254430294037, + 0.02107180841267109, + 0.0295890960842371, + 0.1326955109834671, + 0.12734025716781616, + 0.10500721633434296, + 0.09109827876091003, + 0.07653193175792694, + 0.06674057245254517, + 0.06286616623401642, + 0.05514088645577431, + 0.03707997500896454, + 0.028924724087119102, + 0.027746524661779404, + 0.026825517416000366 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 9.408127326082649e-07, + "first_to_last_ratio": 0.26109600872884425, + "imbalance_abs_log_ratio": 1.3428670897636954, + "last_quartile_mean": 3.6033210051300557e-06, + "mean": 2.51254786931554e-06, + "normalized": [ + 0.9411291774986253, + 0.42452308852469994, + 0.5064277702253434, + 0.31422637836100054, + 0.14909909853668837, + 0.18364790421678406, + 0.20923427416444018, + 0.2672778667582645, + 0.28101884819319983, + 0.26433726849666683, + 0.32343818509729044, + 0.3930866637355723, + 0.38182199462519517, + 0.3573030049702274, + 0.36652953981026026, + 0.20759778048664343, + 0.36120097522274675, + 0.33747924881801133, + 0.4694224480590536, + 0.6406335998556872, + 3.722312978049966, + 3.6582343696129027, + 3.0403294690381584, + 2.726645745510304, + 2.2400013253030986, + 1.9540567521771424, + 1.8942350408219104, + 1.6614529947683714, + 1.1379024352010345, + 0.8900222502819433, + 0.8554212967076426, + 0.8399502268711246 + ], + "peak_layer": 21, + "peak_normalized": 3.722312978049966, + "population_cv": 1.0300414744999016, + "rest_mean": 1.5459126466844787e-06, + "spike_contrast": 5.0018208261035415, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.732378071523271e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.364632109674858e-06, + 1.0666345815479872e-06, + 1.2724240150419064e-06, + 7.895088174336706e-07, + 3.7461862234522414e-07, + 4.614241504441452e-07, + 5.257111297396477e-07, + 6.715484346386802e-07, + 7.060733082653314e-07, + 6.641600407419901e-07, + 8.126539228214824e-07, + 9.876490594251663e-07, + 9.593460390533437e-07, + 8.977409038379847e-07, + 9.209230142914748e-07, + 5.215993610363512e-07, + 9.075347406906076e-07, + 8.479327675559034e-07, + 1.17944637167966e-06, + 1.6096225863293512e-06, + 9.352489541925024e-06, + 9.191488970827777e-06, + 7.638973329449072e-06, + 6.850827958260197e-06, + 5.628110557154287e-06, + 4.9096611292043235e-06, + 4.759356215799926e-06, + 4.174480181973195e-06, + 2.8590343390533235e-06, + 2.236223508589319e-06, + 2.1492869564099237e-06, + 2.1104151528561488e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.0381400045389455e-05, + "first_to_last_ratio": 0.30181441355404504, + "imbalance_abs_log_ratio": 1.1979429751604156, + "last_quartile_mean": 3.439663441895391e-05, + "mean": 2.6535070020372586e-05, + "normalized": [ + 0.9040898455603302, + 0.4673694653321381, + 0.5438132272984187, + 0.2908698623251208, + 0.18839899103808294, + 0.1956218714505659, + 0.25253197694418594, + 0.28717026631709835, + 0.3611321614066451, + 0.34954594583315046, + 0.4132621282293166, + 0.48655346340205047, + 0.47102858233475486, + 0.4382954282683324, + 0.4460853575084453, + 0.28452002254446673, + 0.469204525002259, + 0.546001458578785, + 0.6236410135514943, + 0.8373328706810375, + 3.7465220405244266, + 3.619599622084698, + 2.9108020234237175, + 2.49644206069555, + 2.1762350991842796, + 1.8187600325437407, + 1.7041392558246975, + 1.4815869865618754, + 0.9869504724512707, + 0.7532641117134347, + 0.7324670886872183, + 0.7167627426984122 + ], + "peak_layer": 21, + "peak_normalized": 3.7465220405244266, + "population_cv": 0.9710857842550003, + "rest_mean": 1.6756797608481805e-05, + "spike_contrast": 4.734660100234446, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.93377410445828e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.3990087356651202e-05, + 1.2401681487972382e-05, + 1.4430122064368334e-05, + 7.718252163613215e-06, + 4.999180418963078e-06, + 5.190840056457091e-06, + 6.700953690597089e-06, + 7.620083124493249e-06, + 9.582667189533822e-06, + 9.27522614802001e-06, + 1.0965939509333111e-05, + 1.29107302200282e-05, + 1.2498776413849555e-05, + 1.163019987870939e-05, + 1.1836906196549535e-05, + 7.5497587204154115e-06, + 1.2450374924810603e-05, + 1.4488186934613623e-05, + 1.6548357962165028e-05, + 2.2218686353880912e-05, + 9.941422467818484e-05, + 9.604632941773161e-05, + 7.723833550699055e-05, + 6.624326488235965e-05, + 5.774655073764734e-05, + 4.8260924813803285e-05, + 4.521945447777398e-05, + 3.931401442969218e-05, + 2.6188799893134274e-05, + 1.9987915948149748e-05, + 1.9436065485933796e-05, + 1.9019349565496668e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 7.0662427162915264e-06, + "first_to_last_ratio": 0.31298041774626, + "imbalance_abs_log_ratio": 1.1616146535101273, + "last_quartile_mean": 2.2577267827728065e-05, + "mean": 1.7418877341413008e-05, + "normalized": [ + 0.9686515201468897, + 0.49383920743252663, + 0.5592845449086765, + 0.2807880642348226, + 0.19794654052965874, + 0.19434619484963173, + 0.2653113252926559, + 0.28515895873794617, + 0.3736472447718833, + 0.36301406000122344, + 0.43599250394692873, + 0.5281282188816567, + 0.5269171867525786, + 0.48757686495182617, + 0.47960088642402976, + 0.274044916832133, + 0.5050358505640102, + 0.5616893266095021, + 0.6389811522580301, + 0.8090338932687061, + 3.6651836739778836, + 3.5584695736966103, + 2.7908959611190953, + 2.3873587945098413, + 2.235209216498694, + 1.7539959875660012, + 1.6837970592001588, + 1.45698578925774, + 1.0345422359131593, + 0.7653898492004969, + 0.7229649825149719, + 0.716218415150031 + ], + "peak_layer": 21, + "peak_normalized": 3.6651836739778836, + "population_cv": 0.9404784507863737, + "rest_mean": 1.1201552794133292e-05, + "spike_contrast": 4.552264389936214, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.099242989672348e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.6872822016011924e-05, + 8.602124580647796e-06, + 9.74210888671223e-06, + 4.891012849839171e-06, + 3.4480065096431645e-06, + 3.3852925298560876e-06, + 4.6214254325605e-06, + 4.967148925061338e-06, + 6.508515525638359e-06, + 6.323297384369653e-06, + 7.594499948027078e-06, + 9.1994006652385e-06, + 9.178305845125578e-06, + 8.493041605106555e-06, + 8.354109013453126e-06, + 4.773554792336654e-06, + 8.797157533990685e-06, + 9.783997484191786e-06, + 1.1130334314657375e-05, + 1.4092462151893415e-05, + 6.384338485077024e-05, + 6.19845450273715e-05, + 4.861427441937849e-05, + 4.158511001151055e-05, + 3.8934835174586624e-05, + 3.055264096474275e-05, + 2.9329854442039505e-05, + 2.5379056751262397e-05, + 1.802056431188248e-05, + 1.3332231901586056e-05, + 1.2593238352565095e-05, + 1.2475720723159611e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.965646217489848e-05, + "first_to_last_ratio": 0.25778316490635544, + "imbalance_abs_log_ratio": 1.3556364935327478, + "last_quartile_mean": 0.00011504421627250849, + "mean": 8.427746251982171e-05, + "normalized": [ + 0.7921668867268374, + 0.4122773826527116, + 0.48829879548757044, + 0.2751554675194965, + 0.1662779686404814, + 0.1822655064180769, + 0.22566214189861117, + 0.2730221051205942, + 0.330970122717494, + 0.3135140758471058, + 0.3728659820070459, + 0.42617092604953055, + 0.39288617322467556, + 0.3613498654548339, + 0.3791111655490991, + 0.2670862108876096, + 0.42548587073411964, + 0.4864503548031842, + 0.5713051825384278, + 0.8003801258416829, + 3.770712978765767, + 3.65288415518208, + 3.0183549283675815, + 2.6948254474092166, + 2.183148265157847, + 1.929015794151022, + 1.9467070562963738, + 1.6704513131036254, + 0.9796587997179057, + 0.7674221880006324, + 0.7441547396212447, + 0.6999620241075153 + ], + "peak_layer": 21, + "peak_normalized": 3.770712978765767, + "population_cv": 1.0201901284069992, + "rest_mean": 5.206497519759944e-05, + "spike_contrast": 4.95966613024965, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000258224894059822, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.676181510556489e-05, + 3.4745691664284095e-05, + 4.11525834351778e-05, + 2.3189404600998387e-05, + 1.401348526997026e-05, + 1.53608743858058e-05, + 1.901823270600289e-05, + 2.3009610231383704e-05, + 2.7893322112504393e-05, + 2.6422170776640996e-05, + 3.1424198823515326e-05, + 3.591660424717702e-05, + 3.311144973849878e-05, + 3.045364974241238e-05, + 3.1950527045410126e-05, + 2.2509348127641715e-05, + 3.5858869523508474e-05, + 4.099680154467933e-05, + 4.814815110876225e-05, + 6.745400605723262e-05, + 0.00031778612174093723, + 0.0003078558074776083, + 0.000254379294347018, + 0.000227113050641492, + 0.00018399019609205425, + 0.00016257255629170686, + 0.0001640635309740901, + 0.00014078139793127775, + 8.256315777543932e-05, + 6.467639468610287e-05, + 6.271547317737713e-05, + 5.899102325201966e-05 + ] + } + } + } + } + } + }, + "step": 2000 + }, + { + "modes": { + "learned": { + "bits_per_byte": 1.9288425276664853, + "forward": { + "activation_sha256": { + "attention_branch_output": "41db358e51c17755a85e4f4c84288ab48da8d214d2518c64ab3d3ea8eb9826e1", + "mlp_branch_output": "6c58c506475022833bc648a96c247244a57dd96d762e84fedda83974a9fa4cd0", + "post_attention_state": "90bf4c661b8e06ef22fe9277f79a81ef01617138f2a24fe93a0093039186433f", + "post_mlp_state": "25421e314b847c78a706fbfc95b3ae6bf0cea5ddc086f201de56fb1ddc434ca3", + "pre_attention_input": "36b6c956f67457ef008d5eacc14cf9477822b0791655ecd816dc4fb89c68ffd5", + "pre_mlp_input": "c1e7ba7b2fabbdfa228a4a3c33df9fd2cb9236dae2c5351a1ecf233f5b8dce87" + }, + "logits_sha256": "562529f2e73d225129c831e3649b7f416f6e2834513ba88186693cdef5a65cc3", + "loss_nats": 1.3369717597961426, + "mixer_summary_sha256": "900263e7a76f1202bf5b394a28abd810902e027421616ad02b8950572b564b66" + }, + "loss_nats": 1.3369717597961426, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.2508508265018463, + "group": 1, + "latest_source_mass": 0.07481259852647781, + "layer": 1, + "max_source_mass": 0.9251874089241028, + "mixer_index": 1, + "normalized_entropy": 0.36190124339711144, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9251874089241028, + "median": 0.939024806022644, + "p05": 0.8262566328048706, + "p95": 0.9732407331466675 + }, + { + "label": "current_group_partial", + "mean": 0.07481259852647781, + "median": 0.06097518280148506, + "p05": 0.026759283617138863, + "p95": 0.1737433522939682 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5796181559562683, + "group": 1, + "latest_source_mass": 0.3836008310317993, + "layer": 2, + "max_source_mass": 0.6163992285728455, + "mixer_index": 2, + "normalized_entropy": 0.8362122392073141, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6163992285728455, + "median": 0.6123820543289185, + "p05": 0.30569732189178467, + "p95": 0.9099069833755493 + }, + { + "label": "current_group_partial", + "mean": 0.3836008310317993, + "median": 0.38761788606643677, + "p05": 0.09009299427270889, + "p95": 0.6943026781082153 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3817020058631897, + "group": 1, + "latest_source_mass": 0.13486965000629425, + "layer": 2, + "max_source_mass": 0.8651303648948669, + "mixer_index": 3, + "normalized_entropy": 0.5506795909561938, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8651303648948669, + "median": 0.882187008857727, + "p05": 0.7400804758071899, + "p95": 0.931220531463623 + }, + { + "label": "current_group_partial", + "mean": 0.13486965000629425, + "median": 0.11781297624111176, + "p05": 0.06877946853637695, + "p95": 0.25991958379745483 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6884468793869019, + "group": 1, + "latest_source_mass": 0.5067460536956787, + "layer": 3, + "max_source_mass": 0.5067460536956787, + "mixer_index": 4, + "normalized_entropy": 0.9932188988069657, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4932539165019989, + "median": 0.4924398362636566, + "p05": 0.4183998703956604, + "p95": 0.5739172101020813 + }, + { + "label": "current_group_partial", + "mean": 0.5067460536956787, + "median": 0.507560133934021, + "p05": 0.4260827600955963, + "p95": 0.5816001892089844 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.2797870337963104, + "group": 1, + "latest_source_mass": 0.08572537451982498, + "layer": 3, + "max_source_mass": 0.9142746329307556, + "mixer_index": 5, + "normalized_entropy": 0.40364736616296987, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9142746329307556, + "median": 0.9290047883987427, + "p05": 0.8227594494819641, + "p95": 0.965082049369812 + }, + { + "label": "current_group_partial", + "mean": 0.08572537451982498, + "median": 0.07099512964487076, + "p05": 0.03491789847612381, + "p95": 0.1772405207157135 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6642253994941711, + "group": 1, + "latest_source_mass": 0.39495500922203064, + "layer": 4, + "max_source_mass": 0.605044960975647, + "mixer_index": 6, + "normalized_entropy": 0.9582746898827313, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.605044960975647, + "median": 0.6064984202384949, + "p05": 0.5077860951423645, + "p95": 0.6980465054512024 + }, + { + "label": "current_group_partial", + "mean": 0.39495500922203064, + "median": 0.39350154995918274, + "p05": 0.3019534945487976, + "p95": 0.4922138750553131 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5038842558860779, + "group": 1, + "latest_source_mass": 0.2243884801864624, + "layer": 4, + "max_source_mass": 0.7756115198135376, + "mixer_index": 7, + "normalized_entropy": 0.7269513171488701, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7756115198135376, + "median": 0.786786675453186, + "p05": 0.6057990789413452, + "p95": 0.9406262636184692 + }, + { + "label": "current_group_partial", + "mean": 0.2243884801864624, + "median": 0.21321335434913635, + "p05": 0.059373728930950165, + "p95": 0.3942009508609772 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6434851884841919, + "group": 2, + "latest_source_mass": 0.4625895023345947, + "layer": 5, + "max_source_mass": 0.5374104976654053, + "mixer_index": 8, + "normalized_entropy": 0.9283528903116436, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5374104976654053, + "median": 0.522662878036499, + "p05": 0.32466351985931396, + "p95": 0.7907323837280273 + }, + { + "label": "completed_group_1", + "mean": 0.4625895023345947, + "median": 0.47733718156814575, + "p05": 0.20926763117313385, + "p95": 0.675336480140686 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0016276836395264, + "group": 2, + "latest_source_mass": 0.37876495718955994, + "layer": 5, + "max_source_mass": 0.4353652000427246, + "mixer_index": 9, + "normalized_entropy": 0.911720808124073, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4353652000427246, + "median": 0.43071597814559937, + "p05": 0.24926689267158508, + "p95": 0.665458083152771 + }, + { + "label": "completed_group_1", + "mean": 0.18586981296539307, + "median": 0.18826425075531006, + "p05": 0.1286066770553589, + "p95": 0.24030350148677826 + }, + { + "label": "current_group_partial", + "mean": 0.37876495718955994, + "median": 0.3880766034126282, + "p05": 0.176315575838089, + "p95": 0.5597050189971924 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0655895471572876, + "group": 2, + "latest_source_mass": 0.4483259618282318, + "layer": 6, + "max_source_mass": 0.4483259618282318, + "mixer_index": 10, + "normalized_entropy": 0.9699414053060913, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2726097106933594, + "median": 0.2658982276916504, + "p05": 0.22330613434314728, + "p95": 0.3535451292991638 + }, + { + "label": "completed_group_1", + "mean": 0.2790643572807312, + "median": 0.2796473205089569, + "p05": 0.2485380917787552, + "p95": 0.30792567133903503 + }, + { + "label": "current_group_partial", + "mean": 0.4483259618282318, + "median": 0.4539736807346344, + "p05": 0.378553181886673, + "p95": 0.4919973611831665 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0300546884536743, + "group": 2, + "latest_source_mass": 0.3101818859577179, + "layer": 6, + "max_source_mass": 0.40803641080856323, + "mixer_index": 11, + "normalized_entropy": 0.9375961830014203, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40803641080856323, + "median": 0.39597707986831665, + "p05": 0.17321011424064636, + "p95": 0.7176744341850281 + }, + { + "label": "completed_group_1", + "mean": 0.2817816734313965, + "median": 0.2903806269168854, + "p05": 0.1267791986465454, + "p95": 0.37898188829421997 + }, + { + "label": "current_group_partial", + "mean": 0.3101818859577179, + "median": 0.312465637922287, + "p05": 0.13594681024551392, + "p95": 0.466165691614151 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0874333381652832, + "group": 2, + "latest_source_mass": 0.35985422134399414, + "layer": 7, + "max_source_mass": 0.36608952283859253, + "mixer_index": 12, + "normalized_entropy": 0.9898244807398074, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.36608952283859253, + "median": 0.37737005949020386, + "p05": 0.2977083623409271, + "p95": 0.4045135974884033 + }, + { + "label": "completed_group_1", + "mean": 0.27405625581741333, + "median": 0.2681817412376404, + "p05": 0.25118786096572876, + "p95": 0.3135649561882019 + }, + { + "label": "current_group_partial", + "mean": 0.35985422134399414, + "median": 0.35611432790756226, + "p05": 0.33197838068008423, + "p95": 0.3942449688911438 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9672750234603882, + "group": 2, + "latest_source_mass": 0.2810039520263672, + "layer": 7, + "max_source_mass": 0.5343947410583496, + "mixer_index": 13, + "normalized_entropy": 0.8804516692900397, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5343947410583496, + "median": 0.5141075849533081, + "p05": 0.3664405345916748, + "p95": 0.8041804432868958 + }, + { + "label": "completed_group_1", + "mean": 0.1846013069152832, + "median": 0.18730750679969788, + "p05": 0.07676203548908234, + "p95": 0.26666438579559326 + }, + { + "label": "current_group_partial", + "mean": 0.2810039520263672, + "median": 0.2932106554508209, + "p05": 0.11621598154306412, + "p95": 0.3943445682525635 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0731852054595947, + "group": 2, + "latest_source_mass": 0.37608763575553894, + "layer": 8, + "max_source_mass": 0.3801807165145874, + "mixer_index": 14, + "normalized_entropy": 0.976855271444905, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3801807165145874, + "median": 0.39405983686447144, + "p05": 0.2808566689491272, + "p95": 0.44335222244262695 + }, + { + "label": "completed_group_1", + "mean": 0.24373164772987366, + "median": 0.2364288568496704, + "p05": 0.20917385816574097, + "p95": 0.29448947310447693 + }, + { + "label": "current_group_partial", + "mean": 0.37608763575553894, + "median": 0.3712879717350006, + "p05": 0.3317960500717163, + "p95": 0.4302467107772827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9721636772155762, + "group": 2, + "latest_source_mass": 0.29772043228149414, + "layer": 8, + "max_source_mass": 0.49137914180755615, + "mixer_index": 15, + "normalized_entropy": 0.8849015137034084, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.49137914180755615, + "median": 0.4575059711933136, + "p05": 0.2624281048774719, + "p95": 0.8368043899536133 + }, + { + "label": "completed_group_1", + "mean": 0.2109004259109497, + "median": 0.20731960237026215, + "p05": 0.05826364830136299, + "p95": 0.34210139513015747 + }, + { + "label": "current_group_partial", + "mean": 0.29772043228149414, + "median": 0.3127709627151489, + "p05": 0.10320040583610535, + "p95": 0.44153448939323425 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0864832401275635, + "group": 3, + "latest_source_mass": 0.3128502070903778, + "layer": 9, + "max_source_mass": 0.39626434445381165, + "mixer_index": 16, + "normalized_entropy": 0.9889596642367338, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.39626434445381165, + "median": 0.3972475528717041, + "p05": 0.3466483950614929, + "p95": 0.44265681505203247 + }, + { + "label": "completed_group_1", + "mean": 0.29088544845581055, + "median": 0.289419949054718, + "p05": 0.26393142342567444, + "p95": 0.3211052417755127 + }, + { + "label": "completed_group_2", + "mean": 0.3128502070903778, + "median": 0.31452691555023193, + "p05": 0.2603488564491272, + "p95": 0.36055514216423035 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3038380146026611, + "group": 3, + "latest_source_mass": 0.29599112272262573, + "layer": 9, + "max_source_mass": 0.35696184635162354, + "mixer_index": 17, + "normalized_entropy": 0.9405203188948855, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35696184635162354, + "median": 0.36454010009765625, + "p05": 0.19070445001125336, + "p95": 0.5144885778427124 + }, + { + "label": "completed_group_1", + "mean": 0.1472446471452713, + "median": 0.14020296931266785, + "p05": 0.08932732790708542, + "p95": 0.22387003898620605 + }, + { + "label": "completed_group_2", + "mean": 0.19980241358280182, + "median": 0.1939706951379776, + "p05": 0.11784223467111588, + "p95": 0.2938849925994873 + }, + { + "label": "current_group_partial", + "mean": 0.29599112272262573, + "median": 0.2978004813194275, + "p05": 0.23555772006511688, + "p95": 0.3528674244880676 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3611116409301758, + "group": 3, + "latest_source_mass": 0.2453548014163971, + "layer": 10, + "max_source_mass": 0.3264957070350647, + "mixer_index": 18, + "normalized_entropy": 0.9818345072331021, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3264957070350647, + "median": 0.33824437856674194, + "p05": 0.23342323303222656, + "p95": 0.3961111009120941 + }, + { + "label": "completed_group_1", + "mean": 0.22676822543144226, + "median": 0.225631982088089, + "p05": 0.20189295709133148, + "p95": 0.25452396273612976 + }, + { + "label": "completed_group_2", + "mean": 0.20138128101825714, + "median": 0.1983025223016739, + "p05": 0.17473457753658295, + "p95": 0.23882760107517242 + }, + { + "label": "current_group_partial", + "mean": 0.2453548014163971, + "median": 0.24060846865177155, + "p05": 0.1962868720293045, + "p95": 0.3133445978164673 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3161137104034424, + "group": 3, + "latest_source_mass": 0.2900698184967041, + "layer": 10, + "max_source_mass": 0.3259783387184143, + "mixer_index": 19, + "normalized_entropy": 0.9493753616225099, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3259783387184143, + "median": 0.35052698850631714, + "p05": 0.10893625020980835, + "p95": 0.48943614959716797 + }, + { + "label": "completed_group_1", + "mean": 0.17862649261951447, + "median": 0.18309253454208374, + "p05": 0.1213371679186821, + "p95": 0.21994648873806 + }, + { + "label": "completed_group_2", + "mean": 0.20532533526420593, + "median": 0.19941335916519165, + "p05": 0.12927237153053284, + "p95": 0.29834914207458496 + }, + { + "label": "current_group_partial", + "mean": 0.2900698184967041, + "median": 0.2715068459510803, + "p05": 0.20594194531440735, + "p95": 0.42361485958099365 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3568220138549805, + "group": 3, + "latest_source_mass": 0.27865856885910034, + "layer": 11, + "max_source_mass": 0.3223523199558258, + "mixer_index": 20, + "normalized_entropy": 0.9787401953787784, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3223523199558258, + "median": 0.31804126501083374, + "p05": 0.2675665020942688, + "p95": 0.3895469605922699 + }, + { + "label": "completed_group_1", + "mean": 0.21851874887943268, + "median": 0.21804103255271912, + "p05": 0.18837261199951172, + "p95": 0.251605361700058 + }, + { + "label": "completed_group_2", + "mean": 0.1804703176021576, + "median": 0.1816348135471344, + "p05": 0.15208400785923004, + "p95": 0.2051190286874771 + }, + { + "label": "current_group_partial", + "mean": 0.27865856885910034, + "median": 0.2795049548149109, + "p05": 0.22788430750370026, + "p95": 0.3272843062877655 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2413711547851562, + "group": 3, + "latest_source_mass": 0.26951953768730164, + "layer": 11, + "max_source_mass": 0.4272037148475647, + "mixer_index": 21, + "normalized_entropy": 0.8954600044555754, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4272037148475647, + "median": 0.4580039381980896, + "p05": 0.18249742686748505, + "p95": 0.6239475607872009 + }, + { + "label": "completed_group_1", + "mean": 0.16667886078357697, + "median": 0.1573459506034851, + "p05": 0.0859292671084404, + "p95": 0.26250067353248596 + }, + { + "label": "completed_group_2", + "mean": 0.1365978717803955, + "median": 0.13323482871055603, + "p05": 0.07157228887081146, + "p95": 0.2005852907896042 + }, + { + "label": "current_group_partial", + "mean": 0.26951953768730164, + "median": 0.2574368417263031, + "p05": 0.1767870932817459, + "p95": 0.40349671244621277 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3540384769439697, + "group": 3, + "latest_source_mass": 0.2906486392021179, + "layer": 12, + "max_source_mass": 0.32447606325149536, + "mixer_index": 22, + "normalized_entropy": 0.9767322979299551, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32447606325149536, + "median": 0.31636133790016174, + "p05": 0.2827298045158386, + "p95": 0.3818347454071045 + }, + { + "label": "completed_group_1", + "mean": 0.2030642032623291, + "median": 0.20119589567184448, + "p05": 0.18187163770198822, + "p95": 0.22999082505702972 + }, + { + "label": "completed_group_2", + "mean": 0.18181107938289642, + "median": 0.18250997364521027, + "p05": 0.15624107420444489, + "p95": 0.2048231065273285 + }, + { + "label": "current_group_partial", + "mean": 0.2906486392021179, + "median": 0.2923159897327423, + "p05": 0.24348178505897522, + "p95": 0.33497530221939087 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.1544246673583984, + "group": 3, + "latest_source_mass": 0.22926729917526245, + "layer": 12, + "max_source_mass": 0.5128031969070435, + "mixer_index": 23, + "normalized_entropy": 0.8327413713389263, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5128031969070435, + "median": 0.4827004075050354, + "p05": 0.32023805379867554, + "p95": 0.7668037414550781 + }, + { + "label": "completed_group_1", + "mean": 0.13654223084449768, + "median": 0.13301728665828705, + "p05": 0.05933574587106705, + "p95": 0.21417926251888275 + }, + { + "label": "completed_group_2", + "mean": 0.12138727307319641, + "median": 0.12862378358840942, + "p05": 0.05767478048801422, + "p95": 0.1747671365737915 + }, + { + "label": "current_group_partial", + "mean": 0.22926729917526245, + "median": 0.22523194551467896, + "p05": 0.10545835644006729, + "p95": 0.37030228972435 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3712997436523438, + "group": 4, + "latest_source_mass": 0.22271712124347687, + "layer": 13, + "max_source_mass": 0.31037354469299316, + "mixer_index": 24, + "normalized_entropy": 0.9891836698697716, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31037354469299316, + "median": 0.3152464032173157, + "p05": 0.24687863886356354, + "p95": 0.3615240454673767 + }, + { + "label": "completed_group_1", + "mean": 0.24219870567321777, + "median": 0.24224354326725006, + "p05": 0.20525576174259186, + "p95": 0.2801668643951416 + }, + { + "label": "completed_group_2", + "mean": 0.2247106432914734, + "median": 0.2232801616191864, + "p05": 0.19794036448001862, + "p95": 0.25516608357429504 + }, + { + "label": "completed_group_3", + "mean": 0.22271712124347687, + "median": 0.22098374366760254, + "p05": 0.185325026512146, + "p95": 0.2633301615715027 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5205519199371338, + "group": 4, + "latest_source_mass": 0.2427731454372406, + "layer": 13, + "max_source_mass": 0.330753892660141, + "mixer_index": 25, + "normalized_entropy": 0.9447720276686312, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.330753892660141, + "median": 0.3369896411895752, + "p05": 0.22567971050739288, + "p95": 0.46019247174263 + }, + { + "label": "completed_group_1", + "mean": 0.1283297836780548, + "median": 0.13036663830280304, + "p05": 0.09653771668672562, + "p95": 0.15339873731136322 + }, + { + "label": "completed_group_2", + "mean": 0.12636876106262207, + "median": 0.1281418800354004, + "p05": 0.095677949488163, + "p95": 0.1535145789384842 + }, + { + "label": "completed_group_3", + "mean": 0.17177443206310272, + "median": 0.16995373368263245, + "p05": 0.12080889940261841, + "p95": 0.22754061222076416 + }, + { + "label": "current_group_partial", + "mean": 0.2427731454372406, + "median": 0.2397468388080597, + "p05": 0.19260814785957336, + "p95": 0.3004864454269409 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.518624186515808, + "group": 4, + "latest_source_mass": 0.3593595027923584, + "layer": 14, + "max_source_mass": 0.3593595027923584, + "mixer_index": 26, + "normalized_entropy": 0.9435742595494434, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2150207757949829, + "median": 0.21538454294204712, + "p05": 0.16830593347549438, + "p95": 0.263691782951355 + }, + { + "label": "completed_group_1", + "mean": 0.17106646299362183, + "median": 0.17181023955345154, + "p05": 0.13603076338768005, + "p95": 0.20341315865516663 + }, + { + "label": "completed_group_2", + "mean": 0.13557285070419312, + "median": 0.1350601464509964, + "p05": 0.11946140229701996, + "p95": 0.153931125998497 + }, + { + "label": "completed_group_3", + "mean": 0.11898040026426315, + "median": 0.11543577164411545, + "p05": 0.10078167170286179, + "p95": 0.14709262549877167 + }, + { + "label": "current_group_partial", + "mean": 0.3593595027923584, + "median": 0.36046332120895386, + "p05": 0.3041363060474396, + "p95": 0.40930092334747314 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4772696495056152, + "group": 4, + "latest_source_mass": 0.3085501194000244, + "layer": 14, + "max_source_mass": 0.3085501194000244, + "mixer_index": 27, + "normalized_entropy": 0.9178792410024722, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30324018001556396, + "median": 0.3082737922668457, + "p05": 0.16577009856700897, + "p95": 0.4784274101257324 + }, + { + "label": "completed_group_1", + "mean": 0.13238272070884705, + "median": 0.12282240390777588, + "p05": 0.08379106968641281, + "p95": 0.2037200629711151 + }, + { + "label": "completed_group_2", + "mean": 0.0979682207107544, + "median": 0.09772157669067383, + "p05": 0.0601615384221077, + "p95": 0.13479545712471008 + }, + { + "label": "completed_group_3", + "mean": 0.15785875916481018, + "median": 0.1573975682258606, + "p05": 0.11552698910236359, + "p95": 0.20258599519729614 + }, + { + "label": "current_group_partial", + "mean": 0.3085501194000244, + "median": 0.3097953200340271, + "p05": 0.21463434398174286, + "p95": 0.40537965297698975 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4979676008224487, + "group": 4, + "latest_source_mass": 0.3820047378540039, + "layer": 15, + "max_source_mass": 0.3820047378540039, + "mixer_index": 28, + "normalized_entropy": 0.9307396012294349, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20932656526565552, + "median": 0.20591703057289124, + "p05": 0.1635243147611618, + "p95": 0.2720984220504761 + }, + { + "label": "completed_group_1", + "mean": 0.1654297560453415, + "median": 0.16333067417144775, + "p05": 0.1427382081747055, + "p95": 0.19703316688537598 + }, + { + "label": "completed_group_2", + "mean": 0.1272144615650177, + "median": 0.12634585797786713, + "p05": 0.10835681855678558, + "p95": 0.14949949085712433 + }, + { + "label": "completed_group_3", + "mean": 0.1160244345664978, + "median": 0.11398661881685257, + "p05": 0.09442843496799469, + "p95": 0.14416006207466125 + }, + { + "label": "current_group_partial", + "mean": 0.3820047378540039, + "median": 0.38286471366882324, + "p05": 0.31784284114837646, + "p95": 0.439454048871994 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.49334716796875, + "group": 4, + "latest_source_mass": 0.31850871443748474, + "layer": 15, + "max_source_mass": 0.31850871443748474, + "mixer_index": 29, + "normalized_entropy": 0.927868764884645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28671571612358093, + "median": 0.27600669860839844, + "p05": 0.15595941245555878, + "p95": 0.4432004988193512 + }, + { + "label": "completed_group_1", + "mean": 0.13947689533233643, + "median": 0.13652008771896362, + "p05": 0.10096997767686844, + "p95": 0.18479548394680023 + }, + { + "label": "completed_group_2", + "mean": 0.11353637278079987, + "median": 0.11486998945474625, + "p05": 0.07941771298646927, + "p95": 0.14473722875118256 + }, + { + "label": "completed_group_3", + "mean": 0.14176231622695923, + "median": 0.14077864587306976, + "p05": 0.10397820174694061, + "p95": 0.18330445885658264 + }, + { + "label": "current_group_partial", + "mean": 0.31850871443748474, + "median": 0.31773626804351807, + "p05": 0.22442427277565002, + "p95": 0.41696855425834656 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5120060443878174, + "group": 4, + "latest_source_mass": 0.28145554661750793, + "layer": 16, + "max_source_mass": 0.28788846731185913, + "mixer_index": 30, + "normalized_entropy": 0.9394621766434421, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21571135520935059, + "median": 0.21688061952590942, + "p05": 0.16800610721111298, + "p95": 0.2583240270614624 + }, + { + "label": "completed_group_1", + "mean": 0.28788846731185913, + "median": 0.2848382592201233, + "p05": 0.23045042157173157, + "p95": 0.35751083493232727 + }, + { + "label": "completed_group_2", + "mean": 0.131511390209198, + "median": 0.12990695238113403, + "p05": 0.11185698211193085, + "p95": 0.15568122267723083 + }, + { + "label": "completed_group_3", + "mean": 0.08343322575092316, + "median": 0.0825304388999939, + "p05": 0.05796157196164131, + "p95": 0.1136971041560173 + }, + { + "label": "current_group_partial", + "mean": 0.28145554661750793, + "median": 0.28277498483657837, + "p05": 0.23862068355083466, + "p95": 0.32226449251174927 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5766459703445435, + "group": 4, + "latest_source_mass": 0.20234227180480957, + "layer": 16, + "max_source_mass": 0.2565065026283264, + "mixer_index": 31, + "normalized_entropy": 0.9796252208077026, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2565065026283264, + "median": 0.2449997365474701, + "p05": 0.18619969487190247, + "p95": 0.3817925453186035 + }, + { + "label": "completed_group_1", + "mean": 0.22934341430664062, + "median": 0.2310539335012436, + "p05": 0.19203530251979828, + "p95": 0.2589260935783386 + }, + { + "label": "completed_group_2", + "mean": 0.1593727320432663, + "median": 0.16161103546619415, + "p05": 0.12202638387680054, + "p95": 0.19153417646884918 + }, + { + "label": "completed_group_3", + "mean": 0.1524350941181183, + "median": 0.14918631315231323, + "p05": 0.12257804721593857, + "p95": 0.19255484640598297 + }, + { + "label": "current_group_partial", + "mean": 0.20234227180480957, + "median": 0.19873467087745667, + "p05": 0.1501774936914444, + "p95": 0.2577142119407654 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5796699523925781, + "group": 5, + "latest_source_mass": 0.2423015832901001, + "layer": 17, + "max_source_mass": 0.2423015832901001, + "mixer_index": 32, + "normalized_entropy": 0.9815041264956277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23561140894889832, + "median": 0.23130320012569427, + "p05": 0.19288715720176697, + "p95": 0.2910196781158447 + }, + { + "label": "completed_group_1", + "mean": 0.21223093569278717, + "median": 0.20132887363433838, + "p05": 0.16995927691459656, + "p95": 0.29450756311416626 + }, + { + "label": "completed_group_2", + "mean": 0.14396421611309052, + "median": 0.1432584673166275, + "p05": 0.1268429160118103, + "p95": 0.1634216159582138 + }, + { + "label": "completed_group_3", + "mean": 0.1658918410539627, + "median": 0.16675393283367157, + "p05": 0.1365433931350708, + "p95": 0.19156640768051147 + }, + { + "label": "completed_group_4", + "mean": 0.2423015832901001, + "median": 0.24500936269760132, + "p05": 0.16469213366508484, + "p95": 0.3121073544025421 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7626371383666992, + "group": 5, + "latest_source_mass": 0.18827059864997864, + "layer": 17, + "max_source_mass": 0.20886825025081635, + "mixer_index": 33, + "normalized_entropy": 0.983746517676336, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16389364004135132, + "median": 0.16299191117286682, + "p05": 0.12742109596729279, + "p95": 0.20235949754714966 + }, + { + "label": "completed_group_1", + "mean": 0.20886825025081635, + "median": 0.2163066267967224, + "p05": 0.15428033471107483, + "p95": 0.24192781746387482 + }, + { + "label": "completed_group_2", + "mean": 0.18027648329734802, + "median": 0.18112054467201233, + "p05": 0.15291966497898102, + "p95": 0.20525163412094116 + }, + { + "label": "completed_group_3", + "mean": 0.1462823748588562, + "median": 0.1460549235343933, + "p05": 0.1107979416847229, + "p95": 0.1837027370929718 + }, + { + "label": "completed_group_4", + "mean": 0.11240866780281067, + "median": 0.11029648035764694, + "p05": 0.09211881458759308, + "p95": 0.13722771406173706 + }, + { + "label": "current_group_partial", + "mean": 0.18827059864997864, + "median": 0.1826695203781128, + "p05": 0.13249869644641876, + "p95": 0.2633744776248932 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6546597480773926, + "group": 5, + "latest_source_mass": 0.35775136947631836, + "layer": 18, + "max_source_mass": 0.35775136947631836, + "mixer_index": 34, + "normalized_entropy": 0.9234831887286026, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17577320337295532, + "median": 0.17441116273403168, + "p05": 0.12769785523414612, + "p95": 0.226454958319664 + }, + { + "label": "completed_group_1", + "mean": 0.14207448065280914, + "median": 0.14135520160198212, + "p05": 0.12424421310424805, + "p95": 0.16169092059135437 + }, + { + "label": "completed_group_2", + "mean": 0.08111411333084106, + "median": 0.08080219477415085, + "p05": 0.071528859436512, + "p95": 0.09159335494041443 + }, + { + "label": "completed_group_3", + "mean": 0.0931762307882309, + "median": 0.09094840288162231, + "p05": 0.0776137039065361, + "p95": 0.11699455231428146 + }, + { + "label": "completed_group_4", + "mean": 0.15011057257652283, + "median": 0.14907215535640717, + "p05": 0.1309027075767517, + "p95": 0.17212367057800293 + }, + { + "label": "current_group_partial", + "mean": 0.35775136947631836, + "median": 0.35810214281082153, + "p05": 0.3160427212715149, + "p95": 0.4006560444831848 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.430190086364746, + "group": 5, + "latest_source_mass": 0.5245762467384338, + "layer": 18, + "max_source_mass": 0.5245762467384338, + "mixer_index": 35, + "normalized_entropy": 0.7982042851884109, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1354856789112091, + "median": 0.1260826736688614, + "p05": 0.08359937369823456, + "p95": 0.2213967740535736 + }, + { + "label": "completed_group_1", + "mean": 0.09479983896017075, + "median": 0.09236767888069153, + "p05": 0.0743592232465744, + "p95": 0.12202530354261398 + }, + { + "label": "completed_group_2", + "mean": 0.07003438472747803, + "median": 0.06948544830083847, + "p05": 0.05187927931547165, + "p95": 0.09030532836914062 + }, + { + "label": "completed_group_3", + "mean": 0.083448626101017, + "median": 0.080234095454216, + "p05": 0.06169745698571205, + "p95": 0.11337985098361969 + }, + { + "label": "completed_group_4", + "mean": 0.09165525436401367, + "median": 0.09136426448822021, + "p05": 0.07385377585887909, + "p95": 0.11038129031658173 + }, + { + "label": "current_group_partial", + "mean": 0.5245762467384338, + "median": 0.5328655242919922, + "p05": 0.42046138644218445, + "p95": 0.60233473777771 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7008471488952637, + "group": 5, + "latest_source_mass": 0.31297987699508667, + "layer": 19, + "max_source_mass": 0.31297987699508667, + "mixer_index": 36, + "normalized_entropy": 0.9492608679378381, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18653985857963562, + "median": 0.18515554070472717, + "p05": 0.14839154481887817, + "p95": 0.2285771518945694 + }, + { + "label": "completed_group_1", + "mean": 0.149653822183609, + "median": 0.14952652156352997, + "p05": 0.1333337277173996, + "p95": 0.16640914976596832 + }, + { + "label": "completed_group_2", + "mean": 0.09563065320253372, + "median": 0.0953449159860611, + "p05": 0.08531444519758224, + "p95": 0.10701718926429749 + }, + { + "label": "completed_group_3", + "mean": 0.09960515052080154, + "median": 0.09781023114919662, + "p05": 0.08538899570703506, + "p95": 0.12114660441875458 + }, + { + "label": "completed_group_4", + "mean": 0.15559062361717224, + "median": 0.15579181909561157, + "p05": 0.14112713932991028, + "p95": 0.16977989673614502 + }, + { + "label": "current_group_partial", + "mean": 0.31297987699508667, + "median": 0.3142726719379425, + "p05": 0.2775314450263977, + "p95": 0.3449920415878296 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.514115333557129, + "group": 5, + "latest_source_mass": 0.4571846127510071, + "layer": 19, + "max_source_mass": 0.4571846127510071, + "mixer_index": 37, + "normalized_entropy": 0.84504385748242, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14724519848823547, + "median": 0.13547402620315552, + "p05": 0.08005326241254807, + "p95": 0.2764008939266205 + }, + { + "label": "completed_group_1", + "mean": 0.13274207711219788, + "median": 0.13091760873794556, + "p05": 0.1022145226597786, + "p95": 0.1677413284778595 + }, + { + "label": "completed_group_2", + "mean": 0.096603162586689, + "median": 0.09685055911540985, + "p05": 0.07085022330284119, + "p95": 0.12084522098302841 + }, + { + "label": "completed_group_3", + "mean": 0.1134733334183693, + "median": 0.11085811257362366, + "p05": 0.08714926987886429, + "p95": 0.14451190829277039 + }, + { + "label": "completed_group_4", + "mean": 0.0527515783905983, + "median": 0.05150189250707626, + "p05": 0.03931276127696037, + "p95": 0.07069683074951172 + }, + { + "label": "current_group_partial", + "mean": 0.4571846127510071, + "median": 0.4572353661060333, + "p05": 0.3533322513103485, + "p95": 0.5581004023551941 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7213332653045654, + "group": 5, + "latest_source_mass": 0.2855108678340912, + "layer": 20, + "max_source_mass": 0.2855108678340912, + "mixer_index": 38, + "normalized_entropy": 0.9606943872026353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19527462124824524, + "median": 0.19608022272586823, + "p05": 0.14261451363563538, + "p95": 0.2492632269859314 + }, + { + "label": "completed_group_1", + "mean": 0.16114139556884766, + "median": 0.16161862015724182, + "p05": 0.14146453142166138, + "p95": 0.18065661191940308 + }, + { + "label": "completed_group_2", + "mean": 0.10406571626663208, + "median": 0.10396719723939896, + "p05": 0.09171589463949203, + "p95": 0.11652514338493347 + }, + { + "label": "completed_group_3", + "mean": 0.10423287749290466, + "median": 0.10206204652786255, + "p05": 0.08988140523433685, + "p95": 0.12570036947727203 + }, + { + "label": "completed_group_4", + "mean": 0.14977452158927917, + "median": 0.14978185296058655, + "p05": 0.13461759686470032, + "p95": 0.1649411916732788 + }, + { + "label": "current_group_partial", + "mean": 0.2855108678340912, + "median": 0.28573572635650635, + "p05": 0.24868330359458923, + "p95": 0.32105109095573425 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.503009557723999, + "group": 5, + "latest_source_mass": 0.44554412364959717, + "layer": 20, + "max_source_mass": 0.44554412364959717, + "mixer_index": 39, + "normalized_entropy": 0.8388456059738542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1677132397890091, + "median": 0.14348560571670532, + "p05": 0.08334699273109436, + "p95": 0.36093956232070923 + }, + { + "label": "completed_group_1", + "mean": 0.12867671251296997, + "median": 0.12582841515541077, + "p05": 0.09450241923332214, + "p95": 0.17028142511844635 + }, + { + "label": "completed_group_2", + "mean": 0.09363879263401031, + "median": 0.0947161316871643, + "p05": 0.06503768265247345, + "p95": 0.11920955032110214 + }, + { + "label": "completed_group_3", + "mean": 0.12237932533025742, + "median": 0.11932459473609924, + "p05": 0.09139132499694824, + "p95": 0.15721726417541504 + }, + { + "label": "completed_group_4", + "mean": 0.04204780235886574, + "median": 0.0400366373360157, + "p05": 0.027764063328504562, + "p95": 0.06483393162488937 + }, + { + "label": "current_group_partial", + "mean": 0.44554412364959717, + "median": 0.44789671897888184, + "p05": 0.2981249690055847, + "p95": 0.563161313533783 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7531253099441528, + "group": 6, + "latest_source_mass": 0.22089648246765137, + "layer": 21, + "max_source_mass": 0.22089648246765137, + "mixer_index": 40, + "normalized_entropy": 0.9784378651557807, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21246549487113953, + "median": 0.2120085060596466, + "p05": 0.16239126026630402, + "p95": 0.2642289698123932 + }, + { + "label": "completed_group_1", + "mean": 0.17559616267681122, + "median": 0.17583107948303223, + "p05": 0.15805834531784058, + "p95": 0.19371670484542847 + }, + { + "label": "completed_group_2", + "mean": 0.11484665423631668, + "median": 0.11523692309856415, + "p05": 0.10031608492136002, + "p95": 0.12835028767585754 + }, + { + "label": "completed_group_3", + "mean": 0.11058571934700012, + "median": 0.10834631323814392, + "p05": 0.09625516831874847, + "p95": 0.13235723972320557 + }, + { + "label": "completed_group_4", + "mean": 0.1656094789505005, + "median": 0.1659335494041443, + "p05": 0.14772318303585052, + "p95": 0.18170902132987976 + }, + { + "label": "completed_group_5", + "mean": 0.22089648246765137, + "median": 0.22141876816749573, + "p05": 0.1899021565914154, + "p95": 0.24976326525211334 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3981997966766357, + "group": 6, + "latest_source_mass": 0.5701477527618408, + "layer": 21, + "max_source_mass": 0.5701477527618408, + "mixer_index": 41, + "normalized_entropy": 0.7185325578138456, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08501994609832764, + "median": 0.07170915603637695, + "p05": 0.0405949205160141, + "p95": 0.19168326258659363 + }, + { + "label": "completed_group_1", + "mean": 0.06276540458202362, + "median": 0.061594001948833466, + "p05": 0.04767107218503952, + "p95": 0.08238202333450317 + }, + { + "label": "completed_group_2", + "mean": 0.0543542318046093, + "median": 0.054001111537218094, + "p05": 0.039771370589733124, + "p95": 0.07025467604398727 + }, + { + "label": "completed_group_3", + "mean": 0.06455785036087036, + "median": 0.06315328925848007, + "p05": 0.046149056404829025, + "p95": 0.08615517616271973 + }, + { + "label": "completed_group_4", + "mean": 0.03336101397871971, + "median": 0.03271401301026344, + "p05": 0.025995943695306778, + "p95": 0.04298749938607216 + }, + { + "label": "completed_group_5", + "mean": 0.12979380786418915, + "median": 0.1275225281715393, + "p05": 0.09634383022785187, + "p95": 0.16848969459533691 + }, + { + "label": "current_group_partial", + "mean": 0.5701477527618408, + "median": 0.5752488374710083, + "p05": 0.4706467390060425, + "p95": 0.6460088491439819 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.6064693927764893, + "group": 6, + "latest_source_mass": 0.4729730188846588, + "layer": 22, + "max_source_mass": 0.4729730188846588, + "mixer_index": 42, + "normalized_entropy": 0.8255619580155779, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12975823879241943, + "median": 0.13364392518997192, + "p05": 0.07449157536029816, + "p95": 0.17710214853286743 + }, + { + "label": "completed_group_1", + "mean": 0.09198253601789474, + "median": 0.09123716503381729, + "p05": 0.07526302337646484, + "p95": 0.11214317381381989 + }, + { + "label": "completed_group_2", + "mean": 0.061632055789232254, + "median": 0.06144782900810242, + "p05": 0.05014941096305847, + "p95": 0.07370813935995102 + }, + { + "label": "completed_group_3", + "mean": 0.06021754443645477, + "median": 0.05830872803926468, + "p05": 0.048907674849033356, + "p95": 0.07788939028978348 + }, + { + "label": "completed_group_4", + "mean": 0.078399658203125, + "median": 0.07768932729959488, + "p05": 0.06658457219600677, + "p95": 0.09283287078142166 + }, + { + "label": "completed_group_5", + "mean": 0.10503692924976349, + "median": 0.1041785180568695, + "p05": 0.08639058470726013, + "p95": 0.12733891606330872 + }, + { + "label": "current_group_partial", + "mean": 0.4729730188846588, + "median": 0.46905672550201416, + "p05": 0.40225255489349365, + "p95": 0.5535125136375427 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3297276496887207, + "group": 6, + "latest_source_mass": 0.6041883230209351, + "layer": 22, + "max_source_mass": 0.6041883230209351, + "mixer_index": 43, + "normalized_entropy": 0.6833448349782582, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09028719365596771, + "median": 0.0814780443906784, + "p05": 0.04628303647041321, + "p95": 0.17968304455280304 + }, + { + "label": "completed_group_1", + "mean": 0.057085148990154266, + "median": 0.05594421550631523, + "p05": 0.040768980979919434, + "p95": 0.07630246877670288 + }, + { + "label": "completed_group_2", + "mean": 0.04790058732032776, + "median": 0.04758875072002411, + "p05": 0.033580001443624496, + "p95": 0.06301991641521454 + }, + { + "label": "completed_group_3", + "mean": 0.05685827136039734, + "median": 0.0552905797958374, + "p05": 0.037766557186841965, + "p95": 0.07916736602783203 + }, + { + "label": "completed_group_4", + "mean": 0.031231366097927094, + "median": 0.030573323369026184, + "p05": 0.023405978456139565, + "p95": 0.041360143572092056 + }, + { + "label": "completed_group_5", + "mean": 0.11244906485080719, + "median": 0.11069123446941376, + "p05": 0.07942862808704376, + "p95": 0.1529570370912552 + }, + { + "label": "current_group_partial", + "mean": 0.6041883230209351, + "median": 0.6044443845748901, + "p05": 0.495843768119812, + "p95": 0.7062365412712097 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.6772459745407104, + "group": 6, + "latest_source_mass": 0.4253680109977722, + "layer": 23, + "max_source_mass": 0.4253680109977722, + "mixer_index": 44, + "normalized_entropy": 0.8619339260628082, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1455874890089035, + "median": 0.15217238664627075, + "p05": 0.08379869163036346, + "p95": 0.19571389257907867 + }, + { + "label": "completed_group_1", + "mean": 0.10527922213077545, + "median": 0.10456782579421997, + "p05": 0.08953425288200378, + "p95": 0.12367816269397736 + }, + { + "label": "completed_group_2", + "mean": 0.06669704616069794, + "median": 0.06652139127254486, + "p05": 0.05353657156229019, + "p95": 0.0801846832036972 + }, + { + "label": "completed_group_3", + "mean": 0.06069611385464668, + "median": 0.058348577469587326, + "p05": 0.048359066247940063, + "p95": 0.08090560138225555 + }, + { + "label": "completed_group_4", + "mean": 0.09951996803283691, + "median": 0.09863626956939697, + "p05": 0.08421580493450165, + "p95": 0.11803246289491653 + }, + { + "label": "completed_group_5", + "mean": 0.09685218334197998, + "median": 0.09553758800029755, + "p05": 0.07873044908046722, + "p95": 0.11939825117588043 + }, + { + "label": "current_group_partial", + "mean": 0.4253680109977722, + "median": 0.4225534200668335, + "p05": 0.3542138934135437, + "p95": 0.5074539184570312 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3742375373840332, + "group": 6, + "latest_source_mass": 0.5837908983230591, + "layer": 23, + "max_source_mass": 0.5837908983230591, + "mixer_index": 45, + "normalized_entropy": 0.706218392483943, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09774194657802582, + "median": 0.0849265456199646, + "p05": 0.05457359179854393, + "p95": 0.19752079248428345 + }, + { + "label": "completed_group_1", + "mean": 0.06367622315883636, + "median": 0.062340207397937775, + "p05": 0.047256506979465485, + "p95": 0.0836888924241066 + }, + { + "label": "completed_group_2", + "mean": 0.05300067365169525, + "median": 0.052215054631233215, + "p05": 0.03693629428744316, + "p95": 0.07218416780233383 + }, + { + "label": "completed_group_3", + "mean": 0.05981069058179855, + "median": 0.05713159963488579, + "p05": 0.039428286254405975, + "p95": 0.08748359233140945 + }, + { + "label": "completed_group_4", + "mean": 0.03241024166345596, + "median": 0.03175225481390953, + "p05": 0.024235714226961136, + "p95": 0.042685288935899734 + }, + { + "label": "completed_group_5", + "mean": 0.10956928879022598, + "median": 0.10649192333221436, + "p05": 0.07568290829658508, + "p95": 0.15524834394454956 + }, + { + "label": "current_group_partial", + "mean": 0.5837908983230591, + "median": 0.5887123346328735, + "p05": 0.4634547531604767, + "p95": 0.6879903078079224 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.672610878944397, + "group": 6, + "latest_source_mass": 0.42961159348487854, + "layer": 24, + "max_source_mass": 0.42961159348487854, + "mixer_index": 46, + "normalized_entropy": 0.8595519581191374, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14811497926712036, + "median": 0.15288400650024414, + "p05": 0.08557615429162979, + "p95": 0.20698775351047516 + }, + { + "label": "completed_group_1", + "mean": 0.10026681423187256, + "median": 0.09941448271274567, + "p05": 0.08483528345823288, + "p95": 0.11770403385162354 + }, + { + "label": "completed_group_2", + "mean": 0.0711982250213623, + "median": 0.07080988585948944, + "p05": 0.05726742744445801, + "p95": 0.08595790714025497 + }, + { + "label": "completed_group_3", + "mean": 0.06714795529842377, + "median": 0.06442591547966003, + "p05": 0.052491698414087296, + "p95": 0.0918450579047203 + }, + { + "label": "completed_group_4", + "mean": 0.08556140959262848, + "median": 0.0845319926738739, + "p05": 0.0714617371559143, + "p95": 0.1041039377450943 + }, + { + "label": "completed_group_5", + "mean": 0.09809904545545578, + "median": 0.09676970541477203, + "p05": 0.07911732792854309, + "p95": 0.12164569646120071 + }, + { + "label": "current_group_partial", + "mean": 0.42961159348487854, + "median": 0.42717063426971436, + "p05": 0.3544589579105377, + "p95": 0.5187017321586609 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4325350522994995, + "group": 6, + "latest_source_mass": 0.5540482997894287, + "layer": 24, + "max_source_mass": 0.5540482997894287, + "mixer_index": 47, + "normalized_entropy": 0.736177388763277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11123399436473846, + "median": 0.09619481861591339, + "p05": 0.04875103384256363, + "p95": 0.24804341793060303 + }, + { + "label": "completed_group_1", + "mean": 0.07182647287845612, + "median": 0.07066938281059265, + "p05": 0.054608479142189026, + "p95": 0.09248427301645279 + }, + { + "label": "completed_group_2", + "mean": 0.05620439723134041, + "median": 0.055612146854400635, + "p05": 0.039178267121315, + "p95": 0.07496444135904312 + }, + { + "label": "completed_group_3", + "mean": 0.06410455703735352, + "median": 0.06079697608947754, + "p05": 0.04396975412964821, + "p95": 0.09297186136245728 + }, + { + "label": "completed_group_4", + "mean": 0.03741488978266716, + "median": 0.03642338514328003, + "p05": 0.027955297380685806, + "p95": 0.05010963976383209 + }, + { + "label": "completed_group_5", + "mean": 0.10516737401485443, + "median": 0.10250729322433472, + "p05": 0.07237984985113144, + "p95": 0.14790719747543335 + }, + { + "label": "current_group_partial", + "mean": 0.5540482997894287, + "median": 0.5591273903846741, + "p05": 0.42828047275543213, + "p95": 0.6615095138549805 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7746202945709229, + "group": 7, + "latest_source_mass": 0.3583717346191406, + "layer": 25, + "max_source_mass": 0.3583717346191406, + "mixer_index": 48, + "normalized_entropy": 0.911974427715716, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15383490920066833, + "median": 0.1525440216064453, + "p05": 0.09242676198482513, + "p95": 0.2355288565158844 + }, + { + "label": "completed_group_1", + "mean": 0.11435382068157196, + "median": 0.11365524679422379, + "p05": 0.09744305908679962, + "p95": 0.13308049738407135 + }, + { + "label": "completed_group_2", + "mean": 0.0908353179693222, + "median": 0.09055374562740326, + "p05": 0.07310554385185242, + "p95": 0.109998419880867 + }, + { + "label": "completed_group_3", + "mean": 0.08329387754201889, + "median": 0.08085054904222488, + "p05": 0.06748563796281815, + "p95": 0.10712415724992752 + }, + { + "label": "completed_group_4", + "mean": 0.08837151527404785, + "median": 0.0870782658457756, + "p05": 0.07312256097793579, + "p95": 0.10766587406396866 + }, + { + "label": "completed_group_5", + "mean": 0.11093883961439133, + "median": 0.11001912504434586, + "p05": 0.08911329507827759, + "p95": 0.1362026184797287 + }, + { + "label": "completed_group_6", + "mean": 0.3583717346191406, + "median": 0.35752072930336, + "p05": 0.2904500365257263, + "p95": 0.4329596757888794 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5557539463043213, + "group": 7, + "latest_source_mass": 0.27552926540374756, + "layer": 25, + "max_source_mass": 0.40426212549209595, + "mixer_index": 49, + "normalized_entropy": 0.7481595010588931, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09889799356460571, + "median": 0.09010560810565948, + "p05": 0.052236538380384445, + "p95": 0.17090652883052826 + }, + { + "label": "completed_group_1", + "mean": 0.05227670818567276, + "median": 0.05044294521212578, + "p05": 0.03345092013478279, + "p95": 0.0764506608247757 + }, + { + "label": "completed_group_2", + "mean": 0.03886467218399048, + "median": 0.03779824823141098, + "p05": 0.024015849456191063, + "p95": 0.05721688270568848 + }, + { + "label": "completed_group_3", + "mean": 0.039664678275585175, + "median": 0.03654132038354874, + "p05": 0.02151499129831791, + "p95": 0.06773733347654343 + }, + { + "label": "completed_group_4", + "mean": 0.02741251140832901, + "median": 0.026939166709780693, + "p05": 0.019440561532974243, + "p95": 0.03718416765332222 + }, + { + "label": "completed_group_5", + "mean": 0.06309205293655396, + "median": 0.05998644977807999, + "p05": 0.03781827166676521, + "p95": 0.09845902025699615 + }, + { + "label": "completed_group_6", + "mean": 0.40426212549209595, + "median": 0.4023381471633911, + "p05": 0.21087685227394104, + "p95": 0.595990002155304 + }, + { + "label": "current_group_partial", + "mean": 0.27552926540374756, + "median": 0.27764469385147095, + "p05": 0.05720072239637375, + "p95": 0.5175203680992126 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7942585945129395, + "group": 7, + "latest_source_mass": 0.322842001914978, + "layer": 26, + "max_source_mass": 0.322842001914978, + "mixer_index": 50, + "normalized_entropy": 0.8628559921254064, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10892649739980698, + "median": 0.11189229786396027, + "p05": 0.03843415528535843, + "p95": 0.1755073070526123 + }, + { + "label": "completed_group_1", + "mean": 0.08669496327638626, + "median": 0.08709807693958282, + "p05": 0.051237381994724274, + "p95": 0.12081066519021988 + }, + { + "label": "completed_group_2", + "mean": 0.06022954732179642, + "median": 0.059791453182697296, + "p05": 0.04038726165890694, + "p95": 0.08276855945587158 + }, + { + "label": "completed_group_3", + "mean": 0.05406671017408371, + "median": 0.051769405603408813, + "p05": 0.03718401864171028, + "p95": 0.07747116684913635 + }, + { + "label": "completed_group_4", + "mean": 0.05429235100746155, + "median": 0.05316123366355896, + "p05": 0.040919527411460876, + "p95": 0.07187244296073914 + }, + { + "label": "completed_group_5", + "mean": 0.07220236957073212, + "median": 0.07145723700523376, + "p05": 0.052494119852781296, + "p95": 0.09450244903564453 + }, + { + "label": "completed_group_6", + "mean": 0.24074554443359375, + "median": 0.23948168754577637, + "p05": 0.15157052874565125, + "p95": 0.33065202832221985 + }, + { + "label": "current_group_partial", + "mean": 0.322842001914978, + "median": 0.31601065397262573, + "p05": 0.15581756830215454, + "p95": 0.5172116756439209 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6406831741333008, + "group": 7, + "latest_source_mass": 0.3283761143684387, + "layer": 26, + "max_source_mass": 0.3283761143684387, + "mixer_index": 51, + "normalized_entropy": 0.7890018263306923, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08106359094381332, + "median": 0.06559887528419495, + "p05": 0.031218135729432106, + "p95": 0.21695302426815033 + }, + { + "label": "completed_group_1", + "mean": 0.06122073531150818, + "median": 0.059359729290008545, + "p05": 0.03888052701950073, + "p95": 0.087535060942173 + }, + { + "label": "completed_group_2", + "mean": 0.04382575303316116, + "median": 0.04286644607782364, + "p05": 0.029065817594528198, + "p95": 0.061862632632255554 + }, + { + "label": "completed_group_3", + "mean": 0.0495278537273407, + "median": 0.046749986708164215, + "p05": 0.02850426360964775, + "p95": 0.07717642933130264 + }, + { + "label": "completed_group_4", + "mean": 0.031025834381580353, + "median": 0.03045039065182209, + "p05": 0.0226125568151474, + "p95": 0.04110690951347351 + }, + { + "label": "completed_group_5", + "mean": 0.08371539413928986, + "median": 0.08070263266563416, + "p05": 0.04838854819536209, + "p95": 0.1307208389043808 + }, + { + "label": "completed_group_6", + "mean": 0.3212447166442871, + "median": 0.3227238059043884, + "p05": 0.1668301224708557, + "p95": 0.46654704213142395 + }, + { + "label": "current_group_partial", + "mean": 0.3283761143684387, + "median": 0.31801193952560425, + "p05": 0.1346617341041565, + "p95": 0.5606002807617188 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8771746158599854, + "group": 7, + "latest_source_mass": 0.2884133756160736, + "layer": 27, + "max_source_mass": 0.2884133756160736, + "mixer_index": 52, + "normalized_entropy": 0.9027301697279486, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11823929846286774, + "median": 0.1204136461019516, + "p05": 0.06847050786018372, + "p95": 0.16903573274612427 + }, + { + "label": "completed_group_1", + "mean": 0.09187045693397522, + "median": 0.0907389223575592, + "p05": 0.06882993876934052, + "p95": 0.11873672157526016 + }, + { + "label": "completed_group_2", + "mean": 0.06966834515333176, + "median": 0.06954923272132874, + "p05": 0.052450068295001984, + "p95": 0.08739262074232101 + }, + { + "label": "completed_group_3", + "mean": 0.05918116122484207, + "median": 0.05728268623352051, + "p05": 0.043897710740566254, + "p95": 0.08088754862546921 + }, + { + "label": "completed_group_4", + "mean": 0.06588791310787201, + "median": 0.06536087393760681, + "p05": 0.05338210612535477, + "p95": 0.08069764077663422 + }, + { + "label": "completed_group_5", + "mean": 0.07802121341228485, + "median": 0.0768343061208725, + "p05": 0.06256727874279022, + "p95": 0.09689624607563019 + }, + { + "label": "completed_group_6", + "mean": 0.22871825098991394, + "median": 0.22734758257865906, + "p05": 0.16477933526039124, + "p95": 0.2942171096801758 + }, + { + "label": "current_group_partial", + "mean": 0.2884133756160736, + "median": 0.2829434871673584, + "p05": 0.17691120505332947, + "p95": 0.41698765754699707 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.612527847290039, + "group": 7, + "latest_source_mass": 0.35910409688949585, + "layer": 27, + "max_source_mass": 0.35910409688949585, + "mixer_index": 53, + "normalized_entropy": 0.775461976193565, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09160975366830826, + "median": 0.08172477036714554, + "p05": 0.04350627586245537, + "p95": 0.18076449632644653 + }, + { + "label": "completed_group_1", + "mean": 0.05898914858698845, + "median": 0.05659175664186478, + "p05": 0.03745892643928528, + "p95": 0.08595352619886398 + }, + { + "label": "completed_group_2", + "mean": 0.039636120200157166, + "median": 0.037070050835609436, + "p05": 0.022959833964705467, + "p95": 0.06498780846595764 + }, + { + "label": "completed_group_3", + "mean": 0.04581688344478607, + "median": 0.0417281799018383, + "p05": 0.02348574623465538, + "p95": 0.07863478362560272 + }, + { + "label": "completed_group_4", + "mean": 0.032106831669807434, + "median": 0.03162588179111481, + "p05": 0.022784041240811348, + "p95": 0.04321740195155144 + }, + { + "label": "completed_group_5", + "mean": 0.06671778857707977, + "median": 0.06372548639774323, + "p05": 0.038862183690071106, + "p95": 0.1047079786658287 + }, + { + "label": "completed_group_6", + "mean": 0.3060194253921509, + "median": 0.30642110109329224, + "p05": 0.14355266094207764, + "p95": 0.46712520718574524 + }, + { + "label": "current_group_partial", + "mean": 0.35910409688949585, + "median": 0.3470078706741333, + "p05": 0.15172286331653595, + "p95": 0.5934250950813293 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8253357410430908, + "group": 7, + "latest_source_mass": 0.2919178009033203, + "layer": 28, + "max_source_mass": 0.2919178009033203, + "mixer_index": 54, + "normalized_entropy": 0.8778009405200828, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10355445742607117, + "median": 0.1021876335144043, + "p05": 0.0484653078019619, + "p95": 0.16425444185733795 + }, + { + "label": "completed_group_1", + "mean": 0.08380813896656036, + "median": 0.08186767250299454, + "p05": 0.05864008516073227, + "p95": 0.11570068448781967 + }, + { + "label": "completed_group_2", + "mean": 0.06609268486499786, + "median": 0.0652528628706932, + "p05": 0.04682006686925888, + "p95": 0.08782888948917389 + }, + { + "label": "completed_group_3", + "mean": 0.07030214369297028, + "median": 0.06750299036502838, + "p05": 0.04772508144378662, + "p95": 0.10195659846067429 + }, + { + "label": "completed_group_4", + "mean": 0.04850628226995468, + "median": 0.047426991164684296, + "p05": 0.03554021194577217, + "p95": 0.0651378333568573 + }, + { + "label": "completed_group_5", + "mean": 0.0775601714849472, + "median": 0.07640299201011658, + "p05": 0.05519672483205795, + "p95": 0.10486777871847153 + }, + { + "label": "completed_group_6", + "mean": 0.25825831294059753, + "median": 0.2556683421134949, + "p05": 0.16664613783359528, + "p95": 0.35751956701278687 + }, + { + "label": "current_group_partial", + "mean": 0.2919178009033203, + "median": 0.2851294279098511, + "p05": 0.14807775616645813, + "p95": 0.4601541757583618 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.714170217514038, + "group": 7, + "latest_source_mass": 0.25887250900268555, + "layer": 28, + "max_source_mass": 0.3351396918296814, + "mixer_index": 55, + "normalized_entropy": 0.8243416240156862, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09696917235851288, + "median": 0.07196179032325745, + "p05": 0.02744467183947563, + "p95": 0.2601720094680786 + }, + { + "label": "completed_group_1", + "mean": 0.07828262448310852, + "median": 0.07698164135217667, + "p05": 0.05464315786957741, + "p95": 0.10479431599378586 + }, + { + "label": "completed_group_2", + "mean": 0.05506835877895355, + "median": 0.054356254637241364, + "p05": 0.03805546090006828, + "p95": 0.07392264902591705 + }, + { + "label": "completed_group_3", + "mean": 0.0616651214659214, + "median": 0.058430496603250504, + "p05": 0.0378233827650547, + "p95": 0.09199885278940201 + }, + { + "label": "completed_group_4", + "mean": 0.03598042577505112, + "median": 0.03550964966416359, + "p05": 0.025762276723980904, + "p95": 0.04832924157381058 + }, + { + "label": "completed_group_5", + "mean": 0.07802211493253708, + "median": 0.07588380575180054, + "p05": 0.05081838741898537, + "p95": 0.1116337925195694 + }, + { + "label": "completed_group_6", + "mean": 0.3351396918296814, + "median": 0.34016722440719604, + "p05": 0.1802617907524109, + "p95": 0.47657349705696106 + }, + { + "label": "current_group_partial", + "mean": 0.25887250900268555, + "median": 0.2348807156085968, + "p05": 0.08903636038303375, + "p95": 0.5087121725082397 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.875918984413147, + "group": 8, + "latest_source_mass": 0.22883272171020508, + "layer": 29, + "max_source_mass": 0.2776731252670288, + "mixer_index": 56, + "normalized_entropy": 0.9021263386407693, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11156761646270752, + "median": 0.10779387503862381, + "p05": 0.05963520333170891, + "p95": 0.17809909582138062 + }, + { + "label": "completed_group_1", + "mean": 0.0880265012383461, + "median": 0.08795224130153656, + "p05": 0.05159609019756317, + "p95": 0.12478911876678467 + }, + { + "label": "completed_group_2", + "mean": 0.07785218209028244, + "median": 0.0769609659910202, + "p05": 0.052540622651576996, + "p95": 0.1063356101512909 + }, + { + "label": "completed_group_3", + "mean": 0.07930341362953186, + "median": 0.076581209897995, + "p05": 0.051584642380476, + "p95": 0.11473879218101501 + }, + { + "label": "completed_group_4", + "mean": 0.04990004748106003, + "median": 0.048692211508750916, + "p05": 0.03663463890552521, + "p95": 0.06705896556377411 + }, + { + "label": "completed_group_5", + "mean": 0.08684436976909637, + "median": 0.08598007261753082, + "p05": 0.06167534738779068, + "p95": 0.11696360260248184 + }, + { + "label": "completed_group_6", + "mean": 0.2776731252670288, + "median": 0.27646756172180176, + "p05": 0.18715886771678925, + "p95": 0.37483638525009155 + }, + { + "label": "completed_group_7", + "mean": 0.22883272171020508, + "median": 0.21553385257720947, + "p05": 0.11123093962669373, + "p95": 0.38216158747673035 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.731968641281128, + "group": 8, + "latest_source_mass": 0.3927639126777649, + "layer": 29, + "max_source_mass": 0.3927639126777649, + "mixer_index": 57, + "normalized_entropy": 0.788252898290834, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.06617744266986847, + "median": 0.05556660145521164, + "p05": 0.026227911934256554, + "p95": 0.14206086099147797 + }, + { + "label": "completed_group_1", + "mean": 0.05556279420852661, + "median": 0.052200887352228165, + "p05": 0.03175564482808113, + "p95": 0.09087387472391129 + }, + { + "label": "completed_group_2", + "mean": 0.043194323778152466, + "median": 0.04139059782028198, + "p05": 0.026660459116101265, + "p95": 0.06614840030670166 + }, + { + "label": "completed_group_3", + "mean": 0.045391663908958435, + "median": 0.04136885702610016, + "p05": 0.02558080479502678, + "p95": 0.0755799189209938 + }, + { + "label": "completed_group_4", + "mean": 0.026374589651823044, + "median": 0.025806495919823647, + "p05": 0.01769660972058773, + "p95": 0.03712644428014755 + }, + { + "label": "completed_group_5", + "mean": 0.054588086903095245, + "median": 0.052078139036893845, + "p05": 0.03126347064971924, + "p95": 0.08616672456264496 + }, + { + "label": "completed_group_6", + "mean": 0.21148523688316345, + "median": 0.20441749691963196, + "p05": 0.11578395217657089, + "p95": 0.3317175507545471 + }, + { + "label": "completed_group_7", + "mean": 0.10446193814277649, + "median": 0.08533146977424622, + "p05": 0.03682969510555267, + "p95": 0.23046134412288666 + }, + { + "label": "current_group_partial", + "mean": 0.3927639126777649, + "median": 0.398543119430542, + "p05": 0.17646673321723938, + "p95": 0.5867685079574585 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9649744033813477, + "group": 8, + "latest_source_mass": 0.18465125560760498, + "layer": 30, + "max_source_mass": 0.27242881059646606, + "mixer_index": 58, + "normalized_entropy": 0.8942983906376845, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07813189923763275, + "median": 0.05827457085251808, + "p05": 0.030488120391964912, + "p95": 0.23055212199687958 + }, + { + "label": "completed_group_1", + "mean": 0.08751582354307175, + "median": 0.08534294366836548, + "p05": 0.05223194509744644, + "p95": 0.12876519560813904 + }, + { + "label": "completed_group_2", + "mean": 0.05837182328104973, + "median": 0.05781412869691849, + "p05": 0.042913295328617096, + "p95": 0.07544869184494019 + }, + { + "label": "completed_group_3", + "mean": 0.07608449459075928, + "median": 0.0739547461271286, + "p05": 0.05176331102848053, + "p95": 0.10407407581806183 + }, + { + "label": "completed_group_4", + "mean": 0.031109986826777458, + "median": 0.029160991311073303, + "p05": 0.02102847397327423, + "p95": 0.04819672927260399 + }, + { + "label": "completed_group_5", + "mean": 0.09630586206912994, + "median": 0.09455011785030365, + "p05": 0.06405375152826309, + "p95": 0.1344345211982727 + }, + { + "label": "completed_group_6", + "mean": 0.27242881059646606, + "median": 0.2743605375289917, + "p05": 0.1689583659172058, + "p95": 0.373348206281662 + }, + { + "label": "completed_group_7", + "mean": 0.1154000535607338, + "median": 0.1035175770521164, + "p05": 0.04644457623362541, + "p95": 0.2236812710762024 + }, + { + "label": "current_group_partial", + "mean": 0.18465125560760498, + "median": 0.17620417475700378, + "p05": 0.09633602201938629, + "p95": 0.30151551961898804 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.8779311180114746, + "group": 8, + "latest_source_mass": 0.24353644251823425, + "layer": 30, + "max_source_mass": 0.26414474844932556, + "mixer_index": 59, + "normalized_entropy": 0.8546832842586183, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07922039180994034, + "median": 0.06023593619465828, + "p05": 0.03731776028871536, + "p95": 0.21911568939685822 + }, + { + "label": "completed_group_1", + "mean": 0.064256951212883, + "median": 0.06290853023529053, + "p05": 0.041900213807821274, + "p95": 0.08994727581739426 + }, + { + "label": "completed_group_2", + "mean": 0.04481552541255951, + "median": 0.04388217628002167, + "p05": 0.029372509568929672, + "p95": 0.06349608302116394 + }, + { + "label": "completed_group_3", + "mean": 0.05368471518158913, + "median": 0.05044667050242424, + "p05": 0.03311370313167572, + "p95": 0.08277972787618637 + }, + { + "label": "completed_group_4", + "mean": 0.03291679173707962, + "median": 0.03245514631271362, + "p05": 0.023155367001891136, + "p95": 0.044260501861572266 + }, + { + "label": "completed_group_5", + "mean": 0.07845190167427063, + "median": 0.07614757120609283, + "p05": 0.04700833186507225, + "p95": 0.11541540920734406 + }, + { + "label": "completed_group_6", + "mean": 0.26414474844932556, + "median": 0.2620946168899536, + "p05": 0.13117244839668274, + "p95": 0.4018894135951996 + }, + { + "label": "completed_group_7", + "mean": 0.13897252082824707, + "median": 0.11859199404716492, + "p05": 0.050003524869680405, + "p95": 0.2953890562057495 + }, + { + "label": "current_group_partial", + "mean": 0.24353644251823425, + "median": 0.23090004920959473, + "p05": 0.0989413633942604, + "p95": 0.43275272846221924 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9959204196929932, + "group": 8, + "latest_source_mass": 0.177350252866745, + "layer": 31, + "max_source_mass": 0.23861892521381378, + "mixer_index": 60, + "normalized_entropy": 0.9083825296150313, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10073552280664444, + "median": 0.10023926943540573, + "p05": 0.015173540450632572, + "p95": 0.2112843543291092 + }, + { + "label": "completed_group_1", + "mean": 0.0823616161942482, + "median": 0.07981257140636444, + "p05": 0.05091645568609238, + "p95": 0.12525078654289246 + }, + { + "label": "completed_group_2", + "mean": 0.0686727911233902, + "median": 0.06696957349777222, + "p05": 0.045773059129714966, + "p95": 0.09787814319133759 + }, + { + "label": "completed_group_3", + "mean": 0.0658436119556427, + "median": 0.06227286905050278, + "p05": 0.04276903346180916, + "p95": 0.09779240190982819 + }, + { + "label": "completed_group_4", + "mean": 0.048376429826021194, + "median": 0.04572375863790512, + "p05": 0.033916816115379333, + "p95": 0.07051397114992142 + }, + { + "label": "completed_group_5", + "mean": 0.07448098063468933, + "median": 0.07275891304016113, + "p05": 0.051099833101034164, + "p95": 0.10395565629005432 + }, + { + "label": "completed_group_6", + "mean": 0.23861892521381378, + "median": 0.238796204328537, + "p05": 0.14983810484409332, + "p95": 0.3338800072669983 + }, + { + "label": "completed_group_7", + "mean": 0.14355985820293427, + "median": 0.12910249829292297, + "p05": 0.051254600286483765, + "p95": 0.2900521159172058 + }, + { + "label": "current_group_partial", + "mean": 0.177350252866745, + "median": 0.17761719226837158, + "p05": 0.060382649302482605, + "p95": 0.30869320034980774 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9298079013824463, + "group": 8, + "latest_source_mass": 0.2151678204536438, + "layer": 31, + "max_source_mass": 0.2151678204536438, + "mixer_index": 61, + "normalized_entropy": 0.8782934258463589, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09570661187171936, + "median": 0.09196878969669342, + "p05": 0.026315581053495407, + "p95": 0.18887758255004883 + }, + { + "label": "completed_group_1", + "mean": 0.07176163792610168, + "median": 0.06893502175807953, + "p05": 0.04786262661218643, + "p95": 0.10555325448513031 + }, + { + "label": "completed_group_2", + "mean": 0.05499213933944702, + "median": 0.05241403728723526, + "p05": 0.03552260994911194, + "p95": 0.08411884307861328 + }, + { + "label": "completed_group_3", + "mean": 0.05746772885322571, + "median": 0.05466804280877113, + "p05": 0.03474465757608414, + "p95": 0.08745065331459045 + }, + { + "label": "completed_group_4", + "mean": 0.03367581218481064, + "median": 0.03265039622783661, + "p05": 0.023353515192866325, + "p95": 0.047148022800683975 + }, + { + "label": "completed_group_5", + "mean": 0.06890923529863358, + "median": 0.0663483589887619, + "p05": 0.03880944103002548, + "p95": 0.1076190173625946 + }, + { + "label": "completed_group_6", + "mean": 0.21427617967128754, + "median": 0.2086736112833023, + "p05": 0.11234137415885925, + "p95": 0.3368087410926819 + }, + { + "label": "completed_group_7", + "mean": 0.18804283440113068, + "median": 0.1678360402584076, + "p05": 0.06922265142202377, + "p95": 0.37698543071746826 + }, + { + "label": "current_group_partial", + "mean": 0.2151678204536438, + "median": 0.21195992827415466, + "p05": 0.07000727951526642, + "p95": 0.3764724135398865 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.8635971546173096, + "group": 8, + "latest_source_mass": 0.10272546857595444, + "layer": 32, + "max_source_mass": 0.3632875978946686, + "mixer_index": 62, + "normalized_entropy": 0.8481596163814172, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07685244083404541, + "median": 0.050476111471652985, + "p05": 0.021709207445383072, + "p95": 0.25511038303375244 + }, + { + "label": "completed_group_1", + "mean": 0.07133443653583527, + "median": 0.06903944909572601, + "p05": 0.041264068335294724, + "p95": 0.1117594763636589 + }, + { + "label": "completed_group_2", + "mean": 0.05962139368057251, + "median": 0.05772271752357483, + "p05": 0.03670363873243332, + "p95": 0.08935610204935074 + }, + { + "label": "completed_group_3", + "mean": 0.08040933310985565, + "median": 0.07558005303144455, + "p05": 0.04971299692988396, + "p95": 0.123878113925457 + }, + { + "label": "completed_group_4", + "mean": 0.033340416848659515, + "median": 0.030976884067058563, + "p05": 0.01889173313975334, + "p95": 0.056192222982645035 + }, + { + "label": "completed_group_5", + "mean": 0.10067726671695709, + "median": 0.09931844472885132, + "p05": 0.05756794661283493, + "p95": 0.14904096722602844 + }, + { + "label": "completed_group_6", + "mean": 0.3632875978946686, + "median": 0.36857712268829346, + "p05": 0.18867944180965424, + "p95": 0.5085643529891968 + }, + { + "label": "completed_group_7", + "mean": 0.11175162345170975, + "median": 0.09184789657592773, + "p05": 0.03606988489627838, + "p95": 0.2585130035877228 + }, + { + "label": "current_group_partial", + "mean": 0.10272546857595444, + "median": 0.09300123900175095, + "p05": 0.0489085391163826, + "p95": 0.18737691640853882 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0009379386901855, + "group": 8, + "latest_source_mass": 0.13621056079864502, + "layer": 32, + "max_source_mass": 0.2692604660987854, + "mixer_index": 63, + "normalized_entropy": 0.9106661009208262, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09139549732208252, + "median": 0.07199805974960327, + "p05": 0.048967160284519196, + "p95": 0.1950986087322235 + }, + { + "label": "completed_group_1", + "mean": 0.08807492256164551, + "median": 0.08772338181734085, + "p05": 0.06209781393408775, + "p95": 0.11598332226276398 + }, + { + "label": "completed_group_2", + "mean": 0.06457877159118652, + "median": 0.06372794508934021, + "p05": 0.045170269906520844, + "p95": 0.08764317631721497 + }, + { + "label": "completed_group_3", + "mean": 0.07007957994937897, + "median": 0.06752179563045502, + "p05": 0.04661180451512337, + "p95": 0.10126036405563354 + }, + { + "label": "completed_group_4", + "mean": 0.03462113440036774, + "median": 0.033816177397966385, + "p05": 0.025839729234576225, + "p95": 0.04629064351320267 + }, + { + "label": "completed_group_5", + "mean": 0.09155916422605515, + "median": 0.08975613117218018, + "p05": 0.06288522481918335, + "p95": 0.12600551545619965 + }, + { + "label": "completed_group_6", + "mean": 0.2692604660987854, + "median": 0.2713700532913208, + "p05": 0.16808731853961945, + "p95": 0.3636854290962219 + }, + { + "label": "completed_group_7", + "mean": 0.15421991050243378, + "median": 0.14455348253250122, + "p05": 0.07559763640165329, + "p95": 0.26658111810684204 + }, + { + "label": "current_group_partial", + "mean": 0.13621056079864502, + "median": 0.12935234606266022, + "p05": 0.07068746536970139, + "p95": 0.22442279756069183 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.7290174961090088, + "group": null, + "latest_source_mass": 0.2353290170431137, + "layer": null, + "max_source_mass": 0.3520195186138153, + "mixer_index": null, + "normalized_entropy": 0.7869097742412674, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10267993062734604, + "median": 0.10614241659641266, + "p05": 0.037625476717948914, + "p95": 0.15687566995620728 + }, + { + "label": "completed_group_1", + "mean": 0.05008202791213989, + "median": 0.04896092414855957, + "p05": 0.03316216915845871, + "p95": 0.06852905452251434 + }, + { + "label": "completed_group_2", + "mean": 0.044380687177181244, + "median": 0.0435444712638855, + "p05": 0.02717534825205803, + "p95": 0.06371284276247025 + }, + { + "label": "completed_group_3", + "mean": 0.0326685830950737, + "median": 0.03019065037369728, + "p05": 0.017900623381137848, + "p95": 0.05591970682144165 + }, + { + "label": "completed_group_4", + "mean": 0.033161722123622894, + "median": 0.032340701669454575, + "p05": 0.02229827456176281, + "p95": 0.04706215858459473 + }, + { + "label": "completed_group_5", + "mean": 0.04359141364693642, + "median": 0.04160827398300171, + "p05": 0.02683408558368683, + "p95": 0.06707148253917694 + }, + { + "label": "completed_group_6", + "mean": 0.1060870811343193, + "median": 0.09384027123451233, + "p05": 0.048520199954509735, + "p95": 0.2056540995836258 + }, + { + "label": "completed_group_7", + "mean": 0.3520195186138153, + "median": 0.3484790027141571, + "p05": 0.12008242309093475, + "p95": 0.6019007563591003 + }, + { + "label": "completed_group_8", + "mean": 0.2353290170431137, + "median": 0.22978201508522034, + "p05": 0.03305097296833992, + "p95": 0.4762387275695801 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "f261d623683eac3593609972ee3faff36fe6f0b7a8891b85f65ae72db324d1ad", + "optimizer_state_before": "f261d623683eac3593609972ee3faff36fe6f0b7a8891b85f65ae72db324d1ad", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "41db358e51c17755a85e4f4c84288ab48da8d214d2518c64ab3d3ea8eb9826e1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.08832799643278122, + 0.17041416466236115, + 0.08745641261339188, + 0.05030680447816849, + 0.12814420461654663, + 0.05527547001838684, + 0.06746833771467209, + 0.04974694177508354, + 0.03680809214711189, + 0.03558321297168732, + 0.051462169736623764, + 0.0526844747364521, + 0.05210389941930771, + 0.03620988130569458, + 0.04957791045308113, + 0.4858623445034027, + 0.04707946628332138, + 0.04491531476378441, + 0.07858023792505264, + 0.09729185700416565, + 0.05887215584516525, + 0.026784580200910568, + 0.041033755987882614, + 0.032232560217380524, + 0.02874751202762127, + 0.022572210058569908, + 0.02659173309803009, + 0.025225715711712837, + 0.041177187114953995, + 0.027006076648831367, + 0.023083338513970375, + 0.016772178933024406 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.076440735749202e-06, + "first_to_last_ratio": 0.7397477609382365, + "imbalance_abs_log_ratio": 0.30144601448810004, + "last_quartile_mean": 1.0917830593371036e-05, + "mean": 1.1650229552628844e-05, + "normalized": [ + 0.7822818865880429, + 0.8682868511675877, + 0.7339307440237431, + 0.6974478440894981, + 0.7421854363314211, + 0.6291858737475481, + 0.5631776493231546, + 0.5294481610694797, + 0.6111720271757957, + 0.5670561579456124, + 0.5320779151344551, + 0.4996505790449965, + 0.4443598163721978, + 0.4189457172671887, + 0.40342707684344364, + 0.4824682206447709, + 1.039372314743425, + 1.0567625256698632, + 1.0003136048176742, + 0.9195164682282294, + 3.600772368765321, + 3.0127235740722966, + 2.4106853870817164, + 1.9576767978188092, + 1.5527140323483644, + 1.4071601191923482, + 1.1966803370484358, + 0.9452267132705859, + 0.8812449826684232, + 0.5867822018999592, + 0.49102686555744746, + 0.4362397500481644 + ], + "peak_layer": 21, + "peak_normalized": 3.600772368765321, + "population_cv": 0.7485297797974629, + "rest_mean": 8.399137136115718e-06, + "spike_contrast": 3.4772772641388783, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.920612860179972e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.113763553614262e-06, + 1.0115741133631673e-05, + 8.550461643608287e-06, + 8.125427484628744e-06, + 8.646630703879055e-06, + 7.330159860430285e-06, + 6.56114889352466e-06, + 6.168192612676648e-06, + 7.120294412743533e-06, + 6.606334409298142e-06, + 6.198829851200571e-06, + 5.821043941978132e-06, + 5.176893864700105e-06, + 4.88081377625349e-06, + 4.700018052972155e-06, + 5.6208655223599635e-06, + 1.2108926057408098e-05, + 1.2311526006669737e-05, + 1.1653883120743558e-05, + 1.071257793228142e-05, + 4.194982466287911e-05, + 3.5098921216558665e-05, + 2.8085038138669915e-05, + 2.2807384084444493e-05, + 1.8089474906446412e-05, + 1.639373840589542e-05, + 1.3941600627731532e-05, + 1.101210818887921e-05, + 1.0266706340189558e-05, + 6.8361473495315295e-06, + 5.720575700252084e-06, + 5.082293228042545e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.205728376087791e-05, + "first_to_last_ratio": 0.7358290958532654, + "imbalance_abs_log_ratio": 0.3067573939385225, + "last_quartile_mean": 4.356620843282144e-05, + "mean": 4.66029975427773e-05, + "normalized": [ + 0.770903229800344, + 0.8488940585533936, + 0.7281730770967723, + 0.696963880781291, + 0.7270770698623731, + 0.6281766634906372, + 0.5677765947233064, + 0.5350777965526399, + 0.616871083452953, + 0.5727615541231824, + 0.5392590563455532, + 0.5043753474566816, + 0.4510025663848085, + 0.42369383655582077, + 0.4063875388466082, + 0.48098941018748065, + 1.0458107275691728, + 1.050722619820089, + 1.0070247957719578, + 0.911755381743329, + 3.606423358142869, + 3.0176573271540965, + 2.418448938919519, + 1.9650777400909958, + 1.5495404924169884, + 1.402027598502842, + 1.194110810724892, + 0.9430892449785634, + 0.8795982640712019, + 0.5857415506262685, + 0.4894539193353971, + 0.43513446591797145 + ], + "peak_layer": 21, + "peak_normalized": 3.606423358142869, + "population_cv": 0.7497897507905983, + "rest_mean": 3.355908113546859e-05, + "spike_contrast": 3.4875849451832757, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011704014614224433, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.592640132410452e-05, + 3.956100772484206e-05, + 3.393504812265746e-05, + 3.248060602345504e-05, + 3.3883970900205895e-05, + 2.927491550508421e-05, + 2.6460091248736717e-05, + 2.493622923793737e-05, + 2.8748041586368345e-05, + 2.669240529939998e-05, + 2.513108847779222e-05, + 2.350540307816118e-05, + 2.1018071493017487e-05, + 1.9745402823900804e-05, + 1.89388774742838e-05, + 2.2415548301069066e-05, + 4.873791476711631e-05, + 4.896682366961613e-05, + 4.693037408287637e-05, + 4.2490533814998344e-05, + 0.0001680701388977468, + 0.00014063187700230628, + 0.00011270696995779872, + 9.157851309282705e-05, + 7.221323176054284e-05, + 6.533868872793391e-05, + 5.564914317801595e-05, + 4.395078576635569e-05, + 4.0991915739141405e-05, + 2.7297312044538558e-05, + 2.281001980009023e-05, + 2.0278570445952937e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.02842872031033039, + "first_to_last_ratio": 0.7358291075435154, + "imbalance_abs_log_ratio": 0.30675737805134173, + "last_quartile_mean": 0.0386349493637681, + "mean": 0.04132800444494933, + "normalized": [ + 0.7709032599004878, + 0.8488941256873797, + 0.7281730724499529, + 0.6969638541985573, + 0.7270771099736086, + 0.6281766484573852, + 0.567776573083768, + 0.5350777977388507, + 0.6168710678618747, + 0.5727615189938741, + 0.5392590137957409, + 0.5043753443878958, + 0.45100255319043453, + 0.4236938111733711, + 0.4063875464463718, + 0.48098938815921266, + 1.0458106889833942, + 1.0507225767377182, + 1.007024785618777, + 0.9117554181054791, + 3.606423512151686, + 3.0176575247118804, + 2.4184488631852656, + 1.9650776212623908, + 1.549540498537689, + 1.4020275638202186, + 1.1941108129352203, + 0.9430892093010953, + 0.8795982916909686, + 0.5857415330277667, + 0.4894539484537472, + 0.43513446597793715 + ], + "peak_layer": 21, + "peak_normalized": 3.606423512151686, + "population_cv": 0.7497897765581046, + "rest_mean": 0.029760528631784296, + "spike_contrast": 3.4875850197496177, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1037923738360405, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03185989335179329, + 0.0350831001996994, + 0.03009393997490406, + 0.028804125264286995, + 0.03004864603281021, + 0.025961287319660187, + 0.023465072736144066, + 0.022113697603344917, + 0.025494050234556198, + 0.023671090602874756, + 0.02228649891912937, + 0.020844826474785805, + 0.018639035522937775, + 0.017510419711470604, + 0.016795186325907707, + 0.019878331571817398, + 0.04322126880288124, + 0.04342426732182503, + 0.04161832481622696, + 0.037681031972169876, + 0.14904628694057465, + 0.12471376359462738, + 0.0999496653676033, + 0.08121273666620255, + 0.06403941661119461, + 0.05794300138950348, + 0.04935021698474884, + 0.038975995033979416, + 0.036352042108774185, + 0.024207528680562973, + 0.020228154957294464, + 0.017983239144086838 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3043631952314172e-06, + "first_to_last_ratio": 0.4450420118390764, + "imbalance_abs_log_ratio": 0.8095865926449729, + "last_quartile_mean": 2.9308765476798726e-06, + "mean": 2.6055976825034577e-06, + "normalized": [ + 0.6733728960131794, + 0.5812204606869391, + 0.5698542538139965, + 0.5614717618725491, + 0.4098041093483211, + 0.40675520855306163, + 0.3998664066116846, + 0.40245795919812427, + 0.45755685594196943, + 0.4432237760005925, + 0.4415823924604673, + 0.4405822214062051, + 0.4093562292335019, + 0.4106227286093283, + 0.4071803565159256, + 0.41488511618036494, + 0.769918701482759, + 0.7873489825883292, + 0.8102338457371544, + 0.8263187848592529, + 3.7447318793466984, + 3.357046502905009, + 2.8666278561167187, + 2.409272840740863, + 1.8742389642782162, + 1.717309102182846, + 1.4585974041085068, + 1.1705649823440567, + 1.0363341993226067, + 0.6723831965836182, + 0.5682358642164441, + 0.5010441607407099 + ], + "peak_layer": 21, + "peak_normalized": 3.7447318793466984, + "population_cv": 0.8923650037371901, + "rest_mean": 1.7127541190752204e-06, + "spike_contrast": 4.336263356368996, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.426952925015939e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.754538857312582e-06, + 1.5144266853894806e-06, + 1.4848109231024864e-06, + 1.4629695215262473e-06, + 1.067784637598379e-06, + 1.059840428752068e-06, + 1.0418909823783906e-06, + 1.0486435257917037e-06, + 1.192209083455964e-06, + 1.1548628435775754e-06, + 1.150586058429326e-06, + 1.1479800150482333e-06, + 1.0666176422091667e-06, + 1.069917630047712e-06, + 1.0609481932988274e-06, + 1.0810236972247367e-06, + 2.0060983842995483e-06, + 2.051514684353606e-06, + 2.1111434307385935e-06, + 2.1530543108383426e-06, + 9.757264706422575e-06, + 8.74711258802563e-06, + 7.469278898497578e-06, + 6.277595730352914e-06, + 4.883512701781001e-06, + 4.474616616789717e-06, + 3.8005180158506846e-06, + 3.050021405215375e-06, + 2.7002699880540604e-06, + 1.7519600987725426e-06, + 1.4805940509177162e-06, + 1.3055195040578838e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.5097264369833283e-05, + "first_to_last_ratio": 0.8841933392684389, + "imbalance_abs_log_ratio": 0.12307953067826505, + "last_quartile_mean": 2.838436262209143e-05, + "mean": 3.3539646210556384e-05, + "normalized": [ + 0.8485407160857603, + 0.9216195218127955, + 0.8021469941710103, + 0.7692195219321104, + 0.7681705820049233, + 0.6786898940432919, + 0.6159832336652391, + 0.5819215114063201, + 0.6596195359678714, + 0.6147763643099993, + 0.5770323156855243, + 0.5403848244462823, + 0.5142674622180825, + 0.4828105450759033, + 0.4626075097666633, + 0.534068298132254, + 1.1134406485128994, + 1.1076985687998104, + 1.0384059475615184, + 0.9464901526906941, + 3.6165010328459855, + 2.9556041233139223, + 2.271107394153319, + 1.8085505389490877, + 1.4615469366449991, + 1.285848929384096, + 1.0756954067402729, + 0.8610270657251896, + 0.7574296454091144, + 0.5047961437234808, + 0.42883702412472346, + 0.3951616106968552 + ], + "peak_layer": 21, + "peak_normalized": 3.6165010328459855, + "population_cv": 0.7191375453041929, + "rest_mean": 2.4703427630743977e-05, + "spike_contrast": 3.289228837233073, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.125522654154338e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.8459755412768573e-05, + 3.0910792702343315e-05, + 2.690372639335692e-05, + 2.57993506238563e-05, + 2.5764169549802318e-05, + 2.276301893289201e-05, + 2.0659859728766605e-05, + 1.9517441614880227e-05, + 2.2123405869933777e-05, + 2.06193817575695e-05, + 1.9353459720150568e-05, + 1.812431582948193e-05, + 1.724834874039516e-05, + 1.6193294868571684e-05, + 1.5515692211920395e-05, + 1.791246177162975e-05, + 3.734440542757511e-05, + 3.715181810548529e-05, + 3.482776810415089e-05, + 3.1744944863021374e-05, + 0.00012129616516176611, + 9.912991663441062e-05, + 7.617213850608096e-05, + 6.065814523026347e-05, + 4.901976717519574e-05, + 4.312691817176528e-05, + 3.60784433723893e-05, + 2.887854316213634e-05, + 2.540392233640887e-05, + 1.6930684068938717e-05, + 1.4383042071131058e-05, + 1.3253580618766136e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.942075255101372e-05, + "first_to_last_ratio": 1.0607277160429414, + "imbalance_abs_log_ratio": 0.058955197142597754, + "last_quartile_mean": 1.8308895164409478e-05, + "mean": 2.3988490454485145e-05, + "normalized": [ + 0.9265433805953981, + 1.002916928598659, + 0.8771624828869276, + 0.838576400799728, + 0.8117319176145469, + 0.7227362607416208, + 0.665755087924021, + 0.6312677145244906, + 0.7026744795970906, + 0.6544976481800655, + 0.6129505366421577, + 0.5743798475555982, + 0.5940726516503032, + 0.5567087586484061, + 0.5341993766607235, + 0.5982231505815051, + 1.1866035009261522, + 1.176017666141157, + 1.071729859322026, + 0.9847732063269867, + 3.624279119252218, + 2.891470983126283, + 2.056678834001149, + 1.5981569839009522, + 1.3288427129095406, + 1.1368223852010402, + 0.9591780585040989, + 0.8005681249434506, + 0.6527448176244227, + 0.45705341091844826, + 0.40201406892297076, + 0.3686696447778616 + ], + "peak_layer": 21, + "peak_normalized": 3.624279119252218, + "population_cv": 0.6854481321415927, + "rest_mean": 1.8213991131267028e-05, + "spike_contrast": 3.0290333624437817, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.517078679986298e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2226377041079104e-05, + 2.405846316833049e-05, + 2.104180384776555e-05, + 2.0116181985940784e-05, + 1.947222335729748e-05, + 1.7337351891910657e-05, + 1.59704595716903e-05, + 1.5143159544095397e-05, + 1.6856100046425126e-05, + 1.5700410585850477e-05, + 1.4703758097311947e-05, + 1.3778505490336102e-05, + 1.4250906133383978e-05, + 1.3354602742765564e-05, + 1.2814636647817679e-05, + 1.4350470337376464e-05, + 2.846482675522566e-05, + 2.8210888558533043e-05, + 2.5709181500133127e-05, + 2.3623222659807652e-05, + 8.694098505657166e-05, + 6.936202407814562e-05, + 4.93366205773782e-05, + 3.833737355307676e-05, + 3.187693073414266e-05, + 2.727065293584019e-05, + 2.300923370057717e-05, + 1.9204420823371038e-05, + 1.565836282679811e-05, + 1.0964021385007072e-05, + 9.643710654927418e-06, + 8.843828254612163e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.355073401209665e-05, + "first_to_last_ratio": 0.6873255585991813, + "imbalance_abs_log_ratio": 0.37494721458370633, + "last_quartile_mean": 9.246089166481397e-05, + "mean": 9.620298442314379e-05, + "normalized": [ + 0.7444695267276318, + 0.8149049645336431, + 0.7060285527098226, + 0.6746516185572761, + 0.6906865904744257, + 0.6051414541820805, + 0.5346022124048582, + 0.5142354572810733, + 0.6084072519080361, + 0.5653159734269397, + 0.535822410124007, + 0.506437491196043, + 0.4288114445296347, + 0.40180191593012804, + 0.38634873785526624, + 0.45058433694504346, + 1.02746448873708, + 1.018965798576808, + 0.9889253452805591, + 0.8860995894165328, + 3.6016343169500162, + 3.0479477917075943, + 2.516639360612916, + 2.0552564610408948, + 1.5982436897490269, + 1.4392591156998307, + 1.2480975795145037, + 1.003440959836939, + 0.8890976144939841, + 0.5932797044328455, + 0.47882480290953644, + 0.4385734422550231 + ], + "peak_layer": 21, + "peak_normalized": 3.6016343169500162, + "population_cv": 0.7690776651523741, + "rest_mean": 6.834074156358838e-05, + "spike_contrast": 3.6092540148285686, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000246659095864743, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.162019028328359e-05, + 7.839628960937262e-05, + 6.792205385863781e-05, + 6.490349915111437e-05, + 6.644611130468547e-05, + 5.821641389047727e-05, + 5.143032831256278e-05, + 4.9470985686639324e-05, + 5.853059337823652e-05, + 5.4385083785746247e-05, + 5.154771497473121e-05, + 4.872079807682894e-05, + 4.1252940718550235e-05, + 3.8654543459415436e-05, + 3.716790160979144e-05, + 4.334755794843659e-05, + 9.884515020530671e-05, + 9.802755084820092e-05, + 9.513756958767772e-05, + 8.524542499799281e-05, + 0.00034648797009140253, + 0.0002932216739282012, + 0.00024210821720771492, + 0.00019772180530708283, + 0.0001537558127893135, + 0.00013846102228853852, + 0.00012007071200059727, + 9.653401502873749e-05, + 8.553384395781904e-05, + 5.707527816412039e-05, + 4.606437505572103e-05, + 4.219207403366454e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "6c58c506475022833bc648a96c247244a57dd96d762e84fedda83974a9fa4cd0", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.11756844818592072, + 0.05185626819729805, + 0.0881219431757927, + 0.03964918479323387, + 0.022397184744477272, + 0.022468235343694687, + 0.031668391078710556, + 0.045926205813884735, + 0.03180550038814545, + 0.03912416473031044, + 0.06056958809494972, + 0.10351218283176422, + 0.05620818957686424, + 0.0675361230969429, + 0.0823286920785904, + 0.07062793523073196, + 0.0371064618229866, + 0.023919809609651566, + 0.035764746367931366, + 0.07103991508483887, + 0.045838613063097, + 0.05455324053764343, + 0.06383209675550461, + 0.07182251662015915, + 0.05687069892883301, + 0.052836012095212936, + 0.06559387594461441, + 0.07689779996871948, + 0.07740812748670578, + 0.08842137455940247, + 0.11899886280298233, + 0.16321057081222534 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.0315435866396e-06, + "first_to_last_ratio": 0.8686691420702204, + "imbalance_abs_log_ratio": 0.14079296031078245, + "last_quartile_mean": 9.24580280070586e-06, + "mean": 1.0551741240760748e-05, + "normalized": [ + 0.859577091723593, + 0.9553086695692968, + 0.8084573574612413, + 0.7695612835108373, + 0.8176461339830406, + 0.6924183904282428, + 0.6104697187838303, + 0.5758263608607518, + 0.6429532324009292, + 0.6044828265447498, + 0.5630258916216904, + 0.536585037633965, + 0.4709644027945216, + 0.4456596693190924, + 0.44879673586224333, + 0.5476213352179609, + 1.1393797585544587, + 1.1777628931854884, + 1.083279918764496, + 1.0241341623776496, + 3.41283327136807, + 2.7698759447615164, + 2.2051169112537026, + 1.8283846589145576, + 1.5645585305615886, + 1.3333451203358413, + 1.0503960349342405, + 0.9149561789360716, + 0.6506532695639259, + 0.5429656631634003, + 0.4819381667747187, + 0.47106537883428756 + ], + "peak_layer": 21, + "peak_normalized": 3.41283327136807, + "population_cv": 0.680677072069579, + "rest_mean": 7.901781120583311e-06, + "spike_contrast": 3.1463192298451848, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4861525889718906e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.07003504835302e-06, + 1.0080169886350632e-05, + 8.530632840120234e-06, + 8.120211532514077e-06, + 8.627590432297438e-06, + 7.3062196861428674e-06, + 6.441518507926958e-06, + 6.075970759411575e-06, + 6.784276138205314e-06, + 6.378346370183863e-06, + 5.940903520240681e-06, + 5.661906470777467e-06, + 4.96949451189721e-06, + 4.702485512098065e-06, + 4.735587026516441e-06, + 5.7783586271398235e-06, + 1.2022440387227107e-05, + 1.2427449291863013e-05, + 1.1430489394115284e-05, + 1.080639867723221e-05, + 3.601133357733488e-05, + 2.9227014238131233e-05, + 2.3267823053174652e-05, + 1.9292641809443012e-05, + 1.6508816770510748e-05, + 1.4069112694414798e-05, + 1.1083507160947192e-05, + 9.654380846768618e-06, + 6.865524937893497e-06, + 5.7292331803182606e-06, + 5.08528682985343e-06, + 4.970559984940337e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.186719573022856e-05, + "first_to_last_ratio": 0.8636501027700138, + "imbalance_abs_log_ratio": 0.14658756584787153, + "last_quartile_mean": 3.689827121888811e-05, + "mean": 4.217961037511486e-05, + "normalized": [ + 0.8482760786319284, + 0.9347808205670737, + 0.803242608202134, + 0.7694642925784047, + 0.8009222324808606, + 0.6911336504850758, + 0.615226970004603, + 0.5810481970164953, + 0.646501974290034, + 0.6090268670825646, + 0.5690153717762457, + 0.5404692649235632, + 0.47664103577546924, + 0.448947205348827, + 0.4514020012299935, + 0.547017773515084, + 1.1465663590184285, + 1.1708024389092253, + 1.0904267732668775, + 1.0148176111319622, + 3.42036094521711, + 2.784326838457232, + 2.216199364233774, + 1.8250690425828833, + 1.5605387928833803, + 1.3332716055000426, + 1.0480080487898067, + 0.9133789058697885, + 0.6504212469789677, + 0.541778233519404, + 0.48088076923057743, + 0.47003668050218345 + ], + "peak_layer": 21, + "peak_normalized": 3.42036094521711, + "population_cv": 0.6827034882268642, + "rest_mean": 3.154645087811837e-05, + "spike_contrast": 3.157206877049384, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.95986716588959e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.577995448722504e-05, + 3.9428690797649324e-05, + 3.388046025065705e-05, + 3.2455704058520496e-05, + 3.3782587706809863e-05, + 2.9151748094591312e-05, + 2.5950033887056634e-05, + 2.450838655931875e-05, + 2.726920138229616e-05, + 2.5688515961519442e-05, + 2.4000846678973176e-05, + 2.2796783014200628e-05, + 2.0104533177800477e-05, + 1.8936418200610206e-05, + 1.9039960534428246e-05, + 2.3072996555129066e-05, + 4.8361722292611375e-05, + 4.938399069942534e-05, + 4.599377643899061e-05, + 4.280461143935099e-05, + 0.00014426949201151729, + 0.00011744182120310143, + 9.347842569695786e-05, + 7.698070112382993e-05, + 6.582291825907305e-05, + 5.623687684419565e-05, + 4.420457116793841e-05, + 3.852596637443639e-05, + 2.7434514777269214e-05, + 2.2851994799566455e-05, + 2.028336348303128e-05, + 1.9825964045594446e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.02826014719903469, + "first_to_last_ratio": 0.863650085818606, + "imbalance_abs_log_ratio": 0.14658758547550527, + "last_quartile_mean": 0.03272175579331815, + "mean": 0.037405300070531666, + "normalized": [ + 0.8482760062421019, + 0.9347808255690103, + 0.8032425824902223, + 0.7694642654684261, + 0.8009222247907771, + 0.6911336564434044, + 0.6152269813998814, + 0.5810482026183381, + 0.6465019891547976, + 0.6090268951969023, + 0.5690153774396436, + 0.5404693059390998, + 0.476641019679117, + 0.44894721167248924, + 0.45140201935344615, + 0.5470177669830626, + 1.1465664364462826, + 1.1708023901135867, + 1.0904266972771928, + 1.0148176048594157, + 3.420360920331411, + 2.7843268581630904, + 2.2161993983606085, + 1.8250690648858443, + 1.5605389254466793, + 1.3332715485871998, + 1.0480080268879794, + 0.9133788783268157, + 0.6504212061158645, + 0.5417782512035506, + 0.4808807753553657, + 0.47003668719839264 + ], + "peak_layer": 21, + "peak_normalized": 3.420360920331411, + "population_cv": 0.6827034910812323, + "rest_mean": 0.02797570769433622, + "spike_contrast": 3.157206954942155, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08832509890198707, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03173001855611801, + 0.03496575728058815, + 0.030045529827475548, + 0.028782041743397713, + 0.029958736151456833, + 0.025852061808109283, + 0.02301274985074997, + 0.02173428237438202, + 0.024182600900530815, + 0.022780833765864372, + 0.0212841909378767, + 0.02021641656756401, + 0.01782890036702156, + 0.016793005168437958, + 0.016884827986359596, + 0.020461363717913628, + 0.04288766160607338, + 0.043794214725494385, + 0.04078773781657219, + 0.03795955702662468, + 0.1279396265745163, + 0.10414858162403107, + 0.0828976035118103, + 0.06826725602149963, + 0.05837242677807808, + 0.04987142235040665, + 0.039201054722070694, + 0.03416521102190018, + 0.024329200387001038, + 0.020265378057956696, + 0.017987489700317383, + 0.017581863328814507 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2999315259776267e-06, + "first_to_last_ratio": 0.5262833924647308, + "imbalance_abs_log_ratio": 0.641915442383951, + "last_quartile_mean": 2.4700219398710033e-06, + "mean": 2.3013876315758353e-06, + "normalized": [ + 0.7510790889350714, + 0.657054866040563, + 0.6465905227557444, + 0.6364494482049028, + 0.464747216363479, + 0.46266301233609036, + 0.448491551252894, + 0.45169899557263793, + 0.49533187678596646, + 0.4887005700041262, + 0.4851101332493999, + 0.48095555698641873, + 0.4488186731288716, + 0.4438844284018987, + 0.4569450978053562, + 0.47536237557813144, + 0.861237602967154, + 0.8798991600628995, + 0.8809450411197095, + 0.9018750120700136, + 3.4686861617847566, + 3.081635479089349, + 2.692581952941867, + 2.3530557260816174, + 1.9593302158594676, + 1.6585873452103246, + 1.316811871407005, + 1.144956403072183, + 0.7621399776403528, + 0.6320030705342806, + 0.5650087375706647, + 0.5473628291868035 + ], + "peak_layer": 21, + "peak_normalized": 3.4686861617847566, + "population_cv": 0.8118793877537595, + "rest_mean": 1.57216401964467e-06, + "spike_contrast": 3.9685395785957933, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.239195136004127e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.72852412561042e-06, + 1.512137941972469e-06, + 1.4880554317642236e-06, + 1.4647168882220285e-06, + 1.0695634955482092e-06, + 1.0647669341778965e-06, + 1.0321529089196702e-06, + 1.039534481606097e-06, + 1.1399506547604688e-06, + 1.1246894473515567e-06, + 1.1164264606122742e-06, + 1.1068651701862109e-06, + 1.0329057431590627e-06, + 1.021550133373239e-06, + 1.0516077963984571e-06, + 1.0939930916720186e-06, + 1.982041567316628e-06, + 2.024989044002723e-06, + 2.027396021730965e-06, + 2.0755639980052365e-06, + 7.982791430549696e-06, + 7.092037776601501e-06, + 6.19667480350472e-06, + 5.41529334441293e-06, + 4.50917832495179e-06, + 3.817052402155241e-06, + 3.0304945539683104e-06, + 2.6349885047238786e-06, + 1.7539795180709916e-06, + 1.4544840496455436e-06, + 1.3003041203774046e-06, + 1.2596940450748662e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.497316063454491e-05, + "first_to_last_ratio": 1.0253055639724644, + "imbalance_abs_log_ratio": 0.024990679356707225, + "last_quartile_mean": 2.4356798121516476e-05, + "mean": 3.0721413736500836e-05, + "normalized": [ + 0.923683121360003, + 1.0034279851985808, + 0.8748807920505384, + 0.8394613337717378, + 0.8363423131230837, + 0.7383758864764528, + 0.6615002397989881, + 0.6254563024059205, + 0.6885636855662547, + 0.64879661636469, + 0.6045999907493074, + 0.5741721615335839, + 0.5386709353912388, + 0.5062958349979282, + 0.5053995265849414, + 0.597134898616305, + 1.2072446578739298, + 1.220368251949879, + 1.111346881203051, + 1.0425813513547888, + 3.4234834365844424, + 2.7178133929240076, + 2.0856692963667, + 1.6821068182789498, + 1.431433074359949, + 1.2175281064911556, + 0.9576212846278308, + 0.8451805251137674, + 0.5586565169962346, + 0.4724016049341192, + 0.4327639669778174, + 0.42703920997382316 + ], + "peak_layer": 21, + "peak_normalized": 3.4234834365844424, + "population_cv": 0.6561034075590804, + "rest_mean": 2.3506994895999218e-05, + "spike_contrast": 2.964193244754933, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.967927547520957e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.8376851332723163e-05, + 3.082672628806904e-05, + 2.687757478270214e-05, + 2.5789438950596377e-05, + 2.569361822679639e-05, + 2.268395110149868e-05, + 2.032222255365923e-05, + 1.921490184031427e-05, + 2.1153649868210778e-05, + 1.993194928218145e-05, + 1.8574166460894048e-05, + 1.763938053045422e-05, + 1.6548732673982158e-05, + 1.5554123820038512e-05, + 1.5526587958447635e-05, + 1.8344828276894987e-05, + 3.70882626157254e-05, + 3.749143797904253e-05, + 3.414214734220877e-05, + 3.2029573048930615e-05, + 0.00010517425107536837, + 8.349506970262155e-05, + 6.407470937119797e-05, + 5.1676699513336644e-05, + 4.397564771352336e-05, + 3.740418469533324e-05, + 2.9419479687931016e-05, + 2.5965140594053082e-05, + 1.7162717995233834e-05, + 1.451284515496809e-05, + 1.3295120879774913e-05, + 1.3119248251314275e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9352088997948158e-05, + "first_to_last_ratio": 1.206212898548031, + "imbalance_abs_log_ratio": 0.18748561551764262, + "last_quartile_mean": 1.6043676055232936e-05, + "mean": 2.226409807803975e-05, + "normalized": [ + 0.9991480788422865, + 1.0765252138266244, + 0.9446533114437263, + 0.905168326838966, + 0.8728412921796674, + 0.7752388554010682, + 0.7090929337562585, + 0.6709804778017834, + 0.7282270427975684, + 0.6827339620063341, + 0.6396496998383137, + 0.608460304703459, + 0.6143468304484543, + 0.5792646484687448, + 0.5732065515361786, + 0.6525851461240976, + 1.266901747363313, + 1.2797303701596927, + 1.1381896960520796, + 1.069494553362227, + 3.415424819879391, + 2.644513480032256, + 1.8913997913758187, + 1.49736286595504, + 1.261891215081716, + 1.0849304046051615, + 0.8879356350587702, + 0.8045755676621354, + 0.5007272020530341, + 0.43401113301181293, + 0.3983632971667951, + 0.3924255451672251 + ], + "peak_layer": 21, + "peak_normalized": 3.415424819879391, + "population_cv": 0.6261266628879907, + "rest_mean": 1.755516532958373e-05, + "spike_contrast": 2.716712376347249, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.769233491970226e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.2245130821829662e-05, + 2.396786294411868e-05, + 2.1031853975728154e-05, + 2.015275640587788e-05, + 1.9433024135651067e-05, + 1.725999391055666e-05, + 1.5787314623594284e-05, + 1.493877516622888e-05, + 1.6213318303925917e-05, + 1.5200455891317688e-05, + 1.4241223652788904e-05, + 1.3546819900511764e-05, + 1.3677878087037243e-05, + 1.2896804946649354e-05, + 1.2761926882376429e-05, + 1.452921969757881e-05, + 2.820642475853674e-05, + 2.849204247468151e-05, + 2.5340767024317756e-05, + 2.3811331629985943e-05, + 7.604135316796601e-05, + 5.8877707488136366e-05, + 4.211031045997515e-05, + 3.33374337060377e-05, + 2.809486977639608e-05, + 2.415499693597667e-05, + 1.976908606593497e-05, + 1.791314934962429e-05, + 1.1148239536851179e-05, + 9.662866432336159e-06, + 8.86919951881282e-06, + 8.737000825931318e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.326739594442188e-05, + "first_to_last_ratio": 0.8019224850951497, + "imbalance_abs_log_ratio": 0.22074332778778838, + "last_quartile_mean": 7.88946526881773e-05, + "mean": 8.741268038647831e-05, + "normalized": [ + 0.8172201386791746, + 0.8930527530237261, + 0.7770273032061067, + 0.7443277001726077, + 0.7577746159675975, + 0.6634374473413018, + 0.577056141658992, + 0.5603298181762961, + 0.6302796253573144, + 0.6008518970750966, + 0.5697352095332306, + 0.5465079619158248, + 0.4488951312608189, + 0.42194694599696664, + 0.4248763842791633, + 0.5137640768693087, + 1.120298638062262, + 1.1322557804193147, + 1.0880678250613065, + 0.9947818548229707, + 3.4370126991938204, + 2.832083111262142, + 2.310584831224533, + 1.9174011752223596, + 1.605064988302073, + 1.3860757842433205, + 1.1026642419836798, + 0.9897246325205491, + 0.6559259444095199, + 0.5273199913410769, + 0.48280183533337045, + 0.4708535160841745 + ], + "peak_layer": 21, + "peak_normalized": 3.4370126991938204, + "population_cv": 0.7050175147633709, + "rest_mean": 6.441943265532177e-05, + "spike_contrast": 3.2843539505659556, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002115762181347236, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.143540278775617e-05, + 7.806413486832753e-05, + 6.792203930672258e-05, + 6.506367935799062e-05, + 6.623911031056195e-05, + 5.7992845540866256e-05, + 5.044202407589182e-05, + 4.8979931307258084e-05, + 5.509443144546822e-05, + 5.252207483863458e-05, + 4.980208177585155e-05, + 4.777172580361366e-05, + 3.9239126635948196e-05, + 3.688351353048347e-05, + 3.713958358275704e-05, + 4.490949504543096e-05, + 9.792830678634346e-05, + 9.897351264953613e-05, + 9.511092503089458e-05, + 8.69565483299084e-05, + 0.00030043849255889654, + 0.00024755997583270073, + 0.00020197441335767508, + 0.00016760517610237002, + 0.00014030303282197565, + 0.00012116059951949865, + 9.638683695811778e-05, + 8.651448297314346e-05, + 5.733624493586831e-05, + 4.6094453864498064e-05, + 4.220300252200104e-05, + 4.115856791031547e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90bf4c661b8e06ef22fe9277f79a81ef01617138f2a24fe93a0093039186433f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.08832799643278122, + 0.2790028154850006, + 0.315830796957016, + 0.3664044439792633, + 0.12814420461654663, + 0.14859555661678314, + 0.17815031111240387, + 0.19290222227573395, + 0.03680809214711189, + 0.06554166972637177, + 0.11251214146614075, + 0.16864995658397675, + 0.05210389941930771, + 0.09766767174005508, + 0.16688434779644012, + 0.6339711546897888, + 0.04707946628332138, + 0.084575355052948, + 0.13482175767421722, + 0.19051826000213623, + 0.05887215584516525, + 0.08281512558460236, + 0.11754488199949265, + 0.14818532764911652, + 0.02874751202762127, + 0.07124026119709015, + 0.10666517913341522, + 0.14956052601337433, + 0.041177187114953995, + 0.09537366032600403, + 0.15850447118282318, + 0.25068944692611694 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.076430106029875e-06, + "first_to_last_ratio": 0.7397413644921954, + "imbalance_abs_log_ratio": 0.3014546613188623, + "last_quartile_mean": 1.0917910628904792e-05, + "mean": 1.1650247159877836e-05, + "normalized": [ + 0.7822789087791068, + 0.8682934236309049, + 0.7339372072758686, + 0.6974498346185574, + 0.7421498092330202, + 0.629192183036434, + 0.5631748074843903, + 0.5294525913634219, + 0.6111628284442171, + 0.5670595555698634, + 0.5320827317867908, + 0.49964759901627454, + 0.444357193138454, + 0.41894363987490607, + 0.40343368829278586, + 0.4824716290086071, + 1.0393637959957764, + 1.0567609285637538, + 1.0003416021830591, + 0.9195068034895976, + 3.60074631727536, + 3.012709340630306, + 2.4107007920054846, + 1.9576641588898815, + 1.5527213659541896, + 1.4071726690364275, + 1.1966858667382718, + 0.9452322326544652, + 0.8812437288920071, + 0.5867980603612143, + 0.4910267089580212, + 0.4362379978185813 + ], + "peak_layer": 21, + "peak_normalized": 3.60074631727536, + "population_cv": 0.7485264383518611, + "rest_mean": 8.399162854604785e-06, + "spike_contrast": 3.477263498033052, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.9206102408352307e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.113742635236122e-06, + 1.0115832992596552e-05, + 8.550549864594359e-06, + 8.125462954922114e-06, + 8.646228707220871e-06, + 7.33024444343755e-06, + 6.561125701409765e-06, + 6.168253548821667e-06, + 7.120198006305145e-06, + 6.606383976759389e-06, + 6.198895334819099e-06, + 5.821018021379132e-06, + 5.176871127332561e-06, + 4.880796950601507e-06, + 4.700102181232069e-06, + 5.6209137255791575e-06, + 1.210884511237964e-05, + 1.2311526006669737e-05, + 1.1654226909740828e-05, + 1.0712481525843032e-05, + 4.194958455627784e-05, + 3.5098808439215645e-05, + 2.8085260055377148e-05, + 2.2807271307101473e-05, + 1.8089587683789432e-05, + 1.6393909390899353e-05, + 1.3941686120233499e-05, + 1.1012189133907668e-05, + 1.026670724968426e-05, + 6.83634243614506e-06, + 5.720582521462347e-06, + 5.08228049511672e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.205732150490803e-05, + "first_to_last_ratio": 0.735826267840192, + "imbalance_abs_log_ratio": 0.30676123724741555, + "last_quartile_mean": 4.356642716629722e-05, + "mean": 4.660306160531036e-05, + "normalized": [ + 0.7709022481473525, + 0.8488925013124051, + 0.728178711481676, + 0.6969576144155101, + 0.727058115881797, + 0.6281832549972298, + 0.5677856111509606, + 0.5350832279959402, + 0.6168642246183335, + 0.5727581516676233, + 0.5392630769055691, + 0.5043755908780991, + 0.4510033125216739, + 0.4236961034310519, + 0.40639369363456396, + 0.48099062251322566, + 1.0458012494557996, + 1.050717740676697, + 1.0070350428730397, + 0.9117479614221499, + 3.6063815548246874, + 3.0176519299457474, + 2.418472936497765, + 1.9650819083638658, + 1.5495511646994138, + 1.402015679141591, + 1.1941147897920268, + 0.9430952864984172, + 0.8796006458403054, + 0.5857526110303165, + 0.4894569545079926, + 0.4351364828771731 + ], + "peak_layer": 21, + "peak_normalized": 3.6063815548246874, + "population_cv": 0.749785809803623, + "rest_mean": 3.355914170107873e-05, + "spike_contrast": 3.4875811226244497, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011704022908816114, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.5926404962083325e-05, + 3.956098953494802e-05, + 3.3935357350856066e-05, + 3.2480358640896156e-05, + 3.3883134165080264e-05, + 2.9275262932060286e-05, + 2.6460547815077007e-05, + 2.493651663826313e-05, + 2.87477614620002e-05, + 2.6692283427109942e-05, + 2.513131039449945e-05, + 2.3505446733906865e-05, + 2.101813515764661e-05, + 1.9745535610127263e-05, + 1.893919034046121e-05, + 2.2415635612560436e-05, + 4.873754005529918e-05, + 4.896666359854862e-05, + 4.6930916141718626e-05, + 4.249024641467258e-05, + 0.00016806842177174985, + 0.00014063181879464537, + 0.0001127082432503812, + 9.157883323496208e-05, + 7.22138283890672e-05, + 6.53382230666466e-05, + 5.564940511249006e-05, + 4.395112773636356e-05, + 4.099208308616653e-05, + 2.7297865017317235e-05, + 2.2810192604083568e-05, + 2.0278692318242975e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.028428752906620502, + "first_to_last_ratio": 0.7358262440829839, + "imbalance_abs_log_ratio": 0.30676126953385263, + "last_quartile_mean": 0.038635144010186195, + "mean": 0.041328060964588076, + "normalized": [ + 0.7709022056247067, + 0.8488925140550104, + 0.7281787018637822, + 0.6969575828113492, + 0.7270581328057514, + 0.6281832258802316, + 0.5677856218053148, + 0.5350832405294478, + 0.6168642299621969, + 0.5727581216485953, + 0.5392630987758338, + 0.5043756010763449, + 0.4510033335681608, + 0.42369611619969416, + 0.4063937060687683, + 0.4809906232945811, + 1.0458012363337954, + 1.050717714484636, + 1.0070350364208112, + 0.9117480417171357, + 3.606381442591284, + 3.0176517752977294, + 2.4184729581554, + 1.965081964731739, + 1.549551179216697, + 1.4020157310844736, + 1.1941147685354327, + 0.9430953109844836, + 0.8796006943450475, + 0.5857526303889504, + 0.4894569748028903, + 0.43513648493972445 + ], + "peak_layer": 21, + "peak_normalized": 3.606381442591284, + "population_cv": 0.7497857870552702, + "rest_mean": 0.029760582648493624, + "spike_contrast": 3.4875810429320246, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10379244387149811, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03185989335179329, + 0.03508308157324791, + 0.030094213783740997, + 0.028803905472159386, + 0.030047902837395668, + 0.02596159465610981, + 0.023465478792786598, + 0.022113952785730362, + 0.02549380250275135, + 0.0236709825694561, + 0.02228669822216034, + 0.02084486559033394, + 0.0186390932649374, + 0.017510538920760155, + 0.016795463860034943, + 0.019878409802913666, + 0.04322093725204468, + 0.043424125760793686, + 0.04161880537867546, + 0.03768077865242958, + 0.14904475212097168, + 0.12471369653940201, + 0.09995079785585403, + 0.08121302723884583, + 0.06403994560241699, + 0.05794259160757065, + 0.049350447952747345, + 0.03897630050778389, + 0.036352191120386124, + 0.02420802041888237, + 0.0202283076941967, + 0.017983347177505493 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3043483590990945e-06, + "first_to_last_ratio": 0.44502800577020923, + "imbalance_abs_log_ratio": 0.8096180644811009, + "last_quartile_mean": 2.930935451672667e-06, + "mean": 2.605597909877133e-06, + "normalized": [ + 0.6733235333569506, + 0.5812250349347619, + 0.5698763690235182, + 0.5614353676156453, + 0.4097737931419157, + 0.4067861516118697, + 0.3999128395484859, + 0.40242406582797585, + 0.45755336910444566, + 0.4431962056791131, + 0.44158440461941995, + 0.4405653847296191, + 0.40935929136700205, + 0.41062613968643547, + 0.4071873893299861, + 0.4148719468145246, + 0.769932770989047, + 0.7873496119897289, + 0.8102137044210286, + 0.8263661841129266, + 3.7447615712230267, + 3.356866796136387, + 2.866602125013433, + 2.4093180075356178, + 1.8743363613537902, + 1.7172763157635853, + 1.4586587976158965, + 1.1705806748956349, + 1.0363602879478928, + 0.6724110186074196, + 0.5682209361980447, + 0.5010435498048719 + ], + "peak_layer": 21, + "peak_normalized": 3.7447615712230267, + "population_cv": 0.8923578131661756, + "rest_mean": 1.7127574707316213e-06, + "spike_contrast": 4.3362451533143265, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.426936281262897e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.7544103911859565e-06, + 1.514438736194279e-06, + 1.484868676016049e-06, + 1.4628748203904252e-06, + 1.0677057389330002e-06, + 1.0599211464068503e-06, + 1.0420120588605641e-06, + 1.0485553048056317e-06, + 1.192200102195784e-06, + 1.154791107182973e-06, + 1.1505914017106988e-06, + 1.1479362456157105e-06, + 1.066625713974645e-06, + 1.069926611307892e-06, + 1.0609666105665383e-06, + 1.0809894774865825e-06, + 2.00613521883497e-06, + 2.0515165033430094e-06, + 2.1110911347932415e-06, + 2.1531780021177838e-06, + 9.757342922966927e-06, + 8.746645107748918e-06, + 7.469212505384348e-06, + 6.2777139646641444e-06, + 4.8837669055501465e-06, + 4.4745315790351015e-06, + 3.800678314291872e-06, + 3.0500625598506304e-06, + 2.7003382001566933e-06, + 1.7520327446618467e-06, + 1.4805552837060532e-06, + 1.3055180261289934e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.509732189537317e-05, + "first_to_last_ratio": 0.8841928656854868, + "imbalance_abs_log_ratio": 0.12308006628858677, + "last_quartile_mean": 2.8384442884998862e-05, + "mean": 3.3539693617967714e-05, + "normalized": [ + 0.848541794522451, + 0.9216211477618174, + 0.8021519345590349, + 0.7692168076440962, + 0.7681649405673239, + 0.6786894770735635, + 0.6159871355761651, + 0.581923997146414, + 0.6596177901053007, + 0.614775007235569, + 0.5770344286988468, + 0.5403836267561264, + 0.5142676030584072, + 0.48281214046187954, + 0.46261206234129265, + 0.5340701464704227, + 1.1134364714672835, + 1.1076980877777178, + 1.0384077338398015, + 0.946489031789269, + 3.6164685871229025, + 2.9556055859795665, + 2.2711256606434183, + 1.8085464640638436, + 1.461551487329098, + 1.285844508644236, + 1.0756934524037012, + 0.8610315974843117, + 0.7574319915412695, + 0.5048016671103398, + 0.4288378822917964, + 0.39515975053273394 + ], + "peak_layer": 21, + "peak_normalized": 3.6164685871229025, + "population_cv": 0.7191344912178012, + "rest_mean": 2.4703477754007542e-05, + "spike_contrast": 3.289223488792826, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.125525928335265e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.845983181032352e-05, + 3.0910890927771106e-05, + 2.6903930120170116e-05, + 2.5799296054174192e-05, + 2.576401675469242e-05, + 2.2763037122786045e-05, + 2.0660019799834117e-05, + 1.9517552573233843e-05, + 2.2123378585092723e-05, + 2.0619365386664867e-05, + 1.935355794557836e-05, + 1.81243012775667e-05, + 1.7248377844225615e-05, + 1.6193371266126633e-05, + 1.5515866834903136e-05, + 1.791254908312112e-05, + 3.734431811608374e-05, + 3.715185448527336e-05, + 3.4827877243515104e-05, + 3.174495213897899e-05, + 0.00012129524839110672, + 9.913010580930859e-05, + 7.617285882588476e-05, + 6.065809429856017e-05, + 4.901998909190297e-05, + 4.312683086027391e-05, + 3.607842882047407e-05, + 2.8878735975013115e-05, + 2.5404036932741292e-05, + 1.6930893252720125e-05, + 1.4383091183844954e-05, + 1.3253536963020451e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9421660454099765e-05, + "first_to_last_ratio": 1.0607859196904783, + "imbalance_abs_log_ratio": 0.059010067068080395, + "last_quartile_mean": 1.8308746462025738e-05, + "mean": 2.3987808219771978e-05, + "normalized": [ + 0.9266617896291613, + 1.0030511591186864, + 0.8771969847201996, + 0.8385340512941075, + 0.8118692035934721, + 0.7226894791926554, + 0.6658428001735601, + 0.6313316969985058, + 0.7028068439710677, + 0.6545495519368416, + 0.6129768036688611, + 0.5743184579423999, + 0.5940844670146944, + 0.5568005354338362, + 0.5341777923418284, + 0.598202628858524, + 1.1864309920816327, + 1.1759385817920434, + 1.0718528525675306, + 0.9849245133243942, + 3.623573548579859, + 2.8913148104249835, + 2.0566096304938517, + 1.5982435366367922, + 1.3288207525445885, + 1.1368754947964976, + 0.9592605424407418, + 0.8006210740556025, + 0.6527473063645078, + 0.4570466562772806, + 0.40203858319370883, + 0.36860687853758356 + ], + "peak_layer": 21, + "peak_normalized": 3.623573548579859, + "population_cv": 0.6853417288273663, + "rest_mean": 1.8214242825504494e-05, + "spike_contrast": 3.0286771663970296, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.516506134881638e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.222858529421501e-05, + 2.4060998839559034e-05, + 2.1042033040430397e-05, + 2.011459400819149e-05, + 1.947496275533922e-05, + 1.733573662932031e-05, + 1.5972109395079315e-05, + 1.514426367066335e-05, + 1.685879578872118e-05, + 1.5701209122198634e-05, + 1.470397000957746e-05, + 1.3776641026197467e-05, + 1.425078426109394e-05, + 1.3356424460653216e-05, + 1.2813754437956959e-05, + 1.4349569937621709e-05, + 2.8459879104048014e-05, + 2.820818917825818e-05, + 2.5711400667205453e-05, + 2.3626180336577818e-05, + 8.692158735357225e-05, + 6.935630517546088e-05, + 4.933355739922263e-05, + 3.833835944533348e-05, + 3.187549737049267e-05, + 2.727115133893676e-05, + 2.301055792486295e-05, + 1.920514478115365e-05, + 1.565797720104456e-05, + 1.0963547538267449e-05, + 9.64402443059953e-06, + 8.842071110848337e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.355676441671676e-05, + "first_to_last_ratio": 0.6873951646261188, + "imbalance_abs_log_ratio": 0.3748459488854143, + "last_quartile_mean": 9.246030185749987e-05, + "mean": 9.619895513424126e-05, + "normalized": [ + 0.7445812595177274, + 0.8150771297110402, + 0.7062879022236439, + 0.6744850418983919, + 0.6907271675605413, + 0.6051874865339609, + 0.5348077152621196, + 0.5142895188349452, + 0.60851218903597, + 0.5653097004083167, + 0.535848710379822, + 0.5063380285002478, + 0.4287242733552982, + 0.4018182537611802, + 0.38641419591620674, + 0.45067226392677057, + 1.0274243261167273, + 1.0190208819675328, + 0.9889455130861956, + 0.8859005728221988, + 3.6013359024505665, + 3.047775942336502, + 2.5162213793615953, + 2.0552047393489516, + 1.5986593070507766, + 1.4391617767539748, + 1.2481945560010297, + 1.0031304566212418, + 0.8891777390557095, + 0.5933080709297001, + 0.47895592769362005, + 0.4385020715774952 + ], + "peak_layer": 21, + "peak_normalized": 3.6013359024505665, + "population_cv": 0.7690005057883333, + "rest_mean": 6.833974745219436e-05, + "spike_contrast": 3.6090077270160457, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00024663867661729456, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.16279391781427e-05, + 7.84095682320185e-05, + 6.79441582178697e-05, + 6.488475628430024e-05, + 6.644723180215806e-05, + 5.821840386488475e-05, + 5.144794340594672e-05, + 4.947411434841342e-05, + 5.853823677171022e-05, + 5.438220250653103e-05, + 5.154808604856953e-05, + 4.870918928645551e-05, + 4.124282713746652e-05, + 3.8654496165690944e-05, + 3.717264189617708e-05, + 4.335420089773834e-05, + 9.883714665193111e-05, + 9.802874410524964e-05, + 9.513552504358813e-05, + 8.522270945832133e-05, + 0.0003464447509031743, + 0.00029319286113604903, + 0.00024205786758102477, + 0.0001977085485123098, + 0.00015378935495391488, + 0.00013844585919287056, + 0.00012007501209154725, + 9.650010179029778e-05, + 8.553796942578629e-05, + 5.707561649614945e-05, + 4.607505979947746e-05, + 4.218344110995531e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "25421e314b847c78a706fbfc95b3ae6bf0cea5ddc086f201de56fb1ddc434ca3", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.15143299102783203, + 0.30262914299964905, + 0.3626292943954468, + 0.38562148809432983, + 0.13662196695804596, + 0.15442071855068207, + 0.18618692457675934, + 0.2074802666902542, + 0.04909220710396767, + 0.0881529152393341, + 0.14954733848571777, + 0.23951733112335205, + 0.08217982947826385, + 0.14409078657627106, + 0.22605396807193756, + 0.6592875719070435, + 0.06389845162630081, + 0.09458807110786438, + 0.14630787074565887, + 0.21908092498779297, + 0.07773751765489578, + 0.10736925154924393, + 0.1424422264099121, + 0.17246152460575104, + 0.06572399288415909, + 0.10123347491025925, + 0.14551502466201782, + 0.19983991980552673, + 0.09023923426866531, + 0.15600618720054626, + 0.24885882437229156, + 0.3860291540622711 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.031635957195249e-06, + "first_to_last_ratio": 0.8686801313140773, + "imbalance_abs_log_ratio": 0.1407803097240232, + "last_quartile_mean": 9.245792170986533e-06, + "mean": 1.055174838882067e-05, + "normalized": [ + 0.8595768541961558, + 0.9553206928959411, + 0.8084617228322365, + 0.7695638651625307, + 0.8176598882474133, + 0.6924422710967179, + 0.6104757697647815, + 0.57582984949779, + 0.6429573651142346, + 0.604473194319946, + 0.5630245620804605, + 0.5365777355393115, + 0.4709580501875008, + 0.44566609052837647, + 0.4487992331310253, + 0.5476071301474617, + 1.1393753665683581, + 1.1777433050985566, + 1.0832692726387225, + 1.0241332100182778, + 3.4128037221928054, + 2.7698985473907003, + 2.205139206917057, + 1.828377559139235, + 1.564558332621775, + 1.3333393040458241, + 1.05039290994071, + 0.9149440953502781, + 0.6506508463364419, + 0.5429766729169746, + 0.48193486661167245, + 0.47106850747072765 + ], + "peak_layer": 21, + "peak_normalized": 3.4128037221928054, + "population_cv": 0.6806759200046126, + "rest_mean": 7.901783326950087e-06, + "spike_contrast": 3.146322633035019, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.486155972292181e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.070038686331827e-06, + 1.0080303582071792e-05, + 8.530684681318235e-06, + 8.12024427432334e-06, + 8.627741408417933e-06, + 7.306476618396118e-06, + 6.441586720029591e-06, + 6.076011686673155e-06, + 6.784324341424508e-06, + 6.378249054250773e-06, + 5.940893515798962e-06, + 5.661833256453974e-06, + 4.969430847268086e-06, + 4.702556452684803e-06, + 4.735616585094249e-06, + 5.778212653240189e-06, + 1.2022402188449632e-05, + 1.2427251022018027e-05, + 1.143038480222458e-05, + 1.0806395948748104e-05, + 3.601104617700912e-05, + 2.9227272534626536e-05, + 2.326807407371234e-05, + 1.929257996380329e-05, + 1.6508825865457766e-05, + 1.4069060853216797e-05, + 1.1083481695095543e-05, + 9.654259883973282e-06, + 6.865504019515356e-06, + 5.729353233618895e-06, + 5.085255452286219e-06, + 4.970596364728408e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.186757840012433e-05, + "first_to_last_ratio": 0.8636564556008468, + "imbalance_abs_log_ratio": 0.146580210082559, + "last_quartile_mean": 3.689844288601307e-05, + "mean": 4.2179749868864747e-05, + "normalized": [ + 0.8482764645044092, + 0.9347954102594115, + 0.8032484042203742, + 0.7694648528419803, + 0.8009406285876599, + 0.6911447334785021, + 0.6152281265998635, + 0.5810488197746441, + 0.646507771172811, + 0.6090191173677569, + 0.569016724323795, + 0.5404629494292167, + 0.4766369582304063, + 0.4489483081063157, + 0.4514009827584627, + 0.5470082020108817, + 1.146557047216301, + 1.170778557053383, + 1.090417647125437, + 1.014806923801809, + 3.420348943656972, + 2.7843447126541196, + 2.216221532283493, + 1.825052484420704, + 1.5605363919661974, + 1.3332759073938238, + 1.048016916561615, + 0.913369157756135, + 0.6504165515961319, + 0.5417902416971032, + 0.48087879077488155, + 0.470039740375403 + ], + "peak_layer": 21, + "peak_normalized": 3.420348943656972, + "population_cv": 0.6827037752697015, + "rest_mean": 3.154654101914881e-05, + "spike_contrast": 3.1572107254761765, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.959907765733078e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.57800890924409e-05, + 3.942943658330478e-05, + 3.388081677258015e-05, + 3.245583502575755e-05, + 3.3783475373638794e-05, + 2.915231198130641e-05, + 2.5950168492272496e-05, + 2.450849387969356e-05, + 2.7269536076346412e-05, + 2.568827403592877e-05, + 2.400098310317844e-05, + 2.2796592020313255e-05, + 2.010442767641507e-05, + 1.8936527339974418e-05, + 1.9039980543311685e-05, + 2.3072669137036428e-05, + 4.836148946196772e-05, + 4.9383146688342094e-05, + 4.5993543608346954e-05, + 4.280430221115239e-05, + 0.00014426946290768683, + 0.00011744296352844685, + 9.347966988570988e-05, + 7.698025729041547e-05, + 6.582303467439488e-05, + 5.6237244280055165e-05, + 4.4205091398907825e-05, + 3.852568261208944e-05, + 2.7434407456894405e-05, + 2.285257687617559e-05, + 2.028334711212665e-05, + 1.9826158677460626e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.028260487830266356, + "first_to_last_ratio": 0.8636564890006067, + "imbalance_abs_log_ratio": 0.1465801714100533, + "last_quartile_mean": 0.032721907598897815, + "mean": 0.037405423703603446, + "normalized": [ + 0.848276489049398, + 0.9347954633352487, + 0.803248442735902, + 0.7694648593773151, + 0.8009406413292095, + 0.6911447672537372, + 0.6152281348927102, + 0.5810488217298512, + 0.6465077699087762, + 0.6090191058798913, + 0.5690167334680125, + 0.5404629383268489, + 0.47663695447049736, + 0.4489483172026723, + 0.45140097553658615, + 0.5470081907747307, + 1.1465570696285177, + 1.1707785023145068, + 1.0904177151979917, + 1.0148068808397375, + 3.4203488185556594, + 2.7843447444483567, + 2.216221552644854, + 1.8250524758566367, + 1.5605363569241573, + 1.3332759059523371, + 1.048016912432173, + 0.9133691867299689, + 0.6504165665258815, + 0.5417902042388995, + 0.48087878757067437, + 0.47003971486826046 + ], + "peak_layer": 21, + "peak_normalized": 3.4203488185556594, + "population_cv": 0.6827037640207692, + "rest_mean": 0.02797578799503821, + "spike_contrast": 3.1572106760860907, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08832545652985573, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03173014149069786, + 0.034966420382261276, + 0.030045848339796066, + 0.028782159090042114, + 0.029959524050354958, + 0.025852562859654427, + 0.02301286906003952, + 0.02173437736928463, + 0.024182897061109543, + 0.02278061769902706, + 0.0212843120098114, + 0.02021624520421028, + 0.0178288072347641, + 0.016793102025985718, + 0.01688484475016594, + 0.020461073145270348, + 0.042887452989816666, + 0.043793465942144394, + 0.04078753665089607, + 0.03795928135514259, + 0.1279395967721939, + 0.10414959490299225, + 0.08289870619773865, + 0.068266861140728, + 0.05837252363562584, + 0.04987175017595291, + 0.0392015166580677, + 0.03416496142745018, + 0.024329107254743576, + 0.020265892148017883, + 0.01798747479915619, + 0.017582034692168236 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2999394698454125e-06, + "first_to_last_ratio": 0.5262796475240541, + "imbalance_abs_log_ratio": 0.6419225582345641, + "last_quartile_mean": 2.4700546106259935e-06, + "mean": 2.301371839763533e-06, + "normalized": [ + 0.7510621611549569, + 0.6571077368974771, + 0.6465980717778745, + 0.6364463561264533, + 0.46473272036308516, + 0.46266426050499954, + 0.44850722565952067, + 0.4517147907876578, + 0.49535804892346047, + 0.4887073813963764, + 0.4851234901505736, + 0.4809132614348466, + 0.44883904274174174, + 0.4438839175271594, + 0.45695317328219487, + 0.4753797657574617, + 0.8612276060345527, + 0.8798755580967531, + 0.8808719479357286, + 0.901864305995852, + 3.468670048732778, + 3.0815001270843183, + 2.6925377905168015, + 2.353088273209896, + 1.9593845634770406, + 1.6586749992669176, + 1.3167743728360517, + 1.144966828441851, + 0.7621392794267813, + 0.6320454943721241, + 0.5650235813213862, + 0.547363818765327 + ], + "peak_layer": 21, + "peak_normalized": 3.468670048732778, + "population_cv": 0.8118673460429436, + "rest_mean": 1.5721624996095435e-06, + "spike_contrast": 3.9684843507872736, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.2391022765950765e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.7284733075939585e-06, + 1.5122492413865984e-06, + 1.488062594035e-06, + 1.4646997215095325e-06, + 1.0695227956603048e-06, + 1.0647625003912253e-06, + 1.0321818990632892e-06, + 1.0395636991233914e-06, + 1.1400030643926584e-06, + 1.1246974054301973e-06, + 1.1164495390403317e-06, + 1.1067602372349938e-06, + 1.0329455335522653e-06, + 1.0215419479209231e-06, + 1.0516191650822293e-06, + 1.094025606107607e-06, + 1.9820049601548817e-06, + 2.02492083190009e-06, + 2.027213895416935e-06, + 2.0755251171067357e-06, + 7.982699571584817e-06, + 7.091677616699599e-06, + 6.196530648594489e-06, + 5.415331088443054e-06, + 4.509272457653424e-06, + 3.817227934632683e-06, + 3.0303874609671766e-06, + 2.63499441643944e-06, + 1.753965875650465e-06, + 1.454571702197427e-06, + 1.3003293588553788e-06, + 1.2596876786119537e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.4973254312499193e-05, + "first_to_last_ratio": 1.0253056867861459, + "imbalance_abs_log_ratio": 0.024990799139217344, + "last_quartile_mean": 2.4356886569876224e-05, + "mean": 3.072147035254602e-05, + "normalized": [ + 0.9236862150549096, + 1.003433477922723, + 0.8748851598635943, + 0.8394575367992361, + 0.8363452717314914, + 0.7383780782804547, + 0.6614970076240289, + 0.6254576365443, + 0.6885640744783226, + 0.6487931115557062, + 0.6046011264871656, + 0.5741697415936372, + 0.5386709492382923, + 0.50629821766184, + 0.5054001346288222, + 0.597129298278525, + 1.2072440909181623, + 1.2203532137980602, + 1.1113429384319282, + 1.0425775353104243, + 3.4234650488600016, + 2.717828515395231, + 2.0856806102438092, + 1.6821053762082474, + 1.4314304363976806, + 1.2175336783265207, + 0.9576282827857749, + 0.8451765399742887, + 0.5586553690400654, + 0.4724074841866198, + 0.4327641759989182, + 0.42703966638121854 + ], + "peak_layer": 21, + "peak_normalized": 3.4234650488600016, + "population_cv": 0.6561028335210565, + "rest_mean": 2.350703370110649e-05, + "spike_contrast": 2.9641948514771395, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.967942827031947e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.837699867086485e-05, + 3.082695184275508e-05, + 2.68777585006319e-05, + 2.5789369828999043e-05, + 2.5693756469991058e-05, + 2.268406024086289e-05, + 2.032216070801951e-05, + 1.9214978237869218e-05, + 2.1153700799914077e-05, + 1.993187834159471e-05, + 1.8574235582491383e-05, + 1.7639338693697937e-05, + 1.654876359680202e-05, + 1.555422568344511e-05, + 1.5526635252172127e-05, + 1.8344690033700317e-05, + 3.70883135474287e-05, + 3.7491045077331364e-05, + 3.414208913454786e-05, + 3.20295148412697e-05, + 0.00010517388000153005, + 8.349568815901875e-05, + 6.407517503248528e-05, + 5.167675044503994e-05, + 4.397564771352336e-05, + 3.740442480193451e-05, + 2.941974889836274e-05, + 2.5965066015487537e-05, + 1.7162714357255027e-05, + 1.4513052519760095e-05, + 1.3295151802594773e-05, + 1.311928645009175e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9351784885657253e-05, + "first_to_last_ratio": 1.2061546872991604, + "imbalance_abs_log_ratio": 0.18743735483861235, + "last_quartile_mean": 1.604419821887859e-05, + "mean": 2.2264120815407296e-05, + "normalized": [ + 0.9988983621888606, + 1.0764878394047224, + 0.9445299593886946, + 0.9051803111065253, + 0.8729387681656537, + 0.7752191091718748, + 0.7092152505168287, + 0.6710625142970259, + 0.7281617557104753, + 0.68279470352268, + 0.6397189413546165, + 0.6085099699564221, + 0.6142476722616719, + 0.5792470223407198, + 0.5731808432472965, + 0.6526049864885849, + 1.2669803566037365, + 1.2796741605033877, + 1.1379908185023777, + 1.069524915820068, + 3.416144871325449, + 2.6441962324509487, + 1.8911944255720232, + 1.497254472533639, + 1.262212643271896, + 1.084844818314985, + 0.8878285176224576, + 0.8046030960517679, + 0.5007719935996711, + 0.43406073132059764, + 0.3983201201335122, + 0.39239981725082984 + ], + "peak_layer": 21, + "peak_normalized": 3.416144871325449, + "population_cv": 0.6261751709955039, + "rest_mean": 1.755484478322884e-05, + "spike_contrast": 2.716868874553423, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.769421138917096e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.2239593818085268e-05, + 2.3967055312823504e-05, + 2.1029129129601642e-05, + 2.015304380620364e-05, + 1.9435214198892936e-05, + 1.7259571905015036e-05, + 1.5790054021636024e-05, + 1.494061689299997e-05, + 1.6211881302297115e-05, + 1.5201823771349154e-05, + 1.4242779798223637e-05, + 1.3547939488489646e-05, + 1.3675684385816567e-05, + 1.2896425687358715e-05, + 1.2761367543134838e-05, + 1.45296762639191e-05, + 2.820820373017341e-05, + 2.849082011380233e-05, + 2.5336365069961175e-05, + 2.381203194090631e-05, + 7.605746213812381e-05, + 5.8870704378932714e-05, + 4.2105781176360324e-05, + 3.333505446789786e-05, + 2.8102054784540087e-05, + 2.4153116100933403e-05, + 1.976672137971036e-05, + 1.791378053894732e-05, + 1.1149248166475445e-05, + 9.663980563345831e-06, + 8.868247277860064e-06, + 8.736436939216219e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.32552014394605e-05, + "first_to_last_ratio": 0.8017636202998193, + "imbalance_abs_log_ratio": 0.22094145234011275, + "last_quartile_mean": 7.889507560321363e-05, + "mean": 8.741289354929904e-05, + "normalized": [ + 0.8167213061822914, + 0.8928199264535667, + 0.7770023518116121, + 0.7441008131359753, + 0.7576510760758486, + 0.6634661276454551, + 0.5770587298246667, + 0.5602754300157883, + 0.6301536495616524, + 0.6007155884581821, + 0.569798578315965, + 0.5464900651195925, + 0.44900852862753954, + 0.42205691313896365, + 0.42493631904268764, + 0.5138066897392368, + 1.1202499594926825, + 1.1323452455529242, + 1.0885892297612487, + 0.9948773152900628, + 3.437247368841268, + 2.8322050553727807, + 2.3105289217473306, + 1.9174427790824178, + 1.6051556310819608, + 1.3857412887469938, + 1.1028008079933802, + 0.9896415626766426, + 0.6559632164073802, + 0.5273861687480318, + 0.4828880564820797, + 0.470875299573792 + ], + "peak_layer": 21, + "peak_normalized": 3.437247368841268, + "population_cv": 0.7050518993663308, + "rest_mean": 6.441818806234989e-05, + "spike_contrast": 3.284542914712745, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00021158430317882448, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.139197259675711e-05, + 7.804397318977863e-05, + 6.792002386646345e-05, + 6.504400516860187e-05, + 6.622847286053002e-05, + 5.799549398943782e-05, + 5.04423733218573e-05, + 4.897529652225785e-05, + 5.508355388883501e-05, + 5.25102877872996e-05, + 4.980774247087538e-05, + 4.777027788804844e-05, + 3.9249134715646505e-05, + 3.689321601996198e-05, + 3.714491322170943e-05, + 4.491332947509363e-05, + 9.792429045774043e-05, + 9.898157441057265e-05, + 9.515673446003348e-05, + 8.696510485606268e-05, + 0.00030045973835512996, + 0.0002475712390150875, + 0.0002019700186792761, + 0.0001676092215348035, + 0.00014031129830982536, + 0.00012113165576010942, + 9.639900963520631e-05, + 8.650743257021531e-05, + 5.733964280807413e-05, + 4.610035102814436e-05, + 4.2210642277495936e-05, + 4.1160572436638176e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "36b6c956f67457ef008d5eacc14cf9477822b0791655ecd816dc4fb89c68ffd5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059407927095890045, + 0.07886742800474167, + 0.16177938878536224, + 0.15622827410697937, + 0.1895449161529541, + 0.15217198431491852, + 0.14916078746318817, + 0.14915435016155243, + 0.16036193072795868, + 0.12247956544160843, + 0.12010690569877625, + 0.12369577586650848, + 0.14881008863449097, + 0.10321364551782608, + 0.1168464794754982, + 0.16267083585262299, + 0.21974295377731323, + 0.1438056081533432, + 0.14797262847423553, + 0.15148812532424927, + 0.1692686378955841, + 0.07994630187749863, + 0.09600856900215149, + 0.09550122171640396, + 0.10489360988140106, + 0.07298607379198074, + 0.08083610236644745, + 0.0843365490436554, + 0.09205732494592667, + 0.07988084107637405, + 0.08835897594690323, + 0.09466564655303955 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 5.231516780668244e-06, + "first_to_last_ratio": 1.2181687678086963, + "imbalance_abs_log_ratio": 0.1973487211082494, + "last_quartile_mean": 4.29457470829675e-06, + "mean": 7.48252770677027e-06, + "normalized": [ + 0.7049335753209298, + 0.8672771403140204, + 0.8924172675590143, + 0.5058818096791461, + 0.5579230679041902, + 0.8745502145818771, + 0.7289352734533796, + 0.46139662247417734, + 0.4048700191822373, + 0.5891204135113416, + 0.5910949790604852, + 0.5127332908040538, + 0.2632672019864895, + 0.3082220550136737, + 0.22207938427345525, + 1.138921629601989, + 0.7844735000286154, + 1.3148028500337634, + 2.424187191214654, + 2.3081308703788386, + 3.6462484228858485, + 3.003416179964025, + 2.691692507346097, + 1.611848128667073, + 0.9027728276579748, + 0.8071999655073968, + 0.8661618055353845, + 0.5565823202716984, + 0.6137168515164881, + 0.45586796048570105, + 0.23946068136284482, + 0.14981399242313648 + ], + "peak_layer": 21, + "peak_normalized": 3.6462484228858485, + "population_cv": 0.8597567898445678, + "rest_mean": 5.5825260090016485e-06, + "spike_contrast": 3.178227355521773, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7742536874720828e-05, + "top_five_layers": [ + 21, + 22, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.274685008771485e-06, + 6.489425231848145e-06, + 6.6775369305105414e-06, + 3.785274657275295e-06, + 4.174674813839374e-06, + 6.543846211570781e-06, + 5.454278380057076e-06, + 3.4524130114732543e-06, + 3.0294511361717014e-06, + 4.408109816722572e-06, + 4.422884558152873e-06, + 3.836541054624831e-06, + 1.969904133147793e-06, + 2.306280066477484e-06, + 1.6617151459286106e-06, + 8.52201264933683e-06, + 5.869844699191162e-06, + 9.838048754318152e-06, + 1.8139047824661247e-05, + 1.7270653188461438e-05, + 2.728315485001076e-05, + 2.2473144781542942e-05, + 2.0140663764323108e-05, + 1.2060698281857185e-05, + 6.755022695870139e-06, + 6.039896106813103e-06, + 6.4810797084646765e-06, + 4.164642632531468e-06, + 4.592153345583938e-06, + 3.4110446449631127e-06, + 1.7917711829795735e-06, + 1.12098734916799e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.9753211972783902e-05, + "first_to_last_ratio": 1.1556699430673998, + "imbalance_abs_log_ratio": 0.14468021310494286, + "last_quartile_mean": 1.709243377945313e-05, + "mean": 2.9343291515715464e-05, + "normalized": [ + 0.6423992673864061, + 0.7671882759686803, + 0.8850259786401377, + 0.5070019793112642, + 0.5368844244513357, + 0.8343894815896736, + 0.7353960442078943, + 0.4771259191368489, + 0.4241382669659466, + 0.5855916700343277, + 0.5778160836542935, + 0.5253742530620523, + 0.25966342686064026, + 0.31295329557343554, + 0.2348431281691379, + 1.1373249210235588, + 0.8069155942709877, + 1.3369118805785314, + 2.3460920286428437, + 2.4540482840769475, + 3.5680018751807268, + 3.0328373621092837, + 2.734987268865684, + 1.6170984312591716, + 0.9302523641112638, + 0.8177707797658776, + 0.8816015913798516, + 0.5588721385124307, + 0.6102613176308225, + 0.4675196143273077, + 0.24642719158815024, + 0.14728586166448673 + ], + "peak_layer": 21, + "peak_normalized": 3.5680018751807268, + "population_cv": 0.8610884962987538, + "rest_mean": 2.1862733067084468e-05, + "spike_contrast": 3.1898256692946436, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.973830713832285e-05, + "top_five_layers": [ + 21, + 22, + 23, + 20, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.8850108972401358e-05, + 2.251182922918815e-05, + 2.5969575290218927e-05, + 1.4877106877975166e-05, + 1.5753956176922657e-05, + 2.4483733795932494e-05, + 2.157894050469622e-05, + 1.400044493493624e-05, + 1.244561281055212e-05, + 1.7183187082991935e-05, + 1.6955025785136968e-05, + 1.541620986245107e-05, + 7.619379630341427e-06, + 9.183079782815184e-06, + 6.891070370329544e-06, + 3.337285670568235e-05, + 2.3677559511270374e-05, + 3.9229395042639226e-05, + 6.884206231916323e-05, + 7.200985419331118e-05, + 0.00010469691915204749, + 8.899343083612621e-05, + 8.025352872209623e-05, + 4.7450990678044036e-05, + 2.72966663033003e-05, + 2.3996086383704096e-05, + 2.586909249657765e-05, + 1.6399148080381565e-05, + 1.790707574400585e-05, + 1.3718564332521055e-05, + 7.230984920170158e-06, + 4.321851974964375e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.017517346306703985, + "first_to_last_ratio": 1.1556699645631487, + "imbalance_abs_log_ratio": 0.1446802317051919, + "last_quartile_mean": 0.015157741261646152, + "mean": 0.026021924597444013, + "normalized": [ + 0.642399269560858, + 0.7671882665980466, + 0.8850260023995508, + 0.5070019973604465, + 0.5368843965740409, + 0.8343894944034302, + 0.7353960815145425, + 0.47712593296252226, + 0.4241382778148869, + 0.5855916847742128, + 0.5778160385433538, + 0.5253742512409212, + 0.25966343220529986, + 0.31295327953435204, + 0.23484312335084737, + 1.1373249608588862, + 0.8069155777916533, + 1.3369118107572777, + 2.346092040189918, + 2.45404832250407, + 3.5680018220110936, + 3.0328371668368583, + 2.73498744787703, + 1.6170984888723343, + 0.9302523630738624, + 0.8177707318164164, + 0.8816015513659362, + 0.5588721480759673, + 0.6102613538594246, + 0.4675196323727517, + 0.24642719276174477, + 0.14728586013746342 + ], + "peak_layer": 21, + "peak_normalized": 3.5680018220110936, + "population_cv": 0.861088491107969, + "rest_mean": 0.01938809050867955, + "spike_contrast": 3.189825663805616, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.061844628676772116, + "top_five_layers": [ + 21, + 22, + 23, + 20, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.01671646535396576, + 0.019963715225458145, + 0.023030079901218414, + 0.013193167746067047, + 0.01397076528519392, + 0.021712420508265495, + 0.019136421382427216, + 0.012415735051035881, + 0.011036894284188747, + 0.015238222666084766, + 0.015035885386168957, + 0.013671249151229858, + 0.006756942253559828, + 0.008143646642565727, + 0.0061110700480639935, + 0.029595384374260902, + 0.02099749632179737, + 0.03478901833295822, + 0.06104983016848564, + 0.06385906040668488, + 0.09284627437591553, + 0.07892026007175446, + 0.07116963714361191, + 0.04208001494407654, + 0.02420695684850216, + 0.021279968321323395, + 0.022940969094634056, + 0.014542928896844387, + 0.015880174934864044, + 0.01216576062142849, + 0.0064125098288059235, + 0.003832661546766758 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.912403284620723e-06, + "first_to_last_ratio": 0.7440148464151746, + "imbalance_abs_log_ratio": 0.29569428948903514, + "last_quartile_mean": 2.570383230704465e-06, + "mean": 2.2519678122279174e-06, + "normalized": [ + 3.7363448723044836, + 1.2999770872722904, + 0.2528355179896465, + 0.3447212899074926, + 0.28850832739316085, + 0.26077899448522845, + 0.3004387502401656, + 0.310109614891652, + 0.26155308064966604, + 0.34461906116108865, + 0.3430527653161843, + 0.4158029818571386, + 0.21524181782036755, + 0.3598800713242707, + 0.3675641897039758, + 0.3564396575202899, + 0.3380877296575, + 0.6093802060812419, + 0.9402690126586363, + 1.079633600656378, + 1.9776909756331749, + 3.4168782728753038, + 2.651381140535597, + 2.3976565605938904, + 1.2926094488981195, + 1.6689818693255922, + 1.661631597217477, + 1.2056041530585988, + 1.4143405120828008, + 0.8908828858291652, + 0.5365608273760537, + 0.460543127683369 + ], + "peak_layer": 1, + "peak_normalized": 3.7363448723044836, + "population_cv": 0.9346042008561078, + "rest_mean": 1.6901254972018016e-06, + "spike_contrast": 3.127528885943908, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.285916313368944e-06, + "top_five_layers": [ + 1, + 22, + 23, + 24, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.414128387812525e-06, + 2.927506557171e-06, + 5.693774483006564e-07, + 7.763012490613619e-07, + 6.497114668491122e-07, + 5.872659016858961e-07, + 6.765783950868354e-07, + 6.983568709983956e-07, + 5.890091188121005e-07, + 7.760710332149756e-07, + 7.725437853878248e-07, + 9.363749313706649e-07, + 4.847176455768931e-07, + 8.104383368845447e-07, + 8.277427241409896e-07, + 8.026906357372354e-07, + 7.613626848979038e-07, + 1.3723046095037716e-06, + 2.117455551342573e-06, + 2.4313001176778926e-06, + 4.453696419659536e-06, + 7.694699888816103e-06, + 5.970824986434309e-06, + 5.399445399234537e-06, + 2.910914872700232e-06, + 3.758493448913214e-06, + 3.741940872714622e-06, + 2.714981746976264e-06, + 3.1850493087404175e-06, + 2.0062395833519986e-06, + 1.208317712553253e-06, + 1.037128299685719e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.3613449709737324e-05, + "first_to_last_ratio": 1.2961798669265885, + "imbalance_abs_log_ratio": 0.25942137450895547, + "last_quartile_mean": 1.0502747386453848e-05, + "mean": 1.94846966365958e-05, + "normalized": [ + 0.6224917639630821, + 0.8091508021640391, + 0.9260582357582028, + 0.5438708025741, + 0.539202130448075, + 0.86685282634047, + 0.7739497051591201, + 0.5078152293450356, + 0.44530646133543333, + 0.6024460206063752, + 0.6204465476276101, + 0.5023513612748098, + 0.2919364277081767, + 0.34034090010393764, + 0.2666101183752142, + 1.157137994849184, + 0.8274903470382167, + 1.369617562543775, + 2.446429377746549, + 2.1498756230367086, + 3.341616124095529, + 3.2302364132353425, + 2.8911802848363917, + 1.6153833341689192, + 0.9062162391904391, + 0.7992110395710725, + 0.8759188816263488, + 0.4509435491843002, + 0.543934283818765, + 0.3834922528262834, + 0.21307834519941646, + 0.13940901424907648 + ], + "peak_layer": 21, + "peak_normalized": 3.341616124095529, + "population_cv": 0.8584035053051066, + "rest_mean": 1.4444198734930035e-05, + "spike_contrast": 3.2333662920774784, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.670338530559093e-05, + "top_five_layers": [ + 21, + 22, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.2129063179600053e-05, + 1.5766057913424447e-05, + 1.8043963791569695e-05, + 1.0597157597658224e-05, + 1.0506189937586896e-05, + 1.689036434981972e-05, + 1.5080175217008218e-05, + 9.89462569123134e-06, + 8.676661309436895e-06, + 1.1738477951439563e-05, + 1.208921275974717e-05, + 9.788163879420608e-06, + 5.688292731065303e-06, + 6.6314391915511806e-06, + 5.194817276787944e-06, + 2.2546482796315104e-05, + 1.612339838175103e-05, + 2.668658271431923e-05, + 4.7667934268247336e-05, + 4.1889674321282655e-05, + 6.511037645395845e-05, + 6.294017657637596e-05, + 5.633377077174373e-05, + 3.147525421809405e-05, + 1.7657348507782444e-05, + 1.557238465466071e-05, + 1.7067013686755672e-05, + 8.786498256085906e-06, + 1.0598394510452636e-05, + 7.472230208804831e-06, + 4.1517669160384685e-06, + 2.716342351050116e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.224245218319993e-06, + "first_to_last_ratio": 1.42352192746992, + "imbalance_abs_log_ratio": 0.3531340315301507, + "last_quartile_mean": 6.47987575064235e-06, + "mean": 1.2729493800378577e-05, + "normalized": [ + 0.575325501428242, + 0.8765327600629662, + 0.9593474390198574, + 0.5853400578880529, + 0.5587901873677211, + 0.889540337545407, + 0.8146648699468889, + 0.5375441750172566, + 0.47404798886848176, + 0.6359131178887726, + 0.6792320083196889, + 0.46547624987616953, + 0.32872806943308774, + 0.37175338073287834, + 0.3016769923893083, + 1.1851024317290795, + 0.8670161235264493, + 1.4111720806791088, + 2.506650783858839, + 1.9418762691937648, + 2.8977084750056923, + 3.415371148318928, + 3.0397221818581524, + 1.609113290765392, + 0.8688260399415566, + 0.8080203647056072, + 0.8707912839378436, + 0.3953034784106991, + 0.49891926408419757, + 0.31272264819953055, + 0.18262343848706983, + 0.13514756151330948 + ], + "peak_layer": 22, + "peak_normalized": 3.415371148318928, + "population_cv": 0.8455005471645312, + "rest_mean": 9.509053913663794e-06, + "spike_contrast": 3.16749378666982, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.0119869188638403e-05, + "top_five_layers": [ + 22, + 23, + 21, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.3236024036305025e-06, + 1.1157818335050251e-05, + 1.221200727741234e-05, + 7.451082637999207e-06, + 7.11311622580979e-06, + 1.1323398211970925e-05, + 1.0370271411375143e-05, + 6.842665243311785e-06, + 6.0343909353832714e-06, + 8.094852091744542e-06, + 8.64627963892417e-06, + 5.92527703702217e-06, + 4.184541921858909e-06, + 4.732232355308952e-06, + 3.840195404336555e-06, + 1.5085754057508893e-05, + 1.1036676369258203e-05, + 1.7963506252272055e-05, + 3.19083956128452e-05, + 2.471910192980431e-05, + 3.688636206788942e-05, + 4.347594585851766e-05, + 3.8694124668836594e-05, + 2.0483197658904828e-05, + 1.1059715689043514e-05, + 1.0285690223099664e-05, + 1.108473225031048e-05, + 5.032013177697081e-06, + 6.350989679049235e-06, + 3.980801011493895e-06, + 2.3247039280249737e-06, + 1.7203600464199553e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.924221596207644e-05, + "first_to_last_ratio": 1.220088035622856, + "imbalance_abs_log_ratio": 0.19892301648833594, + "last_quartile_mean": 3.216342986434029e-05, + "mean": 5.811260362520443e-05, + "normalized": [ + 0.6785990759991457, + 0.7772257654622007, + 0.8732309863789576, + 0.5020979620112224, + 0.5168098626008664, + 0.8345517579225961, + 0.7379646492413534, + 0.4817512656456255, + 0.4163654546730875, + 0.5357163632735232, + 0.5807450859096301, + 0.520383542426531, + 0.2612071324961203, + 0.31411274991679294, + 0.24095333784569256, + 1.12561496002337, + 0.8021979217859124, + 1.3082348163567306, + 2.4237584214732073, + 2.344022957737727, + 3.760809922544462, + 3.1425833141182378, + 2.8316122260106797, + 1.5617115119945912, + 0.9609041629692773, + 0.7679731552856547, + 0.8714908320820871, + 0.46503806554022425, + 0.5652008886197345, + 0.42614135692315014, + 0.23070316107128952, + 0.14028733366031948 + ], + "peak_layer": 21, + "peak_normalized": 3.760809922544462, + "population_cv": 0.8951816421708676, + "rest_mean": 4.249189027581856e-05, + "spike_contrast": 3.352744601078911, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014246445571188814, + "top_five_layers": [ + 21, + 22, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.943515912396833e-05, + 4.516661283560097e-05, + 5.0745726184686646e-05, + 2.9178219847381115e-05, + 3.003316669492051e-05, + 4.849797551287338e-05, + 4.288504715077579e-05, + 2.79958203464048e-05, + 2.4196080630645156e-05, + 3.113187267445028e-05, + 3.374860898475163e-05, + 3.0240842534112744e-05, + 1.5179426554823294e-05, + 1.8253909729537554e-05, + 1.4002425814396702e-05, + 6.541241600643843e-05, + 4.661780985770747e-05, + 7.60249313316308e-05, + 0.00014085091243032366, + 0.00013621727703139186, + 0.00021855045633856207, + 0.00018262369849253446, + 0.0001645523589104414, + 9.075512207346037e-05, + 5.584064274444245e-05, + 4.462891956791282e-05, + 5.064460128778592e-05, + 2.7024572773370892e-05, + 3.284529520897195e-05, + 2.476418376318179e-05, + 1.340676135441754e-05, + 8.152462214638945e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c1e7ba7b2fabbdfa228a4a3c33df9fd2cb9236dae2c5351a1ecf233f5b8dce87", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05594591051340103, + 0.07073435187339783, + 0.06750936061143875, + 0.10916222631931305, + 0.11270010471343994, + 0.14727257192134857, + 0.11801524460315704, + 0.13372685015201569, + 0.09399056434631348, + 0.10827244818210602, + 0.10214843600988388, + 0.09725081920623779, + 0.0902416780591011, + 0.09460548311471939, + 0.11164536327123642, + 0.19675607979297638, + 0.15372900664806366, + 0.10524243861436844, + 0.10666082799434662, + 0.11902136355638504, + 0.05537491664290428, + 0.06347332894802094, + 0.07961282879114151, + 0.09120825678110123, + 0.07538540661334991, + 0.07336290925741196, + 0.07823792845010757, + 0.08682125806808472, + 0.057621169835329056, + 0.0771951824426651, + 0.09140052646398544, + 0.09293081611394882 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.0007654601149625e-06, + "first_to_last_ratio": 0.21469374652673343, + "imbalance_abs_log_ratio": 1.5385427010854755, + "last_quartile_mean": 1.397695791638398e-05, + "mean": 9.062807375670445e-06, + "normalized": [ + 0.6788277855296613, + 0.33022065578659243, + 0.5185396106172449, + 0.3147344808808441, + 0.1493975773164771, + 0.16583194042116398, + 0.21736502256384724, + 0.2739446631895577, + 0.2986054592892529, + 0.2841589081967956, + 0.3703811746673963, + 0.4629251306676604, + 0.33656653078263976, + 0.3194237016069162, + 0.33703323000014157, + 0.2868358429246865, + 0.3379218451804822, + 0.44189350931307353, + 0.5391000167958766, + 0.7544514236626646, + 3.62784685261383, + 3.4055741874667143, + 2.8075043335903698, + 2.4030544923752126, + 2.128695565263529, + 2.0503888456814066, + 2.0115823114995672, + 1.62589115957208, + 1.6316185991465908, + 1.0929664148973441, + 0.9374347986692717, + 0.8592839298311081 + ], + "peak_layer": 21, + "peak_normalized": 3.62784685261383, + "population_cv": 0.9767141058146375, + "rest_mean": 5.916779522757915e-06, + "spike_contrast": 4.402962402299404, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.605135778139811e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.15208546150825e-06, + 2.9927261948614614e-06, + 4.699424607679248e-06, + 2.852377974704723e-06, + 1.3539614656110643e-06, + 1.5029029327706667e-06, + 1.9699373297044076e-06, + 2.4827077140798792e-06, + 2.7062037588621024e-06, + 2.5752774490683805e-06, + 3.3566932415851625e-06, + 4.195401288598077e-06, + 3.0502376375807216e-06, + 2.8948754788871156e-06, + 3.0544672426913166e-06, + 2.599537992864498e-06, + 3.0625205909018405e-06, + 4.0047957554634195e-06, + 4.885759608441731e-06, + 6.837447926955065e-06, + 3.287847721367143e-05, + 3.086406286456622e-05, + 2.5443870981689543e-05, + 2.1778419977636077e-05, + 1.929195786942728e-05, + 1.8582279153633863e-05, + 1.8230583009426482e-05, + 1.4735138393007219e-05, + 1.4787045074626803e-05, + 9.905344086291734e-06, + 8.495791007590014e-06, + 7.787524737068452e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.1920566521439468e-05, + "first_to_last_ratio": 0.21385698745540163, + "imbalance_abs_log_ratio": 1.5424477702795347, + "last_quartile_mean": 5.574083252213313e-05, + "mean": 3.626819514579438e-05, + "normalized": [ + 0.6714239045350167, + 0.3268984674053377, + 0.5163313168674706, + 0.31628734338757886, + 0.14365858953035543, + 0.1607028969713726, + 0.2191876706792618, + 0.2749357246554526, + 0.3063367204191196, + 0.290288819397969, + 0.37595208658458074, + 0.4631048978627428, + 0.3453629561715752, + 0.32617163789727366, + 0.3475938493090438, + 0.3029470728685741, + 0.34742911900824086, + 0.44840259940070093, + 0.5403498924783475, + 0.7515227531851133, + 3.6289752746901156, + 3.3976055612895637, + 2.7998333290588935, + 2.4034438268675102, + 2.124889978824914, + 2.0413065915513173, + 2.0135506517890525, + 1.6239375357758048, + 1.6208285992904423, + 1.0902980270168148, + 0.928318564252677, + 0.8521237409777672 + ], + "peak_layer": 21, + "peak_normalized": 3.6289752746901156, + "population_cv": 0.9738982135151035, + "rest_mean": 2.3702275703490834e-05, + "spike_contrast": 4.3930026566562255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010412416013423353, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4351333195227198e-05, + 1.185601740871789e-05, + 1.8726404960034415e-05, + 1.1471171092125587e-05, + 5.210237759456504e-06, + 5.82840402785223e-06, + 7.94954121374758e-06, + 9.971422514354344e-06, + 1.1110279956483282e-05, + 1.0528251550567802e-05, + 1.363510364171816e-05, + 1.679597880865913e-05, + 1.2525691090559121e-05, + 1.1829656614281703e-05, + 1.2606601558218244e-05, + 1.0987343557644635e-05, + 1.2600627087522298e-05, + 1.6262752978946082e-05, + 1.959751534741372e-05, + 2.7256373869022354e-05, + 0.00013161638344172388, + 0.00012322502152528614, + 0.00010154490155400708, + 8.71685697347857e-05, + 7.706592441536486e-05, + 7.403450581477955e-05, + 7.302784797502682e-05, + 5.8897283452097327e-05, + 5.8784527936950326e-05, + 3.954314161092043e-05, + 3.366843884577975e-05, + 3.0904990126146004e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.010571277292910963, + "first_to_last_ratio": 0.2138569831347789, + "imbalance_abs_log_ratio": 1.5424477904828622, + "last_quartile_mean": 0.04943152726627886, + "mean": 0.032162998293642886, + "normalized": [ + 0.6714238750017825, + 0.3268984397015616, + 0.5163313088425289, + 0.31628733258408726, + 0.14365859237299086, + 0.16070289962904324, + 0.21918766128478195, + 0.2749357162222242, + 0.3063367194831541, + 0.29028882869370426, + 0.3759520938745508, + 0.46310490191839176, + 0.345362968353024, + 0.3261716356763452, + 0.3475938513217391, + 0.302947090830093, + 0.34742911872813526, + 0.4484025830910049, + 0.5403498943254139, + 0.7515227169830379, + 3.6289754691077154, + 3.39760554441039, + 2.7998334990791727, + 2.4034437339282073, + 2.12489003938986, + 2.0413065340618877, + 2.013550612247264, + 1.6239375562448826, + 1.6208285723970963, + 1.0902980031467058, + 0.9283185200253238, + 0.8521236870439002 + ], + "peak_layer": 21, + "peak_normalized": 3.6289754691077154, + "population_cv": 0.9738982344432222, + "rest_mean": 0.021019414843370516, + "spike_contrast": 4.393002831581538, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09233834892511368, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.021595004945993423, + 0.01051403395831585, + 0.016606763005256653, + 0.010172748938202858, + 0.004620491061359644, + 0.0051686870865523815, + 0.0070497323758900166, + 0.00884275697171688, + 0.009852707386016846, + 0.009336559101939201, + 0.012091746553778648, + 0.01489484217017889, + 0.011107908561825752, + 0.010490657761693, + 0.011179660446941853, + 0.009743686765432358, + 0.011174362152814865, + 0.014421971514821053, + 0.0173792727291584, + 0.024171223863959312, + 0.11671873182058334, + 0.10927718132734299, + 0.09005104005336761, + 0.07730195671319962, + 0.06834283471107483, + 0.06565453857183456, + 0.06476182490587234, + 0.052230700850486755, + 0.05213070660829544, + 0.035067252814769745, + 0.029857506975531578, + 0.027406852692365646 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.322070665338742e-07, + "first_to_last_ratio": 0.19582650195201706, + "imbalance_abs_log_ratio": 1.630526205889656, + "last_quartile_mean": 3.7390601335118845e-06, + "mean": 2.320639892694487e-06, + "normalized": [ + 0.6549870344311939, + 0.3169062144097135, + 0.5191630414175221, + 0.3243434965301332, + 0.105717119181206, + 0.15182526527542842, + 0.19569754579298845, + 0.2555160344471036, + 0.23140272407874934, + 0.23882207606587388, + 0.31847255362240917, + 0.41661698388421226, + 0.3042701489276803, + 0.32496723001951616, + 0.2926718756416169, + 0.14809689981232488, + 0.29560203174175614, + 0.3094570768459675, + 0.46180841517164245, + 0.6023706663610227, + 3.6555712774602895, + 3.5328017931896913, + 2.9040888216519436, + 2.54906806729493, + 2.239269068685756, + 2.2005088234873713, + 2.152636739293442, + 1.694221234427147, + 1.6655655501398623, + 1.1343787866859985, + 0.9462733663708415, + 0.8569020376546662 + ], + "peak_layer": 21, + "peak_normalized": 3.6555712774602895, + "population_cv": 1.0356066456325392, + "rest_mean": 1.471388915037837e-06, + "spike_contrast": 4.6939290499295305, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.9065951720403975e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.519989041298686e-06, + 7.354252034019737e-07, + 1.204790464726102e-06, + 7.526844569838431e-07, + 2.453313641126442e-07, + 3.5233176731708227e-07, + 4.541435316696152e-07, + 5.929607027610473e-07, + 5.370023927753209e-07, + 5.542200369745842e-07, + 7.390601126644469e-07, + 9.668179927757592e-07, + 7.061014457576675e-07, + 7.541319178017147e-07, + 6.791860300836561e-07, + 3.436795736888598e-07, + 6.859858672214614e-07, + 7.181384376053757e-07, + 1.0716910310293315e-06, + 1.3978853985463502e-06, + 8.483264537062496e-06, + 8.198360774258617e-06, + 6.739344371453626e-06, + 5.91546904615825e-06, + 5.196537131268997e-06, + 5.106588560011005e-06, + 4.995494691684144e-06, + 3.931677383661736e-06, + 3.865177859552205e-06, + 2.6324846658098977e-06, + 2.1959597233944805e-06, + 1.988561052712612e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 8.062098231675918e-06, + "first_to_last_ratio": 0.23172405564077758, + "imbalance_abs_log_ratio": 1.4622080305731233, + "last_quartile_mean": 3.479180531940074e-05, + "mean": 2.387608699905286e-05, + "normalized": [ + 0.7258751481874687, + 0.3420421656378502, + 0.5451754824595273, + 0.28689678416588577, + 0.14555997121980288, + 0.15910389252688525, + 0.22744868868375562, + 0.26921093638205285, + 0.331277324094237, + 0.3124442700204022, + 0.40511967495320234, + 0.5118199353326568, + 0.40767330553010017, + 0.38094082583923083, + 0.3799217810408602, + 0.2343632382817309, + 0.37127597519882294, + 0.47558084035172465, + 0.5644703292999084, + 0.7840592083426426, + 3.8152513101159804, + 3.5348515650624037, + 2.7948696249240457, + 2.3373113491061237, + 2.169069307958383, + 1.9545846045943898, + 1.8833096737193575, + 1.503620407983867, + 1.520809170499446, + 0.9931364071654021, + 0.8480889104968432, + 0.7848378908250101 + ], + "peak_layer": 21, + "peak_normalized": 3.8152513101159804, + "population_cv": 0.9831166322047221, + "rest_mean": 1.5341400049456518e-05, + "spike_contrast": 4.560431010294372, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.996339652687311e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.733105818857439e-05, + 8.16662850411376e-06, + 1.3016657248954289e-05, + 6.84997257849318e-06, + 3.475402536423644e-06, + 3.7987783798598684e-06, + 5.4305846788338386e-06, + 6.427703738154378e-06, + 7.909606210887432e-06, + 7.459946573362686e-06, + 9.672672604210675e-06, + 1.2220257303852122e-05, + 9.733643310028128e-06, + 9.09537629922852e-06, + 9.07104549696669e-06, + 5.595677066594362e-06, + 8.864617484505288e-06, + 1.1355009519320447e-05, + 1.3477342690748628e-05, + 1.8720265870797448e-05, + 9.109327220357955e-05, + 8.439842349616811e-05, + 6.673055031569675e-05, + 5.580584911513142e-05, + 5.178888750378974e-05, + 4.666783206630498e-05, + 4.4966065615881234e-05, + 3.5900571674574167e-05, + 3.631097206380218e-05, + 2.3712211259407923e-05, + 2.024904460995458e-05, + 1.873885776149109e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 5.398108925191991e-06, + "first_to_last_ratio": 0.2553243719405187, + "imbalance_abs_log_ratio": 1.3652204953642635, + "last_quartile_mean": 2.114216079007747e-05, + "mean": 1.5431339967619806e-05, + "normalized": [ + 0.775450361876563, + 0.35993336560900263, + 0.5716811811488363, + 0.2647889720822481, + 0.15443931558124985, + 0.1609106807603714, + 0.24114708063049767, + 0.27016631249985124, + 0.3542374939344869, + 0.3341618028764158, + 0.4390379760950059, + 0.5611696202016426, + 0.4905651293202641, + 0.44854876717590725, + 0.427299912149135, + 0.17007282318732395, + 0.4083388005340233, + 0.5099761158699698, + 0.6066217231343173, + 0.8107855537457127, + 4.065115409247974, + 3.706799247664954, + 2.7077946969814244, + 2.2003225686125982, + 2.1672422326648038, + 1.812476034013852, + 1.6953296147818004, + 1.3306288007797171, + 1.4218486794014777, + 0.9438307759525454, + 0.8255782054336522, + 0.7637007460523768 + ], + "peak_layer": 21, + "peak_normalized": 4.065115409247974, + "population_cv": 0.9906146212599648, + "rest_mean": 9.80331644010827e-06, + "spike_contrast": 4.674200541839902, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.58226670161821e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1966238162131049e-05, + 5.554254130402114e-06, + 8.821806659398135e-06, + 4.08604864787776e-06, + 2.3832055831007892e-06, + 2.4830674192344304e-06, + 3.7212225834082346e-06, + 4.169028215983417e-06, + 5.466359198180726e-06, + 5.156564384378726e-06, + 6.774944267817773e-06, + 8.659599188831635e-06, + 7.57007728680037e-06, + 6.921708518348169e-06, + 6.593810212507378e-06, + 2.6244515538564883e-06, + 6.301214853010606e-06, + 7.869614819355775e-06, + 9.360986041428987e-06, + 1.2511507520684972e-05, + 6.27301778877154e-05, + 5.720087938243523e-05, + 4.1784900531638414e-05, + 3.395392559468746e-05, + 3.344345168443397e-05, + 2.7968933864030987e-05, + 2.6161207642871886e-05, + 2.0533385395538062e-05, + 2.1941030354355462e-05, + 1.4564573575626127e-05, + 1.2739777957904153e-05, + 1.1784925845859107e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.3460118995899393e-05, + "first_to_last_ratio": 0.19614811350083294, + "imbalance_abs_log_ratio": 1.6288852240081404, + "last_quartile_mean": 0.00011960410211031558, + "mean": 7.52558852070706e-05, + "normalized": [ + 0.658313079608564, + 0.30463294507451294, + 0.5014382981404909, + 0.28041349059373355, + 0.1296990900515029, + 0.1558393800657072, + 0.20228299557148446, + 0.2612847052904527, + 0.3002182827700767, + 0.28296321216930564, + 0.37117751666594523, + 0.4617148213254951, + 0.32553402471276316, + 0.31924765387609977, + 0.3311215231692876, + 0.2688995234151016, + 0.3256293540572768, + 0.42908371238624793, + 0.5189360670040057, + 0.7298065590677655, + 3.5899584767580643, + 3.2958212345047615, + 2.8004713972940176, + 2.44112076202934, + 2.203847377095527, + 2.1149132200283227, + 2.164777621743948, + 1.7470912286112248, + 1.6405526614360855, + 1.0907160128127615, + 0.9220261935993064, + 0.8304675790708215 + ], + "peak_layer": 21, + "peak_normalized": 3.5899584767580643, + "population_cv": 0.985057448082149, + "rest_mean": 4.9247397631309774e-05, + "spike_contrast": 4.379961754142385, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002157017181161791, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.9541933549335226e-05, + 2.292542194481939e-05, + 3.7736183003289625e-05, + 2.1102765458635986e-05, + 9.760619832377415e-06, + 1.1727830496965908e-05, + 1.5222985894070007e-05, + 1.9663211787701584e-05, + 2.2593192625208758e-05, + 2.129464701283723e-05, + 2.793329258565791e-05, + 3.474675759207457e-05, + 2.449835119477939e-05, + 2.4025264792726375e-05, + 2.4918843337218277e-05, + 2.023627166636288e-05, + 2.4505525288986973e-05, + 3.2291074603563175e-05, + 3.9052993088262156e-05, + 5.492223863257095e-05, + 0.00027016550302505493, + 0.00024802994448691607, + 0.00021075195400044322, + 0.00018370870384387672, + 0.0001658524852246046, + 0.00015915966650936753, + 0.00016291225620079786, + 0.00013147889694664627, + 0.00012346124276518822, + 8.208279905375093e-05, + 6.938789738342166e-05, + 6.249757279874757e-05 + ] + } + } + } + } + } + }, + "step": 4000 + }, + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 1.877417995231997, + "forward": { + "activation_sha256": { + "attention_branch_output": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", + "mlp_branch_output": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", + "post_attention_state": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", + "post_mlp_state": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", + "pre_attention_input": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", + "pre_mlp_input": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a" + }, + "logits_sha256": "5f6747d51014b7e09e13dda43380fde8b7ba2972518bba8957bd86ff4ed7f868", + "loss_nats": 1.3013269901275635, + "mixer_summary_sha256": "9abaf96810a98e69acfbc813a9b903aa15d6279ddc462a3c26f1a1846711526f" + }, + "loss_nats": 1.3013269901275635, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.2200722098350525, + "group": 1, + "latest_source_mass": 0.06159520894289017, + "layer": 1, + "max_source_mass": 0.9384047985076904, + "mixer_index": 1, + "normalized_entropy": 0.3174970857665056, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9384047985076904, + "median": 0.949082612991333, + "p05": 0.8598991632461548, + "p95": 0.9761404991149902 + }, + { + "label": "current_group_partial", + "mean": 0.06159520894289017, + "median": 0.05091731622815132, + "p05": 0.02385946735739708, + "p95": 0.14010077714920044 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5651201605796814, + "group": 1, + "latest_source_mass": 0.3442652225494385, + "layer": 2, + "max_source_mass": 0.6557347774505615, + "mixer_index": 2, + "normalized_entropy": 0.815296053174681, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6557347774505615, + "median": 0.6636689305305481, + "p05": 0.3472050428390503, + "p95": 0.9190996289253235 + }, + { + "label": "current_group_partial", + "mean": 0.3442652225494385, + "median": 0.3363310396671295, + "p05": 0.08090042322874069, + "p95": 0.6527949571609497 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4018177092075348, + "group": 1, + "latest_source_mass": 0.14357808232307434, + "layer": 2, + "max_source_mass": 0.856421947479248, + "mixer_index": 3, + "normalized_entropy": 0.5797004164150741, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.856421947479248, + "median": 0.8660546541213989, + "p05": 0.7540444135665894, + "p95": 0.9155591726303101 + }, + { + "label": "current_group_partial", + "mean": 0.14357808232307434, + "median": 0.13394534587860107, + "p05": 0.08444082736968994, + "p95": 0.24595558643341064 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.687358021736145, + "group": 1, + "latest_source_mass": 0.5312775373458862, + "layer": 3, + "max_source_mass": 0.5312775373458862, + "mixer_index": 4, + "normalized_entropy": 0.9916480092739848, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46872249245643616, + "median": 0.4674533009529114, + "p05": 0.3992259204387665, + "p95": 0.5431661605834961 + }, + { + "label": "current_group_partial", + "mean": 0.5312775373458862, + "median": 0.5325466990470886, + "p05": 0.45683377981185913, + "p95": 0.6007740497589111 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.2678314447402954, + "group": 1, + "latest_source_mass": 0.07946884632110596, + "layer": 3, + "max_source_mass": 0.920531153678894, + "mixer_index": 5, + "normalized_entropy": 0.38639909712095066, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.920531153678894, + "median": 0.9314625263214111, + "p05": 0.8444089889526367, + "p95": 0.9650533199310303 + }, + { + "label": "current_group_partial", + "mean": 0.07946884632110596, + "median": 0.06853749603033066, + "p05": 0.03494669869542122, + "p95": 0.1555909812450409 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.657238245010376, + "group": 1, + "latest_source_mass": 0.3777551054954529, + "layer": 4, + "max_source_mass": 0.6222448348999023, + "mixer_index": 6, + "normalized_entropy": 0.948194356759035, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6222448348999023, + "median": 0.6208515167236328, + "p05": 0.5314525961875916, + "p95": 0.7065955400466919 + }, + { + "label": "current_group_partial", + "mean": 0.3777551054954529, + "median": 0.3791484832763672, + "p05": 0.2934044897556305, + "p95": 0.4685474634170532 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5044373869895935, + "group": 1, + "latest_source_mass": 0.2207653522491455, + "layer": 4, + "max_source_mass": 0.7792346477508545, + "mixer_index": 7, + "normalized_entropy": 0.7277493166488734, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7792346477508545, + "median": 0.7866228222846985, + "p05": 0.6320486664772034, + "p95": 0.9385759830474854 + }, + { + "label": "current_group_partial", + "mean": 0.2207653522491455, + "median": 0.21337716281414032, + "p05": 0.06142403930425644, + "p95": 0.367951363325119 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.650418758392334, + "group": 2, + "latest_source_mass": 0.4602207541465759, + "layer": 5, + "max_source_mass": 0.5397792458534241, + "mixer_index": 8, + "normalized_entropy": 0.9383559172337771, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5397792458534241, + "median": 0.5213949680328369, + "p05": 0.34510529041290283, + "p95": 0.7807042598724365 + }, + { + "label": "completed_group_1", + "mean": 0.4602207541465759, + "median": 0.4786049723625183, + "p05": 0.2192957103252411, + "p95": 0.6548947095870972 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0032464265823364, + "group": 2, + "latest_source_mass": 0.37668097019195557, + "layer": 5, + "max_source_mass": 0.4337552785873413, + "mixer_index": 9, + "normalized_entropy": 0.913194251448444, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4337552785873413, + "median": 0.42766237258911133, + "p05": 0.2500072717666626, + "p95": 0.6592159271240234 + }, + { + "label": "completed_group_1", + "mean": 0.1895637810230255, + "median": 0.18978442251682281, + "p05": 0.1284591555595398, + "p95": 0.25696656107902527 + }, + { + "label": "current_group_partial", + "mean": 0.37668097019195557, + "median": 0.3803442120552063, + "p05": 0.1759825050830841, + "p95": 0.568291425704956 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0385558605194092, + "group": 2, + "latest_source_mass": 0.4949595034122467, + "layer": 6, + "max_source_mass": 0.4949595034122467, + "mixer_index": 10, + "normalized_entropy": 0.9453342832879565, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24478834867477417, + "median": 0.23865866661071777, + "p05": 0.19890648126602173, + "p95": 0.32154381275177 + }, + { + "label": "completed_group_1", + "mean": 0.26025211811065674, + "median": 0.26050686836242676, + "p05": 0.23401768505573273, + "p95": 0.2860541343688965 + }, + { + "label": "current_group_partial", + "mean": 0.4949595034122467, + "median": 0.5017017722129822, + "p05": 0.42678141593933105, + "p95": 0.5358227491378784 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.031773328781128, + "group": 2, + "latest_source_mass": 0.33116668462753296, + "layer": 6, + "max_source_mass": 0.40370824933052063, + "mixer_index": 11, + "normalized_entropy": 0.9391605568439314, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40370824933052063, + "median": 0.38690465688705444, + "p05": 0.18056227266788483, + "p95": 0.7100913524627686 + }, + { + "label": "completed_group_1", + "mean": 0.2651250660419464, + "median": 0.2730395793914795, + "p05": 0.12547528743743896, + "p95": 0.35344114899635315 + }, + { + "label": "current_group_partial", + "mean": 0.33116668462753296, + "median": 0.33928292989730835, + "p05": 0.1502782255411148, + "p95": 0.47667479515075684 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0838522911071777, + "group": 2, + "latest_source_mass": 0.38127952814102173, + "layer": 7, + "max_source_mass": 0.38127952814102173, + "mixer_index": 12, + "normalized_entropy": 0.9865648712351232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35612422227859497, + "median": 0.36401283740997314, + "p05": 0.2946957051753998, + "p95": 0.39828816056251526 + }, + { + "label": "completed_group_1", + "mean": 0.2625962495803833, + "median": 0.2590169608592987, + "p05": 0.24180665612220764, + "p95": 0.2929879426956177 + }, + { + "label": "current_group_partial", + "mean": 0.38127952814102173, + "median": 0.37846946716308594, + "p05": 0.3489896059036255, + "p95": 0.41648316383361816 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.96053147315979, + "group": 2, + "latest_source_mass": 0.3146248161792755, + "layer": 7, + "max_source_mass": 0.5271762013435364, + "mixer_index": 13, + "normalized_entropy": 0.874313425279704, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5271762013435364, + "median": 0.5114221572875977, + "p05": 0.3548204302787781, + "p95": 0.7728348970413208 + }, + { + "label": "completed_group_1", + "mean": 0.1581989824771881, + "median": 0.1618250608444214, + "p05": 0.07541970163583755, + "p95": 0.21691158413887024 + }, + { + "label": "current_group_partial", + "mean": 0.3146248161792755, + "median": 0.32365772128105164, + "p05": 0.14578065276145935, + "p95": 0.4483411908149719 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0704245567321777, + "group": 2, + "latest_source_mass": 0.38260507583618164, + "layer": 8, + "max_source_mass": 0.38260507583618164, + "mixer_index": 14, + "normalized_entropy": 0.9743424206822726, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3788670599460602, + "median": 0.39538437128067017, + "p05": 0.27489906549453735, + "p95": 0.43731769919395447 + }, + { + "label": "completed_group_1", + "mean": 0.23852786421775818, + "median": 0.2294245958328247, + "p05": 0.20573674142360687, + "p95": 0.29735344648361206 + }, + { + "label": "current_group_partial", + "mean": 0.38260507583618164, + "median": 0.37669745087623596, + "p05": 0.34335845708847046, + "p95": 0.435106486082077 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9674104452133179, + "group": 2, + "latest_source_mass": 0.3281455934047699, + "layer": 8, + "max_source_mass": 0.48829177021980286, + "mixer_index": 15, + "normalized_entropy": 0.8805749354816949, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48829177021980286, + "median": 0.44956350326538086, + "p05": 0.2823859453201294, + "p95": 0.8209846615791321 + }, + { + "label": "completed_group_1", + "mean": 0.18356263637542725, + "median": 0.18210050463676453, + "p05": 0.05149933695793152, + "p95": 0.2906108498573303 + }, + { + "label": "current_group_partial", + "mean": 0.3281455934047699, + "median": 0.34843379259109497, + "p05": 0.12376503646373749, + "p95": 0.46937939524650574 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0832805633544922, + "group": 3, + "latest_source_mass": 0.31498202681541443, + "layer": 9, + "max_source_mass": 0.40388280153274536, + "mixer_index": 16, + "normalized_entropy": 0.9860444622076776, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40388280153274536, + "median": 0.403384804725647, + "p05": 0.3489384353160858, + "p95": 0.4601682126522064 + }, + { + "label": "completed_group_1", + "mean": 0.2811351716518402, + "median": 0.28186845779418945, + "p05": 0.25417307019233704, + "p95": 0.30605652928352356 + }, + { + "label": "completed_group_2", + "mean": 0.31498202681541443, + "median": 0.3153840899467468, + "p05": 0.2613564431667328, + "p95": 0.3658811151981354 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2917464971542358, + "group": 3, + "latest_source_mass": 0.33769795298576355, + "layer": 9, + "max_source_mass": 0.33769795298576355, + "mixer_index": 17, + "normalized_entropy": 0.9317981327650527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3349480926990509, + "median": 0.3352771997451782, + "p05": 0.18474003672599792, + "p95": 0.48752525448799133 + }, + { + "label": "completed_group_1", + "mean": 0.1244347020983696, + "median": 0.11896269023418427, + "p05": 0.07986846566200256, + "p95": 0.1843445897102356 + }, + { + "label": "completed_group_2", + "mean": 0.20291924476623535, + "median": 0.20060715079307556, + "p05": 0.1253921389579773, + "p95": 0.2836313247680664 + }, + { + "label": "current_group_partial", + "mean": 0.33769795298576355, + "median": 0.33908897638320923, + "p05": 0.26979348063468933, + "p95": 0.4016563892364502 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3533791303634644, + "group": 3, + "latest_source_mass": 0.28473395109176636, + "layer": 10, + "max_source_mass": 0.32115638256073, + "mixer_index": 18, + "normalized_entropy": 0.976256679908994, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32115638256073, + "median": 0.3355404734611511, + "p05": 0.22348493337631226, + "p95": 0.3863494396209717 + }, + { + "label": "completed_group_1", + "mean": 0.20602071285247803, + "median": 0.2052205502986908, + "p05": 0.1802767962217331, + "p95": 0.23218519985675812 + }, + { + "label": "completed_group_2", + "mean": 0.18808895349502563, + "median": 0.18706050515174866, + "p05": 0.16633892059326172, + "p95": 0.2130376398563385 + }, + { + "label": "current_group_partial", + "mean": 0.28473395109176636, + "median": 0.275795578956604, + "p05": 0.22748424112796783, + "p95": 0.3660258948802948 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.308367371559143, + "group": 3, + "latest_source_mass": 0.29719752073287964, + "layer": 10, + "max_source_mass": 0.3113970160484314, + "mixer_index": 19, + "normalized_entropy": 0.9437875593046517, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3113970160484314, + "median": 0.33913683891296387, + "p05": 0.0915825292468071, + "p95": 0.47554588317871094 + }, + { + "label": "completed_group_1", + "mean": 0.16179001331329346, + "median": 0.16501447558403015, + "p05": 0.10682816803455353, + "p95": 0.20702867209911346 + }, + { + "label": "completed_group_2", + "mean": 0.2296154499053955, + "median": 0.2261994481086731, + "p05": 0.14510810375213623, + "p95": 0.32595163583755493 + }, + { + "label": "current_group_partial", + "mean": 0.29719752073287964, + "median": 0.2761377692222595, + "p05": 0.194716677069664, + "p95": 0.4515363574028015 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3437637090682983, + "group": 3, + "latest_source_mass": 0.3098427951335907, + "layer": 11, + "max_source_mass": 0.3217787742614746, + "mixer_index": 20, + "normalized_entropy": 0.969320619599697, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3217787742614746, + "median": 0.31754863262176514, + "p05": 0.25967735052108765, + "p95": 0.40468457341194153 + }, + { + "label": "completed_group_1", + "mean": 0.1920534074306488, + "median": 0.1915990114212036, + "p05": 0.16391108930110931, + "p95": 0.22057634592056274 + }, + { + "label": "completed_group_2", + "mean": 0.1763250231742859, + "median": 0.1779440939426422, + "p05": 0.14714840054512024, + "p95": 0.1996835470199585 + }, + { + "label": "current_group_partial", + "mean": 0.3098427951335907, + "median": 0.31089526414871216, + "p05": 0.2459525167942047, + "p95": 0.36900240182876587 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2280659675598145, + "group": 3, + "latest_source_mass": 0.29383569955825806, + "layer": 11, + "max_source_mass": 0.42534634470939636, + "mixer_index": 21, + "normalized_entropy": 0.8858623406415255, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42534634470939636, + "median": 0.4574331045150757, + "p05": 0.17354995012283325, + "p95": 0.6023476123809814 + }, + { + "label": "completed_group_1", + "mean": 0.14848899841308594, + "median": 0.14150214195251465, + "p05": 0.07509882748126984, + "p95": 0.23531809449195862 + }, + { + "label": "completed_group_2", + "mean": 0.13232898712158203, + "median": 0.13064970076084137, + "p05": 0.07000484317541122, + "p95": 0.19124652445316315 + }, + { + "label": "current_group_partial", + "mean": 0.29383569955825806, + "median": 0.27858495712280273, + "p05": 0.187642902135849, + "p95": 0.4577033817768097 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3462648391723633, + "group": 3, + "latest_source_mass": 0.3101534843444824, + "layer": 12, + "max_source_mass": 0.3221249580383301, + "mixer_index": 22, + "normalized_entropy": 0.9711248035985732, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3221249580383301, + "median": 0.3120178282260895, + "p05": 0.2763444483280182, + "p95": 0.3882659375667572 + }, + { + "label": "completed_group_1", + "mean": 0.18557845056056976, + "median": 0.1851869523525238, + "p05": 0.16533124446868896, + "p95": 0.20681798458099365 + }, + { + "label": "completed_group_2", + "mean": 0.18214309215545654, + "median": 0.18278372287750244, + "p05": 0.15564095973968506, + "p95": 0.2072875201702118 + }, + { + "label": "current_group_partial", + "mean": 0.3101534843444824, + "median": 0.31112557649612427, + "p05": 0.2608441412448883, + "p95": 0.36039745807647705 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.147221326828003, + "group": 3, + "latest_source_mass": 0.2585829496383667, + "layer": 12, + "max_source_mass": 0.5053804516792297, + "mixer_index": 23, + "normalized_entropy": 0.8275452595084083, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5053804516792297, + "median": 0.48097139596939087, + "p05": 0.30125147104263306, + "p95": 0.7477225065231323 + }, + { + "label": "completed_group_1", + "mean": 0.12089012563228607, + "median": 0.11902710795402527, + "p05": 0.05662901699542999, + "p95": 0.1844676434993744 + }, + { + "label": "completed_group_2", + "mean": 0.11514648050069809, + "median": 0.11994394659996033, + "p05": 0.05982710421085358, + "p95": 0.16130289435386658 + }, + { + "label": "current_group_partial", + "mean": 0.2585829496383667, + "median": 0.25007814168930054, + "p05": 0.12433226406574249, + "p95": 0.42273372411727905 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3729398250579834, + "group": 4, + "latest_source_mass": 0.22271886467933655, + "layer": 13, + "max_source_mass": 0.30785465240478516, + "mixer_index": 24, + "normalized_entropy": 0.9903667385250569, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30785465240478516, + "median": 0.3129041790962219, + "p05": 0.25217515230178833, + "p95": 0.3537779450416565 + }, + { + "label": "completed_group_1", + "mean": 0.23079943656921387, + "median": 0.22906282544136047, + "p05": 0.19741898775100708, + "p95": 0.26931890845298767 + }, + { + "label": "completed_group_2", + "mean": 0.23862707614898682, + "median": 0.23815912008285522, + "p05": 0.21215641498565674, + "p95": 0.2663886845111847 + }, + { + "label": "completed_group_3", + "mean": 0.22271886467933655, + "median": 0.22075486183166504, + "p05": 0.186451256275177, + "p95": 0.2638319730758667 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5232064723968506, + "group": 4, + "latest_source_mass": 0.24461600184440613, + "layer": 13, + "max_source_mass": 0.3136816620826721, + "mixer_index": 25, + "normalized_entropy": 0.9464213938474744, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3136816620826721, + "median": 0.326117604970932, + "p05": 0.20238417387008667, + "p95": 0.4362232983112335 + }, + { + "label": "completed_group_1", + "mean": 0.11285123229026794, + "median": 0.11373943835496902, + "p05": 0.08906605839729309, + "p95": 0.13395145535469055 + }, + { + "label": "completed_group_2", + "mean": 0.13196846842765808, + "median": 0.1334267258644104, + "p05": 0.10183953493833542, + "p95": 0.16085752844810486 + }, + { + "label": "completed_group_3", + "mean": 0.19688265025615692, + "median": 0.1940557211637497, + "p05": 0.13830327987670898, + "p95": 0.2671412229537964 + }, + { + "label": "current_group_partial", + "mean": 0.24461600184440613, + "median": 0.2419893592596054, + "p05": 0.18994617462158203, + "p95": 0.3077915906906128 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4876129627227783, + "group": 4, + "latest_source_mass": 0.396677166223526, + "layer": 14, + "max_source_mass": 0.396677166223526, + "mixer_index": 26, + "normalized_entropy": 0.9243059028433878, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2035508155822754, + "median": 0.20008960366249084, + "p05": 0.16096048057079315, + "p95": 0.255715548992157 + }, + { + "label": "completed_group_1", + "mean": 0.15372203290462494, + "median": 0.15584711730480194, + "p05": 0.12048184871673584, + "p95": 0.18010932207107544 + }, + { + "label": "completed_group_2", + "mean": 0.13167491555213928, + "median": 0.13113251328468323, + "p05": 0.11609883606433868, + "p95": 0.14900363981723785 + }, + { + "label": "completed_group_3", + "mean": 0.11437506973743439, + "median": 0.11201842874288559, + "p05": 0.09515875577926636, + "p95": 0.1399364471435547 + }, + { + "label": "current_group_partial", + "mean": 0.396677166223526, + "median": 0.39533013105392456, + "p05": 0.3409242630004883, + "p95": 0.4550648033618927 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4587602615356445, + "group": 4, + "latest_source_mass": 0.33461427688598633, + "layer": 14, + "max_source_mass": 0.33461427688598633, + "mixer_index": 27, + "normalized_entropy": 0.906378711639412, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28519001603126526, + "median": 0.28675830364227295, + "p05": 0.1500462144613266, + "p95": 0.45491817593574524 + }, + { + "label": "completed_group_1", + "mean": 0.10532153397798538, + "median": 0.09640030562877655, + "p05": 0.06345808506011963, + "p95": 0.1692449301481247 + }, + { + "label": "completed_group_2", + "mean": 0.09675674885511398, + "median": 0.09718051552772522, + "p05": 0.058796193450689316, + "p95": 0.13363884389400482 + }, + { + "label": "completed_group_3", + "mean": 0.17811740934848785, + "median": 0.1759425848722458, + "p05": 0.1284305453300476, + "p95": 0.23726706206798553 + }, + { + "label": "current_group_partial", + "mean": 0.33461427688598633, + "median": 0.33546411991119385, + "p05": 0.23162710666656494, + "p95": 0.4484652876853943 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4565696716308594, + "group": 4, + "latest_source_mass": 0.4221859574317932, + "layer": 15, + "max_source_mass": 0.4221859574317932, + "mixer_index": 28, + "normalized_entropy": 0.9050176216042753, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20232918858528137, + "median": 0.1998670995235443, + "p05": 0.15170496702194214, + "p95": 0.26285088062286377 + }, + { + "label": "completed_group_1", + "mean": 0.14928826689720154, + "median": 0.14749380946159363, + "p05": 0.12863489985466003, + "p95": 0.17874351143836975 + }, + { + "label": "completed_group_2", + "mean": 0.12029382586479187, + "median": 0.11950792372226715, + "p05": 0.10211217403411865, + "p95": 0.14088989794254303 + }, + { + "label": "completed_group_3", + "mean": 0.1059027910232544, + "median": 0.10411787033081055, + "p05": 0.08704904466867447, + "p95": 0.13138693571090698 + }, + { + "label": "current_group_partial", + "mean": 0.4221859574317932, + "median": 0.4195709228515625, + "p05": 0.35523390769958496, + "p95": 0.4969034492969513 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4788098335266113, + "group": 4, + "latest_source_mass": 0.34753501415252686, + "layer": 15, + "max_source_mass": 0.34753501415252686, + "mixer_index": 29, + "normalized_entropy": 0.9188362111403675, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2697942852973938, + "median": 0.25651901960372925, + "p05": 0.146310955286026, + "p95": 0.42084938287734985 + }, + { + "label": "completed_group_1", + "mean": 0.11948063224554062, + "median": 0.11549928039312363, + "p05": 0.0849025622010231, + "p95": 0.16021209955215454 + }, + { + "label": "completed_group_2", + "mean": 0.11090938746929169, + "median": 0.11184529960155487, + "p05": 0.08139486610889435, + "p95": 0.13886672258377075 + }, + { + "label": "completed_group_3", + "mean": 0.15228068828582764, + "median": 0.15164361894130707, + "p05": 0.11261995136737823, + "p95": 0.19522227346897125 + }, + { + "label": "current_group_partial", + "mean": 0.34753501415252686, + "median": 0.3453425168991089, + "p05": 0.25332537293434143, + "p95": 0.44237542152404785 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5018762350082397, + "group": 4, + "latest_source_mass": 0.31088152527809143, + "layer": 16, + "max_source_mass": 0.31088152527809143, + "mixer_index": 30, + "normalized_entropy": 0.9331681721954809, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.208971306681633, + "median": 0.20881521701812744, + "p05": 0.16508029401302338, + "p95": 0.2516583502292633 + }, + { + "label": "completed_group_1", + "mean": 0.27331358194351196, + "median": 0.26948875188827515, + "p05": 0.23055149614810944, + "p95": 0.33120542764663696 + }, + { + "label": "completed_group_2", + "mean": 0.12766750156879425, + "median": 0.1261082887649536, + "p05": 0.10924770683050156, + "p95": 0.1494429111480713 + }, + { + "label": "completed_group_3", + "mean": 0.07916608452796936, + "median": 0.07811431586742401, + "p05": 0.0561223030090332, + "p95": 0.10760308802127838 + }, + { + "label": "current_group_partial", + "mean": 0.31088152527809143, + "median": 0.3123162090778351, + "p05": 0.2707694172859192, + "p95": 0.3467172384262085 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5627843141555786, + "group": 4, + "latest_source_mass": 0.22652018070220947, + "layer": 16, + "max_source_mass": 0.27452588081359863, + "mixer_index": 31, + "normalized_entropy": 0.9710124895666443, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27452588081359863, + "median": 0.2678954601287842, + "p05": 0.19414913654327393, + "p95": 0.3916982412338257 + }, + { + "label": "completed_group_1", + "mean": 0.21448183059692383, + "median": 0.21617943048477173, + "p05": 0.17793960869312286, + "p95": 0.24279263615608215 + }, + { + "label": "completed_group_2", + "mean": 0.143874391913414, + "median": 0.1446641981601715, + "p05": 0.11340779811143875, + "p95": 0.17275942862033844 + }, + { + "label": "completed_group_3", + "mean": 0.14059773087501526, + "median": 0.13659776747226715, + "p05": 0.11039599776268005, + "p95": 0.18301524221897125 + }, + { + "label": "current_group_partial", + "mean": 0.22652018070220947, + "median": 0.22071748971939087, + "p05": 0.16668984293937683, + "p95": 0.293569415807724 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.580754280090332, + "group": 5, + "latest_source_mass": 0.24107450246810913, + "layer": 17, + "max_source_mass": 0.2518634498119354, + "mixer_index": 32, + "normalized_entropy": 0.9821778571747528, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2518634498119354, + "median": 0.24593469500541687, + "p05": 0.21242231130599976, + "p95": 0.3016672730445862 + }, + { + "label": "completed_group_1", + "mean": 0.18758200109004974, + "median": 0.1801876276731491, + "p05": 0.15723729133605957, + "p95": 0.24892441928386688 + }, + { + "label": "completed_group_2", + "mean": 0.13903990387916565, + "median": 0.13839681446552277, + "p05": 0.12513971328735352, + "p95": 0.15491701662540436 + }, + { + "label": "completed_group_3", + "mean": 0.18044015765190125, + "median": 0.1819951832294464, + "p05": 0.14810611307621002, + "p95": 0.20801132917404175 + }, + { + "label": "completed_group_4", + "mean": 0.24107450246810913, + "median": 0.24384509027004242, + "p05": 0.1871698796749115, + "p95": 0.2886185050010681 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7564777135849, + "group": 5, + "latest_source_mass": 0.2205006182193756, + "layer": 17, + "max_source_mass": 0.2205006182193756, + "mixer_index": 33, + "normalized_entropy": 0.9803088772521708, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15440437197685242, + "median": 0.15525281429290771, + "p05": 0.11778812110424042, + "p95": 0.18865147233009338 + }, + { + "label": "completed_group_1", + "mean": 0.19411899149417877, + "median": 0.2002902328968048, + "p05": 0.14482440054416656, + "p95": 0.2256385236978531 + }, + { + "label": "completed_group_2", + "mean": 0.17806550860404968, + "median": 0.1781877875328064, + "p05": 0.15101563930511475, + "p95": 0.20418882369995117 + }, + { + "label": "completed_group_3", + "mean": 0.14696575701236725, + "median": 0.14620032906532288, + "p05": 0.10836094617843628, + "p95": 0.1874053031206131 + }, + { + "label": "completed_group_4", + "mean": 0.10594475269317627, + "median": 0.10578415542840958, + "p05": 0.08951808512210846, + "p95": 0.1216079369187355 + }, + { + "label": "current_group_partial", + "mean": 0.2205006182193756, + "median": 0.21638520061969757, + "p05": 0.16007332503795624, + "p95": 0.2944811284542084 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5899518728256226, + "group": 5, + "latest_source_mass": 0.4157395362854004, + "layer": 18, + "max_source_mass": 0.4157395362854004, + "mixer_index": 34, + "normalized_entropy": 0.8873690359290373, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1636897772550583, + "median": 0.16218292713165283, + "p05": 0.1178273856639862, + "p95": 0.21766462922096252 + }, + { + "label": "completed_group_1", + "mean": 0.11789833009243011, + "median": 0.11790023744106293, + "p05": 0.1032494530081749, + "p95": 0.13267967104911804 + }, + { + "label": "completed_group_2", + "mean": 0.07030131667852402, + "median": 0.0698043555021286, + "p05": 0.06258689612150192, + "p95": 0.07938572019338608 + }, + { + "label": "completed_group_3", + "mean": 0.09238425642251968, + "median": 0.09013497084379196, + "p05": 0.0768275037407875, + "p95": 0.1164383590221405 + }, + { + "label": "completed_group_4", + "mean": 0.1399867981672287, + "median": 0.1400289535522461, + "p05": 0.12679682672023773, + "p95": 0.15317322313785553 + }, + { + "label": "current_group_partial", + "mean": 0.4157395362854004, + "median": 0.4171099066734314, + "p05": 0.367803692817688, + "p95": 0.4605288803577423 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.2894535064697266, + "group": 5, + "latest_source_mass": 0.5977407693862915, + "layer": 18, + "max_source_mass": 0.5977407693862915, + "mixer_index": 35, + "normalized_entropy": 0.7196577044045218, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1212274506688118, + "median": 0.10892187058925629, + "p05": 0.07249757647514343, + "p95": 0.2145335078239441 + }, + { + "label": "completed_group_1", + "mean": 0.06944876164197922, + "median": 0.0678764283657074, + "p05": 0.05350388586521149, + "p95": 0.08999023586511612 + }, + { + "label": "completed_group_2", + "mean": 0.05726535990834236, + "median": 0.0559837631881237, + "p05": 0.0409373864531517, + "p95": 0.07825973629951477 + }, + { + "label": "completed_group_3", + "mean": 0.09096579253673553, + "median": 0.08796361088752747, + "p05": 0.06407731771469116, + "p95": 0.12661314010620117 + }, + { + "label": "completed_group_4", + "mean": 0.06335186958312988, + "median": 0.06283852458000183, + "p05": 0.05032140761613846, + "p95": 0.0781375840306282 + }, + { + "label": "current_group_partial", + "mean": 0.5977407693862915, + "median": 0.6085352301597595, + "p05": 0.4878399074077606, + "p95": 0.6737192869186401 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6581614017486572, + "group": 5, + "latest_source_mass": 0.35773903131484985, + "layer": 19, + "max_source_mass": 0.35773903131484985, + "mixer_index": 36, + "normalized_entropy": 0.9254374988530375, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18132120370864868, + "median": 0.1806161105632782, + "p05": 0.1410265862941742, + "p95": 0.22331592440605164 + }, + { + "label": "completed_group_1", + "mean": 0.12711825966835022, + "median": 0.12722507119178772, + "p05": 0.11291128396987915, + "p95": 0.14178316295146942 + }, + { + "label": "completed_group_2", + "mean": 0.08560527861118317, + "median": 0.08513904362916946, + "p05": 0.07705061137676239, + "p95": 0.09564155340194702 + }, + { + "label": "completed_group_3", + "mean": 0.09748885035514832, + "median": 0.0953599065542221, + "p05": 0.08346174657344818, + "p95": 0.1194010004401207 + }, + { + "label": "completed_group_4", + "mean": 0.15072739124298096, + "median": 0.150638610124588, + "p05": 0.13762864470481873, + "p95": 0.1634974181652069 + }, + { + "label": "current_group_partial", + "mean": 0.35773903131484985, + "median": 0.35794761776924133, + "p05": 0.32225728034973145, + "p95": 0.39409786462783813 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3699336051940918, + "group": 5, + "latest_source_mass": 0.5385624766349792, + "layer": 19, + "max_source_mass": 0.5385624766349792, + "mixer_index": 37, + "normalized_entropy": 0.7645745027284836, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13604649901390076, + "median": 0.11998516321182251, + "p05": 0.06740576773881912, + "p95": 0.26897257566452026 + }, + { + "label": "completed_group_1", + "mean": 0.10151474922895432, + "median": 0.0991508960723877, + "p05": 0.07438431680202484, + "p95": 0.13600951433181763 + }, + { + "label": "completed_group_2", + "mean": 0.07715044170618057, + "median": 0.0767412930727005, + "p05": 0.054624155163764954, + "p95": 0.09943743050098419 + }, + { + "label": "completed_group_3", + "mean": 0.11300846189260483, + "median": 0.11003986746072769, + "p05": 0.08064281940460205, + "p95": 0.1516837179660797 + }, + { + "label": "completed_group_4", + "mean": 0.033717282116413116, + "median": 0.03286907076835632, + "p05": 0.024589763954281807, + "p95": 0.04592139273881912 + }, + { + "label": "current_group_partial", + "mean": 0.5385624766349792, + "median": 0.5415897369384766, + "p05": 0.4144754409790039, + "p95": 0.6506978273391724 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6926933526992798, + "group": 5, + "latest_source_mass": 0.3198680281639099, + "layer": 20, + "max_source_mass": 0.3198680281639099, + "mixer_index": 38, + "normalized_entropy": 0.9447101476341264, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19425281882286072, + "median": 0.19403350353240967, + "p05": 0.139950692653656, + "p95": 0.2494640350341797 + }, + { + "label": "completed_group_1", + "mean": 0.14001017808914185, + "median": 0.14058321714401245, + "p05": 0.12015820294618607, + "p95": 0.158815398812294 + }, + { + "label": "completed_group_2", + "mean": 0.09434760361909866, + "median": 0.09424005448818207, + "p05": 0.08333250135183334, + "p95": 0.10548526048660278 + }, + { + "label": "completed_group_3", + "mean": 0.10331346094608307, + "median": 0.10153411328792572, + "p05": 0.08690286427736282, + "p95": 0.12658309936523438 + }, + { + "label": "completed_group_4", + "mean": 0.14820793271064758, + "median": 0.14894580841064453, + "p05": 0.13278263807296753, + "p95": 0.16199614107608795 + }, + { + "label": "current_group_partial", + "mean": 0.3198680281639099, + "median": 0.3201481103897095, + "p05": 0.2800636887550354, + "p95": 0.3604968786239624 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4148362874984741, + "group": 5, + "latest_source_mass": 0.4996800422668457, + "layer": 20, + "max_source_mass": 0.4996800422668457, + "mixer_index": 39, + "normalized_entropy": 0.789635166883214, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16652248799800873, + "median": 0.14292314648628235, + "p05": 0.08186839520931244, + "p95": 0.350705623626709 + }, + { + "label": "completed_group_1", + "mean": 0.10329679399728775, + "median": 0.10057851672172546, + "p05": 0.07478320598602295, + "p95": 0.13858266174793243 + }, + { + "label": "completed_group_2", + "mean": 0.08063912391662598, + "median": 0.0807541012763977, + "p05": 0.056308042258024216, + "p95": 0.10469165444374084 + }, + { + "label": "completed_group_3", + "mean": 0.1165856122970581, + "median": 0.11373114585876465, + "p05": 0.08339665830135345, + "p95": 0.15619920194149017 + }, + { + "label": "completed_group_4", + "mean": 0.03327595070004463, + "median": 0.032142095267772675, + "p05": 0.022759035229682922, + "p95": 0.04896518588066101 + }, + { + "label": "current_group_partial", + "mean": 0.4996800422668457, + "median": 0.5001581907272339, + "p05": 0.3463003933429718, + "p95": 0.6266770362854004 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.740808129310608, + "group": 6, + "latest_source_mass": 0.24231041967868805, + "layer": 21, + "max_source_mass": 0.24231041967868805, + "mixer_index": 40, + "normalized_entropy": 0.9715635157550481, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21459883451461792, + "median": 0.21599377691745758, + "p05": 0.1570315659046173, + "p95": 0.27208006381988525 + }, + { + "label": "completed_group_1", + "mean": 0.15640269219875336, + "median": 0.1569681316614151, + "p05": 0.13785089552402496, + "p95": 0.1749057024717331 + }, + { + "label": "completed_group_2", + "mean": 0.10570268332958221, + "median": 0.10564640164375305, + "p05": 0.09223169088363647, + "p95": 0.11907710134983063 + }, + { + "label": "completed_group_3", + "mean": 0.11097013205289841, + "median": 0.10862545669078827, + "p05": 0.09454305469989777, + "p95": 0.13635265827178955 + }, + { + "label": "completed_group_4", + "mean": 0.17001524567604065, + "median": 0.17010903358459473, + "p05": 0.15088173747062683, + "p95": 0.18767674267292023 + }, + { + "label": "completed_group_5", + "mean": 0.24231041967868805, + "median": 0.24274763464927673, + "p05": 0.2087050974369049, + "p95": 0.27555277943611145 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3179612159729004, + "group": 6, + "latest_source_mass": 0.603651762008667, + "layer": 21, + "max_source_mass": 0.603651762008667, + "mixer_index": 41, + "normalized_entropy": 0.6772980841960945, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08068332821130753, + "median": 0.06917878985404968, + "p05": 0.039396293461322784, + "p95": 0.17595769464969635 + }, + { + "label": "completed_group_1", + "mean": 0.049232251942157745, + "median": 0.04861311614513397, + "p05": 0.03742668777704239, + "p95": 0.06363335251808167 + }, + { + "label": "completed_group_2", + "mean": 0.045073024928569794, + "median": 0.04482679069042206, + "p05": 0.03333880379796028, + "p95": 0.057495348155498505 + }, + { + "label": "completed_group_3", + "mean": 0.06111945956945419, + "median": 0.059528060257434845, + "p05": 0.042710162699222565, + "p95": 0.08362975716590881 + }, + { + "label": "completed_group_4", + "mean": 0.028317855671048164, + "median": 0.027924545109272003, + "p05": 0.022549759596586227, + "p95": 0.03542562201619148 + }, + { + "label": "completed_group_5", + "mean": 0.13192236423492432, + "median": 0.12939827144145966, + "p05": 0.10056710243225098, + "p95": 0.17195004224777222 + }, + { + "label": "current_group_partial", + "mean": 0.603651762008667, + "median": 0.6094026565551758, + "p05": 0.5065152645111084, + "p95": 0.6787958145141602 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.5319744348526, + "group": 6, + "latest_source_mass": 0.5132461190223694, + "layer": 22, + "max_source_mass": 0.5132461190223694, + "mixer_index": 42, + "normalized_entropy": 0.7872791226235868, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12406684458255768, + "median": 0.1289612352848053, + "p05": 0.06980825215578079, + "p95": 0.16972985863685608 + }, + { + "label": "completed_group_1", + "mean": 0.07462044060230255, + "median": 0.07392798364162445, + "p05": 0.06016036123037338, + "p95": 0.0917147696018219 + }, + { + "label": "completed_group_2", + "mean": 0.05378449708223343, + "median": 0.053309448063373566, + "p05": 0.04370884597301483, + "p95": 0.06546851992607117 + }, + { + "label": "completed_group_3", + "mean": 0.05575074627995491, + "median": 0.053552836179733276, + "p05": 0.0437602773308754, + "p95": 0.07444392144680023 + }, + { + "label": "completed_group_4", + "mean": 0.07357081025838852, + "median": 0.07329098880290985, + "p05": 0.06283499300479889, + "p95": 0.08542977273464203 + }, + { + "label": "completed_group_5", + "mean": 0.10496053099632263, + "median": 0.1038457602262497, + "p05": 0.08522139489650726, + "p95": 0.1291121542453766 + }, + { + "label": "current_group_partial", + "mean": 0.5132461190223694, + "median": 0.5090781450271606, + "p05": 0.4398295283317566, + "p95": 0.59932541847229 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.2461028099060059, + "group": 6, + "latest_source_mass": 0.6375080347061157, + "layer": 22, + "max_source_mass": 0.6375080347061157, + "mixer_index": 43, + "normalized_entropy": 0.640370168432985, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08556011319160461, + "median": 0.07947102189064026, + "p05": 0.043556369841098785, + "p95": 0.16172736883163452 + }, + { + "label": "completed_group_1", + "mean": 0.0446893647313118, + "median": 0.04398339241743088, + "p05": 0.03128381073474884, + "p95": 0.05990619957447052 + }, + { + "label": "completed_group_2", + "mean": 0.04084867238998413, + "median": 0.04042873531579971, + "p05": 0.02942952886223793, + "p95": 0.05392487719655037 + }, + { + "label": "completed_group_3", + "mean": 0.05386577919125557, + "median": 0.052273523062467575, + "p05": 0.033789411187171936, + "p95": 0.076903335750103 + }, + { + "label": "completed_group_4", + "mean": 0.025119207799434662, + "median": 0.024625632911920547, + "p05": 0.019138969480991364, + "p95": 0.03265690058469772 + }, + { + "label": "completed_group_5", + "mean": 0.11240880191326141, + "median": 0.11003861576318741, + "p05": 0.07852104306221008, + "p95": 0.155371755361557 + }, + { + "label": "current_group_partial", + "mean": 0.6375080347061157, + "median": 0.6392797231674194, + "p05": 0.5317062735557556, + "p95": 0.7386638522148132 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.6323832273483276, + "group": 6, + "latest_source_mass": 0.4532663822174072, + "layer": 23, + "max_source_mass": 0.4532663822174072, + "mixer_index": 44, + "normalized_entropy": 0.8388790346464895, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1418324112892151, + "median": 0.14852997660636902, + "p05": 0.07787702977657318, + "p95": 0.1923113763332367 + }, + { + "label": "completed_group_1", + "mean": 0.08991365134716034, + "median": 0.08918912708759308, + "p05": 0.0752246081829071, + "p95": 0.10654127597808838 + }, + { + "label": "completed_group_2", + "mean": 0.06188817322254181, + "median": 0.06161077320575714, + "p05": 0.04994015768170357, + "p95": 0.07481888681650162 + }, + { + "label": "completed_group_3", + "mean": 0.05739111453294754, + "median": 0.0548475980758667, + "p05": 0.04428064078092575, + "p95": 0.07816634327173233 + }, + { + "label": "completed_group_4", + "mean": 0.09848874807357788, + "median": 0.09768162667751312, + "p05": 0.08320866525173187, + "p95": 0.11606857925653458 + }, + { + "label": "completed_group_5", + "mean": 0.0972195416688919, + "median": 0.09613433480262756, + "p05": 0.07767169177532196, + "p95": 0.12004032731056213 + }, + { + "label": "current_group_partial", + "mean": 0.4532663822174072, + "median": 0.45108917355537415, + "p05": 0.37653055787086487, + "p95": 0.5389904975891113 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.301480770111084, + "group": 6, + "latest_source_mass": 0.6134276390075684, + "layer": 23, + "max_source_mass": 0.6134276390075684, + "mixer_index": 45, + "normalized_entropy": 0.6688288103861927, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09513165056705475, + "median": 0.08380547165870667, + "p05": 0.055099427700042725, + "p95": 0.1882638931274414 + }, + { + "label": "completed_group_1", + "mean": 0.050291214138269424, + "median": 0.04947858303785324, + "p05": 0.03757611662149429, + "p95": 0.06516233086585999 + }, + { + "label": "completed_group_2", + "mean": 0.045042067766189575, + "median": 0.0443364642560482, + "p05": 0.031873252242803574, + "p95": 0.06115761026740074 + }, + { + "label": "completed_group_3", + "mean": 0.05902485549449921, + "median": 0.05624900758266449, + "p05": 0.038259170949459076, + "p95": 0.08772744238376617 + }, + { + "label": "completed_group_4", + "mean": 0.02615261636674404, + "median": 0.02565642073750496, + "p05": 0.01966264471411705, + "p95": 0.03415588289499283 + }, + { + "label": "completed_group_5", + "mean": 0.1109299436211586, + "median": 0.10745325684547424, + "p05": 0.07642066478729248, + "p95": 0.15984633564949036 + }, + { + "label": "current_group_partial", + "mean": 0.6134276390075684, + "median": 0.6200529336929321, + "p05": 0.49638131260871887, + "p95": 0.7104787826538086 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.629331111907959, + "group": 6, + "latest_source_mass": 0.45539987087249756, + "layer": 24, + "max_source_mass": 0.45539987087249756, + "mixer_index": 46, + "normalized_entropy": 0.837310557580963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14790503680706024, + "median": 0.15420395135879517, + "p05": 0.08257082104682922, + "p95": 0.20396333932876587 + }, + { + "label": "completed_group_1", + "mean": 0.08658000826835632, + "median": 0.08578156679868698, + "p05": 0.07299856841564178, + "p95": 0.10278668999671936 + }, + { + "label": "completed_group_2", + "mean": 0.06608415395021439, + "median": 0.06551258265972137, + "p05": 0.054121918976306915, + "p95": 0.07912646979093552 + }, + { + "label": "completed_group_3", + "mean": 0.06427294760942459, + "median": 0.06133077293634415, + "p05": 0.04914242774248123, + "p95": 0.09066928923130035 + }, + { + "label": "completed_group_4", + "mean": 0.0819322019815445, + "median": 0.08114765584468842, + "p05": 0.06886082887649536, + "p95": 0.09821749478578568 + }, + { + "label": "completed_group_5", + "mean": 0.0978257954120636, + "median": 0.09643188118934631, + "p05": 0.07740232348442078, + "p95": 0.12209523469209671 + }, + { + "label": "current_group_partial", + "mean": 0.45539987087249756, + "median": 0.45274412631988525, + "p05": 0.3780696392059326, + "p95": 0.5462272763252258 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.338958501815796, + "group": 6, + "latest_source_mass": 0.5946891903877258, + "layer": 24, + "max_source_mass": 0.5946891903877258, + "mixer_index": 47, + "normalized_entropy": 0.6880885545850224, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10803166776895523, + "median": 0.09507323801517487, + "p05": 0.04510519653558731, + "p95": 0.2335781455039978 + }, + { + "label": "completed_group_1", + "mean": 0.0559723861515522, + "median": 0.05525296553969383, + "p05": 0.042058296501636505, + "p95": 0.07207933813333511 + }, + { + "label": "completed_group_2", + "mean": 0.047874726355075836, + "median": 0.04723185673356056, + "p05": 0.03294869884848595, + "p95": 0.06497423350811005 + }, + { + "label": "completed_group_3", + "mean": 0.060293328016996384, + "median": 0.05662515014410019, + "p05": 0.040346790105104446, + "p95": 0.08978720009326935 + }, + { + "label": "completed_group_4", + "mean": 0.029215168207883835, + "median": 0.028496116399765015, + "p05": 0.021991699934005737, + "p95": 0.039112430065870285 + }, + { + "label": "completed_group_5", + "mean": 0.10392352938652039, + "median": 0.10063231736421585, + "p05": 0.06900570541620255, + "p95": 0.1502695530653 + }, + { + "label": "current_group_partial", + "mean": 0.5946891903877258, + "median": 0.600748598575592, + "p05": 0.46449366211891174, + "p95": 0.7016042470932007 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7611865997314453, + "group": 7, + "latest_source_mass": 0.3679775297641754, + "layer": 25, + "max_source_mass": 0.3679775297641754, + "mixer_index": 48, + "normalized_entropy": 0.9050708742058073, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1573912650346756, + "median": 0.15898014605045319, + "p05": 0.09474922716617584, + "p95": 0.2298184633255005 + }, + { + "label": "completed_group_1", + "mean": 0.10317619889974594, + "median": 0.10270150750875473, + "p05": 0.08812867850065231, + "p95": 0.11994096636772156 + }, + { + "label": "completed_group_2", + "mean": 0.08533225208520889, + "median": 0.08495655655860901, + "p05": 0.07036984711885452, + "p95": 0.1013721376657486 + }, + { + "label": "completed_group_3", + "mean": 0.08218386024236679, + "median": 0.07942138612270355, + "p05": 0.06530377268791199, + "p95": 0.1088358536362648 + }, + { + "label": "completed_group_4", + "mean": 0.09052872657775879, + "median": 0.08931052684783936, + "p05": 0.07639768719673157, + "p95": 0.10804671049118042 + }, + { + "label": "completed_group_5", + "mean": 0.11341015249490738, + "median": 0.1123356968164444, + "p05": 0.09177497029304504, + "p95": 0.13803066313266754 + }, + { + "label": "completed_group_6", + "mean": 0.3679775297641754, + "median": 0.3671194612979889, + "p05": 0.2961871325969696, + "p95": 0.4462817311286926 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.476489782333374, + "group": 7, + "latest_source_mass": 0.3396714925765991, + "layer": 25, + "max_source_mass": 0.3836256265640259, + "mixer_index": 49, + "normalized_entropy": 0.7100414956318613, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09070298075675964, + "median": 0.0818760022521019, + "p05": 0.04576021432876587, + "p95": 0.1613437533378601 + }, + { + "label": "completed_group_1", + "mean": 0.03713828697800636, + "median": 0.035560622811317444, + "p05": 0.02337636426091194, + "p95": 0.054472144693136215 + }, + { + "label": "completed_group_2", + "mean": 0.030013620853424072, + "median": 0.028701743111014366, + "p05": 0.01836019568145275, + "p95": 0.04582716524600983 + }, + { + "label": "completed_group_3", + "mean": 0.03533655032515526, + "median": 0.031870774924755096, + "p05": 0.018502909690141678, + "p95": 0.061826568096876144 + }, + { + "label": "completed_group_4", + "mean": 0.020730752497911453, + "median": 0.020491186529397964, + "p05": 0.014567261561751366, + "p95": 0.027856238186359406 + }, + { + "label": "completed_group_5", + "mean": 0.06278066337108612, + "median": 0.059403590857982635, + "p05": 0.03608490154147148, + "p95": 0.10122650861740112 + }, + { + "label": "completed_group_6", + "mean": 0.3836256265640259, + "median": 0.3773127794265747, + "p05": 0.1874689906835556, + "p95": 0.5895293951034546 + }, + { + "label": "current_group_partial", + "mean": 0.3396714925765991, + "median": 0.34821513295173645, + "p05": 0.09212851524353027, + "p95": 0.587702751159668 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7213938236236572, + "group": 7, + "latest_source_mass": 0.37924307584762573, + "layer": 26, + "max_source_mass": 0.37924307584762573, + "mixer_index": 50, + "normalized_entropy": 0.8278154442529139, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10584281384944916, + "median": 0.10982948541641235, + "p05": 0.03868259862065315, + "p95": 0.16580678522586823 + }, + { + "label": "completed_group_1", + "mean": 0.06887699663639069, + "median": 0.06925956159830093, + "p05": 0.04082746431231499, + "p95": 0.09546782076358795 + }, + { + "label": "completed_group_2", + "mean": 0.05207539349794388, + "median": 0.051871635019779205, + "p05": 0.035383015871047974, + "p95": 0.07048650085926056 + }, + { + "label": "completed_group_3", + "mean": 0.050678279250860214, + "median": 0.04831986129283905, + "p05": 0.03378130868077278, + "p95": 0.07452918589115143 + }, + { + "label": "completed_group_4", + "mean": 0.04768219590187073, + "median": 0.04721469432115555, + "p05": 0.03680850565433502, + "p95": 0.060343313962221146 + }, + { + "label": "completed_group_5", + "mean": 0.06809806823730469, + "median": 0.06714263558387756, + "p05": 0.04759174585342407, + "p95": 0.09223967790603638 + }, + { + "label": "completed_group_6", + "mean": 0.2275032103061676, + "median": 0.22425106167793274, + "p05": 0.14320722222328186, + "p95": 0.3203037679195404 + }, + { + "label": "current_group_partial", + "mean": 0.37924307584762573, + "median": 0.3716055750846863, + "p05": 0.20950253307819366, + "p95": 0.5737906098365784 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5405172109603882, + "group": 7, + "latest_source_mass": 0.4017985463142395, + "layer": 26, + "max_source_mass": 0.4017985463142395, + "mixer_index": 51, + "normalized_entropy": 0.7408321802188831, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.074287548661232, + "median": 0.06164507195353508, + "p05": 0.028203874826431274, + "p95": 0.19368377327919006 + }, + { + "label": "completed_group_1", + "mean": 0.04491765797138214, + "median": 0.04365704953670502, + "p05": 0.02776508405804634, + "p95": 0.06526201963424683 + }, + { + "label": "completed_group_2", + "mean": 0.035157643258571625, + "median": 0.03418130800127983, + "p05": 0.022541508078575134, + "p95": 0.050758615136146545 + }, + { + "label": "completed_group_3", + "mean": 0.046252522617578506, + "median": 0.04353209584951401, + "p05": 0.026215802878141403, + "p95": 0.07329559326171875 + }, + { + "label": "completed_group_4", + "mean": 0.023427214473485947, + "median": 0.023055266588926315, + "p05": 0.016644639894366264, + "p95": 0.03150089830160141 + }, + { + "label": "completed_group_5", + "mean": 0.07749318331480026, + "median": 0.07363636791706085, + "p05": 0.04229677468538284, + "p95": 0.12579958140850067 + }, + { + "label": "completed_group_6", + "mean": 0.29666566848754883, + "median": 0.2920765280723572, + "p05": 0.14379237592220306, + "p95": 0.45965293049812317 + }, + { + "label": "current_group_partial", + "mean": 0.4017985463142395, + "median": 0.4004032611846924, + "p05": 0.18456703424453735, + "p95": 0.6275562047958374 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8063910007476807, + "group": 7, + "latest_source_mass": 0.3437207341194153, + "layer": 27, + "max_source_mass": 0.3437207341194153, + "mixer_index": 52, + "normalized_entropy": 0.868690446228377, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11433833092451096, + "median": 0.11714513599872589, + "p05": 0.06115472689270973, + "p95": 0.16258974373340607 + }, + { + "label": "completed_group_1", + "mean": 0.07374332845211029, + "median": 0.0731307864189148, + "p05": 0.05447638779878616, + "p95": 0.09509895741939545 + }, + { + "label": "completed_group_2", + "mean": 0.05932536721229553, + "median": 0.05929289758205414, + "p05": 0.04452420026063919, + "p95": 0.07498160749673843 + }, + { + "label": "completed_group_3", + "mean": 0.05389238893985748, + "median": 0.051720209419727325, + "p05": 0.03926049917936325, + "p95": 0.07536239922046661 + }, + { + "label": "completed_group_4", + "mean": 0.06126017868518829, + "median": 0.06108957529067993, + "p05": 0.0506296306848526, + "p95": 0.07238088548183441 + }, + { + "label": "completed_group_5", + "mean": 0.07349519431591034, + "median": 0.07249070703983307, + "p05": 0.05706139653921127, + "p95": 0.09332700818777084 + }, + { + "label": "completed_group_6", + "mean": 0.22022445499897003, + "median": 0.21959853172302246, + "p05": 0.15368635952472687, + "p95": 0.2904376685619354 + }, + { + "label": "current_group_partial", + "mean": 0.3437207341194153, + "median": 0.33986932039260864, + "p05": 0.22015926241874695, + "p95": 0.4802429676055908 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.4859801530838013, + "group": 7, + "latest_source_mass": 0.4424910843372345, + "layer": 27, + "max_source_mass": 0.4424910843372345, + "mixer_index": 53, + "normalized_entropy": 0.7146053992378076, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08387120068073273, + "median": 0.07644708454608917, + "p05": 0.0386628657579422, + "p95": 0.15876422822475433 + }, + { + "label": "completed_group_1", + "mean": 0.041406162083148956, + "median": 0.0394824743270874, + "p05": 0.025755178183317184, + "p95": 0.06136181950569153 + }, + { + "label": "completed_group_2", + "mean": 0.032444946467876434, + "median": 0.0307333804666996, + "p05": 0.018246563151478767, + "p95": 0.052875272929668427 + }, + { + "label": "completed_group_3", + "mean": 0.03686721250414848, + "median": 0.03247762471437454, + "p05": 0.017809681594371796, + "p95": 0.06780916452407837 + }, + { + "label": "completed_group_4", + "mean": 0.025052767246961594, + "median": 0.024561133235692978, + "p05": 0.017057543620467186, + "p95": 0.034822143614292145 + }, + { + "label": "completed_group_5", + "mean": 0.06012478470802307, + "median": 0.056938059628009796, + "p05": 0.03254516050219536, + "p95": 0.09905251860618591 + }, + { + "label": "completed_group_6", + "mean": 0.27774184942245483, + "median": 0.2747848629951477, + "p05": 0.11654777824878693, + "p95": 0.4562506675720215 + }, + { + "label": "current_group_partial", + "mean": 0.4424910843372345, + "median": 0.4387660026550293, + "p05": 0.21194897592067719, + "p95": 0.6782112717628479 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7777256965637207, + "group": 7, + "latest_source_mass": 0.3409850299358368, + "layer": 28, + "max_source_mass": 0.3409850299358368, + "mixer_index": 54, + "normalized_entropy": 0.8549053488311195, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10586117208003998, + "median": 0.10652337223291397, + "p05": 0.05095779523253441, + "p95": 0.1629783660173416 + }, + { + "label": "completed_group_1", + "mean": 0.07087038457393646, + "median": 0.06906798481941223, + "p05": 0.04985840618610382, + "p95": 0.09773125499486923 + }, + { + "label": "completed_group_2", + "mean": 0.05850498005747795, + "median": 0.05767729878425598, + "p05": 0.04179685190320015, + "p95": 0.07779045403003693 + }, + { + "label": "completed_group_3", + "mean": 0.06360965222120285, + "median": 0.06068425625562668, + "p05": 0.04235687851905823, + "p95": 0.09371945261955261 + }, + { + "label": "completed_group_4", + "mean": 0.04672512412071228, + "median": 0.04617916792631149, + "p05": 0.0356140062212944, + "p95": 0.05914818495512009 + }, + { + "label": "completed_group_5", + "mean": 0.07485505938529968, + "median": 0.07274073362350464, + "p05": 0.052147433161735535, + "p95": 0.10333258658647537 + }, + { + "label": "completed_group_6", + "mean": 0.2385886013507843, + "median": 0.2345724105834961, + "p05": 0.15389031171798706, + "p95": 0.33735349774360657 + }, + { + "label": "current_group_partial", + "mean": 0.3409850299358368, + "median": 0.33667099475860596, + "p05": 0.18655306100845337, + "p95": 0.5089476704597473 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6318418979644775, + "group": 7, + "latest_source_mass": 0.3292362689971924, + "layer": 28, + "max_source_mass": 0.3292362689971924, + "mixer_index": 55, + "normalized_entropy": 0.7847500712360619, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09244748950004578, + "median": 0.06782519817352295, + "p05": 0.025312144309282303, + "p95": 0.24087309837341309 + }, + { + "label": "completed_group_1", + "mean": 0.05762210115790367, + "median": 0.056058838963508606, + "p05": 0.03864594176411629, + "p95": 0.08028930425643921 + }, + { + "label": "completed_group_2", + "mean": 0.045099519193172455, + "median": 0.04426075518131256, + "p05": 0.03003283590078354, + "p95": 0.06244165822863579 + }, + { + "label": "completed_group_3", + "mean": 0.056236449629068375, + "median": 0.052902162075042725, + "p05": 0.03282861411571503, + "p95": 0.08620619773864746 + }, + { + "label": "completed_group_4", + "mean": 0.029373466968536377, + "median": 0.02887408435344696, + "p05": 0.020401213318109512, + "p95": 0.04018516466021538 + }, + { + "label": "completed_group_5", + "mean": 0.07237439602613449, + "median": 0.06993190199136734, + "p05": 0.04555661231279373, + "p95": 0.10748817771673203 + }, + { + "label": "completed_group_6", + "mean": 0.31761035323143005, + "median": 0.3173876404762268, + "p05": 0.16091161966323853, + "p95": 0.4724525213241577 + }, + { + "label": "current_group_partial", + "mean": 0.3292362689971924, + "median": 0.3128150701522827, + "p05": 0.12605780363082886, + "p95": 0.5847846865653992 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8334553241729736, + "group": 8, + "latest_source_mass": 0.270294725894928, + "layer": 29, + "max_source_mass": 0.27155569195747375, + "mixer_index": 56, + "normalized_entropy": 0.8817056346252721, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1157165914773941, + "median": 0.113271564245224, + "p05": 0.06316040456295013, + "p95": 0.17676270008087158 + }, + { + "label": "completed_group_1", + "mean": 0.07161639630794525, + "median": 0.07118864357471466, + "p05": 0.04294680058956146, + "p95": 0.102086141705513 + }, + { + "label": "completed_group_2", + "mean": 0.06577154994010925, + "median": 0.06490545719861984, + "p05": 0.04486069828271866, + "p95": 0.09022088348865509 + }, + { + "label": "completed_group_3", + "mean": 0.07425849884748459, + "median": 0.07179903984069824, + "p05": 0.045271825045347214, + "p95": 0.11117851734161377 + }, + { + "label": "completed_group_4", + "mean": 0.04881400242447853, + "median": 0.04833249747753143, + "p05": 0.0363147035241127, + "p95": 0.0629924014210701 + }, + { + "label": "completed_group_5", + "mean": 0.08197253197431564, + "median": 0.08029600977897644, + "p05": 0.05680408701300621, + "p95": 0.11223416775465012 + }, + { + "label": "completed_group_6", + "mean": 0.27155569195747375, + "median": 0.2687028646469116, + "p05": 0.18294592201709747, + "p95": 0.3740781545639038 + }, + { + "label": "completed_group_7", + "mean": 0.270294725894928, + "median": 0.2569413185119629, + "p05": 0.13936233520507812, + "p95": 0.43686625361442566 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6227548122406006, + "group": 8, + "latest_source_mass": 0.45173269510269165, + "layer": 29, + "max_source_mass": 0.45173269510269165, + "mixer_index": 57, + "normalized_entropy": 0.7385475426494315, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0605507455766201, + "median": 0.05293343961238861, + "p05": 0.02206447720527649, + "p95": 0.1254502236843109 + }, + { + "label": "completed_group_1", + "mean": 0.040063515305519104, + "median": 0.03803131729364395, + "p05": 0.022060411050915718, + "p95": 0.06445677578449249 + }, + { + "label": "completed_group_2", + "mean": 0.032619886100292206, + "median": 0.030926726758480072, + "p05": 0.020269516855478287, + "p95": 0.050711989402770996 + }, + { + "label": "completed_group_3", + "mean": 0.03868011012673378, + "median": 0.0344192236661911, + "p05": 0.020861119031906128, + "p95": 0.06798171997070312 + }, + { + "label": "completed_group_4", + "mean": 0.020290488377213478, + "median": 0.01974661275744438, + "p05": 0.014590941369533539, + "p95": 0.027847949415445328 + }, + { + "label": "completed_group_5", + "mean": 0.04760073497891426, + "median": 0.044880062341690063, + "p05": 0.027415042743086815, + "p95": 0.07675685733556747 + }, + { + "label": "completed_group_6", + "mean": 0.19551873207092285, + "median": 0.18908299505710602, + "p05": 0.10506269335746765, + "p95": 0.30520933866500854 + }, + { + "label": "completed_group_7", + "mean": 0.11294309794902802, + "median": 0.09412655979394913, + "p05": 0.03902517631649971, + "p95": 0.24146248400211334 + }, + { + "label": "current_group_partial", + "mean": 0.45173269510269165, + "median": 0.4618796110153198, + "p05": 0.24265648424625397, + "p95": 0.6204620599746704 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9787883758544922, + "group": 8, + "latest_source_mass": 0.1984800100326538, + "layer": 30, + "max_source_mass": 0.24502354860305786, + "mixer_index": 58, + "normalized_entropy": 0.9005854004479842, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0788881778717041, + "median": 0.055554650723934174, + "p05": 0.03367501497268677, + "p95": 0.21530622243881226 + }, + { + "label": "completed_group_1", + "mean": 0.08108963817358017, + "median": 0.07848536968231201, + "p05": 0.05139587074518204, + "p95": 0.12043403089046478 + }, + { + "label": "completed_group_2", + "mean": 0.057054273784160614, + "median": 0.056182146072387695, + "p05": 0.04127398133277893, + "p95": 0.07570968568325043 + }, + { + "label": "completed_group_3", + "mean": 0.08211587369441986, + "median": 0.0802752673625946, + "p05": 0.055154237896203995, + "p95": 0.113825723528862 + }, + { + "label": "completed_group_4", + "mean": 0.026040805503726006, + "median": 0.025504354387521744, + "p05": 0.01917124353349209, + "p95": 0.03502596169710159 + }, + { + "label": "completed_group_5", + "mean": 0.10537257045507431, + "median": 0.10299044847488403, + "p05": 0.07248030602931976, + "p95": 0.14682361483573914 + }, + { + "label": "completed_group_6", + "mean": 0.24502354860305786, + "median": 0.24406442046165466, + "p05": 0.1495005488395691, + "p95": 0.3413054645061493 + }, + { + "label": "completed_group_7", + "mean": 0.12593510746955872, + "median": 0.11572076380252838, + "p05": 0.051278628408908844, + "p95": 0.2299422323703766 + }, + { + "label": "current_group_partial", + "mean": 0.1984800100326538, + "median": 0.19106052815914154, + "p05": 0.11614266037940979, + "p95": 0.3049595355987549 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7590525150299072, + "group": 8, + "latest_source_mass": 0.3258843719959259, + "layer": 30, + "max_source_mass": 0.3258843719959259, + "mixer_index": 59, + "normalized_entropy": 0.800579300438408, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07131639868021011, + "median": 0.052984386682510376, + "p05": 0.03260878100991249, + "p95": 0.19837424159049988 + }, + { + "label": "completed_group_1", + "mean": 0.044687941670417786, + "median": 0.04335416108369827, + "p05": 0.027018578723073006, + "p95": 0.06556333601474762 + }, + { + "label": "completed_group_2", + "mean": 0.033290669322013855, + "median": 0.03222261369228363, + "p05": 0.01970716379582882, + "p95": 0.05042369291186333 + }, + { + "label": "completed_group_3", + "mean": 0.043597716838121414, + "median": 0.03997292369604111, + "p05": 0.024005703628063202, + "p95": 0.07244924455881119 + }, + { + "label": "completed_group_4", + "mean": 0.02658817172050476, + "median": 0.02605675905942917, + "p05": 0.017710059881210327, + "p95": 0.03680335357785225 + }, + { + "label": "completed_group_5", + "mean": 0.06922741234302521, + "median": 0.06634850800037384, + "p05": 0.03838749974966049, + "p95": 0.10990548133850098 + }, + { + "label": "completed_group_6", + "mean": 0.23554196953773499, + "median": 0.2292930632829666, + "p05": 0.10460641980171204, + "p95": 0.3925905227661133 + }, + { + "label": "completed_group_7", + "mean": 0.14986534416675568, + "median": 0.12543796002864838, + "p05": 0.04750998318195343, + "p95": 0.3270293176174164 + }, + { + "label": "current_group_partial", + "mean": 0.3258843719959259, + "median": 0.31322309374809265, + "p05": 0.1389845609664917, + "p95": 0.5583901405334473 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9736928939819336, + "group": 8, + "latest_source_mass": 0.20976582169532776, + "layer": 31, + "max_source_mass": 0.22560220956802368, + "mixer_index": 60, + "normalized_entropy": 0.8982663467084998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09849467873573303, + "median": 0.09779177606105804, + "p05": 0.017307927832007408, + "p95": 0.19556622207164764 + }, + { + "label": "completed_group_1", + "mean": 0.068942591547966, + "median": 0.0661388635635376, + "p05": 0.044743385165929794, + "p95": 0.10368303209543228 + }, + { + "label": "completed_group_2", + "mean": 0.0599900484085083, + "median": 0.05786054581403732, + "p05": 0.040265996009111404, + "p95": 0.08643331378698349 + }, + { + "label": "completed_group_3", + "mean": 0.06335455179214478, + "median": 0.05964011698961258, + "p05": 0.04099462926387787, + "p95": 0.09641734510660172 + }, + { + "label": "completed_group_4", + "mean": 0.04671281576156616, + "median": 0.04506188631057739, + "p05": 0.03403949737548828, + "p95": 0.06450185924768448 + }, + { + "label": "completed_group_5", + "mean": 0.0713595524430275, + "median": 0.06962447613477707, + "p05": 0.048271458595991135, + "p95": 0.10105480253696442 + }, + { + "label": "completed_group_6", + "mean": 0.22560220956802368, + "median": 0.22400957345962524, + "p05": 0.13957585394382477, + "p95": 0.31992682814598083 + }, + { + "label": "completed_group_7", + "mean": 0.1557777225971222, + "median": 0.1399117112159729, + "p05": 0.06281191110610962, + "p95": 0.2967374622821808 + }, + { + "label": "current_group_partial", + "mean": 0.20976582169532776, + "median": 0.2105630338191986, + "p05": 0.08139262348413467, + "p95": 0.34620192646980286 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.809161901473999, + "group": 8, + "latest_source_mass": 0.288146048784256, + "layer": 31, + "max_source_mass": 0.288146048784256, + "mixer_index": 61, + "normalized_entropy": 0.8233850650202157, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08854076266288757, + "median": 0.08406365662813187, + "p05": 0.020090684294700623, + "p95": 0.1807844340801239 + }, + { + "label": "completed_group_1", + "mean": 0.05356752499938011, + "median": 0.05036265030503273, + "p05": 0.031541675329208374, + "p95": 0.08521421998739243 + }, + { + "label": "completed_group_2", + "mean": 0.04243364930152893, + "median": 0.03939494490623474, + "p05": 0.023794323205947876, + "p95": 0.0721740797162056 + }, + { + "label": "completed_group_3", + "mean": 0.04702243581414223, + "median": 0.043501369655132294, + "p05": 0.02430840954184532, + "p95": 0.0788305327296257 + }, + { + "label": "completed_group_4", + "mean": 0.02648795023560524, + "median": 0.025087328627705574, + "p05": 0.01698113977909088, + "p95": 0.040197182446718216 + }, + { + "label": "completed_group_5", + "mean": 0.05820769816637039, + "median": 0.05441046878695488, + "p05": 0.02834305912256241, + "p95": 0.10125020891427994 + }, + { + "label": "completed_group_6", + "mean": 0.18489578366279602, + "median": 0.17386877536773682, + "p05": 0.07876831293106079, + "p95": 0.32456493377685547 + }, + { + "label": "completed_group_7", + "mean": 0.21069815754890442, + "median": 0.19004324078559875, + "p05": 0.07121473550796509, + "p95": 0.42417171597480774 + }, + { + "label": "current_group_partial", + "mean": 0.288146048784256, + "median": 0.28367581963539124, + "p05": 0.0962119847536087, + "p95": 0.5029257535934448 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.8544633388519287, + "group": 8, + "latest_source_mass": 0.12106549739837646, + "layer": 32, + "max_source_mass": 0.3525030016899109, + "mixer_index": 62, + "normalized_entropy": 0.8440026376822011, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07977832853794098, + "median": 0.053830958902835846, + "p05": 0.02079753391444683, + "p95": 0.2620829939842224 + }, + { + "label": "completed_group_1", + "mean": 0.05858254432678223, + "median": 0.056158941239118576, + "p05": 0.03545547276735306, + "p95": 0.09179478138685226 + }, + { + "label": "completed_group_2", + "mean": 0.053744811564683914, + "median": 0.05185101926326752, + "p05": 0.03345105051994324, + "p95": 0.08065605163574219 + }, + { + "label": "completed_group_3", + "mean": 0.07626347988843918, + "median": 0.07124648243188858, + "p05": 0.043040163815021515, + "p95": 0.12173676490783691 + }, + { + "label": "completed_group_4", + "mean": 0.031090255826711655, + "median": 0.02958102896809578, + "p05": 0.01873353309929371, + "p95": 0.04807420074939728 + }, + { + "label": "completed_group_5", + "mean": 0.0994817242026329, + "median": 0.0973888486623764, + "p05": 0.0575672909617424, + "p95": 0.14856761693954468 + }, + { + "label": "completed_group_6", + "mean": 0.3525030016899109, + "median": 0.3572760224342346, + "p05": 0.18325307965278625, + "p95": 0.5024369955062866 + }, + { + "label": "completed_group_7", + "mean": 0.1274903416633606, + "median": 0.1073581799864769, + "p05": 0.039848171174526215, + "p95": 0.2741084098815918 + }, + { + "label": "current_group_partial", + "mean": 0.12106549739837646, + "median": 0.10882791876792908, + "p05": 0.0503288172185421, + "p95": 0.22616314888000488 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9456019401550293, + "group": 8, + "latest_source_mass": 0.18013209104537964, + "layer": 32, + "max_source_mass": 0.2524128556251526, + "mixer_index": 63, + "normalized_entropy": 0.885481602665194, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08823961019515991, + "median": 0.07105386257171631, + "p05": 0.04482517018914223, + "p95": 0.1774362027645111 + }, + { + "label": "completed_group_1", + "mean": 0.06737187504768372, + "median": 0.06628458946943283, + "p05": 0.04582075774669647, + "p95": 0.09290395677089691 + }, + { + "label": "completed_group_2", + "mean": 0.05197303742170334, + "median": 0.051264435052871704, + "p05": 0.035227708518505096, + "p95": 0.07133325934410095 + }, + { + "label": "completed_group_3", + "mean": 0.0631948709487915, + "median": 0.06016676500439644, + "p05": 0.03969947248697281, + "p95": 0.09776238352060318 + }, + { + "label": "completed_group_4", + "mean": 0.02739548496901989, + "median": 0.026907969266176224, + "p05": 0.020028207451105118, + "p95": 0.036658842116594315 + }, + { + "label": "completed_group_5", + "mean": 0.08291903138160706, + "median": 0.08066994696855545, + "p05": 0.05307728424668312, + "p95": 0.12080682814121246 + }, + { + "label": "completed_group_6", + "mean": 0.2524128556251526, + "median": 0.25180506706237793, + "p05": 0.14340050518512726, + "p95": 0.36010295152664185 + }, + { + "label": "completed_group_7", + "mean": 0.186361163854599, + "median": 0.17697101831436157, + "p05": 0.08843781799077988, + "p95": 0.3156980276107788 + }, + { + "label": "current_group_partial", + "mean": 0.18013209104537964, + "median": 0.16958200931549072, + "p05": 0.08391770720481873, + "p95": 0.3105517029762268 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.596132755279541, + "group": null, + "latest_source_mass": 0.29075440764427185, + "layer": null, + "max_source_mass": 0.3670029938220978, + "mixer_index": null, + "normalized_entropy": 0.7264313223797062, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09668220579624176, + "median": 0.1013428270816803, + "p05": 0.02658359333872795, + "p95": 0.15407970547676086 + }, + { + "label": "completed_group_1", + "mean": 0.037084318697452545, + "median": 0.035886794328689575, + "p05": 0.02348068356513977, + "p95": 0.053765345364809036 + }, + { + "label": "completed_group_2", + "mean": 0.03457995876669884, + "median": 0.03355112299323082, + "p05": 0.019709739834070206, + "p95": 0.0531856045126915 + }, + { + "label": "completed_group_3", + "mean": 0.026012206450104713, + "median": 0.023950476199388504, + "p05": 0.013590577989816666, + "p95": 0.04525115340948105 + }, + { + "label": "completed_group_4", + "mean": 0.0258483849465847, + "median": 0.02512776106595993, + "p05": 0.016705092042684555, + "p95": 0.03687155246734619 + }, + { + "label": "completed_group_5", + "mean": 0.037477150559425354, + "median": 0.03506678715348244, + "p05": 0.02071693353354931, + "p95": 0.061963990330696106 + }, + { + "label": "completed_group_6", + "mean": 0.0845583900809288, + "median": 0.07240844517946243, + "p05": 0.03393660485744476, + "p95": 0.1794772744178772 + }, + { + "label": "completed_group_7", + "mean": 0.3670029938220978, + "median": 0.3608853220939636, + "p05": 0.1103469580411911, + "p95": 0.6465575098991394 + }, + { + "label": "completed_group_8", + "mean": 0.29075440764427185, + "median": 0.28802984952926636, + "p05": 0.039541684091091156, + "p95": 0.5658512711524963 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "optimizer_state_before": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.09253507107496262, + 0.19514523446559906, + 0.07917427271604538, + 0.04215213656425476, + 0.12552575767040253, + 0.05027730390429497, + 0.054851483553647995, + 0.036243390291929245, + 0.031880609691143036, + 0.035054463893175125, + 0.050445348024368286, + 0.05600574240088463, + 0.053015924990177155, + 0.032114963978528976, + 0.048427484929561615, + 0.5742782950401306, + 0.04075472429394722, + 0.03505101054906845, + 0.06577442586421967, + 0.08553697913885117, + 0.0493597537279129, + 0.023477228358387947, + 0.03512806445360184, + 0.030729109421372414, + 0.023035654798150063, + 0.019568851217627525, + 0.02376176416873932, + 0.02176014520227909, + 0.03424545377492905, + 0.021422436460852623, + 0.019477492198348045, + 0.017491821199655533 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0284768109158904e-05, + "first_to_last_ratio": 0.5012331703882884, + "imbalance_abs_log_ratio": 0.6906838762102308, + "last_quartile_mean": 2.051892954568757e-05, + "mean": 1.8776570954059935e-05, + "normalized": [ + 0.5986193859992756, + 0.6558081812372716, + 0.5520121545916151, + 0.5273260046333214, + 0.6138572699796184, + 0.5188085183079506, + 0.470653216662131, + 0.4448732456112806, + 0.571885288304143, + 0.5260631358207655, + 0.4944655013148357, + 0.4611973943964093, + 0.3985954392877791, + 0.37569126542434245, + 0.3598786676337444, + 0.4301976753792103, + 1.0911627323935584, + 1.0892352977591147, + 1.0129363601224668, + 0.9043812768757663, + 3.7939202517290083, + 3.07802379201043, + 2.391002620377405, + 1.8970509950687215, + 1.8215330675860186, + 1.6199762600873888, + 1.323734904300092, + 0.9702906094541006, + 1.1932084612277043, + 0.7841779822795031, + 0.5845545188198558, + 0.4448785253251717 + ], + "peak_layer": 21, + "peak_normalized": 3.7939202517290083, + "population_cv": 0.8048941588500481, + "rest_mean": 1.3225986583199335e-05, + "spike_contrast": 3.6859047338391244, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.8749726556707176e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.124001937569119e-05, + 1.2313828847254626e-05, + 1.0364895388192963e-05, + 9.901374141918495e-06, + 1.152613458543783e-05, + 9.741444955579937e-06, + 8.837253517413046e-06, + 8.353194061783142e-06, + 1.0738044693425763e-05, + 9.877661796053872e-06, + 9.284366569772828e-06, + 8.659705599711742e-06, + 7.4842555477516726e-06, + 7.054193702060729e-06, + 6.757287337677553e-06, + 8.077637176029384e-06, + 2.0488294467213564e-05, + 2.0452103854040615e-05, + 1.9019471437786706e-05, + 1.698117921478115e-05, + 7.123681280063465e-05, + 5.779473212896846e-05, + 4.489483035285957e-05, + 3.562011261237785e-05, + 3.420214488869533e-05, + 3.0417599191423506e-05, + 2.4855202354956418e-05, + 1.8218730474472977e-05, + 2.2404363335226662e-05, + 1.4724173524882644e-05, + 1.0975929399137385e-05, + 8.353293196705636e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.085704995304695e-05, + "first_to_last_ratio": 0.49797744750683937, + "imbalance_abs_log_ratio": 0.6972004891137765, + "last_quartile_mean": 8.204598452721257e-05, + "mean": 7.523339269255302e-05, + "normalized": [ + 0.5949378857857258, + 0.650065068423859, + 0.5492544663710863, + 0.5251722584489763, + 0.6053425775579288, + 0.5124227888322227, + 0.46580426292226623, + 0.44156615556169815, + 0.5688035499575106, + 0.5274402954719286, + 0.49517820262129336, + 0.4612679954527406, + 0.39847860202951046, + 0.37431010299451944, + 0.3578776058560047, + 0.4280250030645366, + 1.0938473267662936, + 1.088989782662427, + 1.0233362931504402, + 0.9048102634375241, + 3.7996932997999537, + 3.0906905066941115, + 2.4067960338193375, + 1.9114675410445925, + 1.823315112659501, + 1.6164295945753033, + 1.3192322668021113, + 0.9632233265477733, + 1.1927048821627753, + 0.7801549776691259, + 0.5836505536753793, + 0.44571141718154267 + ], + "peak_layer": 21, + "peak_normalized": 3.7996932997999537, + "population_cv": 0.808655698028205, + "rest_mean": 5.285295608498725e-05, + "spike_contrast": 3.710062121371229, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001960877503734082, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.475919558899477e-05, + 4.890660056844354e-05, + 4.1322276956634596e-05, + 3.951049075112678e-05, + 4.554197585093789e-05, + 3.855130489682779e-05, + 3.504403503029607e-05, + 3.3220519981114194e-05, + 4.27930208388716e-05, + 3.9681122871115804e-05, + 3.7253936170600355e-05, + 3.47027562384028e-05, + 2.997889714606572e-05, + 2.816061896737665e-05, + 2.6924346457235515e-05, + 3.2201773137785494e-05, + 8.229384548030794e-05, + 8.192839595722035e-05, + 7.698906119912863e-05, + 6.80719458614476e-05, + 0.0002858638181351125, + 0.00023252313258126378, + 0.00018107143114320934, + 0.00014380618813447654, + 0.0001371741818729788, + 0.00012160948244854808, + 9.925031918101013e-05, + 7.246655877679586e-05, + 8.973123476607725e-05, + 5.869370579603128e-05, + 4.391001129988581e-05, + 3.3532382076373324e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.036232439801096916, + "first_to_last_ratio": 0.49797744231506097, + "imbalance_abs_log_ratio": 0.6972004995395066, + "last_quartile_mean": 0.07275919895619154, + "mean": 0.06671772431582212, + "normalized": [ + 0.5949378741845537, + 0.6500650614940244, + 0.5492544412180356, + 0.5251722869429579, + 0.6053425762620699, + 0.5124227830545606, + 0.46580425469520476, + 0.4415661488209036, + 0.5688035576730777, + 0.5274403134409381, + 0.495178210228144, + 0.46126797835287947, + 0.3984786148158961, + 0.37431010925252656, + 0.35787762380853266, + 0.428024993742463, + 1.0938473087830916, + 1.0889897486949978, + 1.0233363030489668, + 0.9048102958095574, + 3.7996934319878424, + 3.0906904475080834, + 2.406796061359444, + 1.9114674273546057, + 1.8233150739769923, + 1.6164296712797974, + 1.3192322675305295, + 0.9632232990014142, + 1.1927048767720632, + 0.7801549942808673, + 0.5836505631099284, + 0.4457114015150505 + ], + "peak_layer": 21, + "peak_normalized": 3.7996934319878424, + "population_cv": 0.8086557050931692, + "rest_mean": 0.046870529651641846, + "spike_contrast": 3.710062096477808, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.17389257550239562, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03969290107488632, + 0.04337086156010628, + 0.03664500638842583, + 0.0350382998585701, + 0.04038707911968231, + 0.0341876819729805, + 0.031077399849891663, + 0.02946028858423233, + 0.03794927895069122, + 0.03518961742520332, + 0.03303716331720352, + 0.03077474981546402, + 0.026585586369037628, + 0.02497311867773533, + 0.02387678064405918, + 0.028556853532791138, + 0.07297900319099426, + 0.07265491783618927, + 0.06827466934919357, + 0.060366883873939514, + 0.2535068988800049, + 0.20620383322238922, + 0.1605759561061859, + 0.12752875685691833, + 0.1216474324464798, + 0.1078445091843605, + 0.08801617473363876, + 0.06426406651735306, + 0.07957455515861511, + 0.052050165832042694, + 0.038939837366342545, + 0.029736850410699844 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.7214510705798602e-06, + "first_to_last_ratio": 0.31570143383120763, + "imbalance_abs_log_ratio": 1.1529583416761842, + "last_quartile_mean": 5.452781920212146e-06, + "mean": 4.312751560320294e-06, + "normalized": [ + 0.5189290406874598, + 0.4629408602327028, + 0.4379369350734736, + 0.4223325243325377, + 0.3554137514884539, + 0.3403071611993441, + 0.32823921317250115, + 0.32713072242218383, + 0.4058203853809248, + 0.3945172118813988, + 0.39884681315808357, + 0.39929763261093665, + 0.39159574260871993, + 0.3879244716451889, + 0.3882468093992228, + 0.3995901564951789, + 0.788703492444173, + 0.816182126959875, + 0.8217008297860132, + 0.8161870300366048, + 3.918660813887919, + 3.3485633171709597, + 2.7353702011780077, + 2.2808471855925627, + 2.1314258695348536, + 1.89580066182392, + 1.5684885519233662, + 1.157279421610267, + 1.35167586969034, + 0.8709255585803598, + 0.6496847126841314, + 0.4894349253083356 + ], + "peak_layer": 21, + "peak_normalized": 3.918660813887919, + "population_cv": 0.9292175371812949, + "rest_mean": 2.808900300503107e-06, + "spike_contrast": 4.426482620656244, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2433548363333102e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2380120299203554e-06, + 1.996548917304608e-06, + 1.8887132000600104e-06, + 1.8214152532891603e-06, + 1.5328112112911185e-06, + 1.4676602404506411e-06, + 1.4156141787680099e-06, + 1.4108335335549782e-06, + 1.7502025002613664e-06, + 1.7014547211147146e-06, + 1.7201272157763015e-06, + 1.7220714880750165e-06, + 1.688855149950541e-06, + 1.673021870374214e-06, + 1.6744120330258738e-06, + 1.7233330709132133e-06, + 3.401482217668672e-06, + 3.5199907415517373e-06, + 3.5437915357761085e-06, + 3.5200118873035535e-06, + 1.6900210539461114e-05, + 1.4441521670960356e-05, + 1.1796972103184089e-05, + 9.836727258516476e-06, + 9.192310244543478e-06, + 8.176117262337357e-06, + 6.764501449652016e-06, + 4.991058631276246e-06, + 5.8294422160543036e-06, + 3.75608556169027e-06, + 2.8019287583447294e-06, + 2.110811237798771e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.160356504849915e-05, + "first_to_last_ratio": 0.6176036367578212, + "imbalance_abs_log_ratio": 0.48190839175616645, + "last_quartile_mean": 5.117127420817269e-05, + "mean": 5.2418822178879054e-05, + "normalized": [ + 0.666322518769789, + 0.714984736429167, + 0.6173684650819016, + 0.5916946676135548, + 0.6555071657760245, + 0.5687098563341721, + 0.5173612203652723, + 0.49129061612094416, + 0.619582115965453, + 0.5740227294981308, + 0.5363326789838647, + 0.4988165849512774, + 0.45986272133852035, + 0.431797325881486, + 0.41220412812819657, + 0.4796096795486855, + 1.1736071025949124, + 1.1561730056375625, + 1.0641195939880703, + 0.948668636490457, + 3.882987390089663, + 3.0644523599328255, + 2.292877320973713, + 1.7720443276024598, + 1.7296067191133486, + 1.4826105850124727, + 1.1786273763067439, + 0.8692520371850805, + 1.00316951798014, + 0.6504669042458316, + 0.4961137877350426, + 0.39975612432523655 + ], + "peak_layer": 21, + "peak_normalized": 3.882987390089663, + "population_cv": 0.786177948625012, + "rest_mean": 3.738827217603102e-05, + "spike_contrast": 3.5728795266419584, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013358379219425843, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.492784162517637e-05, + 3.747865775949322e-05, + 3.23617277899757e-05, + 3.101593756582588e-05, + 3.4360913559794426e-05, + 2.9811100830556825e-05, + 2.7119465812575072e-05, + 2.5752875444595702e-05, + 3.24777647620067e-05, + 3.008959538419731e-05, + 2.811392732837703e-05, + 2.6147377866436727e-05, + 2.410546221653931e-05, + 2.2634307242697105e-05, + 2.1607254893751815e-05, + 2.514057450753171e-05, + 6.151910201879218e-05, + 6.0605227190535516e-05, + 5.577989577432163e-05, + 4.9728092562872916e-05, + 0.00020354162552393973, + 0.00016063498333096504, + 0.00012018992856610566, + 9.288847650168464e-05, + 9.066394704859704e-05, + 7.771670061629266e-05, + 6.178225885378197e-05, + 4.55651679658331e-05, + 5.2584964578272775e-05, + 3.40967089869082e-05, + 2.6005700419773348e-05, + 2.0954745195922442e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.434365933368099e-05, + "first_to_last_ratio": 0.8577466133816546, + "imbalance_abs_log_ratio": 0.15344654550942588, + "last_quartile_mean": 2.8380944854689005e-05, + "mean": 3.523111084291486e-05, + "normalized": [ + 0.7673952990085477, + 0.811220254411165, + 0.7193610364184987, + 0.6887351358833893, + 0.7328492204226176, + 0.6490237652719044, + 0.5963486702580318, + 0.5628308587943773, + 0.6991412032912354, + 0.6479259522662333, + 0.6038428974789878, + 0.5611268039454765, + 0.5540897621717866, + 0.518717032406164, + 0.49549402526296327, + 0.5544919614528937, + 1.3117045453116274, + 1.2768062454810614, + 1.1497523702928032, + 1.0122574224190877, + 4.0186385000439255, + 3.041265540561009, + 2.0735563094276896, + 1.5089063120619997, + 1.5078974578960394, + 1.239262930352535, + 0.9604083073922689, + 0.7448019221905203, + 0.7558275520339612, + 0.5149618128538204, + 0.3996625343446833, + 0.32169635859269563 + ], + "peak_layer": 21, + "peak_normalized": 4.0186385000439255, + "population_cv": 0.7633311043023641, + "rest_mean": 2.5901046110784927e-05, + "spike_contrast": 3.30540550486754, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.561346039641649e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7036188839701936e-05, + 2.8580190701177344e-05, + 2.534388841013424e-05, + 2.4264903913717717e-05, + 2.5819092115852982e-05, + 2.286582821398042e-05, + 2.1010026102885604e-05, + 1.982915637199767e-05, + 2.4631521228002384e-05, + 2.2827151042292826e-05, + 2.1274056052789092e-05, + 1.976912062673364e-05, + 1.952119782799855e-05, + 1.8274977264809422e-05, + 1.7456804926041514e-05, + 1.953536775545217e-05, + 4.621280822902918e-05, + 4.4983302359469235e-05, + 4.0507053199689835e-05, + 3.5662953450810164e-05, + 0.00014158109843265265, + 0.00010714716336224228, + 7.30536921764724e-05, + 5.316044553183019e-05, + 5.3124902478884906e-05, + 4.366060966276564e-05, + 3.383625153219327e-05, + 2.624019907671027e-05, + 2.6628644263837487e-05, + 1.814267670852132e-05, + 1.4080555047257803e-05, + 1.1333720067341346e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.111996157822432e-05, + "first_to_last_ratio": 0.45955600820785986, + "imbalance_abs_log_ratio": 0.777494455152158, + "last_quartile_mean": 0.0001765181177688646, + "mean": 0.00015741029130822426, + "normalized": [ + 0.5705309506751165, + 0.6126723076713019, + 0.5240846716131147, + 0.5073054874330758, + 0.5540338647553013, + 0.48682190889438537, + 0.44234061084833826, + 0.4249374177361315, + 0.550364784679922, + 0.5081370833366782, + 0.48009264279277103, + 0.45140806153022744, + 0.3632347905144417, + 0.3493062409166391, + 0.33465617521315694, + 0.40047146041522963, + 1.021224360197918, + 1.022384831920215, + 0.9906834216965009, + 0.8745459299517545, + 3.8481386526758086, + 3.158088682741086, + 2.5497611412614916, + 2.003665132352204, + 1.879007746207825, + 1.6962363619877694, + 1.389737503600916, + 1.0270562090093074, + 1.1914290365360227, + 0.7723867467392906, + 0.5792090660062168, + 0.43604671808984163 + ], + "peak_layer": 21, + "peak_normalized": 3.8481386526758086, + "population_cv": 0.8430355170943084, + "rest_mean": 0.00010821280456083413, + "spike_contrast": 3.9096733650063507, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000423076719744131, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.980744314612821e-05, + 9.644092642702162e-05, + 8.249632082879543e-05, + 7.985510455910116e-05, + 8.72106320457533e-05, + 7.663077849429101e-05, + 6.962896441109478e-05, + 6.688952271360904e-05, + 8.663308108225465e-05, + 7.998600631253794e-05, + 7.557152275694534e-05, + 7.105627446435392e-05, + 5.717689418816008e-05, + 5.498439713846892e-05, + 5.267832602839917e-05, + 6.30383292445913e-05, + 0.00016075122402980924, + 0.00016093389422167093, + 0.0001559437660034746, + 0.00013766252959612757, + 0.0006057366263121367, + 0.0004971156595274806, + 0.00040135864401236176, + 0.0003153975121676922, + 0.00029577515670098364, + 0.0002670050598680973, + 0.00021875898528378457, + 0.00016166921705007553, + 0.0001875431917142123, + 0.00012158162280684337, + 9.117346780840307e-05, + 6.863824091851711e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.116500623524189, + 0.0468219518661499, + 0.09375449270009995, + 0.041188232600688934, + 0.024237047880887985, + 0.022429833188652992, + 0.03027583472430706, + 0.04546776041388512, + 0.030174262821674347, + 0.03663193807005882, + 0.05748353153467178, + 0.10596779733896255, + 0.05394246056675911, + 0.06537146866321564, + 0.08060523867607117, + 0.06690707802772522, + 0.033449217677116394, + 0.022565271705389023, + 0.036697570234537125, + 0.07367803901433945, + 0.042231619358062744, + 0.052135858684778214, + 0.06269875913858414, + 0.0728035643696785, + 0.051585447043180466, + 0.04862172156572342, + 0.05972515419125557, + 0.06787826865911484, + 0.06439156830310822, + 0.07232685387134552, + 0.09627991914749146, + 0.1271906942129135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0227459597444977e-05, + "first_to_last_ratio": 0.6251750002011836, + "imbalance_abs_log_ratio": 0.4697236681166162, + "last_quartile_mean": 1.635935473132122e-05, + "mean": 1.6436911991490888e-05, + "normalized": [ + 0.6817536191781184, + 0.7484899831245089, + 0.6296147869866243, + 0.6007633380256553, + 0.7005091083036028, + 0.5900178858271928, + 0.5278659818185938, + 0.49878628647857515, + 0.6254280014067066, + 0.5828037509404734, + 0.5411654694992786, + 0.5069220940770104, + 0.4384272646939669, + 0.410733592032448, + 0.41012977661150796, + 0.4993830470114355, + 1.2341367480513281, + 1.240449185242777, + 1.1141663566211946, + 1.0232697399583384, + 3.628191161581746, + 2.8189670579566104, + 2.212584875069539, + 1.7731887390792915, + 1.8705442026109216, + 1.5370133298176474, + 1.119075341410653, + 0.8897518054493918, + 0.8979584939233574, + 0.6704127897581839, + 0.5087700320942508, + 0.46872615535907086 + ], + "peak_layer": 21, + "peak_normalized": 3.628191161581746, + "population_cv": 0.7362867375552162, + "rest_mean": 1.1990741886283675e-05, + "spike_contrast": 3.3731216085867595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.044623055960983e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1205924238311127e-05, + 1.2302863979130052e-05, + 1.0348922842240427e-05, + 9.874694114841986e-06, + 1.1514206562424079e-05, + 9.698072062747087e-06, + 8.676486686454155e-06, + 8.198506293410901e-06, + 1.0280105016136076e-05, + 9.579493962519336e-06, + 8.895089194993488e-06, + 8.332233846886083e-06, + 7.206390364444815e-06, + 6.751191904186271e-06, + 6.741267043253174e-06, + 8.208315193769522e-06, + 2.0285397113184445e-05, + 2.0389154087752104e-05, + 1.8313454347662628e-05, + 1.6819394659250975e-05, + 5.963625881122425e-05, + 4.6335113438544795e-05, + 3.636806286522187e-05, + 2.9145747248549014e-05, + 3.074597043450922e-05, + 2.5263752831961028e-05, + 1.839414289861452e-05, + 1.4624772120441776e-05, + 1.475966473662993e-05, + 1.1019516023225151e-05, + 8.362608241441194e-06, + 7.704410563746933e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.0602649278298486e-05, + "first_to_last_ratio": 0.6220533452330901, + "imbalance_abs_log_ratio": 0.4747294258801449, + "last_quartile_mean": 6.527197319883271e-05, + "mean": 6.584073776139121e-05, + "normalized": [ + 0.6772234395168817, + 0.7419070719489791, + 0.6265330495692852, + 0.5983454392285444, + 0.6910105284953505, + 0.5826444544736231, + 0.5218070132506913, + 0.49396695197265295, + 0.6208398201935033, + 0.583554602034964, + 0.5417867192724627, + 0.5073857163005694, + 0.43782857851808293, + 0.40867327334274134, + 0.40807683163251224, + 0.49794392992767184, + 1.2379798176145815, + 1.2443074207246856, + 1.1312899909561613, + 1.0270568775609965, + 3.6326519816528213, + 2.8432167741930536, + 2.2290444258353594, + 1.784033212733279, + 1.868354361784077, + 1.5302682478219727, + 1.1091440983756962, + 0.8793208988009685, + 0.8936974950846793, + 0.6693107032572058, + 0.5098445844785217, + 0.4709516894474245 + ], + "peak_layer": 21, + "peak_normalized": 3.6326519816528213, + "population_cv": 0.7406565006814017, + "rest_mean": 4.789962258655578e-05, + "spike_contrast": 3.3971616250516083, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016272275970550255, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.458889088709839e-05, + 4.8847708967514336e-05, + 4.125139821553603e-05, + 3.939550515497103e-05, + 4.549664299702272e-05, + 3.836174073512666e-05, + 3.435615872149356e-05, + 3.252314854762517e-05, + 4.087655179318972e-05, + 3.8421665522037074e-05, + 3.567163730622269e-05, + 3.340664989082143e-05, + 2.8826956622651778e-05, + 2.6907349820248783e-05, + 2.686807965801563e-05, + 3.27849957102444e-05, + 8.150950452545658e-05, + 8.19261185824871e-05, + 7.448496762663126e-05, + 6.762218254152685e-05, + 0.00023917648650240153, + 0.0001871994900284335, + 0.00014676192949991673, + 0.00011746206291718408, + 0.00012301382957957685, + 0.00010075399040943012, + 7.30268657207489e-05, + 5.7895136706065387e-05, + 5.884170241188258e-05, + 4.406791049405001e-05, + 3.3568543585715815e-05, + 3.100780668319203e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.036006835056468844, + "first_to_last_ratio": 0.6220533380595731, + "imbalance_abs_log_ratio": 0.47472943741214146, + "last_quartile_mean": 0.05788383865728974, + "mean": 0.05838822451187298, + "normalized": [ + 0.6772234369424218, + 0.741907072488623, + 0.6265330654654753, + 0.5983454331639448, + 0.6910105564841633, + 0.582644442563147, + 0.5218069879368077, + 0.4939669174779353, + 0.620839814266596, + 0.5835546430674807, + 0.541786737574626, + 0.5073857075992703, + 0.43782855000599696, + 0.4086732953261738, + 0.40807680965990323, + 0.4979439245673007, + 1.2379798733919256, + 1.2443073810897243, + 1.1312900315759395, + 1.0270568889341698, + 3.632651939357648, + 2.8432167817930876, + 2.229044412718128, + 1.7840331838058763, + 1.8683543984231819, + 1.5302683134059842, + 1.1091440736440559, + 0.8793209345809946, + 0.893697457757885, + 0.6693107027108595, + 0.5098445760442807, + 0.4709516561763931 + ], + "peak_layer": 21, + "peak_normalized": 3.632651939357648, + "population_cv": 0.7406564998970533, + "rest_mean": 0.04247786428917338, + "spike_contrast": 3.3971616070921584, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14430416971445084, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03954187408089638, + 0.04331863671541214, + 0.03658215329051018, + 0.0349363274872303, + 0.04034687951207161, + 0.03401957452297211, + 0.030467383563518524, + 0.02884185127913952, + 0.036249734461307526, + 0.034072719514369965, + 0.03163396567106247, + 0.02962535060942173, + 0.025564031675457954, + 0.023861708119511604, + 0.023826880380511284, + 0.029074061661958694, + 0.07228344678878784, + 0.0726528987288475, + 0.06605401635169983, + 0.05996802821755409, + 0.21210409700870514, + 0.16601037979125977, + 0.1301499456167221, + 0.10416653007268906, + 0.10908989608287811, + 0.0893496498465538, + 0.0647609531879425, + 0.051341988146305084, + 0.05218140780925751, + 0.03907986357808113, + 0.029768919572234154, + 0.02749803103506565 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.7200502355763092e-06, + "first_to_last_ratio": 0.39238338914104137, + "imbalance_abs_log_ratio": 0.9355158836317786, + "last_quartile_mean": 4.383595950230301e-06, + "mean": 3.6741805224949076e-06, + "normalized": [ + 0.6063468981334846, + 0.5443420367186595, + 0.5139094665495357, + 0.4967956421023835, + 0.4188390484506744, + 0.4022862351030342, + 0.3820107165872017, + 0.3806321225478404, + 0.45980638040996796, + 0.4539689687950794, + 0.4522322819061978, + 0.45153818886200525, + 0.44692886894308675, + 0.4403179668717643, + 0.4568379816001622, + 0.47173827055358036, + 0.9154474599445339, + 0.922288261771489, + 0.8927585509522445, + 0.8903920996318164, + 3.60619457117351, + 3.0673607840777577, + 2.5879499263363996, + 2.1944271419318873, + 2.2559707186149534, + 1.8903392611722498, + 1.375705409268863, + 1.1270630373807071, + 1.036432964427479, + 0.7543267801682417, + 0.5745703581277657, + 0.5302416008854436 + ], + "peak_layer": 21, + "peak_normalized": 3.60619457117351, + "population_cv": 0.8429606581046323, + "rest_mean": 2.488658121061841e-06, + "spike_contrast": 4.048768854572246, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0076001490233467e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.227827962997253e-06, + 2.0000109088869067e-06, + 1.8881961523220525e-06, + 1.8253168718729285e-06, + 1.5388902738777688e-06, + 1.4780722494833753e-06, + 1.4035763342690188e-06, + 1.39851113090117e-06, + 1.6894116470211884e-06, + 1.6679639429639792e-06, + 1.661583041823178e-06, + 1.6590328186794068e-06, + 1.6420973452113685e-06, + 1.6178076975847944e-06, + 1.6785052139312029e-06, + 1.733251565383398e-06, + 3.3635192266956437e-06, + 3.3886535675264895e-06, + 3.2801560791995144e-06, + 3.271461309850565e-06, + 1.3249809853732586e-05, + 1.1270037248323206e-05, + 9.50859521253733e-06, + 8.062721462920308e-06, + 8.2888436736539e-06, + 6.945447694306495e-06, + 5.054590019426541e-06, + 4.141033059568144e-06, + 3.808041810771101e-06, + 2.771532763290452e-06, + 2.1110752186359605e-06, + 1.9482033621898154e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.143130106764147e-05, + "first_to_last_ratio": 0.7585265502107228, + "imbalance_abs_log_ratio": 0.27637747723236683, + "last_quartile_mean": 4.143731166550424e-05, + "mean": 4.658882704688949e-05, + "normalized": [ + 0.7474526909468017, + 0.8034952952675303, + 0.6937413721331414, + 0.6643883363040349, + 0.7364124474321028, + 0.6370996067440579, + 0.5721892936725778, + 0.542446525940414, + 0.669091824873211, + 0.627725699423567, + 0.5799860791244953, + 0.5432856872099403, + 0.4993479246608397, + 0.46546811100121066, + 0.46232421378322947, + 0.5482234365904111, + 1.3092639964182107, + 1.3047191024182814, + 1.1584436064552923, + 1.0651739919162182, + 3.6948461619806166, + 2.7963169559172143, + 2.1158202215330553, + 1.6473298337183941, + 1.7175540409463472, + 1.395874122700735, + 1.0016414447628996, + 0.8142117122540828, + 0.7421371569646497, + 0.5662099426381535, + 0.45351577301773555, + 0.4242633912505488 + ], + "peak_layer": 21, + "peak_normalized": 3.6948461619806166, + "population_cv": 0.7186539970377201, + "rest_mean": 3.4558785719801325e-05, + "spike_contrast": 3.2278637078747128, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011155105021316558, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.482294414425269e-05, + 3.7433903344208375e-05, + 3.2320596801582724e-05, + 3.0953073292039335e-05, + 3.430859214859083e-05, + 2.9681723390240222e-05, + 2.6657628040993586e-05, + 2.5271947379224002e-05, + 3.11722033075057e-05, + 2.92450040433323e-05, + 2.7020871129934676e-05, + 2.5311042918474413e-05, + 2.3264034098247066e-05, + 2.1685613319277763e-05, + 2.153914283553604e-05, + 2.554108687036205e-05, + 6.0997073887847364e-05, + 6.0785332607338205e-05, + 5.397052882472053e-05, + 4.9625206884229556e-05, + 0.00017213854880537838, + 0.0001302771270275116, + 9.857358236331493e-05, + 7.674716471228749e-05, + 8.001882815733552e-05, + 6.503213808173314e-05, + 4.6665300033055246e-05, + 3.793316864175722e-05, + 3.457529965089634e-05, + 2.6379057089798152e-05, + 2.1128767912159674e-05, + 1.9765933757298626e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.4235268710981472e-05, + "first_to_last_ratio": 1.025722660756393, + "imbalance_abs_log_ratio": 0.02539739905431976, + "last_quartile_mean": 2.3627506379853003e-05, + "mean": 3.1929538977237826e-05, + "normalized": [ + 0.8468036918455434, + 0.8945762958025398, + 0.792146970968438, + 0.7585176838192365, + 0.8072161456725058, + 0.7130868417798004, + 0.6487939523425942, + 0.6110459875951377, + 0.7424727464984224, + 0.6961736990480094, + 0.6430387303321913, + 0.5996082358384006, + 0.5883303956688428, + 0.5485366222262686, + 0.5418345777794853, + 0.6224185100083337, + 1.4328317480472454, + 1.4116054930014736, + 1.22660698514051, + 1.1171064576533825, + 3.798916633890286, + 2.7250739228804366, + 1.8992806880428186, + 1.414065294394857, + 1.4476049125600117, + 1.1698550635675442, + 0.8584820800626458, + 0.7351807895168184, + 0.5755234543551883, + 0.4483728349764013, + 0.36052571503973785, + 0.32436683964489244 + ], + "peak_layer": 21, + "peak_normalized": 3.798916633890286, + "population_cv": 0.6978402085797324, + "rest_mean": 2.4497121086352746e-05, + "spike_contrast": 2.9417577410010094, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.206459558801725e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.7038051484851167e-05, + 2.856340870494023e-05, + 2.5292887585237622e-05, + 2.4219119950430468e-05, + 2.5774039386305958e-05, + 2.276853410876356e-05, + 2.0715691789519042e-05, + 1.9510416677803732e-05, + 2.37068124988582e-05, + 2.2228505258681253e-05, + 2.0531930204015225e-05, + 1.9145214537275024e-05, + 1.8785118300002068e-05, + 1.7514521459816024e-05, + 1.7300528270425275e-05, + 1.987353607546538e-05, + 4.574965714709833e-05, + 4.507191260927357e-05, + 3.916499554179609e-05, + 3.5668694181367755e-05, + 0.00012129765673307702, + 8.701035403646529e-05, + 6.0643156757578254e-05, + 4.515045293373987e-05, + 4.6221357479225844e-05, + 3.7352932849898934e-05, + 2.7410937036620453e-05, + 2.3473983674193732e-05, + 1.837619856814854e-05, + 1.4316337910713628e-05, + 1.1511419870657846e-05, + 1.0356883649365045e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.03353295850684e-05, + "first_to_last_ratio": 0.568590855165461, + "imbalance_abs_log_ratio": 0.5645941629662194, + "last_quartile_mean": 0.0001412884657838731, + "mean": 0.00013750306106885546, + "normalized": [ + 0.6520467512598511, + 0.6971711299775947, + 0.5978339777797664, + 0.5761185587459762, + 0.6300627892234183, + 0.5553891026221839, + 0.4941772783126335, + 0.4711518985747209, + 0.6059991137617907, + 0.5690214037338702, + 0.5279200902275386, + 0.4965685016208935, + 0.4023636162938652, + 0.37961683338529517, + 0.3803019223025787, + 0.4631494229257258, + 1.1625925589928703, + 1.18192702020461, + 1.08886530788824, + 1.0107340573700256, + 3.686196825835599, + 2.9476998645270003, + 2.331510774085821, + 1.8713443744568392, + 1.9713065042712608, + 1.6029107997806242, + 1.1831279762613542, + 0.9483310133411668, + 0.8858233036848739, + 0.6634841961731901, + 0.4991942965805259, + 0.46605873579829593 + ], + "peak_layer": 21, + "peak_normalized": 3.686196825835599, + "population_cv": 0.7760081772891833, + "rest_mean": 9.773891576548556e-05, + "spike_contrast": 3.6037789344030697, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00035222944570705294, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.965842425823212e-05, + 9.586316446075216e-05, + 8.2204001955688e-05, + 7.921806536614895e-05, + 8.663556218380108e-05, + 7.636770169483498e-05, + 6.795088847866282e-05, + 6.478482828242704e-05, + 8.33267331472598e-05, + 7.824218482710421e-05, + 7.259062840603292e-05, + 6.827968900324777e-05, + 5.532622890314087e-05, + 5.219847662374377e-05, + 5.2292678446974605e-05, + 6.368446338456124e-05, + 0.0001598600356373936, + 0.00016251858323812485, + 0.00014972231292631477, + 0.0001389790268149227, + 0.0005068633472546935, + 0.0004053177544847131, + 0.00032058986835181713, + 0.00025731557980179787, + 0.00027106067864224315, + 0.0002204051415901631, + 0.00016268371837213635, + 0.00013039841724094003, + 0.00012180341582279652, + 9.123110794462264e-05, + 6.864074384793639e-05, + 6.408450281014666e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.09253507107496262, + 0.30484533309936523, + 0.33420246839523315, + 0.38682517409324646, + 0.12552575767040253, + 0.14749252796173096, + 0.1726136952638626, + 0.1892337054014206, + 0.031880609691143036, + 0.06123264133930206, + 0.10403046011924744, + 0.15778414905071259, + 0.053015924990177155, + 0.09510950744152069, + 0.1597633957862854, + 0.7120755910873413, + 0.04075472429394722, + 0.07218439877033234, + 0.11449843645095825, + 0.16539782285690308, + 0.0493597537279129, + 0.07179203629493713, + 0.10295005142688751, + 0.1323672980070114, + 0.023035654798150063, + 0.06294231116771698, + 0.09175154566764832, + 0.1252463310956955, + 0.03424545377492905, + 0.07851024717092514, + 0.12469635903835297, + 0.19299884140491486 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0284754921485728e-05, + "first_to_last_ratio": 0.5012334330235659, + "imbalance_abs_log_ratio": 0.690683352232122, + "last_quartile_mean": 2.0518892483778473e-05, + "mean": 1.877660952231963e-05, + "normalized": [ + 0.5986256157968327, + 0.6557756887645085, + 0.552024728579064, + 0.5273299589895433, + 0.6138703950605617, + 0.5187982010550154, + 0.47064556551807535, + 0.44487320369305056, + 0.571873215148553, + 0.5260659787052139, + 0.4944689419167516, + 0.46120603772309077, + 0.39860670574115153, + 0.37568940388544003, + 0.359876426854885, + 0.4301817276210402, + 1.0911581660743186, + 1.0892455573175825, + 1.0129382513783356, + 0.9043825192325898, + 3.793965158948972, + 3.077978138202815, + 2.391038396737279, + 1.8970614359566813, + 1.8215147947572354, + 1.6199717700575011, + 1.323724822749107, + 0.9702855164110245, + 1.1932092071917983, + 0.7841724480827788, + 0.5845704650360021, + 0.4448715568132013 + ], + "peak_layer": 21, + "peak_normalized": 3.793965158948972, + "population_cv": 0.8048970821977687, + "rest_mean": 1.322599483602163e-05, + "spike_contrast": 3.6859177273800294, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.874992882832885e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1240159437875263e-05, + 1.2313244042161386e-05, + 1.0365152775193565e-05, + 9.90146872936748e-06, + 1.1526404705364257e-05, + 9.741271242091898e-06, + 8.837128007144202e-06, + 8.353210432687774e-06, + 1.0737840057117864e-05, + 9.877735465124715e-06, + 9.284450243285391e-06, + 8.659885679662693e-06, + 7.484482466679765e-06, + 7.054173238429939e-06, + 6.757259143341798e-06, + 8.077354323177133e-06, + 2.048825081146788e-05, + 2.0452338503673673e-05, + 1.901954601635225e-05, + 1.6981237422442064e-05, + 7.123780233087018e-05, + 5.779399361927062e-05, + 4.489559432840906e-05, + 3.562038182280958e-05, + 3.42018720402848e-05, + 3.0417577363550663e-05, + 2.485506411176175e-05, + 1.8218672266812064e-05, + 2.240442336187698e-05, + 1.47240998558118e-05, + 1.0976251360261813e-05, + 8.353179509867914e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.08571245316125e-05, + "first_to_last_ratio": 0.4979792148814041, + "imbalance_abs_log_ratio": 0.6971969400144402, + "last_quartile_mean": 8.204584310078644e-05, + "mean": 7.52336177356483e-05, + "normalized": [ + 0.5949478566233565, + 0.6500556287691993, + 0.5492555313323417, + 0.5251723799753355, + 0.6053417822936014, + 0.5124155017083939, + 0.4658024827358601, + 0.4415692350980013, + 0.568789034243283, + 0.5274402651467243, + 0.4951821372611509, + 0.46126811471013973, + 0.3984797553319404, + 0.3743113527690246, + 0.3578756649483749, + 0.4280192256459612, + 1.0938415402881307, + 1.088997937173476, + 1.0233462881438173, + 0.9048151004160258, + 3.7997527267737836, + 3.0906481863038655, + 2.406800439856087, + 1.911490836810037, + 1.823302308579566, + 1.6164332700326205, + 1.3192176823718622, + 0.9632154163000318, + 1.192695801919089, + 0.7801502262369593, + 0.5836600747187168, + 0.4457062154832425 + ], + "peak_layer": 21, + "peak_normalized": 3.7997527267737836, + "population_cv": 0.8086595618810438, + "rest_mean": 5.285302500447465e-05, + "spike_contrast": 3.710077492582206, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019608881848398596, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.4760079617844895e-05, + 4.890603668172844e-05, + 4.132248068344779e-05, + 3.951061808038503e-05, + 4.554205224849284e-05, + 3.855087197734974e-05, + 3.5044005926465616e-05, + 3.322085103718564e-05, + 4.279205677448772e-05, + 3.968123928643763e-05, + 3.725434362422675e-05, + 3.470286901574582e-05, + 2.9979073588037863e-05, + 2.8160797228338197e-05, + 2.6924280973616987e-05, + 3.2201434805756435e-05, + 8.229365630540997e-05, + 8.192925452021882e-05, + 7.699004345340654e-05, + 6.807251338614151e-05, + 0.0002858691441360861, + 0.00023252064420375973, + 0.00018107230425812304, + 0.0001438083709217608, + 0.00013717362890020013, + 0.00012161012273281813, + 9.924951882567257e-05, + 7.246618042699993e-05, + 8.973082003649324e-05, + 5.869352389709093e-05, + 4.391085894894786e-05, + 3.3532091038068756e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.036232506390661, + "first_to_last_ratio": 0.49797921643881055, + "imbalance_abs_log_ratio": 0.6971969368869876, + "last_quartile_mean": 0.07275907346047461, + "mean": 0.06671792385168374, + "normalized": [ + 0.5949478763664731, + 0.6500556352385921, + 0.5492555345264337, + 0.5251723355462972, + 0.6053418267327986, + 0.512415499380336, + 0.4658024986597426, + 0.44156921137133487, + 0.5687890699892513, + 0.5274402435883548, + 0.4951821454127685, + 0.46126813432265656, + 0.39847974028217725, + 0.37431136283691135, + 0.35787566010736765, + 0.42801924671771074, + 1.093841580569198, + 1.0889979941148478, + 1.0233463082409806, + 0.90481507183944, + 3.799752645338714, + 3.090648148898127, + 2.4068002538924085, + 1.9114909689417183, + 1.8233022505104592, + 1.6164333240978725, + 1.3192177131312344, + 0.9632153929742584, + 1.192695726061282, + 0.7801502600750712, + 0.5836600965214254, + 0.4457062437137565 + ], + "peak_layer": 21, + "peak_normalized": 3.799752645338714, + "population_cv": 0.8086595420067639, + "rest_mean": 0.04687059118791863, + "spike_contrast": 3.7100773817599753, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.17389352023601531, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.0396936871111393, + 0.043370362371206284, + 0.03664518892765045, + 0.035038407891988754, + 0.04038714990019798, + 0.03418729826807976, + 0.031077375635504723, + 0.02946058101952076, + 0.037948425859212875, + 0.03518971800804138, + 0.03303752467036247, + 0.030774852260947227, + 0.026585740968585014, + 0.024973277002573013, + 0.023876721039414406, + 0.02855655550956726, + 0.07297883927822113, + 0.07265568524599075, + 0.06827554106712341, + 0.06036738306283951, + 0.25351160764694214, + 0.20620162785053253, + 0.1605767160654068, + 0.12753070890903473, + 0.1216469407081604, + 0.10784507542848587, + 0.08801546692848206, + 0.0642637312412262, + 0.07957418262958527, + 0.05205000564455986, + 0.038940589874982834, + 0.0297365952283144 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.7214280916277858e-06, + "first_to_last_ratio": 0.3156942462365298, + "imbalance_abs_log_ratio": 1.1529811089991324, + "last_quartile_mean": 5.452833278241087e-06, + "mean": 4.3127742159754234e-06, + "normalized": [ + 0.5189317449354922, + 0.46294464940978675, + 0.43791502232063706, + 0.4223204732934875, + 0.3553613513886626, + 0.34029330041035655, + 0.32825660023687764, + 0.3271476671813151, + 0.4058018309603618, + 0.39452072784651604, + 0.398844507071548, + 0.3992943751785707, + 0.3916207313588865, + 0.38791768893123935, + 0.3882593735909855, + 0.3995747453406645, + 0.7887230737056525, + 0.8161419891665606, + 0.8217969994517562, + 0.8161988751020437, + 3.9185816028423237, + 3.348456311882453, + 2.7353965324598755, + 2.280942122101189, + 2.131436393875469, + 1.8958527027667316, + 1.5685247034794665, + 1.1572372810973706, + 1.3516791024443255, + 0.870914762401272, + 0.6496906841262116, + 0.4894220736419112 + ], + "peak_layer": 21, + "peak_normalized": 3.9185816028423237, + "population_cv": 0.9292103644815702, + "rest_mean": 2.808923749466043e-06, + "spike_contrast": 4.426452208782674, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2433566735126077e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.238035449408926e-06, + 1.9965757473983103e-06, + 1.8886286170527455e-06, + 1.8213728480986902e-06, + 1.5325932736232062e-06, + 1.4676081718789646e-06, + 1.415696601725358e-06, + 1.4109140238360851e-06, + 1.7501316733614658e-06, + 1.7014788227243116e-06, + 1.7201263062815997e-06, + 1.7220664858541568e-06, + 1.6889717926460435e-06, + 1.673001406743424e-06, + 1.6744750155339716e-06, + 1.7232756590601639e-06, + 3.4015845358226215e-06, + 3.519836127452436e-06, + 3.5442249100015033e-06, + 3.520081463648239e-06, + 1.689995769993402e-05, + 1.4441136045206804e-05, + 1.1797147635661531e-05, + 9.837188372330274e-06, + 9.19240392249776e-06, + 8.176384653779678e-06, + 6.764692898286739e-06, + 4.990903107682243e-06, + 5.8294867812946904e-06, + 3.7560587315965677e-06, + 2.8019692308589583e-06, + 2.1107668999320595e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.160366418342164e-05, + "first_to_last_ratio": 0.6176052392742453, + "imbalance_abs_log_ratio": 0.48190579702671593, + "last_quartile_mean": 5.1171301947761094e-05, + "mean": 5.241891676632804e-05, + "normalized": [ + 0.6663285342326818, + 0.7149829604589701, + 0.6173687391102264, + 0.5916959595973353, + 0.6555070239747636, + 0.5687064704554682, + 0.5173636528095495, + 0.4912923321861497, + 0.619570657057656, + 0.5740235328549215, + 0.5363371245546895, + 0.4988167952927593, + 0.459862273249172, + 0.43179654672369233, + 0.4122041130466284, + 0.4796080159925161, + 1.1736023475997315, + 1.156174320077284, + 1.0641178126387796, + 0.9486702559599485, + 3.8830086994480117, + 3.0644404452878833, + 2.2928751268419547, + 1.772053067180281, + 1.7296002668230133, + 1.4826130454616122, + 1.1786287196306404, + 0.8692460269316107, + 1.0031677772103487, + 0.6504658693131994, + 0.49611695253952276, + 0.39975453545899814 + ], + "peak_layer": 21, + "peak_normalized": 3.8830086994480117, + "population_cv": 0.7861794547273071, + "rest_mean": 3.738832122137494e-05, + "spike_contrast": 3.5728839473194793, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013358413270907475, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.492821997497231e-05, + 3.747863229364157e-05, + 3.2361800549551845e-05, + 3.101606125710532e-05, + 3.436096812947653e-05, + 2.9810977139277384e-05, + 2.7119642254547216e-05, + 2.5753011868800968e-05, + 3.247722270316444e-05, + 3.0089691790635698e-05, + 2.8114211090723984e-05, + 2.614743607409764e-05, + 2.4105482225422747e-05, + 2.2634307242697105e-05, + 2.160729309252929e-05, + 2.514053267077543e-05, + 6.151896377559751e-05, + 6.060540545149706e-05, + 5.5779903050279245e-05, + 4.972826718585566e-05, + 0.00020354310981929302, + 0.0001606346486369148, + 0.00012019003042951226, + 9.288910223403946e-05, + 9.06637724256143e-05, + 7.771696982672438e-05, + 6.178244075272232e-05, + 4.5564935135189444e-05, + 5.258496821625158e-05, + 3.409671626286581e-05, + 2.6005913241533563e-05, + 2.0954699721187353e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.4343840550500317e-05, + "first_to_last_ratio": 0.8576976250683997, + "imbalance_abs_log_ratio": 0.1535036599433085, + "last_quartile_mean": 2.8382777145452565e-05, + "mean": 3.5231219840170525e-05, + "normalized": [ + 0.7672930723611481, + 0.8111997257980034, + 0.7192314911950108, + 0.6889029711983062, + 0.73287297469821, + 0.6490049259482068, + 0.596465935802746, + 0.5628171909857288, + 0.6991700183357967, + 0.6477070499213824, + 0.6038389641268231, + 0.5610866035665576, + 0.5540522683260976, + 0.5186653981035969, + 0.4956305510545183, + 0.5544873547004332, + 1.3116879927303091, + 1.2768834577654484, + 1.1499858983830977, + 1.0122913610994995, + 4.018776000950277, + 3.0408232653054914, + 2.07353399227015, + 1.5086765368659305, + 1.5087393189985034, + 1.2391962109666426, + 0.9602587067880377, + 0.7447783979976552, + 0.7557785917490731, + 0.5148267560191684, + 0.39972214388707694, + 0.3216148721010727 + ], + "peak_layer": 21, + "peak_normalized": 4.018776000950277, + "population_cv": 0.7633301669647466, + "rest_mean": 2.5900754365095593e-05, + "spike_contrast": 3.3055304953184197, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.561573340557516e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7032670914195478e-05, + 2.8579555873875506e-05, + 2.5339402782265097e-05, + 2.427089202683419e-05, + 2.582000888651237e-05, + 2.2865235223434865e-05, + 2.1014222511439584e-05, + 1.982873618544545e-05, + 2.4632612621644512e-05, + 2.281950946780853e-05, + 2.127398329321295e-05, + 1.9767765479627997e-05, + 1.951993726834189e-05, + 1.8273214664077386e-05, + 1.7461668903706595e-05, + 1.9535265892045572e-05, + 4.621236803359352e-05, + 4.498616181081161e-05, + 4.051540599903092e-05, + 3.566425948520191e-05, + 0.00014158638077788055, + 0.00010713191295508295, + 7.30531319277361e-05, + 5.315251473803073e-05, + 5.3154726629145443e-05, + 4.365839413367212e-05, + 3.38310856022872e-05, + 2.623945147206541e-05, + 2.6627001716406085e-05, + 1.8137974620913155e-05, + 1.4082698726269882e-05, + 1.1330884262861218e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.112810064631049e-05, + "first_to_last_ratio": 0.4595943192820276, + "imbalance_abs_log_ratio": 0.7774110932187634, + "last_quartile_mean": 0.00017652111273491755, + "mean": 0.00015741467757379723, + "normalized": [ + 0.5707305382524269, + 0.6126189981937518, + 0.5242133090337544, + 0.5073922534137821, + 0.5539956859087919, + 0.4868585867686246, + 0.4423060989265058, + 0.42491050886960396, + 0.550246837152428, + 0.5080246572887676, + 0.4801281677476931, + 0.45131986478447, + 0.36322233500428036, + 0.3493613334959881, + 0.33458611504256075, + 0.4005281085993613, + 1.0209699732329367, + 1.022227385527565, + 0.9907859769111992, + 0.8744049903891328, + 3.8474408994912648, + 3.15824288565664, + 2.5499603975894147, + 2.0045124714088103, + 1.8789611202618999, + 1.6962425295437986, + 1.3895520720535863, + 1.0272169143608445, + 1.191767182381932, + 0.7719440072425139, + 0.579364871029719, + 0.4359629244359521 + ], + "peak_layer": 21, + "peak_normalized": 3.8474408994912648, + "population_cv": 0.8430185744802908, + "rest_mean": 0.00010821315892691793, + "spike_contrast": 3.909902293423383, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00042310287826694546, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.984136366052553e-05, + 9.64352220762521e-05, + 8.251886902144179e-05, + 7.987098797457293e-05, + 8.720705227460712e-05, + 7.663868746021762e-05, + 6.962547195143998e-05, + 6.688715075142682e-05, + 8.661692845635116e-05, + 7.997053762665018e-05, + 7.557922072010115e-05, + 7.104437099769711e-05, + 5.717652675230056e-05, + 5.4994601669022813e-05, + 5.266876542009413e-05, + 6.30490030744113e-05, + 0.0001607156591489911, + 0.00016091359429992735, + 0.00015596425510011613, + 0.0001376441796310246, + 0.0006056436686776578, + 0.0004971537855453789, + 0.0004014011938124895, + 0.00031553968437947333, + 0.0002957760589197278, + 0.0002670134708750993, + 0.00021873589139431715, + 0.00016169901937246323, + 0.00018760164675768465, + 0.00012151531700510532, + 9.120053437072784e-05, + 6.862696318421513e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.15345558524131775, + 0.3249167501926422, + 0.3818626403808594, + 0.40600064396858215, + 0.1351035088300705, + 0.15324510633945465, + 0.18001793324947357, + 0.20391276478767395, + 0.04374876990914345, + 0.08211024105548859, + 0.1380448192358017, + 0.2295975536108017, + 0.08199761807918549, + 0.13892096281051636, + 0.21687114238739014, + 0.7360114455223083, + 0.057135771960020065, + 0.08251863718032837, + 0.12661415338516235, + 0.19619159400463104, + 0.06753352284431458, + 0.095451220870018, + 0.1274663656949997, + 0.15628865361213684, + 0.058436937630176544, + 0.08794140070676804, + 0.12247690558433533, + 0.1640285700559616, + 0.07537907361984253, + 0.12290703505277634, + 0.19157102704048157, + 0.29249706864356995 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0227535881313088e-05, + "first_to_last_ratio": 0.6251784250073547, + "imbalance_abs_log_ratio": 0.4697181899756329, + "last_quartile_mean": 1.635938713206997e-05, + "mean": 1.643692004904551e-05, + "normalized": [ + 0.681769220714557, + 0.748500240032939, + 0.6296107710699952, + 0.6007700154109977, + 0.7004931611617099, + 0.590031263703908, + 0.5278674383583044, + 0.49879356717844686, + 0.6254239322100563, + 0.5827967700199829, + 0.5411623822603087, + 0.5069232288890692, + 0.43843496230947776, + 0.4107323947027216, + 0.41012888390613456, + 0.49939331536990256, + 1.2341393523450959, + 1.24043551870817, + 1.1141674704177404, + 1.0232453347314594, + 3.6281949162463376, + 2.81895018298875, + 2.2126026034623427, + 1.7731690568174834, + 1.8705264645918427, + 1.5370337133448664, + 1.1190747928279094, + 0.8897567918614906, + 0.8979560617670435, + 0.6704137337604921, + 0.5087765332459856, + 0.46872592558447873 + ], + "peak_layer": 21, + "peak_normalized": 3.6281949162463376, + "population_cv": 0.736284928876382, + "rest_mean": 1.19907677395127e-05, + "spike_contrast": 3.373106993578246, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.04461425205227e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1206186172785237e-05, + 1.2303038602112792e-05, + 1.0348861906095408e-05, + 9.87480871117441e-06, + 1.1513950084918179e-05, + 9.698296707938425e-06, + 8.67651488078991e-06, + 8.198629984690342e-06, + 1.0280043170496356e-05, + 9.579383913660422e-06, + 8.895042810763698e-06, + 8.332256584253628e-06, + 7.206520422187168e-06, + 6.751175533281639e-06, + 6.741255674569402e-06, + 8.208487997762859e-06, + 2.0285449863877147e-05, + 2.0388939447002485e-05, + 1.831348163250368e-05, + 1.681900175753981e-05, + 5.963634976069443e-05, + 4.63348587800283e-05, + 3.6368372093420476e-05, + 2.914543802035041e-05, + 3.074569394811988e-05, + 2.5264100258937106e-05, + 1.839414289861452e-05, + 1.462486125092255e-05, + 1.4759631994820666e-05, + 1.1019536941603292e-05, + 8.36271919979481e-06, + 7.704410563746933e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.0602767057862366e-05, + "first_to_last_ratio": 0.6220527400735582, + "imbalance_abs_log_ratio": 0.47473039872247574, + "last_quartile_mean": 6.52722260383598e-05, + "mean": 6.584079096683126e-05, + "normalized": [ + 0.6772375898692375, + 0.7419156446131505, + 0.6265276809050203, + 0.5983548462080928, + 0.6909826192767372, + 0.5826528795658069, + 0.521809962087311, + 0.49396705008933894, + 0.6208428547650034, + 0.583552196572707, + 0.5417821373953783, + 0.5073870191655783, + 0.43782941267698894, + 0.4086714236070873, + 0.40807611508949443, + 0.4979465665227966, + 1.2379953934660572, + 1.244289507430791, + 1.1312943811704865, + 1.027048864560996, + 3.6326329119144347, + 2.8432062989929086, + 2.2290691465672023, + 1.784011106006968, + 1.8683393699616757, + 1.5302817088353375, + 1.1091498325862283, + 0.8793222326330687, + 0.8937044532253541, + 0.6693159640810682, + 0.5098515212827301, + 0.47095130887496306 + ], + "peak_layer": 21, + "peak_normalized": 3.6326329119144347, + "population_cv": 0.7406534740263053, + "rest_mean": 4.7899763524549565e-05, + "spike_contrast": 3.3971428496041334, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016272233915515244, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.458985858946107e-05, + 4.884831287199631e-05, + 4.1251078073401004e-05, + 3.93961563531775e-05, + 4.549484219751321e-05, + 3.83623264497146e-05, + 3.435638063820079e-05, + 3.252318128943443e-05, + 4.087678462383337e-05, + 3.8421538192778826e-05, + 3.567136445781216e-05, + 3.340676266816445e-05, + 2.882703483919613e-05, + 2.6907249775831588e-05, + 2.686805419216398e-05, + 3.278519579907879e-05, + 8.15105959190987e-05, + 8.192500536097214e-05, + 7.448531687259674e-05, + 6.762170960428193e-05, + 0.00023917542421258986, + 0.00018719895160757005, + 0.00014676367572974414, + 0.00011746070231311023, + 0.00012301294191274792, + 0.0001007549581117928, + 7.302730227820575e-05, + 5.789527131128125e-05, + 5.8842208090936765e-05, + 4.4068292481824756e-05, + 3.356902743689716e-05, + 3.100780668319203e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.036006940295919776, + "first_to_last_ratio": 0.6220527516263364, + "imbalance_abs_log_ratio": 0.47473038015045216, + "last_quartile_mean": 0.057884062407538295, + "mean": 0.058388272824231535, + "normalized": [ + 0.6772375510524524, + 0.7419156461027798, + 0.6265276980973783, + 0.5983548273880008, + 0.6909826136489176, + 0.5826528927482733, + 0.5218099376842688, + 0.49396708297188446, + 0.62084280967513, + 0.583552246154482, + 0.5417821421507241, + 0.5073870104266044, + 0.43782939997095355, + 0.4086714259278741, + 0.4080761210926988, + 0.49794657504969064, + 1.2379954375746993, + 1.244289507773514, + 1.1312943272761324, + 1.027048893286559, + 3.6326331106798357, + 2.843206262563162, + 2.229069109978408, + 1.7840111633852547, + 1.868339454059732, + 1.5302815940748427, + 1.109149791313182, + 0.879322184865126, + 0.8937044383290575, + 0.6693159548860595, + 0.5098515233166675, + 0.47095126649565455 + ], + "peak_layer": 21, + "peak_normalized": 3.6326331106798357, + "population_cv": 0.7406534951631871, + "rest_mean": 0.04247798943133266, + "spike_contrast": 3.397142969281968, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14430380314588548, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.039542730897665024, + 0.04331917315721512, + 0.036581870168447495, + 0.03493690490722656, + 0.04034528136253357, + 0.03402009606361389, + 0.030467581003904343, + 0.028841884806752205, + 0.03624993935227394, + 0.03407260775566101, + 0.03163372352719307, + 0.02962545119225979, + 0.025564102455973625, + 0.02386161871254444, + 0.023826859891414642, + 0.02907424047589302, + 0.07228441536426544, + 0.07265191525220871, + 0.0660543218255043, + 0.059967610985040665, + 0.21210317313671112, + 0.16600990295410156, + 0.13015149533748627, + 0.10416533052921295, + 0.10908911377191544, + 0.08935049921274185, + 0.06476134061813354, + 0.05134210363030434, + 0.05218185856938362, + 0.03908020257949829, + 0.029769349843263626, + 0.02749803103506565 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.7200537598682786e-06, + "first_to_last_ratio": 0.39238557455548834, + "imbalance_abs_log_ratio": 0.9355103140576796, + "last_quartile_mean": 4.3835805172420805e-06, + "mean": 3.6742107312193184e-06, + "normalized": [ + 0.6064111606904664, + 0.5442957897497384, + 0.513901156952112, + 0.49677880949687814, + 0.4188469295416997, + 0.4022651669595537, + 0.3819926927365715, + 0.38064734156709856, + 0.45982843639728, + 0.453964184318293, + 0.45225749435035856, + 0.4515574352379086, + 0.44690260682868216, + 0.4403067968449148, + 0.4568377219852153, + 0.47174268441611755, + 0.9154831899852715, + 0.9223194180598117, + 0.8927546763320205, + 0.8904547694224777, + 3.606171110034034, + 3.0673568527539175, + 2.587977908068276, + 2.1944076145236275, + 2.255921476109835, + 1.8902962427769419, + 1.3756718203329616, + 1.127065033693052, + 1.0364317453291698, + 0.7543213208282148, + 0.5745931723466826, + 0.5302372413308171 + ], + "peak_layer": 21, + "peak_normalized": 3.606171110034034, + "population_cv": 0.8429541336830753, + "rest_mean": 2.4886878606964644e-06, + "spike_contrast": 4.048733628339768, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.007603423204273e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.228082394140074e-06, + 1.9998574316559825e-06, + 1.8881811456594733e-06, + 1.8252700328957872e-06, + 1.5389318832603749e-06, + 1.4780069932385231e-06, + 1.4035216509000747e-06, + 1.3985785471959389e-06, + 1.689506575530686e-06, + 1.6679600776114967e-06, + 1.6616893390164478e-06, + 1.6591171743129962e-06, + 1.6420143538198317e-06, + 1.6177799579963903e-06, + 1.6785180605438654e-06, + 1.7332820334559074e-06, + 3.3636781608947786e-06, + 3.388795903447317e-06, + 3.280168812125339e-06, + 3.2717184694774915e-06, + 1.324983259110013e-05, + 1.1270115464867558e-05, + 9.508776201982982e-06, + 8.062716005952097e-06, + 8.28873089631088e-06, + 6.945346740394598e-06, + 5.054508164903382e-06, + 4.141074441577075e-06, + 3.8080686408648035e-06, + 2.771535491774557e-06, + 2.1111763999215327e-06, + 1.9482033621898154e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.1431425668415613e-05, + "first_to_last_ratio": 0.7585286956104332, + "imbalance_abs_log_ratio": 0.27637464885849405, + "last_quartile_mean": 4.1437358731855056e-05, + "mean": 4.658893448095114e-05, + "normalized": [ + 0.7474561991339586, + 0.8034972686588406, + 0.6937382106307133, + 0.6643963308091924, + 0.7364006760736782, + 0.6371054777478163, + 0.5721931279302918, + 0.5424472272278985, + 0.6690921560299369, + 0.6277240957148766, + 0.5799842731563194, + 0.5432849029139358, + 0.49934763211908073, + 0.46546535876668305, + 0.46232326479393904, + 0.5482249444672911, + 1.3092714408814747, + 1.304711330444448, + 1.1584451517697443, + 1.0651690368492541, + 3.6948591936119937, + 2.7963192533283774, + 2.1158287733747994, + 1.6473154151775933, + 1.7175425839388379, + 1.3958754328475529, + 1.0016444448776634, + 0.8142087414676477, + 0.7421362264642747, + 0.5662113699947803, + 0.45351804589774375, + 0.42426241289936223 + ], + "peak_layer": 21, + "peak_normalized": 3.6948591936119937, + "population_cv": 0.7186553093862196, + "rest_mean": 3.455886885435407e-05, + "spike_contrast": 3.2278628485990217, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011155128886457533, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.482318788883276e-05, + 3.743408160516992e-05, + 3.232052404200658e-05, + 3.09535171254538e-05, + 3.430812284932472e-05, + 2.968206536024809e-05, + 2.6657868147594854e-05, + 2.527203832869418e-05, + 3.117229061899707e-05, + 2.9244996767374687e-05, + 2.7020849302061833e-05, + 2.5311064746347256e-05, + 2.3264074116013944e-05, + 2.168553510273341e-05, + 2.153914829250425e-05, + 2.5541216018609703e-05, + 6.0997561377007514e-05, + 6.078511069063097e-05, + 5.3970725275576115e-05, + 4.962509046890773e-05, + 0.00017213955288752913, + 0.000130277534481138, + 9.857420809566975e-05, + 7.674666994716972e-05, + 8.001847891137004e-05, + 6.503234908450395e-05, + 4.666554741561413e-05, + 3.793311771005392e-05, + 3.457533603068441e-05, + 2.63791844190564e-05, + 2.1128922526258975e-05, + 1.9765933757298626e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.4235376486103632e-05, + "first_to_last_ratio": 1.0257631927899071, + "imbalance_abs_log_ratio": 0.025436913861062768, + "last_quartile_mean": 2.3626677830179688e-05, + "mean": 3.192918461536465e-05, + "normalized": [ + 0.8468527407513076, + 0.8944882935849241, + 0.7921265941283154, + 0.7585325396809063, + 0.8071309908556205, + 0.7131774186125125, + 0.6489677316897047, + 0.6110056554397294, + 0.7423413545118936, + 0.6962789572096383, + 0.6429117038614185, + 0.5998452181649871, + 0.5884347987626426, + 0.548496621769699, + 0.5418484530493665, + 0.6223045285782076, + 1.432791592163331, + 1.4116294770658595, + 1.2264918474272424, + 1.117234731658279, + 3.7992903581080144, + 2.725020991299199, + 1.8993221620386231, + 1.4137054453269882, + 1.4476142562135503, + 1.1700817965819004, + 0.8581975882692241, + 0.7352593061462055, + 0.5755171944998035, + 0.4482357862418949, + 0.3604934267212365, + 0.32437043958777473 + ], + "peak_layer": 21, + "peak_normalized": 3.7992903581080144, + "population_cv": 0.6978802845307617, + "rest_mean": 2.4496835302906454e-05, + "spike_contrast": 2.941762477134701, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.206387090263888e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.7039317501476035e-05, + 2.8560281862155534e-05, + 2.5291956262663007e-05, + 2.421932549623307e-05, + 2.57710344158113e-05, + 2.2771173462388106e-05, + 2.072101051453501e-05, + 1.9508912373567e-05, + 2.3702354155830108e-05, + 2.2231619368540123e-05, + 2.0527646483969875e-05, + 1.915256871143356e-05, + 1.878824332379736e-05, + 1.7513049897388555e-05, + 1.7300779290962964e-05, + 1.9869676179951057e-05, + 4.5747867261525244e-05, + 4.5072178181726485e-05, + 3.9160884625744075e-05, + 3.567239400581457e-05, + 0.00012130824325140566, + 8.700769831193611e-05, + 6.064380795578472e-05, + 4.5138462155591697e-05, + 4.6221142838476226e-05, + 3.735975769814104e-05, + 2.7401549232308753e-05, + 2.347623012610711e-05, + 1.837579475250095e-05, + 1.4311803170130588e-05, + 1.1510261174407788e-05, + 1.0356883649365045e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.033554240682861e-05, + "first_to_last_ratio": 0.5685808176961833, + "imbalance_abs_log_ratio": 0.5646118163593131, + "last_quartile_mean": 0.0001412913343301625, + "mean": 0.0001375067947719799, + "normalized": [ + 0.6520214267884701, + 0.6973466566797177, + 0.5977318663617682, + 0.5762119700788582, + 0.6301734142605383, + 0.5553052347453614, + 0.49401564943776766, + 0.47103073872558565, + 0.6061315575728188, + 0.5689597597401734, + 0.527894379296066, + 0.49653787440978014, + 0.4023372137884509, + 0.37963867061667483, + 0.38027646277267696, + 0.46302064917750796, + 1.162580251769062, + 1.1821961108062278, + 1.0889626285084961, + 1.0105664982562483, + 3.6859858284676985, + 2.947740045374168, + 2.3319071788655306, + 1.871247421579714, + 1.9714330950080552, + 1.60290198738071, + 1.1831372292603408, + 0.9483145762048698, + 0.8857899382807543, + 0.6633972344648077, + 0.49916037035168714, + 0.4660460809694134 + ], + "peak_layer": 21, + "peak_normalized": 3.6859858284676985, + "population_cv": 0.7760232497205517, + "rest_mean": 9.774026990204152e-05, + "spike_contrast": 3.6038986737265986, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003522460290696472, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.965737652033567e-05, + 9.588990360498428e-05, + 8.219219307648018e-05, + 7.923306111479178e-05, + 8.665312634548172e-05, + 7.635824294993654e-05, + 6.793050852138549e-05, + 6.47699271212332e-05, + 8.334720769198611e-05, + 7.823583291610703e-05, + 7.258906407514587e-05, + 6.827733159298077e-05, + 5.5324100685538724e-05, + 5.220289676799439e-05, + 5.229059752309695e-05, + 6.36684853816405e-05, + 0.00015986268408596516, + 0.00016255999798886478, + 0.00014973976067267358, + 0.00013895976007916033, + 0.0005068480968475342, + 0.0004053342854604125, + 0.00032065308187156916, + 0.0002573092351667583, + 0.00027108544600196183, + 0.00022040991461835802, + 0.00016268940817099065, + 0.00013039969780948013, + 0.00012180213525425643, + 9.122162737185135e-05, + 6.863794260425493e-05, + 6.408450281014666e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.06279198825359344, + 0.07690861076116562, + 0.18058517575263977, + 0.15882407128810883, + 0.19892723858356476, + 0.15647411346435547, + 0.1533493995666504, + 0.1527898609638214, + 0.16355258226394653, + 0.11789151281118393, + 0.11456070095300674, + 0.12148918211460114, + 0.15094918012619019, + 0.09854138642549515, + 0.11213528364896774, + 0.16254977881908417, + 0.2323070764541626, + 0.1390199512243271, + 0.14812250435352325, + 0.15333011746406555, + 0.1781008094549179, + 0.07593096792697906, + 0.09691466391086578, + 0.09368696808815002, + 0.10724716633558273, + 0.06538715958595276, + 0.07412076741456985, + 0.07655765861272812, + 0.08420819044113159, + 0.07230556756258011, + 0.07723049819469452, + 0.08422473073005676 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.327331502438028e-06, + "first_to_last_ratio": 0.938169966952749, + "imbalance_abs_log_ratio": 0.06382414494874765, + "last_quartile_mean": 7.810238827232752e-06, + "mean": 1.1439842303673231e-05, + "normalized": [ + 1.135812197656909, + 0.7535691222858811, + 0.7074574232777815, + 0.383535267321132, + 0.46428857856750727, + 0.7001642723538085, + 0.5761929500129469, + 0.40305886578711947, + 0.38956488690336305, + 0.6013590312167297, + 0.5758846795304952, + 0.5280893188480897, + 0.29011217837515946, + 0.35128556371858605, + 0.25293197560815023, + 1.0607061392911914, + 0.7223626091760469, + 1.3340410494507338, + 2.2261654108539917, + 2.4044148372801817, + 3.331737764852035, + 3.2127220644280428, + 2.5083141139795067, + 1.6244489369321418, + 0.9349893421747716, + 1.0552287432476088, + 1.0681431895248323, + 0.5998971016831224, + 0.6978207806681275, + 0.5604387184364499, + 0.3068330794026248, + 0.23842980715493145 + ], + "peak_layer": 21, + "peak_normalized": 3.331737764852035, + "population_cv": 0.8302470146151533, + "rest_mean": 8.638262114699399e-06, + "spike_contrast": 3.075662091674845, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6568375324131922e-05, + "top_five_layers": [ + 21, + 22, + 23, + 20, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.2993512427783571e-05, + 8.62071192386793e-06, + 8.093201358860824e-06, + 4.387582976050908e-06, + 5.311388122208882e-06, + 8.009768862393685e-06, + 6.591556484636385e-06, + 4.61092986370204e-06, + 4.45656087322277e-06, + 6.8794524850090966e-06, + 6.5880299189302605e-06, + 6.041258529876359e-06, + 3.3188375709869433e-06, + 4.018651452497579e-06, + 2.893501914513763e-06, + 1.2134310964029282e-05, + 8.263714335043915e-06, + 1.5261219232343137e-05, + 2.5466981242061593e-05, + 2.750612657109741e-05, + 3.8114554627100006e-05, + 3.675303378258832e-05, + 2.8694717912003398e-05, + 1.8583439668873325e-05, + 1.0696130630094558e-05, + 1.2071650417055935e-05, + 1.2219389645906631e-05, + 6.862728241685545e-06, + 7.982959687069524e-06, + 6.411330559785711e-06, + 3.5101220419164747e-06, + 2.727599394347635e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.5874984658003086e-05, + "first_to_last_ratio": 0.8450240698942337, + "imbalance_abs_log_ratio": 0.16839016694872203, + "last_quartile_mean": 3.062041139401117e-05, + "mean": 4.481995506466774e-05, + "normalized": [ + 0.6446676986966234, + 0.6572038842869348, + 0.7204290485695262, + 0.39682677548592965, + 0.4626069284501252, + 0.7305503021209181, + 0.5844980906502212, + 0.4216930592624511, + 0.3971779114095773, + 0.5863820168610077, + 0.5970881316925298, + 0.5421931111926823, + 0.2798279164088101, + 0.3526838924872086, + 0.2505358061975237, + 1.06664298163141, + 0.7393280508622739, + 1.3538526055113962, + 2.339426776387917, + 2.517206586089556, + 3.4365096549417373, + 3.2562764392234147, + 2.588043510314798, + 1.612852686817098, + 0.9533189353197188, + 1.0548076880199933, + 1.0535523324501517, + 0.599042999372865, + 0.6907876896075521, + 0.5683312683754906, + 0.3059266864656475, + 0.239728534836911 + ], + "peak_layer": 21, + "peak_normalized": 3.4365096549417373, + "population_cv": 0.8603945180808654, + "rest_mean": 3.3453944423753354e-05, + "spike_contrast": 3.174406317546299, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010619641252560542, + "top_five_layers": [ + 21, + 22, + 23, + 20, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.8893977287225425e-05, + 2.9455848562065512e-05, + 3.2289597584167495e-05, + 1.778575824573636e-05, + 2.0734021745738573e-05, + 3.274323171353899e-05, + 2.6197178158327006e-05, + 1.8900263967225328e-05, + 1.780149614205584e-05, + 2.6281615646439604e-05, + 2.6761463232105598e-05, + 2.430107088002842e-05, + 1.2541874639282469e-05, + 1.5807276213308796e-05, + 1.1229003575863317e-05, + 4.780689050676301e-05, + 3.31366500176955e-05, + 6.067961294320412e-05, + 0.00010485300299478695, + 0.00011282108607701957, + 0.0001540242083137855, + 0.00014594616368412971, + 0.0001159959938377142, + 7.228798494907096e-05, + 4.272771184332669e-05, + 4.727643317892216e-05, + 4.722016819869168e-05, + 2.6849080313695595e-05, + 3.096107320743613e-05, + 2.547258191043511e-05, + 1.3711620340473019e-05, + 1.0744622159108985e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.022946195444092155, + "first_to_last_ratio": 0.8450240753189588, + "imbalance_abs_log_ratio": 0.16839016052911188, + "last_quartile_mean": 0.02715448720846325, + "mean": 0.0397467844595667, + "normalized": [ + 0.6446677141451772, + 0.6572038849412665, + 0.7204290502088391, + 0.39682678147434747, + 0.46260695282762065, + 0.7305502877937194, + 0.584498096406077, + 0.4216930673859841, + 0.3971779243477385, + 0.5863820273620618, + 0.5970881594088124, + 0.5421931359539319, + 0.2798279255524417, + 0.3526838858930204, + 0.25053580074847415, + 1.0666429892428337, + 0.7393281098238819, + 1.353852553770995, + 2.339426761851287, + 2.517206560586814, + 3.4365097564682574, + 3.2562762221327364, + 2.5880436017094564, + 1.6128525942011334, + 0.953318925735159, + 1.0548076983098749, + 1.053552337934538, + 0.5990430406264167, + 0.6907876804744102, + 0.5683312310796297, + 0.30592670279868195, + 0.23972853880438227 + ], + "peak_layer": 21, + "peak_normalized": 3.4365097564682574, + "population_cv": 0.8603945066377574, + "rest_mean": 0.029667292721569538, + "spike_contrast": 3.1744062637802095, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09417603984475136, + "top_five_layers": [ + 21, + 22, + 23, + 20, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.025623468682169914, + 0.02612174116075039, + 0.02863473817706108, + 0.015772588551044464, + 0.018387138843536377, + 0.029037024825811386, + 0.02323191985487938, + 0.016760943457484245, + 0.01578654535114765, + 0.023306800052523613, + 0.02373233437538147, + 0.021550433710217476, + 0.011122260242700577, + 0.014018050394952297, + 0.009957992471754551, + 0.04239562898874283, + 0.029385915026068687, + 0.05381128564476967, + 0.09298469126224518, + 0.10005086660385132, + 0.13659021258354187, + 0.129426509141922, + 0.10286641120910645, + 0.06410570442676544, + 0.037891361862421036, + 0.04192521423101425, + 0.04187531769275665, + 0.02381003461778164, + 0.02745658904314041, + 0.022589338943362236, + 0.012159602716565132, + 0.009528438560664654 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.3183985646824112e-06, + "first_to_last_ratio": 0.4757249717009145, + "imbalance_abs_log_ratio": 0.7429153822802956, + "last_quartile_mean": 4.873401024951818e-06, + "mean": 3.6109257539607142e-06, + "normalized": [ + 2.4889669214180863, + 1.2151073868036804, + 0.21016319311621645, + 0.26737414578311336, + 0.229833309489438, + 0.21558164250894266, + 0.25728077609312594, + 0.25210078746297815, + 0.3035722943409423, + 0.3462435475099886, + 0.3415130262480174, + 0.41431792433763126, + 0.2453034593327441, + 0.35760301980181497, + 0.36403270780468855, + 0.36703166516115565, + 0.27418249335708944, + 0.6688605260814774, + 0.9706460648458304, + 1.0693924336470282, + 1.9436699367464498, + 3.518245935177478, + 2.6603939519517374, + 2.2215709886204884, + 1.317163303854247, + 1.97794154021279, + 1.9610047181494767, + 1.276948756957183, + 1.7693726874433637, + 1.1962434844187537, + 0.705933272147052, + 0.592404099176991 + ], + "peak_layer": 22, + "peak_normalized": 3.518245935177478, + "population_cv": 0.8711221431824825, + "rest_mean": 2.7200910966274493e-06, + "spike_contrast": 3.096011348297048, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.421432903560344e-06, + "top_five_layers": [ + 22, + 23, + 1, + 24, + 26 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.98747475730488e-06, + 4.387662556837313e-06, + 7.588836865579651e-07, + 9.654681889514904e-07, + 8.299110163534351e-07, + 7.78449305016693e-07, + 9.290217803936685e-07, + 9.103172260438441e-07, + 1.0961770158246509e-06, + 1.2502597428465378e-06, + 1.2331781817920273e-06, + 1.4960712633182993e-06, + 8.857725788402604e-07, + 1.291277953896497e-06, + 1.3144950798960053e-06, + 1.3253240922495024e-06, + 9.900526265482767e-07, + 2.415205699435319e-06, + 3.5049308735324303e-06, + 3.861496679746779e-06, + 7.018447831796948e-06, + 1.2704124856099952e-05, + 9.606485036783852e-06, + 8.021927897061687e-06, + 4.756178896059282e-06, + 7.142200047383085e-06, + 7.081042440404417e-06, + 4.610967152984813e-06, + 6.3890734054439235e-06, + 4.31954640589538e-06, + 2.549072632973548e-06, + 2.139127218470094e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.7170206092487206e-05, + "first_to_last_ratio": 0.8911678909117117, + "imbalance_abs_log_ratio": 0.11522243950697675, + "last_quartile_mean": 1.9267083416707464e-05, + "mean": 2.9415226521223303e-05, + "normalized": [ + 0.5575790030356597, + 0.6724660826854738, + 0.7144899488628155, + 0.417230472852585, + 0.46200781574081473, + 0.7501512042139171, + 0.6407733041668837, + 0.45504858840534107, + 0.42645564325893864, + 0.6277929374359318, + 0.6344293066873767, + 0.5088524459644442, + 0.30594071619112606, + 0.3634377152793242, + 0.2738683738112511, + 1.0754677959704142, + 0.7641943868594835, + 1.3657039182645663, + 2.452287035990965, + 2.1509880308711593, + 3.185322927772963, + 3.4979418159477316, + 2.7764798392086663, + 1.681060761721552, + 0.9405171981782515, + 1.0485813422407628, + 1.0601218645686263, + 0.5229743179246872, + 0.6462290554255541, + 0.525396773887863, + 0.28719893118189943, + 0.20901044539297084 + ], + "peak_layer": 22, + "peak_normalized": 3.4979418159477316, + "population_cv": 0.8613910548049479, + "rest_mean": 2.1700459607624605e-05, + "spike_contrast": 3.2752747702027287, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.107496785465627e-05, + "top_five_layers": [ + 22, + 21, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.6401312677771784e-05, + 1.9780742150032893e-05, + 2.1016883692936972e-05, + 1.2272928870515898e-05, + 1.3590064554591663e-05, + 2.2065867597120814e-05, + 1.8848491890821606e-05, + 1.3385357306106016e-05, + 1.2544289347715676e-05, + 1.8466671463102102e-05, + 1.8661881767911837e-05, + 1.4968009963922668e-05, + 8.999315468827263e-06, + 1.0690602721297182e-05, + 8.055900252657011e-06, + 3.16351288347505e-05, + 2.247895099571906e-05, + 4.017249011667445e-05, + 7.213457865873352e-05, + 6.327180017251521e-05, + 9.369699546368793e-05, + 0.00010289275087416172, + 8.167078340193257e-05, + 4.944878310197964e-05, + 2.7665526431519538e-05, + 3.0844257707940415e-05, + 3.1183724786387756e-05, + 1.5383408026536927e-05, + 1.9008974049938843e-05, + 1.5454665117431432e-05, + 8.448021617368795e-06, + 6.1480895965360105e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.1342676202730217e-05, + "first_to_last_ratio": 0.9298538045037564, + "imbalance_abs_log_ratio": 0.07272790465029173, + "last_quartile_mean": 1.2198343597447092e-05, + "mean": 1.9104475796893894e-05, + "normalized": [ + 0.48611542950870446, + 0.6958890331865999, + 0.7193778246429409, + 0.443536817801586, + 0.4695891169522356, + 0.7667060760609765, + 0.6873473082470735, + 0.48118450474913316, + 0.4543655526511291, + 0.6811299166099339, + 0.6646454976571455, + 0.4541495149539672, + 0.33836611538815065, + 0.39892060594545364, + 0.300624501076897, + 1.0747424125435876, + 0.797130210787533, + 1.3941854251810624, + 2.532754742399452, + 1.9787761975331615, + 2.748320565243497, + 3.7309038097940537, + 2.887865112295946, + 1.7053168420610472, + 0.9364859985915804, + 1.057656867984351, + 1.0602007618551181, + 0.47533215874042833, + 0.6202652229599234, + 0.4947448929841209, + 0.26849791503077114, + 0.1948730485824397 + ], + "peak_layer": 22, + "peak_normalized": 3.7309038097940537, + "population_cv": 0.8569592439250643, + "rest_mean": 1.4145171580484774e-05, + "spike_contrast": 3.2438431944372783, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.588471856550314e-05, + "top_five_layers": [ + 22, + 23, + 21, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.286980457545724e-06, + 1.3294595191837288e-05, + 1.3743336239713244e-05, + 8.473538400721736e-06, + 8.97125391929876e-06, + 1.4647517673438415e-05, + 1.3131410014466383e-05, + 9.19277772482019e-06, + 8.680415703565814e-06, + 1.3012630006414838e-05, + 1.2697703823505435e-05, + 8.676288416609168e-06, + 6.464307261921931e-06, + 7.621169061167166e-06, + 5.74327350477688e-06, + 2.0532390408334322e-05, + 1.5228754818963353e-05, + 2.663518171175383e-05, + 4.8386951675638556e-05, + 3.7803481973242015e-05, + 5.250522372080013e-05, + 7.127696153474972e-05, + 5.517114914255217e-05, + 3.25791843351908e-05, + 1.789107409422286e-05, + 2.0205980035825633e-05, + 2.025457979470957e-05, + 9.08097172214184e-06, + 1.1849841939692851e-05, + 9.451841833651997e-06, + 5.129511919221841e-06, + 3.722947440110147e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 5.0219679451402044e-05, + "first_to_last_ratio": 0.8622296874614376, + "imbalance_abs_log_ratio": 0.14823358504030823, + "last_quartile_mean": 5.8243969306204235e-05, + "mean": 8.878516183585816e-05, + "normalized": [ + 0.5866594709310274, + 0.6211019320430468, + 0.7097025860238361, + 0.39191341369615046, + 0.44573397193374353, + 0.7357376579940132, + 0.6020321723688816, + 0.4321703611028182, + 0.40170670285380555, + 0.5630696276319809, + 0.6035155111168472, + 0.5482618086001224, + 0.2706367956181458, + 0.3291877010861888, + 0.24164854743977685, + 1.100466812721687, + 0.7574718220240377, + 1.3349314458317953, + 2.386976535433516, + 2.2364178778779267, + 3.7197213872776747, + 3.322432289382176, + 2.7119382310195754, + 1.6984839602598483, + 0.9668042280017547, + 1.0009347458556246, + 1.0275821493704995, + 0.5437250890941417, + 0.6536934260198529, + 0.556374259140604, + 0.3002121346940273, + 0.1987553455548734 + ], + "peak_layer": 21, + "peak_normalized": 3.7197213872776747, + "population_cv": 0.8943344260770331, + "rest_mean": 6.438772248252967e-05, + "spike_contrast": 3.4250525696675913, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000220531334343832, + "top_five_layers": [ + 21, + 22, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.2086656069150195e-05, + 5.514463555300608e-05, + 6.301105895545334e-05, + 3.479609586065635e-05, + 3.957456283387728e-05, + 6.532258703373373e-05, + 5.345152385416441e-05, + 3.8370315451174974e-05, + 3.566559462342411e-05, + 4.999222801416181e-05, + 5.3583222324959934e-05, + 4.8677513404982165e-05, + 2.4028531697695144e-05, + 2.9226983315311372e-05, + 2.1454805391840637e-05, + 9.770512406248599e-05, + 6.725225830450654e-05, + 0.00011852210445795208, + 0.00021192809799686074, + 0.00019856072321999818, + 0.0003302560653537512, + 0.00029498268850147724, + 0.0002407798747299239, + 0.0001508001732872799, + 8.58378698467277e-05, + 8.88681533979252e-05, + 9.123404743149877e-05, + 4.827472002943978e-05, + 5.803827662020922e-05, + 4.939777863910422e-05, + 2.665438296389766e-05, + 1.7646525520831347e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0596221424639225, + 0.07634982466697693, + 0.06968431174755096, + 0.11288930475711823, + 0.11885104328393936, + 0.1495160460472107, + 0.11554427444934845, + 0.13194940984249115, + 0.08884616196155548, + 0.10897023230791092, + 0.09843935817480087, + 0.09431816637516022, + 0.09025302529335022, + 0.0895693451166153, + 0.10807885229587555, + 0.2204425185918808, + 0.15380074083805084, + 0.08548793941736221, + 0.09071414917707443, + 0.10860678553581238, + 0.04808543995022774, + 0.05631518363952637, + 0.07238366454839706, + 0.08572373539209366, + 0.06546958535909653, + 0.06323372572660446, + 0.06869969516992569, + 0.07729499042034149, + 0.04753003641963005, + 0.06484207510948181, + 0.07835857570171356, + 0.08203105628490448 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.6136328844804666e-06, + "first_to_last_ratio": 0.1388409422044023, + "imbalance_abs_log_ratio": 1.9744263017511179, + "last_quartile_mean": 2.6027141757367644e-05, + "mean": 1.5468678590480067e-05, + "normalized": [ + 0.478841741648447, + 0.21344980083496803, + 0.36542032760625354, + 0.2153222147003958, + 0.10396918122002749, + 0.11898121205540443, + 0.1673784156250151, + 0.20551445783939368, + 0.2659762543534721, + 0.24072972609830148, + 0.32074315460152014, + 0.3998464374496061, + 0.28459676525425437, + 0.28657748114631404, + 0.27675462844093746, + 0.1575549602617813, + 0.3238731831241911, + 0.4768547933922055, + 0.5952138273554176, + 0.7441668021387481, + 3.7970206059006064, + 3.4384752815376007, + 2.787460393051167, + 2.2747146196513746, + 2.288877373275197, + 2.18568964820933, + 2.055422183181528, + 1.5564826251376447, + 2.0236279536954616, + 1.3877025596275139, + 1.0837206429424298, + 0.8790407486434917 + ], + "peak_layer": 21, + "peak_normalized": 3.7970206059006064, + "population_cv": 1.0347491651618592, + "rest_mean": 9.97641066330996e-06, + "spike_contrast": 4.523362852650103, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.512692539719865e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.4070489972655196e-06, + 3.3017863643181045e-06, + 5.652569598169066e-06, + 3.330750132590765e-06, + 1.6082658476079814e-06, + 1.8404821275908034e-06, + 2.5891229142871452e-06, + 3.1790370940143475e-06, + 4.1143011912936345e-06, + 3.723770760188927e-06, + 4.961472768627573e-06, + 6.185096026456449e-06, + 4.402335889608366e-06, + 4.432974947121693e-06, + 4.281028395780595e-06, + 2.4371670406253543e-06, + 5.0098901738238055e-06, + 7.376313533313805e-06, + 9.207171387970448e-06, + 1.1511277079989668e-05, + 5.873489135410637e-05, + 5.318866897141561e-05, + 4.311832890380174e-05, + 3.518682933645323e-05, + 3.540590842021629e-05, + 3.3809730666689575e-05, + 3.1794665119377896e-05, + 2.4076729459920898e-05, + 3.1302850402425975e-05, + 2.1465924874064513e-05, + 1.6763726307544857e-05, + 1.359759880870115e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.4505445051327115e-05, + "first_to_last_ratio": 0.13916692625915444, + "imbalance_abs_log_ratio": 1.9720811580258348, + "last_quartile_mean": 0.00010423054845887236, + "mean": 6.203866507803468e-05, + "normalized": [ + 0.4777447719884719, + 0.21024222837857554, + 0.3686322036929193, + 0.2152923782079127, + 0.10398351033326099, + 0.1188088861677413, + 0.16855731573325972, + 0.20724253185602967, + 0.2697828383859787, + 0.2445416329818946, + 0.32310538489568486, + 0.39878685324314506, + 0.28439552509962246, + 0.28629442201468447, + 0.27934695856398883, + 0.16785764651758242, + 0.3271642462653634, + 0.4748504817782237, + 0.5983260455063276, + 0.739212269124694, + 3.8074878097059237, + 3.449181173181382, + 2.775295719544032, + 2.2631461806203244, + 2.288355265686405, + 2.179366722064293, + 2.0601866927942867, + 1.5606560046874582, + 2.029916228241676, + 1.373895253382417, + 1.0798399654369917, + 0.8685048539194496 + ], + "peak_layer": 21, + "peak_normalized": 3.8074878097059237, + "population_cv": 1.0347950762894988, + "rest_mean": 4.0018463348840465e-05, + "spike_contrast": 4.521606760318707, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00018094775441568345, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.9638647902174853e-05, + 1.3043147191638127e-05, + 2.286944982188288e-05, + 1.3356451745494269e-05, + 6.450998171203537e-06, + 7.37074469725485e-06, + 1.0457070857228246e-05, + 1.2857050023740157e-05, + 1.6736967154429294e-05, + 1.5171036466199439e-05, + 2.0045026758452877e-05, + 2.4740204025874846e-05, + 1.7643518731347285e-05, + 1.776132376107853e-05, + 1.7330312402918935e-05, + 1.041366431309143e-05, + 2.0296833099564537e-05, + 2.945909000118263e-05, + 3.7119349144632e-05, + 4.585974238580093e-05, + 0.00023621146101504564, + 0.0002139825955964625, + 0.00017217564163729548, + 0.00014040256792213768, + 0.00014196650590747595, + 0.00013520500215236098, + 0.0001278112322324887, + 9.682101517682895e-05, + 0.00012593329302035272, + 8.523462747689337e-05, + 6.699182995362207e-05, + 5.388088175095618e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.012863573734648526, + "first_to_last_ratio": 0.13916692640099965, + "imbalance_abs_log_ratio": 1.9720811570065897, + "last_quartile_mean": 0.09243269264698029, + "mean": 0.05501650820951909, + "normalized": [ + 0.47774476945422384, + 0.21024223314992707, + 0.3686322032999863, + 0.2152923803280601, + 0.10398351730346923, + 0.11880888936358922, + 0.168557316539217, + 0.20724253070496107, + 0.2697828471204318, + 0.24454162479767508, + 0.32310541341967935, + 0.3987868618143491, + 0.28439551814715514, + 0.28629440598800215, + 0.2793469622063591, + 0.16785764609653073, + 0.32716425359298057, + 0.4748504780188051, + 0.5983260263623125, + 0.7392122842371308, + 3.8074877818523305, + 3.449181219200419, + 2.775295592596114, + 2.263146172837106, + 2.2883552995642145, + 2.1793667822100353, + 2.060186762129678, + 1.560655922165235, + 2.029916146426396, + 1.3738953075483749, + 1.0798399969383916, + 0.8685048545868602 + ], + "peak_layer": 21, + "peak_normalized": 3.8074877818523305, + "population_cv": 1.0347950719753642, + "rest_mean": 0.03548877341327844, + "spike_contrast": 4.521606713214234, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1604662761092186, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.02628384903073311, + 0.01156679354608059, + 0.020280856639146805, + 0.011844635009765625, + 0.005720810033380985, + 0.006536450237035751, + 0.009273434989154339, + 0.011401760391891003, + 0.014842510223388672, + 0.013453826308250427, + 0.017776131629943848, + 0.02193986065685749, + 0.015646448358893394, + 0.01575091853737831, + 0.015368694439530373, + 0.009234941564500332, + 0.0179994348436594, + 0.026124615222215652, + 0.0329178087413311, + 0.040668878704309464, + 0.20947468280792236, + 0.1897619068622589, + 0.15268707275390625, + 0.12451039999723434, + 0.12589731812477112, + 0.1199011504650116, + 0.11334428191184998, + 0.08586183935403824, + 0.11167889833450317, + 0.07558692246675491, + 0.05940902605652809, + 0.04778210446238518 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 9.132429070746184e-07, + "first_to_last_ratio": 0.13157058861916074, + "imbalance_abs_log_ratio": 2.028211775804557, + "last_quartile_mean": 6.941087036693716e-06, + "mean": 4.019081015549375e-06, + "normalized": [ + 0.4719080582262415, + 0.19563567332190476, + 0.37589310034262197, + 0.2187020951667458, + 0.08866865629458083, + 0.12479788224062963, + 0.1539116204234896, + 0.18829729202967996, + 0.2114162989880101, + 0.19477558566146955, + 0.27471340176701525, + 0.3560454871229287, + 0.28155198404351195, + 0.2833133731945887, + 0.26477187349304765, + 0.07414785991212178, + 0.28510418059097403, + 0.3651486236923762, + 0.5232000418265924, + 0.5799966034257934, + 4.031742911648876, + 3.4929475225280737, + 2.7997174339527757, + 2.3473254679510194, + 2.3644425871309127, + 2.264940124552736, + 2.1454049683903906, + 1.6160754654331098, + 2.027589421813429, + 1.4023816709528905, + 1.0974092843705072, + 0.8980234495109556 + ], + "peak_layer": 21, + "peak_normalized": 4.031742911648876, + "population_cv": 1.0854863513939026, + "rest_mean": 2.5251475295607194e-06, + "spike_contrast": 4.786382458212523, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2086321839888115e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.8966367179018562e-06, + 7.862756206122867e-07, + 1.5107448234630283e-06, + 8.789814387455408e-07, + 3.563665131878224e-07, + 5.01572799294081e-07, + 6.185832717164885e-07, + 7.567820716758433e-07, + 8.496992336404219e-07, + 7.828188586245233e-07, + 1.1040954177587992e-06, + 1.4309756579677924e-06, + 1.1315802339595393e-06, + 1.1386593996576266e-06, + 1.0641396102073486e-06, + 2.980062561164232e-07, + 1.1458567996669444e-06, + 1.467561901336012e-06, + 2.1027833554398967e-06, + 2.331053337911726e-06, + 1.620390139578376e-05, + 1.4038439076102804e-05, + 1.1252291187702212e-05, + 9.434091225557495e-06, + 9.502886314294301e-06, + 9.102977855945937e-06, + 8.622556379123125e-06, + 6.4951382228173316e-06, + 8.149046152539086e-06, + 5.636285550281173e-06, + 4.410576821101131e-06, + 3.6092289974476444e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 9.655955466314481e-06, + "first_to_last_ratio": 0.1540693296834623, + "imbalance_abs_log_ratio": 1.8703525851251048, + "last_quartile_mean": 6.267279468374909e-05, + "mean": 3.9840255141143643e-05, + "normalized": [ + 0.5186483596750515, + 0.22212115379322156, + 0.38946276978980626, + 0.19702341557402153, + 0.11254194343112874, + 0.1184124498685972, + 0.17702354057452438, + 0.2037008308619059, + 0.29919017744364984, + 0.2686201423560258, + 0.3521019072470715, + 0.44516471741513636, + 0.3377857213600636, + 0.3362681031102567, + 0.3211205247678912, + 0.1386776152858354, + 0.34749360198352186, + 0.5116110525159265, + 0.632501071990601, + 0.7925686032518788, + 4.072805517742206, + 3.6254666562082485, + 2.7879110439949613, + 2.206961143886179, + 2.3448097066416045, + 2.1048264561097376, + 1.9085470681835104, + 1.432339605184221, + 1.8454197739428329, + 1.2178876067716404, + 0.9507433292729381, + 0.7802443897658043 + ], + "peak_layer": 21, + "peak_normalized": 4.072805517742206, + "population_cv": 1.0495741603052116, + "rest_mean": 2.5028601393623155e-05, + "spike_contrast": 4.787450305085089, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011982318537775427, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.066308297798969e-05, + 8.849363439367153e-06, + 1.5516296116402373e-05, + 7.849463145248592e-06, + 4.4836997403763235e-06, + 4.7175822146527935e-06, + 7.0526630224776454e-06, + 8.115493074001279e-06, + 1.191981300507905e-05, + 1.0701895007514395e-05, + 1.4027829820406623e-05, + 1.7735475921654142e-05, + 1.3457469322020188e-05, + 1.3397007023741025e-05, + 1.2793523637810722e-05, + 5.524951575353043e-06, + 1.384423376293853e-05, + 2.038271486526355e-05, + 2.519900408515241e-05, + 3.1576135370414704e-05, + 0.00016226161096710712, + 0.00014443951658904552, + 0.0001110710873035714, + 8.79258950590156e-05, + 9.341781697003171e-05, + 8.385682303924114e-05, + 7.603700214531273e-05, + 5.706477531930432e-05, + 7.352199463639408e-05, + 4.852095298701897e-05, + 3.78778568119742e-05, + 3.108513556071557e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 6.370375842834619e-06, + "first_to_last_ratio": 0.19099125089091262, + "imbalance_abs_log_ratio": 1.6555276588385337, + "last_quartile_mean": 3.3354280958519666e-05, + "mean": 2.3971603710037925e-05, + "normalized": [ + 0.5784128702594641, + 0.24149517133975307, + 0.42604453119872426, + 0.19734517389086656, + 0.13184453780025324, + 0.12733703531507173, + 0.2044711591588218, + 0.2190235425972542, + 0.3481067705913005, + 0.31316189797316313, + 0.3999261751321915, + 0.5288777875214324, + 0.4286351087415631, + 0.4238951258244287, + 0.39498954658816, + 0.11192205557134918, + 0.4033400615894943, + 0.5955935820411911, + 0.7169597117309988, + 0.8670532520551248, + 4.498418466436227, + 3.907133635555405, + 2.7489384705222526, + 2.0558104028032402, + 2.298695941814857, + 1.891274127094267, + 1.6215552432376346, + 1.195951125397394, + 1.5600224249476962, + 1.0366900972214659, + 0.8497028324036099, + 0.6773721356453439 + ], + "peak_layer": 21, + "peak_normalized": 4.498418466436227, + "population_cv": 1.0695380403887063, + "rest_mean": 1.4641325580871327e-05, + "spike_contrast": 5.078440828109265, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.435510560753756e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.3865484106645454e-05, + 5.789026545244269e-06, + 1.0212970664724708e-05, + 4.730680302600376e-06, + 3.160525011480786e-06, + 3.052472948184004e-06, + 4.901501597487368e-06, + 5.2503455663099885e-06, + 8.34467755339574e-06, + 7.506992915295996e-06, + 9.586871783540118e-06, + 1.267804873350542e-05, + 1.0275070962961763e-05, + 1.0161445970879868e-05, + 9.468532880418934e-06, + 2.682951162569225e-06, + 9.668708116805647e-06, + 1.4277333320933394e-05, + 1.7186674085678533e-05, + 2.078465695376508e-05, + 0.00010783430479932576, + 9.366025915369391e-05, + 6.589646363863721e-05, + 4.9281072278972715e-05, + 5.5103428167058155e-05, + 4.533687388175167e-05, + 3.887127968482673e-05, + 2.8668866434600204e-05, + 3.7396239349618554e-05, + 2.485112418071367e-05, + 2.036873956967611e-05, + 1.6237696399912238e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.835175428117509e-05, + "first_to_last_ratio": 0.12494402924893692, + "imbalance_abs_log_ratio": 2.079889407965475, + "last_quartile_mean": 0.00022691563935950398, + "mean": 0.00013135403207797935, + "normalized": [ + 0.45371617384329477, + 0.2008095734378784, + 0.34918619535186396, + 0.18558037039021436, + 0.09416845492909857, + 0.10425965186662237, + 0.1530284666175816, + 0.18598938464103867, + 0.2547076146729275, + 0.23420270109644106, + 0.309427949590735, + 0.3818280972155079, + 0.2607738377460453, + 0.2628749102742167, + 0.25605291926525836, + 0.14748816253206562, + 0.3004644499210074, + 0.4368087743568899, + 0.5630623449909483, + 0.7166583299599412, + 3.751756554644515, + 3.44095092385939, + 2.8305031368360156, + 2.305606684861282, + 2.416018818934567, + 2.2538792330672948, + 2.229722355101626, + 1.6689559515627927, + 2.0203916975982095, + 1.3546173106278736, + 1.030555164602258, + 0.8459538056045994 + ], + "peak_layer": 21, + "peak_normalized": 3.751756554644515, + "population_cv": 1.0575612474642975, + "rest_mean": 8.39457537006604e-05, + "spike_contrast": 4.614393441469027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003873587353155017, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.95974488533102e-05, + 2.637714715092443e-05, + 4.5867014705436304e-05, + 2.4376729925279506e-05, + 1.2369406249490567e-05, + 1.3694925655727275e-05, + 2.0100906112929806e-05, + 2.4430455596302636e-05, + 3.345687218825333e-05, + 3.076346911257133e-05, + 4.064460881636478e-05, + 5.015466012991965e-05, + 3.425369504839182e-05, + 3.452967939665541e-05, + 3.3633583370829e-05, + 1.937316483235918e-05, + 3.9467216993216425e-05, + 5.737659375881776e-05, + 7.39605093258433e-05, + 9.413596126250923e-05, + 0.0004928083508275449, + 0.000451982778031379, + 0.00037179799983277917, + 0.00030285073444247246, + 0.0003173538134433329, + 0.00029605612508021295, + 0.00029288302175700665, + 0.00021922409359831363, + 0.00026538659585639834, + 0.00017793444567359984, + 0.0001353675761492923, + 0.00011111944331787527 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 1.877417995231997, + "forward": { + "activation_sha256": { + "attention_branch_output": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", + "mlp_branch_output": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", + "post_attention_state": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", + "post_mlp_state": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", + "pre_attention_input": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", + "pre_mlp_input": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a" + }, + "logits_sha256": "5f6747d51014b7e09e13dda43380fde8b7ba2972518bba8957bd86ff4ed7f868", + "loss_nats": 1.3013269901275635, + "mixer_summary_sha256": "9abaf96810a98e69acfbc813a9b903aa15d6279ddc462a3c26f1a1846711526f" + }, + "loss_nats": 1.3013269901275635, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.2200722098350525, + "group": 1, + "latest_source_mass": 0.06159520894289017, + "layer": 1, + "max_source_mass": 0.9384047985076904, + "mixer_index": 1, + "normalized_entropy": 0.3174970857665056, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9384047985076904, + "median": 0.949082612991333, + "p05": 0.8598991632461548, + "p95": 0.9761404991149902 + }, + { + "label": "current_group_partial", + "mean": 0.06159520894289017, + "median": 0.05091731622815132, + "p05": 0.02385946735739708, + "p95": 0.14010077714920044 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5651201605796814, + "group": 1, + "latest_source_mass": 0.3442652225494385, + "layer": 2, + "max_source_mass": 0.6557347774505615, + "mixer_index": 2, + "normalized_entropy": 0.815296053174681, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6557347774505615, + "median": 0.6636689305305481, + "p05": 0.3472050428390503, + "p95": 0.9190996289253235 + }, + { + "label": "current_group_partial", + "mean": 0.3442652225494385, + "median": 0.3363310396671295, + "p05": 0.08090042322874069, + "p95": 0.6527949571609497 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4018177092075348, + "group": 1, + "latest_source_mass": 0.14357808232307434, + "layer": 2, + "max_source_mass": 0.856421947479248, + "mixer_index": 3, + "normalized_entropy": 0.5797004164150741, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.856421947479248, + "median": 0.8660546541213989, + "p05": 0.7540444135665894, + "p95": 0.9155591726303101 + }, + { + "label": "current_group_partial", + "mean": 0.14357808232307434, + "median": 0.13394534587860107, + "p05": 0.08444082736968994, + "p95": 0.24595558643341064 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.687358021736145, + "group": 1, + "latest_source_mass": 0.5312775373458862, + "layer": 3, + "max_source_mass": 0.5312775373458862, + "mixer_index": 4, + "normalized_entropy": 0.9916480092739848, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46872249245643616, + "median": 0.4674533009529114, + "p05": 0.3992259204387665, + "p95": 0.5431661605834961 + }, + { + "label": "current_group_partial", + "mean": 0.5312775373458862, + "median": 0.5325466990470886, + "p05": 0.45683377981185913, + "p95": 0.6007740497589111 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.2678314447402954, + "group": 1, + "latest_source_mass": 0.07946884632110596, + "layer": 3, + "max_source_mass": 0.920531153678894, + "mixer_index": 5, + "normalized_entropy": 0.38639909712095066, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.920531153678894, + "median": 0.9314625263214111, + "p05": 0.8444089889526367, + "p95": 0.9650533199310303 + }, + { + "label": "current_group_partial", + "mean": 0.07946884632110596, + "median": 0.06853749603033066, + "p05": 0.03494669869542122, + "p95": 0.1555909812450409 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.657238245010376, + "group": 1, + "latest_source_mass": 0.3777551054954529, + "layer": 4, + "max_source_mass": 0.6222448348999023, + "mixer_index": 6, + "normalized_entropy": 0.948194356759035, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6222448348999023, + "median": 0.6208515167236328, + "p05": 0.5314525961875916, + "p95": 0.7065955400466919 + }, + { + "label": "current_group_partial", + "mean": 0.3777551054954529, + "median": 0.3791484832763672, + "p05": 0.2934044897556305, + "p95": 0.4685474634170532 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5044373869895935, + "group": 1, + "latest_source_mass": 0.2207653522491455, + "layer": 4, + "max_source_mass": 0.7792346477508545, + "mixer_index": 7, + "normalized_entropy": 0.7277493166488734, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7792346477508545, + "median": 0.7866228222846985, + "p05": 0.6320486664772034, + "p95": 0.9385759830474854 + }, + { + "label": "current_group_partial", + "mean": 0.2207653522491455, + "median": 0.21337716281414032, + "p05": 0.06142403930425644, + "p95": 0.367951363325119 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.650418758392334, + "group": 2, + "latest_source_mass": 0.4602207541465759, + "layer": 5, + "max_source_mass": 0.5397792458534241, + "mixer_index": 8, + "normalized_entropy": 0.9383559172337771, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5397792458534241, + "median": 0.5213949680328369, + "p05": 0.34510529041290283, + "p95": 0.7807042598724365 + }, + { + "label": "completed_group_1", + "mean": 0.4602207541465759, + "median": 0.4786049723625183, + "p05": 0.2192957103252411, + "p95": 0.6548947095870972 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0032464265823364, + "group": 2, + "latest_source_mass": 0.37668097019195557, + "layer": 5, + "max_source_mass": 0.4337552785873413, + "mixer_index": 9, + "normalized_entropy": 0.913194251448444, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4337552785873413, + "median": 0.42766237258911133, + "p05": 0.2500072717666626, + "p95": 0.6592159271240234 + }, + { + "label": "completed_group_1", + "mean": 0.1895637810230255, + "median": 0.18978442251682281, + "p05": 0.1284591555595398, + "p95": 0.25696656107902527 + }, + { + "label": "current_group_partial", + "mean": 0.37668097019195557, + "median": 0.3803442120552063, + "p05": 0.1759825050830841, + "p95": 0.568291425704956 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0385558605194092, + "group": 2, + "latest_source_mass": 0.4949595034122467, + "layer": 6, + "max_source_mass": 0.4949595034122467, + "mixer_index": 10, + "normalized_entropy": 0.9453342832879565, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24478834867477417, + "median": 0.23865866661071777, + "p05": 0.19890648126602173, + "p95": 0.32154381275177 + }, + { + "label": "completed_group_1", + "mean": 0.26025211811065674, + "median": 0.26050686836242676, + "p05": 0.23401768505573273, + "p95": 0.2860541343688965 + }, + { + "label": "current_group_partial", + "mean": 0.4949595034122467, + "median": 0.5017017722129822, + "p05": 0.42678141593933105, + "p95": 0.5358227491378784 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.031773328781128, + "group": 2, + "latest_source_mass": 0.33116668462753296, + "layer": 6, + "max_source_mass": 0.40370824933052063, + "mixer_index": 11, + "normalized_entropy": 0.9391605568439314, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40370824933052063, + "median": 0.38690465688705444, + "p05": 0.18056227266788483, + "p95": 0.7100913524627686 + }, + { + "label": "completed_group_1", + "mean": 0.2651250660419464, + "median": 0.2730395793914795, + "p05": 0.12547528743743896, + "p95": 0.35344114899635315 + }, + { + "label": "current_group_partial", + "mean": 0.33116668462753296, + "median": 0.33928292989730835, + "p05": 0.1502782255411148, + "p95": 0.47667479515075684 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0838522911071777, + "group": 2, + "latest_source_mass": 0.38127952814102173, + "layer": 7, + "max_source_mass": 0.38127952814102173, + "mixer_index": 12, + "normalized_entropy": 0.9865648712351232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35612422227859497, + "median": 0.36401283740997314, + "p05": 0.2946957051753998, + "p95": 0.39828816056251526 + }, + { + "label": "completed_group_1", + "mean": 0.2625962495803833, + "median": 0.2590169608592987, + "p05": 0.24180665612220764, + "p95": 0.2929879426956177 + }, + { + "label": "current_group_partial", + "mean": 0.38127952814102173, + "median": 0.37846946716308594, + "p05": 0.3489896059036255, + "p95": 0.41648316383361816 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.96053147315979, + "group": 2, + "latest_source_mass": 0.3146248161792755, + "layer": 7, + "max_source_mass": 0.5271762013435364, + "mixer_index": 13, + "normalized_entropy": 0.874313425279704, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5271762013435364, + "median": 0.5114221572875977, + "p05": 0.3548204302787781, + "p95": 0.7728348970413208 + }, + { + "label": "completed_group_1", + "mean": 0.1581989824771881, + "median": 0.1618250608444214, + "p05": 0.07541970163583755, + "p95": 0.21691158413887024 + }, + { + "label": "current_group_partial", + "mean": 0.3146248161792755, + "median": 0.32365772128105164, + "p05": 0.14578065276145935, + "p95": 0.4483411908149719 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0704245567321777, + "group": 2, + "latest_source_mass": 0.38260507583618164, + "layer": 8, + "max_source_mass": 0.38260507583618164, + "mixer_index": 14, + "normalized_entropy": 0.9743424206822726, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3788670599460602, + "median": 0.39538437128067017, + "p05": 0.27489906549453735, + "p95": 0.43731769919395447 + }, + { + "label": "completed_group_1", + "mean": 0.23852786421775818, + "median": 0.2294245958328247, + "p05": 0.20573674142360687, + "p95": 0.29735344648361206 + }, + { + "label": "current_group_partial", + "mean": 0.38260507583618164, + "median": 0.37669745087623596, + "p05": 0.34335845708847046, + "p95": 0.435106486082077 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9674104452133179, + "group": 2, + "latest_source_mass": 0.3281455934047699, + "layer": 8, + "max_source_mass": 0.48829177021980286, + "mixer_index": 15, + "normalized_entropy": 0.8805749354816949, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48829177021980286, + "median": 0.44956350326538086, + "p05": 0.2823859453201294, + "p95": 0.8209846615791321 + }, + { + "label": "completed_group_1", + "mean": 0.18356263637542725, + "median": 0.18210050463676453, + "p05": 0.05149933695793152, + "p95": 0.2906108498573303 + }, + { + "label": "current_group_partial", + "mean": 0.3281455934047699, + "median": 0.34843379259109497, + "p05": 0.12376503646373749, + "p95": 0.46937939524650574 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0832805633544922, + "group": 3, + "latest_source_mass": 0.31498202681541443, + "layer": 9, + "max_source_mass": 0.40388280153274536, + "mixer_index": 16, + "normalized_entropy": 0.9860444622076776, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40388280153274536, + "median": 0.403384804725647, + "p05": 0.3489384353160858, + "p95": 0.4601682126522064 + }, + { + "label": "completed_group_1", + "mean": 0.2811351716518402, + "median": 0.28186845779418945, + "p05": 0.25417307019233704, + "p95": 0.30605652928352356 + }, + { + "label": "completed_group_2", + "mean": 0.31498202681541443, + "median": 0.3153840899467468, + "p05": 0.2613564431667328, + "p95": 0.3658811151981354 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2917464971542358, + "group": 3, + "latest_source_mass": 0.33769795298576355, + "layer": 9, + "max_source_mass": 0.33769795298576355, + "mixer_index": 17, + "normalized_entropy": 0.9317981327650527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3349480926990509, + "median": 0.3352771997451782, + "p05": 0.18474003672599792, + "p95": 0.48752525448799133 + }, + { + "label": "completed_group_1", + "mean": 0.1244347020983696, + "median": 0.11896269023418427, + "p05": 0.07986846566200256, + "p95": 0.1843445897102356 + }, + { + "label": "completed_group_2", + "mean": 0.20291924476623535, + "median": 0.20060715079307556, + "p05": 0.1253921389579773, + "p95": 0.2836313247680664 + }, + { + "label": "current_group_partial", + "mean": 0.33769795298576355, + "median": 0.33908897638320923, + "p05": 0.26979348063468933, + "p95": 0.4016563892364502 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3533791303634644, + "group": 3, + "latest_source_mass": 0.28473395109176636, + "layer": 10, + "max_source_mass": 0.32115638256073, + "mixer_index": 18, + "normalized_entropy": 0.976256679908994, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32115638256073, + "median": 0.3355404734611511, + "p05": 0.22348493337631226, + "p95": 0.3863494396209717 + }, + { + "label": "completed_group_1", + "mean": 0.20602071285247803, + "median": 0.2052205502986908, + "p05": 0.1802767962217331, + "p95": 0.23218519985675812 + }, + { + "label": "completed_group_2", + "mean": 0.18808895349502563, + "median": 0.18706050515174866, + "p05": 0.16633892059326172, + "p95": 0.2130376398563385 + }, + { + "label": "current_group_partial", + "mean": 0.28473395109176636, + "median": 0.275795578956604, + "p05": 0.22748424112796783, + "p95": 0.3660258948802948 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.308367371559143, + "group": 3, + "latest_source_mass": 0.29719752073287964, + "layer": 10, + "max_source_mass": 0.3113970160484314, + "mixer_index": 19, + "normalized_entropy": 0.9437875593046517, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3113970160484314, + "median": 0.33913683891296387, + "p05": 0.0915825292468071, + "p95": 0.47554588317871094 + }, + { + "label": "completed_group_1", + "mean": 0.16179001331329346, + "median": 0.16501447558403015, + "p05": 0.10682816803455353, + "p95": 0.20702867209911346 + }, + { + "label": "completed_group_2", + "mean": 0.2296154499053955, + "median": 0.2261994481086731, + "p05": 0.14510810375213623, + "p95": 0.32595163583755493 + }, + { + "label": "current_group_partial", + "mean": 0.29719752073287964, + "median": 0.2761377692222595, + "p05": 0.194716677069664, + "p95": 0.4515363574028015 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3437637090682983, + "group": 3, + "latest_source_mass": 0.3098427951335907, + "layer": 11, + "max_source_mass": 0.3217787742614746, + "mixer_index": 20, + "normalized_entropy": 0.969320619599697, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3217787742614746, + "median": 0.31754863262176514, + "p05": 0.25967735052108765, + "p95": 0.40468457341194153 + }, + { + "label": "completed_group_1", + "mean": 0.1920534074306488, + "median": 0.1915990114212036, + "p05": 0.16391108930110931, + "p95": 0.22057634592056274 + }, + { + "label": "completed_group_2", + "mean": 0.1763250231742859, + "median": 0.1779440939426422, + "p05": 0.14714840054512024, + "p95": 0.1996835470199585 + }, + { + "label": "current_group_partial", + "mean": 0.3098427951335907, + "median": 0.31089526414871216, + "p05": 0.2459525167942047, + "p95": 0.36900240182876587 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2280659675598145, + "group": 3, + "latest_source_mass": 0.29383569955825806, + "layer": 11, + "max_source_mass": 0.42534634470939636, + "mixer_index": 21, + "normalized_entropy": 0.8858623406415255, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42534634470939636, + "median": 0.4574331045150757, + "p05": 0.17354995012283325, + "p95": 0.6023476123809814 + }, + { + "label": "completed_group_1", + "mean": 0.14848899841308594, + "median": 0.14150214195251465, + "p05": 0.07509882748126984, + "p95": 0.23531809449195862 + }, + { + "label": "completed_group_2", + "mean": 0.13232898712158203, + "median": 0.13064970076084137, + "p05": 0.07000484317541122, + "p95": 0.19124652445316315 + }, + { + "label": "current_group_partial", + "mean": 0.29383569955825806, + "median": 0.27858495712280273, + "p05": 0.187642902135849, + "p95": 0.4577033817768097 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3462648391723633, + "group": 3, + "latest_source_mass": 0.3101534843444824, + "layer": 12, + "max_source_mass": 0.3221249580383301, + "mixer_index": 22, + "normalized_entropy": 0.9711248035985732, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3221249580383301, + "median": 0.3120178282260895, + "p05": 0.2763444483280182, + "p95": 0.3882659375667572 + }, + { + "label": "completed_group_1", + "mean": 0.18557845056056976, + "median": 0.1851869523525238, + "p05": 0.16533124446868896, + "p95": 0.20681798458099365 + }, + { + "label": "completed_group_2", + "mean": 0.18214309215545654, + "median": 0.18278372287750244, + "p05": 0.15564095973968506, + "p95": 0.2072875201702118 + }, + { + "label": "current_group_partial", + "mean": 0.3101534843444824, + "median": 0.31112557649612427, + "p05": 0.2608441412448883, + "p95": 0.36039745807647705 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.147221326828003, + "group": 3, + "latest_source_mass": 0.2585829496383667, + "layer": 12, + "max_source_mass": 0.5053804516792297, + "mixer_index": 23, + "normalized_entropy": 0.8275452595084083, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5053804516792297, + "median": 0.48097139596939087, + "p05": 0.30125147104263306, + "p95": 0.7477225065231323 + }, + { + "label": "completed_group_1", + "mean": 0.12089012563228607, + "median": 0.11902710795402527, + "p05": 0.05662901699542999, + "p95": 0.1844676434993744 + }, + { + "label": "completed_group_2", + "mean": 0.11514648050069809, + "median": 0.11994394659996033, + "p05": 0.05982710421085358, + "p95": 0.16130289435386658 + }, + { + "label": "current_group_partial", + "mean": 0.2585829496383667, + "median": 0.25007814168930054, + "p05": 0.12433226406574249, + "p95": 0.42273372411727905 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3729398250579834, + "group": 4, + "latest_source_mass": 0.22271886467933655, + "layer": 13, + "max_source_mass": 0.30785465240478516, + "mixer_index": 24, + "normalized_entropy": 0.9903667385250569, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30785465240478516, + "median": 0.3129041790962219, + "p05": 0.25217515230178833, + "p95": 0.3537779450416565 + }, + { + "label": "completed_group_1", + "mean": 0.23079943656921387, + "median": 0.22906282544136047, + "p05": 0.19741898775100708, + "p95": 0.26931890845298767 + }, + { + "label": "completed_group_2", + "mean": 0.23862707614898682, + "median": 0.23815912008285522, + "p05": 0.21215641498565674, + "p95": 0.2663886845111847 + }, + { + "label": "completed_group_3", + "mean": 0.22271886467933655, + "median": 0.22075486183166504, + "p05": 0.186451256275177, + "p95": 0.2638319730758667 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5232064723968506, + "group": 4, + "latest_source_mass": 0.24461600184440613, + "layer": 13, + "max_source_mass": 0.3136816620826721, + "mixer_index": 25, + "normalized_entropy": 0.9464213938474744, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3136816620826721, + "median": 0.326117604970932, + "p05": 0.20238417387008667, + "p95": 0.4362232983112335 + }, + { + "label": "completed_group_1", + "mean": 0.11285123229026794, + "median": 0.11373943835496902, + "p05": 0.08906605839729309, + "p95": 0.13395145535469055 + }, + { + "label": "completed_group_2", + "mean": 0.13196846842765808, + "median": 0.1334267258644104, + "p05": 0.10183953493833542, + "p95": 0.16085752844810486 + }, + { + "label": "completed_group_3", + "mean": 0.19688265025615692, + "median": 0.1940557211637497, + "p05": 0.13830327987670898, + "p95": 0.2671412229537964 + }, + { + "label": "current_group_partial", + "mean": 0.24461600184440613, + "median": 0.2419893592596054, + "p05": 0.18994617462158203, + "p95": 0.3077915906906128 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4876129627227783, + "group": 4, + "latest_source_mass": 0.396677166223526, + "layer": 14, + "max_source_mass": 0.396677166223526, + "mixer_index": 26, + "normalized_entropy": 0.9243059028433878, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2035508155822754, + "median": 0.20008960366249084, + "p05": 0.16096048057079315, + "p95": 0.255715548992157 + }, + { + "label": "completed_group_1", + "mean": 0.15372203290462494, + "median": 0.15584711730480194, + "p05": 0.12048184871673584, + "p95": 0.18010932207107544 + }, + { + "label": "completed_group_2", + "mean": 0.13167491555213928, + "median": 0.13113251328468323, + "p05": 0.11609883606433868, + "p95": 0.14900363981723785 + }, + { + "label": "completed_group_3", + "mean": 0.11437506973743439, + "median": 0.11201842874288559, + "p05": 0.09515875577926636, + "p95": 0.1399364471435547 + }, + { + "label": "current_group_partial", + "mean": 0.396677166223526, + "median": 0.39533013105392456, + "p05": 0.3409242630004883, + "p95": 0.4550648033618927 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4587602615356445, + "group": 4, + "latest_source_mass": 0.33461427688598633, + "layer": 14, + "max_source_mass": 0.33461427688598633, + "mixer_index": 27, + "normalized_entropy": 0.906378711639412, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28519001603126526, + "median": 0.28675830364227295, + "p05": 0.1500462144613266, + "p95": 0.45491817593574524 + }, + { + "label": "completed_group_1", + "mean": 0.10532153397798538, + "median": 0.09640030562877655, + "p05": 0.06345808506011963, + "p95": 0.1692449301481247 + }, + { + "label": "completed_group_2", + "mean": 0.09675674885511398, + "median": 0.09718051552772522, + "p05": 0.058796193450689316, + "p95": 0.13363884389400482 + }, + { + "label": "completed_group_3", + "mean": 0.17811740934848785, + "median": 0.1759425848722458, + "p05": 0.1284305453300476, + "p95": 0.23726706206798553 + }, + { + "label": "current_group_partial", + "mean": 0.33461427688598633, + "median": 0.33546411991119385, + "p05": 0.23162710666656494, + "p95": 0.4484652876853943 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4565696716308594, + "group": 4, + "latest_source_mass": 0.4221859574317932, + "layer": 15, + "max_source_mass": 0.4221859574317932, + "mixer_index": 28, + "normalized_entropy": 0.9050176216042753, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20232918858528137, + "median": 0.1998670995235443, + "p05": 0.15170496702194214, + "p95": 0.26285088062286377 + }, + { + "label": "completed_group_1", + "mean": 0.14928826689720154, + "median": 0.14749380946159363, + "p05": 0.12863489985466003, + "p95": 0.17874351143836975 + }, + { + "label": "completed_group_2", + "mean": 0.12029382586479187, + "median": 0.11950792372226715, + "p05": 0.10211217403411865, + "p95": 0.14088989794254303 + }, + { + "label": "completed_group_3", + "mean": 0.1059027910232544, + "median": 0.10411787033081055, + "p05": 0.08704904466867447, + "p95": 0.13138693571090698 + }, + { + "label": "current_group_partial", + "mean": 0.4221859574317932, + "median": 0.4195709228515625, + "p05": 0.35523390769958496, + "p95": 0.4969034492969513 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4788098335266113, + "group": 4, + "latest_source_mass": 0.34753501415252686, + "layer": 15, + "max_source_mass": 0.34753501415252686, + "mixer_index": 29, + "normalized_entropy": 0.9188362111403675, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2697942852973938, + "median": 0.25651901960372925, + "p05": 0.146310955286026, + "p95": 0.42084938287734985 + }, + { + "label": "completed_group_1", + "mean": 0.11948063224554062, + "median": 0.11549928039312363, + "p05": 0.0849025622010231, + "p95": 0.16021209955215454 + }, + { + "label": "completed_group_2", + "mean": 0.11090938746929169, + "median": 0.11184529960155487, + "p05": 0.08139486610889435, + "p95": 0.13886672258377075 + }, + { + "label": "completed_group_3", + "mean": 0.15228068828582764, + "median": 0.15164361894130707, + "p05": 0.11261995136737823, + "p95": 0.19522227346897125 + }, + { + "label": "current_group_partial", + "mean": 0.34753501415252686, + "median": 0.3453425168991089, + "p05": 0.25332537293434143, + "p95": 0.44237542152404785 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5018762350082397, + "group": 4, + "latest_source_mass": 0.31088152527809143, + "layer": 16, + "max_source_mass": 0.31088152527809143, + "mixer_index": 30, + "normalized_entropy": 0.9331681721954809, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.208971306681633, + "median": 0.20881521701812744, + "p05": 0.16508029401302338, + "p95": 0.2516583502292633 + }, + { + "label": "completed_group_1", + "mean": 0.27331358194351196, + "median": 0.26948875188827515, + "p05": 0.23055149614810944, + "p95": 0.33120542764663696 + }, + { + "label": "completed_group_2", + "mean": 0.12766750156879425, + "median": 0.1261082887649536, + "p05": 0.10924770683050156, + "p95": 0.1494429111480713 + }, + { + "label": "completed_group_3", + "mean": 0.07916608452796936, + "median": 0.07811431586742401, + "p05": 0.0561223030090332, + "p95": 0.10760308802127838 + }, + { + "label": "current_group_partial", + "mean": 0.31088152527809143, + "median": 0.3123162090778351, + "p05": 0.2707694172859192, + "p95": 0.3467172384262085 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5627843141555786, + "group": 4, + "latest_source_mass": 0.22652018070220947, + "layer": 16, + "max_source_mass": 0.27452588081359863, + "mixer_index": 31, + "normalized_entropy": 0.9710124895666443, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27452588081359863, + "median": 0.2678954601287842, + "p05": 0.19414913654327393, + "p95": 0.3916982412338257 + }, + { + "label": "completed_group_1", + "mean": 0.21448183059692383, + "median": 0.21617943048477173, + "p05": 0.17793960869312286, + "p95": 0.24279263615608215 + }, + { + "label": "completed_group_2", + "mean": 0.143874391913414, + "median": 0.1446641981601715, + "p05": 0.11340779811143875, + "p95": 0.17275942862033844 + }, + { + "label": "completed_group_3", + "mean": 0.14059773087501526, + "median": 0.13659776747226715, + "p05": 0.11039599776268005, + "p95": 0.18301524221897125 + }, + { + "label": "current_group_partial", + "mean": 0.22652018070220947, + "median": 0.22071748971939087, + "p05": 0.16668984293937683, + "p95": 0.293569415807724 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.580754280090332, + "group": 5, + "latest_source_mass": 0.24107450246810913, + "layer": 17, + "max_source_mass": 0.2518634498119354, + "mixer_index": 32, + "normalized_entropy": 0.9821778571747528, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2518634498119354, + "median": 0.24593469500541687, + "p05": 0.21242231130599976, + "p95": 0.3016672730445862 + }, + { + "label": "completed_group_1", + "mean": 0.18758200109004974, + "median": 0.1801876276731491, + "p05": 0.15723729133605957, + "p95": 0.24892441928386688 + }, + { + "label": "completed_group_2", + "mean": 0.13903990387916565, + "median": 0.13839681446552277, + "p05": 0.12513971328735352, + "p95": 0.15491701662540436 + }, + { + "label": "completed_group_3", + "mean": 0.18044015765190125, + "median": 0.1819951832294464, + "p05": 0.14810611307621002, + "p95": 0.20801132917404175 + }, + { + "label": "completed_group_4", + "mean": 0.24107450246810913, + "median": 0.24384509027004242, + "p05": 0.1871698796749115, + "p95": 0.2886185050010681 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7564777135849, + "group": 5, + "latest_source_mass": 0.2205006182193756, + "layer": 17, + "max_source_mass": 0.2205006182193756, + "mixer_index": 33, + "normalized_entropy": 0.9803088772521708, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15440437197685242, + "median": 0.15525281429290771, + "p05": 0.11778812110424042, + "p95": 0.18865147233009338 + }, + { + "label": "completed_group_1", + "mean": 0.19411899149417877, + "median": 0.2002902328968048, + "p05": 0.14482440054416656, + "p95": 0.2256385236978531 + }, + { + "label": "completed_group_2", + "mean": 0.17806550860404968, + "median": 0.1781877875328064, + "p05": 0.15101563930511475, + "p95": 0.20418882369995117 + }, + { + "label": "completed_group_3", + "mean": 0.14696575701236725, + "median": 0.14620032906532288, + "p05": 0.10836094617843628, + "p95": 0.1874053031206131 + }, + { + "label": "completed_group_4", + "mean": 0.10594475269317627, + "median": 0.10578415542840958, + "p05": 0.08951808512210846, + "p95": 0.1216079369187355 + }, + { + "label": "current_group_partial", + "mean": 0.2205006182193756, + "median": 0.21638520061969757, + "p05": 0.16007332503795624, + "p95": 0.2944811284542084 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5899518728256226, + "group": 5, + "latest_source_mass": 0.4157395362854004, + "layer": 18, + "max_source_mass": 0.4157395362854004, + "mixer_index": 34, + "normalized_entropy": 0.8873690359290373, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1636897772550583, + "median": 0.16218292713165283, + "p05": 0.1178273856639862, + "p95": 0.21766462922096252 + }, + { + "label": "completed_group_1", + "mean": 0.11789833009243011, + "median": 0.11790023744106293, + "p05": 0.1032494530081749, + "p95": 0.13267967104911804 + }, + { + "label": "completed_group_2", + "mean": 0.07030131667852402, + "median": 0.0698043555021286, + "p05": 0.06258689612150192, + "p95": 0.07938572019338608 + }, + { + "label": "completed_group_3", + "mean": 0.09238425642251968, + "median": 0.09013497084379196, + "p05": 0.0768275037407875, + "p95": 0.1164383590221405 + }, + { + "label": "completed_group_4", + "mean": 0.1399867981672287, + "median": 0.1400289535522461, + "p05": 0.12679682672023773, + "p95": 0.15317322313785553 + }, + { + "label": "current_group_partial", + "mean": 0.4157395362854004, + "median": 0.4171099066734314, + "p05": 0.367803692817688, + "p95": 0.4605288803577423 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.2894535064697266, + "group": 5, + "latest_source_mass": 0.5977407693862915, + "layer": 18, + "max_source_mass": 0.5977407693862915, + "mixer_index": 35, + "normalized_entropy": 0.7196577044045218, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1212274506688118, + "median": 0.10892187058925629, + "p05": 0.07249757647514343, + "p95": 0.2145335078239441 + }, + { + "label": "completed_group_1", + "mean": 0.06944876164197922, + "median": 0.0678764283657074, + "p05": 0.05350388586521149, + "p95": 0.08999023586511612 + }, + { + "label": "completed_group_2", + "mean": 0.05726535990834236, + "median": 0.0559837631881237, + "p05": 0.0409373864531517, + "p95": 0.07825973629951477 + }, + { + "label": "completed_group_3", + "mean": 0.09096579253673553, + "median": 0.08796361088752747, + "p05": 0.06407731771469116, + "p95": 0.12661314010620117 + }, + { + "label": "completed_group_4", + "mean": 0.06335186958312988, + "median": 0.06283852458000183, + "p05": 0.05032140761613846, + "p95": 0.0781375840306282 + }, + { + "label": "current_group_partial", + "mean": 0.5977407693862915, + "median": 0.6085352301597595, + "p05": 0.4878399074077606, + "p95": 0.6737192869186401 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6581614017486572, + "group": 5, + "latest_source_mass": 0.35773903131484985, + "layer": 19, + "max_source_mass": 0.35773903131484985, + "mixer_index": 36, + "normalized_entropy": 0.9254374988530375, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18132120370864868, + "median": 0.1806161105632782, + "p05": 0.1410265862941742, + "p95": 0.22331592440605164 + }, + { + "label": "completed_group_1", + "mean": 0.12711825966835022, + "median": 0.12722507119178772, + "p05": 0.11291128396987915, + "p95": 0.14178316295146942 + }, + { + "label": "completed_group_2", + "mean": 0.08560527861118317, + "median": 0.08513904362916946, + "p05": 0.07705061137676239, + "p95": 0.09564155340194702 + }, + { + "label": "completed_group_3", + "mean": 0.09748885035514832, + "median": 0.0953599065542221, + "p05": 0.08346174657344818, + "p95": 0.1194010004401207 + }, + { + "label": "completed_group_4", + "mean": 0.15072739124298096, + "median": 0.150638610124588, + "p05": 0.13762864470481873, + "p95": 0.1634974181652069 + }, + { + "label": "current_group_partial", + "mean": 0.35773903131484985, + "median": 0.35794761776924133, + "p05": 0.32225728034973145, + "p95": 0.39409786462783813 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3699336051940918, + "group": 5, + "latest_source_mass": 0.5385624766349792, + "layer": 19, + "max_source_mass": 0.5385624766349792, + "mixer_index": 37, + "normalized_entropy": 0.7645745027284836, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13604649901390076, + "median": 0.11998516321182251, + "p05": 0.06740576773881912, + "p95": 0.26897257566452026 + }, + { + "label": "completed_group_1", + "mean": 0.10151474922895432, + "median": 0.0991508960723877, + "p05": 0.07438431680202484, + "p95": 0.13600951433181763 + }, + { + "label": "completed_group_2", + "mean": 0.07715044170618057, + "median": 0.0767412930727005, + "p05": 0.054624155163764954, + "p95": 0.09943743050098419 + }, + { + "label": "completed_group_3", + "mean": 0.11300846189260483, + "median": 0.11003986746072769, + "p05": 0.08064281940460205, + "p95": 0.1516837179660797 + }, + { + "label": "completed_group_4", + "mean": 0.033717282116413116, + "median": 0.03286907076835632, + "p05": 0.024589763954281807, + "p95": 0.04592139273881912 + }, + { + "label": "current_group_partial", + "mean": 0.5385624766349792, + "median": 0.5415897369384766, + "p05": 0.4144754409790039, + "p95": 0.6506978273391724 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6926933526992798, + "group": 5, + "latest_source_mass": 0.3198680281639099, + "layer": 20, + "max_source_mass": 0.3198680281639099, + "mixer_index": 38, + "normalized_entropy": 0.9447101476341264, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19425281882286072, + "median": 0.19403350353240967, + "p05": 0.139950692653656, + "p95": 0.2494640350341797 + }, + { + "label": "completed_group_1", + "mean": 0.14001017808914185, + "median": 0.14058321714401245, + "p05": 0.12015820294618607, + "p95": 0.158815398812294 + }, + { + "label": "completed_group_2", + "mean": 0.09434760361909866, + "median": 0.09424005448818207, + "p05": 0.08333250135183334, + "p95": 0.10548526048660278 + }, + { + "label": "completed_group_3", + "mean": 0.10331346094608307, + "median": 0.10153411328792572, + "p05": 0.08690286427736282, + "p95": 0.12658309936523438 + }, + { + "label": "completed_group_4", + "mean": 0.14820793271064758, + "median": 0.14894580841064453, + "p05": 0.13278263807296753, + "p95": 0.16199614107608795 + }, + { + "label": "current_group_partial", + "mean": 0.3198680281639099, + "median": 0.3201481103897095, + "p05": 0.2800636887550354, + "p95": 0.3604968786239624 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4148362874984741, + "group": 5, + "latest_source_mass": 0.4996800422668457, + "layer": 20, + "max_source_mass": 0.4996800422668457, + "mixer_index": 39, + "normalized_entropy": 0.789635166883214, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16652248799800873, + "median": 0.14292314648628235, + "p05": 0.08186839520931244, + "p95": 0.350705623626709 + }, + { + "label": "completed_group_1", + "mean": 0.10329679399728775, + "median": 0.10057851672172546, + "p05": 0.07478320598602295, + "p95": 0.13858266174793243 + }, + { + "label": "completed_group_2", + "mean": 0.08063912391662598, + "median": 0.0807541012763977, + "p05": 0.056308042258024216, + "p95": 0.10469165444374084 + }, + { + "label": "completed_group_3", + "mean": 0.1165856122970581, + "median": 0.11373114585876465, + "p05": 0.08339665830135345, + "p95": 0.15619920194149017 + }, + { + "label": "completed_group_4", + "mean": 0.03327595070004463, + "median": 0.032142095267772675, + "p05": 0.022759035229682922, + "p95": 0.04896518588066101 + }, + { + "label": "current_group_partial", + "mean": 0.4996800422668457, + "median": 0.5001581907272339, + "p05": 0.3463003933429718, + "p95": 0.6266770362854004 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.740808129310608, + "group": 6, + "latest_source_mass": 0.24231041967868805, + "layer": 21, + "max_source_mass": 0.24231041967868805, + "mixer_index": 40, + "normalized_entropy": 0.9715635157550481, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21459883451461792, + "median": 0.21599377691745758, + "p05": 0.1570315659046173, + "p95": 0.27208006381988525 + }, + { + "label": "completed_group_1", + "mean": 0.15640269219875336, + "median": 0.1569681316614151, + "p05": 0.13785089552402496, + "p95": 0.1749057024717331 + }, + { + "label": "completed_group_2", + "mean": 0.10570268332958221, + "median": 0.10564640164375305, + "p05": 0.09223169088363647, + "p95": 0.11907710134983063 + }, + { + "label": "completed_group_3", + "mean": 0.11097013205289841, + "median": 0.10862545669078827, + "p05": 0.09454305469989777, + "p95": 0.13635265827178955 + }, + { + "label": "completed_group_4", + "mean": 0.17001524567604065, + "median": 0.17010903358459473, + "p05": 0.15088173747062683, + "p95": 0.18767674267292023 + }, + { + "label": "completed_group_5", + "mean": 0.24231041967868805, + "median": 0.24274763464927673, + "p05": 0.2087050974369049, + "p95": 0.27555277943611145 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3179612159729004, + "group": 6, + "latest_source_mass": 0.603651762008667, + "layer": 21, + "max_source_mass": 0.603651762008667, + "mixer_index": 41, + "normalized_entropy": 0.6772980841960945, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08068332821130753, + "median": 0.06917878985404968, + "p05": 0.039396293461322784, + "p95": 0.17595769464969635 + }, + { + "label": "completed_group_1", + "mean": 0.049232251942157745, + "median": 0.04861311614513397, + "p05": 0.03742668777704239, + "p95": 0.06363335251808167 + }, + { + "label": "completed_group_2", + "mean": 0.045073024928569794, + "median": 0.04482679069042206, + "p05": 0.03333880379796028, + "p95": 0.057495348155498505 + }, + { + "label": "completed_group_3", + "mean": 0.06111945956945419, + "median": 0.059528060257434845, + "p05": 0.042710162699222565, + "p95": 0.08362975716590881 + }, + { + "label": "completed_group_4", + "mean": 0.028317855671048164, + "median": 0.027924545109272003, + "p05": 0.022549759596586227, + "p95": 0.03542562201619148 + }, + { + "label": "completed_group_5", + "mean": 0.13192236423492432, + "median": 0.12939827144145966, + "p05": 0.10056710243225098, + "p95": 0.17195004224777222 + }, + { + "label": "current_group_partial", + "mean": 0.603651762008667, + "median": 0.6094026565551758, + "p05": 0.5065152645111084, + "p95": 0.6787958145141602 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.5319744348526, + "group": 6, + "latest_source_mass": 0.5132461190223694, + "layer": 22, + "max_source_mass": 0.5132461190223694, + "mixer_index": 42, + "normalized_entropy": 0.7872791226235868, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12406684458255768, + "median": 0.1289612352848053, + "p05": 0.06980825215578079, + "p95": 0.16972985863685608 + }, + { + "label": "completed_group_1", + "mean": 0.07462044060230255, + "median": 0.07392798364162445, + "p05": 0.06016036123037338, + "p95": 0.0917147696018219 + }, + { + "label": "completed_group_2", + "mean": 0.05378449708223343, + "median": 0.053309448063373566, + "p05": 0.04370884597301483, + "p95": 0.06546851992607117 + }, + { + "label": "completed_group_3", + "mean": 0.05575074627995491, + "median": 0.053552836179733276, + "p05": 0.0437602773308754, + "p95": 0.07444392144680023 + }, + { + "label": "completed_group_4", + "mean": 0.07357081025838852, + "median": 0.07329098880290985, + "p05": 0.06283499300479889, + "p95": 0.08542977273464203 + }, + { + "label": "completed_group_5", + "mean": 0.10496053099632263, + "median": 0.1038457602262497, + "p05": 0.08522139489650726, + "p95": 0.1291121542453766 + }, + { + "label": "current_group_partial", + "mean": 0.5132461190223694, + "median": 0.5090781450271606, + "p05": 0.4398295283317566, + "p95": 0.59932541847229 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.2461028099060059, + "group": 6, + "latest_source_mass": 0.6375080347061157, + "layer": 22, + "max_source_mass": 0.6375080347061157, + "mixer_index": 43, + "normalized_entropy": 0.640370168432985, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08556011319160461, + "median": 0.07947102189064026, + "p05": 0.043556369841098785, + "p95": 0.16172736883163452 + }, + { + "label": "completed_group_1", + "mean": 0.0446893647313118, + "median": 0.04398339241743088, + "p05": 0.03128381073474884, + "p95": 0.05990619957447052 + }, + { + "label": "completed_group_2", + "mean": 0.04084867238998413, + "median": 0.04042873531579971, + "p05": 0.02942952886223793, + "p95": 0.05392487719655037 + }, + { + "label": "completed_group_3", + "mean": 0.05386577919125557, + "median": 0.052273523062467575, + "p05": 0.033789411187171936, + "p95": 0.076903335750103 + }, + { + "label": "completed_group_4", + "mean": 0.025119207799434662, + "median": 0.024625632911920547, + "p05": 0.019138969480991364, + "p95": 0.03265690058469772 + }, + { + "label": "completed_group_5", + "mean": 0.11240880191326141, + "median": 0.11003861576318741, + "p05": 0.07852104306221008, + "p95": 0.155371755361557 + }, + { + "label": "current_group_partial", + "mean": 0.6375080347061157, + "median": 0.6392797231674194, + "p05": 0.5317062735557556, + "p95": 0.7386638522148132 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.6323832273483276, + "group": 6, + "latest_source_mass": 0.4532663822174072, + "layer": 23, + "max_source_mass": 0.4532663822174072, + "mixer_index": 44, + "normalized_entropy": 0.8388790346464895, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1418324112892151, + "median": 0.14852997660636902, + "p05": 0.07787702977657318, + "p95": 0.1923113763332367 + }, + { + "label": "completed_group_1", + "mean": 0.08991365134716034, + "median": 0.08918912708759308, + "p05": 0.0752246081829071, + "p95": 0.10654127597808838 + }, + { + "label": "completed_group_2", + "mean": 0.06188817322254181, + "median": 0.06161077320575714, + "p05": 0.04994015768170357, + "p95": 0.07481888681650162 + }, + { + "label": "completed_group_3", + "mean": 0.05739111453294754, + "median": 0.0548475980758667, + "p05": 0.04428064078092575, + "p95": 0.07816634327173233 + }, + { + "label": "completed_group_4", + "mean": 0.09848874807357788, + "median": 0.09768162667751312, + "p05": 0.08320866525173187, + "p95": 0.11606857925653458 + }, + { + "label": "completed_group_5", + "mean": 0.0972195416688919, + "median": 0.09613433480262756, + "p05": 0.07767169177532196, + "p95": 0.12004032731056213 + }, + { + "label": "current_group_partial", + "mean": 0.4532663822174072, + "median": 0.45108917355537415, + "p05": 0.37653055787086487, + "p95": 0.5389904975891113 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.301480770111084, + "group": 6, + "latest_source_mass": 0.6134276390075684, + "layer": 23, + "max_source_mass": 0.6134276390075684, + "mixer_index": 45, + "normalized_entropy": 0.6688288103861927, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09513165056705475, + "median": 0.08380547165870667, + "p05": 0.055099427700042725, + "p95": 0.1882638931274414 + }, + { + "label": "completed_group_1", + "mean": 0.050291214138269424, + "median": 0.04947858303785324, + "p05": 0.03757611662149429, + "p95": 0.06516233086585999 + }, + { + "label": "completed_group_2", + "mean": 0.045042067766189575, + "median": 0.0443364642560482, + "p05": 0.031873252242803574, + "p95": 0.06115761026740074 + }, + { + "label": "completed_group_3", + "mean": 0.05902485549449921, + "median": 0.05624900758266449, + "p05": 0.038259170949459076, + "p95": 0.08772744238376617 + }, + { + "label": "completed_group_4", + "mean": 0.02615261636674404, + "median": 0.02565642073750496, + "p05": 0.01966264471411705, + "p95": 0.03415588289499283 + }, + { + "label": "completed_group_5", + "mean": 0.1109299436211586, + "median": 0.10745325684547424, + "p05": 0.07642066478729248, + "p95": 0.15984633564949036 + }, + { + "label": "current_group_partial", + "mean": 0.6134276390075684, + "median": 0.6200529336929321, + "p05": 0.49638131260871887, + "p95": 0.7104787826538086 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.629331111907959, + "group": 6, + "latest_source_mass": 0.45539987087249756, + "layer": 24, + "max_source_mass": 0.45539987087249756, + "mixer_index": 46, + "normalized_entropy": 0.837310557580963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14790503680706024, + "median": 0.15420395135879517, + "p05": 0.08257082104682922, + "p95": 0.20396333932876587 + }, + { + "label": "completed_group_1", + "mean": 0.08658000826835632, + "median": 0.08578156679868698, + "p05": 0.07299856841564178, + "p95": 0.10278668999671936 + }, + { + "label": "completed_group_2", + "mean": 0.06608415395021439, + "median": 0.06551258265972137, + "p05": 0.054121918976306915, + "p95": 0.07912646979093552 + }, + { + "label": "completed_group_3", + "mean": 0.06427294760942459, + "median": 0.06133077293634415, + "p05": 0.04914242774248123, + "p95": 0.09066928923130035 + }, + { + "label": "completed_group_4", + "mean": 0.0819322019815445, + "median": 0.08114765584468842, + "p05": 0.06886082887649536, + "p95": 0.09821749478578568 + }, + { + "label": "completed_group_5", + "mean": 0.0978257954120636, + "median": 0.09643188118934631, + "p05": 0.07740232348442078, + "p95": 0.12209523469209671 + }, + { + "label": "current_group_partial", + "mean": 0.45539987087249756, + "median": 0.45274412631988525, + "p05": 0.3780696392059326, + "p95": 0.5462272763252258 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.338958501815796, + "group": 6, + "latest_source_mass": 0.5946891903877258, + "layer": 24, + "max_source_mass": 0.5946891903877258, + "mixer_index": 47, + "normalized_entropy": 0.6880885545850224, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10803166776895523, + "median": 0.09507323801517487, + "p05": 0.04510519653558731, + "p95": 0.2335781455039978 + }, + { + "label": "completed_group_1", + "mean": 0.0559723861515522, + "median": 0.05525296553969383, + "p05": 0.042058296501636505, + "p95": 0.07207933813333511 + }, + { + "label": "completed_group_2", + "mean": 0.047874726355075836, + "median": 0.04723185673356056, + "p05": 0.03294869884848595, + "p95": 0.06497423350811005 + }, + { + "label": "completed_group_3", + "mean": 0.060293328016996384, + "median": 0.05662515014410019, + "p05": 0.040346790105104446, + "p95": 0.08978720009326935 + }, + { + "label": "completed_group_4", + "mean": 0.029215168207883835, + "median": 0.028496116399765015, + "p05": 0.021991699934005737, + "p95": 0.039112430065870285 + }, + { + "label": "completed_group_5", + "mean": 0.10392352938652039, + "median": 0.10063231736421585, + "p05": 0.06900570541620255, + "p95": 0.1502695530653 + }, + { + "label": "current_group_partial", + "mean": 0.5946891903877258, + "median": 0.600748598575592, + "p05": 0.46449366211891174, + "p95": 0.7016042470932007 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7611865997314453, + "group": 7, + "latest_source_mass": 0.3679775297641754, + "layer": 25, + "max_source_mass": 0.3679775297641754, + "mixer_index": 48, + "normalized_entropy": 0.9050708742058073, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1573912650346756, + "median": 0.15898014605045319, + "p05": 0.09474922716617584, + "p95": 0.2298184633255005 + }, + { + "label": "completed_group_1", + "mean": 0.10317619889974594, + "median": 0.10270150750875473, + "p05": 0.08812867850065231, + "p95": 0.11994096636772156 + }, + { + "label": "completed_group_2", + "mean": 0.08533225208520889, + "median": 0.08495655655860901, + "p05": 0.07036984711885452, + "p95": 0.1013721376657486 + }, + { + "label": "completed_group_3", + "mean": 0.08218386024236679, + "median": 0.07942138612270355, + "p05": 0.06530377268791199, + "p95": 0.1088358536362648 + }, + { + "label": "completed_group_4", + "mean": 0.09052872657775879, + "median": 0.08931052684783936, + "p05": 0.07639768719673157, + "p95": 0.10804671049118042 + }, + { + "label": "completed_group_5", + "mean": 0.11341015249490738, + "median": 0.1123356968164444, + "p05": 0.09177497029304504, + "p95": 0.13803066313266754 + }, + { + "label": "completed_group_6", + "mean": 0.3679775297641754, + "median": 0.3671194612979889, + "p05": 0.2961871325969696, + "p95": 0.4462817311286926 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.476489782333374, + "group": 7, + "latest_source_mass": 0.3396714925765991, + "layer": 25, + "max_source_mass": 0.3836256265640259, + "mixer_index": 49, + "normalized_entropy": 0.7100414956318613, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09070298075675964, + "median": 0.0818760022521019, + "p05": 0.04576021432876587, + "p95": 0.1613437533378601 + }, + { + "label": "completed_group_1", + "mean": 0.03713828697800636, + "median": 0.035560622811317444, + "p05": 0.02337636426091194, + "p95": 0.054472144693136215 + }, + { + "label": "completed_group_2", + "mean": 0.030013620853424072, + "median": 0.028701743111014366, + "p05": 0.01836019568145275, + "p95": 0.04582716524600983 + }, + { + "label": "completed_group_3", + "mean": 0.03533655032515526, + "median": 0.031870774924755096, + "p05": 0.018502909690141678, + "p95": 0.061826568096876144 + }, + { + "label": "completed_group_4", + "mean": 0.020730752497911453, + "median": 0.020491186529397964, + "p05": 0.014567261561751366, + "p95": 0.027856238186359406 + }, + { + "label": "completed_group_5", + "mean": 0.06278066337108612, + "median": 0.059403590857982635, + "p05": 0.03608490154147148, + "p95": 0.10122650861740112 + }, + { + "label": "completed_group_6", + "mean": 0.3836256265640259, + "median": 0.3773127794265747, + "p05": 0.1874689906835556, + "p95": 0.5895293951034546 + }, + { + "label": "current_group_partial", + "mean": 0.3396714925765991, + "median": 0.34821513295173645, + "p05": 0.09212851524353027, + "p95": 0.587702751159668 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7213938236236572, + "group": 7, + "latest_source_mass": 0.37924307584762573, + "layer": 26, + "max_source_mass": 0.37924307584762573, + "mixer_index": 50, + "normalized_entropy": 0.8278154442529139, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10584281384944916, + "median": 0.10982948541641235, + "p05": 0.03868259862065315, + "p95": 0.16580678522586823 + }, + { + "label": "completed_group_1", + "mean": 0.06887699663639069, + "median": 0.06925956159830093, + "p05": 0.04082746431231499, + "p95": 0.09546782076358795 + }, + { + "label": "completed_group_2", + "mean": 0.05207539349794388, + "median": 0.051871635019779205, + "p05": 0.035383015871047974, + "p95": 0.07048650085926056 + }, + { + "label": "completed_group_3", + "mean": 0.050678279250860214, + "median": 0.04831986129283905, + "p05": 0.03378130868077278, + "p95": 0.07452918589115143 + }, + { + "label": "completed_group_4", + "mean": 0.04768219590187073, + "median": 0.04721469432115555, + "p05": 0.03680850565433502, + "p95": 0.060343313962221146 + }, + { + "label": "completed_group_5", + "mean": 0.06809806823730469, + "median": 0.06714263558387756, + "p05": 0.04759174585342407, + "p95": 0.09223967790603638 + }, + { + "label": "completed_group_6", + "mean": 0.2275032103061676, + "median": 0.22425106167793274, + "p05": 0.14320722222328186, + "p95": 0.3203037679195404 + }, + { + "label": "current_group_partial", + "mean": 0.37924307584762573, + "median": 0.3716055750846863, + "p05": 0.20950253307819366, + "p95": 0.5737906098365784 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5405172109603882, + "group": 7, + "latest_source_mass": 0.4017985463142395, + "layer": 26, + "max_source_mass": 0.4017985463142395, + "mixer_index": 51, + "normalized_entropy": 0.7408321802188831, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.074287548661232, + "median": 0.06164507195353508, + "p05": 0.028203874826431274, + "p95": 0.19368377327919006 + }, + { + "label": "completed_group_1", + "mean": 0.04491765797138214, + "median": 0.04365704953670502, + "p05": 0.02776508405804634, + "p95": 0.06526201963424683 + }, + { + "label": "completed_group_2", + "mean": 0.035157643258571625, + "median": 0.03418130800127983, + "p05": 0.022541508078575134, + "p95": 0.050758615136146545 + }, + { + "label": "completed_group_3", + "mean": 0.046252522617578506, + "median": 0.04353209584951401, + "p05": 0.026215802878141403, + "p95": 0.07329559326171875 + }, + { + "label": "completed_group_4", + "mean": 0.023427214473485947, + "median": 0.023055266588926315, + "p05": 0.016644639894366264, + "p95": 0.03150089830160141 + }, + { + "label": "completed_group_5", + "mean": 0.07749318331480026, + "median": 0.07363636791706085, + "p05": 0.04229677468538284, + "p95": 0.12579958140850067 + }, + { + "label": "completed_group_6", + "mean": 0.29666566848754883, + "median": 0.2920765280723572, + "p05": 0.14379237592220306, + "p95": 0.45965293049812317 + }, + { + "label": "current_group_partial", + "mean": 0.4017985463142395, + "median": 0.4004032611846924, + "p05": 0.18456703424453735, + "p95": 0.6275562047958374 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8063910007476807, + "group": 7, + "latest_source_mass": 0.3437207341194153, + "layer": 27, + "max_source_mass": 0.3437207341194153, + "mixer_index": 52, + "normalized_entropy": 0.868690446228377, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11433833092451096, + "median": 0.11714513599872589, + "p05": 0.06115472689270973, + "p95": 0.16258974373340607 + }, + { + "label": "completed_group_1", + "mean": 0.07374332845211029, + "median": 0.0731307864189148, + "p05": 0.05447638779878616, + "p95": 0.09509895741939545 + }, + { + "label": "completed_group_2", + "mean": 0.05932536721229553, + "median": 0.05929289758205414, + "p05": 0.04452420026063919, + "p95": 0.07498160749673843 + }, + { + "label": "completed_group_3", + "mean": 0.05389238893985748, + "median": 0.051720209419727325, + "p05": 0.03926049917936325, + "p95": 0.07536239922046661 + }, + { + "label": "completed_group_4", + "mean": 0.06126017868518829, + "median": 0.06108957529067993, + "p05": 0.0506296306848526, + "p95": 0.07238088548183441 + }, + { + "label": "completed_group_5", + "mean": 0.07349519431591034, + "median": 0.07249070703983307, + "p05": 0.05706139653921127, + "p95": 0.09332700818777084 + }, + { + "label": "completed_group_6", + "mean": 0.22022445499897003, + "median": 0.21959853172302246, + "p05": 0.15368635952472687, + "p95": 0.2904376685619354 + }, + { + "label": "current_group_partial", + "mean": 0.3437207341194153, + "median": 0.33986932039260864, + "p05": 0.22015926241874695, + "p95": 0.4802429676055908 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.4859801530838013, + "group": 7, + "latest_source_mass": 0.4424910843372345, + "layer": 27, + "max_source_mass": 0.4424910843372345, + "mixer_index": 53, + "normalized_entropy": 0.7146053992378076, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08387120068073273, + "median": 0.07644708454608917, + "p05": 0.0386628657579422, + "p95": 0.15876422822475433 + }, + { + "label": "completed_group_1", + "mean": 0.041406162083148956, + "median": 0.0394824743270874, + "p05": 0.025755178183317184, + "p95": 0.06136181950569153 + }, + { + "label": "completed_group_2", + "mean": 0.032444946467876434, + "median": 0.0307333804666996, + "p05": 0.018246563151478767, + "p95": 0.052875272929668427 + }, + { + "label": "completed_group_3", + "mean": 0.03686721250414848, + "median": 0.03247762471437454, + "p05": 0.017809681594371796, + "p95": 0.06780916452407837 + }, + { + "label": "completed_group_4", + "mean": 0.025052767246961594, + "median": 0.024561133235692978, + "p05": 0.017057543620467186, + "p95": 0.034822143614292145 + }, + { + "label": "completed_group_5", + "mean": 0.06012478470802307, + "median": 0.056938059628009796, + "p05": 0.03254516050219536, + "p95": 0.09905251860618591 + }, + { + "label": "completed_group_6", + "mean": 0.27774184942245483, + "median": 0.2747848629951477, + "p05": 0.11654777824878693, + "p95": 0.4562506675720215 + }, + { + "label": "current_group_partial", + "mean": 0.4424910843372345, + "median": 0.4387660026550293, + "p05": 0.21194897592067719, + "p95": 0.6782112717628479 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7777256965637207, + "group": 7, + "latest_source_mass": 0.3409850299358368, + "layer": 28, + "max_source_mass": 0.3409850299358368, + "mixer_index": 54, + "normalized_entropy": 0.8549053488311195, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10586117208003998, + "median": 0.10652337223291397, + "p05": 0.05095779523253441, + "p95": 0.1629783660173416 + }, + { + "label": "completed_group_1", + "mean": 0.07087038457393646, + "median": 0.06906798481941223, + "p05": 0.04985840618610382, + "p95": 0.09773125499486923 + }, + { + "label": "completed_group_2", + "mean": 0.05850498005747795, + "median": 0.05767729878425598, + "p05": 0.04179685190320015, + "p95": 0.07779045403003693 + }, + { + "label": "completed_group_3", + "mean": 0.06360965222120285, + "median": 0.06068425625562668, + "p05": 0.04235687851905823, + "p95": 0.09371945261955261 + }, + { + "label": "completed_group_4", + "mean": 0.04672512412071228, + "median": 0.04617916792631149, + "p05": 0.0356140062212944, + "p95": 0.05914818495512009 + }, + { + "label": "completed_group_5", + "mean": 0.07485505938529968, + "median": 0.07274073362350464, + "p05": 0.052147433161735535, + "p95": 0.10333258658647537 + }, + { + "label": "completed_group_6", + "mean": 0.2385886013507843, + "median": 0.2345724105834961, + "p05": 0.15389031171798706, + "p95": 0.33735349774360657 + }, + { + "label": "current_group_partial", + "mean": 0.3409850299358368, + "median": 0.33667099475860596, + "p05": 0.18655306100845337, + "p95": 0.5089476704597473 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6318418979644775, + "group": 7, + "latest_source_mass": 0.3292362689971924, + "layer": 28, + "max_source_mass": 0.3292362689971924, + "mixer_index": 55, + "normalized_entropy": 0.7847500712360619, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09244748950004578, + "median": 0.06782519817352295, + "p05": 0.025312144309282303, + "p95": 0.24087309837341309 + }, + { + "label": "completed_group_1", + "mean": 0.05762210115790367, + "median": 0.056058838963508606, + "p05": 0.03864594176411629, + "p95": 0.08028930425643921 + }, + { + "label": "completed_group_2", + "mean": 0.045099519193172455, + "median": 0.04426075518131256, + "p05": 0.03003283590078354, + "p95": 0.06244165822863579 + }, + { + "label": "completed_group_3", + "mean": 0.056236449629068375, + "median": 0.052902162075042725, + "p05": 0.03282861411571503, + "p95": 0.08620619773864746 + }, + { + "label": "completed_group_4", + "mean": 0.029373466968536377, + "median": 0.02887408435344696, + "p05": 0.020401213318109512, + "p95": 0.04018516466021538 + }, + { + "label": "completed_group_5", + "mean": 0.07237439602613449, + "median": 0.06993190199136734, + "p05": 0.04555661231279373, + "p95": 0.10748817771673203 + }, + { + "label": "completed_group_6", + "mean": 0.31761035323143005, + "median": 0.3173876404762268, + "p05": 0.16091161966323853, + "p95": 0.4724525213241577 + }, + { + "label": "current_group_partial", + "mean": 0.3292362689971924, + "median": 0.3128150701522827, + "p05": 0.12605780363082886, + "p95": 0.5847846865653992 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8334553241729736, + "group": 8, + "latest_source_mass": 0.270294725894928, + "layer": 29, + "max_source_mass": 0.27155569195747375, + "mixer_index": 56, + "normalized_entropy": 0.8817056346252721, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1157165914773941, + "median": 0.113271564245224, + "p05": 0.06316040456295013, + "p95": 0.17676270008087158 + }, + { + "label": "completed_group_1", + "mean": 0.07161639630794525, + "median": 0.07118864357471466, + "p05": 0.04294680058956146, + "p95": 0.102086141705513 + }, + { + "label": "completed_group_2", + "mean": 0.06577154994010925, + "median": 0.06490545719861984, + "p05": 0.04486069828271866, + "p95": 0.09022088348865509 + }, + { + "label": "completed_group_3", + "mean": 0.07425849884748459, + "median": 0.07179903984069824, + "p05": 0.045271825045347214, + "p95": 0.11117851734161377 + }, + { + "label": "completed_group_4", + "mean": 0.04881400242447853, + "median": 0.04833249747753143, + "p05": 0.0363147035241127, + "p95": 0.0629924014210701 + }, + { + "label": "completed_group_5", + "mean": 0.08197253197431564, + "median": 0.08029600977897644, + "p05": 0.05680408701300621, + "p95": 0.11223416775465012 + }, + { + "label": "completed_group_6", + "mean": 0.27155569195747375, + "median": 0.2687028646469116, + "p05": 0.18294592201709747, + "p95": 0.3740781545639038 + }, + { + "label": "completed_group_7", + "mean": 0.270294725894928, + "median": 0.2569413185119629, + "p05": 0.13936233520507812, + "p95": 0.43686625361442566 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6227548122406006, + "group": 8, + "latest_source_mass": 0.45173269510269165, + "layer": 29, + "max_source_mass": 0.45173269510269165, + "mixer_index": 57, + "normalized_entropy": 0.7385475426494315, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0605507455766201, + "median": 0.05293343961238861, + "p05": 0.02206447720527649, + "p95": 0.1254502236843109 + }, + { + "label": "completed_group_1", + "mean": 0.040063515305519104, + "median": 0.03803131729364395, + "p05": 0.022060411050915718, + "p95": 0.06445677578449249 + }, + { + "label": "completed_group_2", + "mean": 0.032619886100292206, + "median": 0.030926726758480072, + "p05": 0.020269516855478287, + "p95": 0.050711989402770996 + }, + { + "label": "completed_group_3", + "mean": 0.03868011012673378, + "median": 0.0344192236661911, + "p05": 0.020861119031906128, + "p95": 0.06798171997070312 + }, + { + "label": "completed_group_4", + "mean": 0.020290488377213478, + "median": 0.01974661275744438, + "p05": 0.014590941369533539, + "p95": 0.027847949415445328 + }, + { + "label": "completed_group_5", + "mean": 0.04760073497891426, + "median": 0.044880062341690063, + "p05": 0.027415042743086815, + "p95": 0.07675685733556747 + }, + { + "label": "completed_group_6", + "mean": 0.19551873207092285, + "median": 0.18908299505710602, + "p05": 0.10506269335746765, + "p95": 0.30520933866500854 + }, + { + "label": "completed_group_7", + "mean": 0.11294309794902802, + "median": 0.09412655979394913, + "p05": 0.03902517631649971, + "p95": 0.24146248400211334 + }, + { + "label": "current_group_partial", + "mean": 0.45173269510269165, + "median": 0.4618796110153198, + "p05": 0.24265648424625397, + "p95": 0.6204620599746704 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9787883758544922, + "group": 8, + "latest_source_mass": 0.1984800100326538, + "layer": 30, + "max_source_mass": 0.24502354860305786, + "mixer_index": 58, + "normalized_entropy": 0.9005854004479842, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0788881778717041, + "median": 0.055554650723934174, + "p05": 0.03367501497268677, + "p95": 0.21530622243881226 + }, + { + "label": "completed_group_1", + "mean": 0.08108963817358017, + "median": 0.07848536968231201, + "p05": 0.05139587074518204, + "p95": 0.12043403089046478 + }, + { + "label": "completed_group_2", + "mean": 0.057054273784160614, + "median": 0.056182146072387695, + "p05": 0.04127398133277893, + "p95": 0.07570968568325043 + }, + { + "label": "completed_group_3", + "mean": 0.08211587369441986, + "median": 0.0802752673625946, + "p05": 0.055154237896203995, + "p95": 0.113825723528862 + }, + { + "label": "completed_group_4", + "mean": 0.026040805503726006, + "median": 0.025504354387521744, + "p05": 0.01917124353349209, + "p95": 0.03502596169710159 + }, + { + "label": "completed_group_5", + "mean": 0.10537257045507431, + "median": 0.10299044847488403, + "p05": 0.07248030602931976, + "p95": 0.14682361483573914 + }, + { + "label": "completed_group_6", + "mean": 0.24502354860305786, + "median": 0.24406442046165466, + "p05": 0.1495005488395691, + "p95": 0.3413054645061493 + }, + { + "label": "completed_group_7", + "mean": 0.12593510746955872, + "median": 0.11572076380252838, + "p05": 0.051278628408908844, + "p95": 0.2299422323703766 + }, + { + "label": "current_group_partial", + "mean": 0.1984800100326538, + "median": 0.19106052815914154, + "p05": 0.11614266037940979, + "p95": 0.3049595355987549 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7590525150299072, + "group": 8, + "latest_source_mass": 0.3258843719959259, + "layer": 30, + "max_source_mass": 0.3258843719959259, + "mixer_index": 59, + "normalized_entropy": 0.800579300438408, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07131639868021011, + "median": 0.052984386682510376, + "p05": 0.03260878100991249, + "p95": 0.19837424159049988 + }, + { + "label": "completed_group_1", + "mean": 0.044687941670417786, + "median": 0.04335416108369827, + "p05": 0.027018578723073006, + "p95": 0.06556333601474762 + }, + { + "label": "completed_group_2", + "mean": 0.033290669322013855, + "median": 0.03222261369228363, + "p05": 0.01970716379582882, + "p95": 0.05042369291186333 + }, + { + "label": "completed_group_3", + "mean": 0.043597716838121414, + "median": 0.03997292369604111, + "p05": 0.024005703628063202, + "p95": 0.07244924455881119 + }, + { + "label": "completed_group_4", + "mean": 0.02658817172050476, + "median": 0.02605675905942917, + "p05": 0.017710059881210327, + "p95": 0.03680335357785225 + }, + { + "label": "completed_group_5", + "mean": 0.06922741234302521, + "median": 0.06634850800037384, + "p05": 0.03838749974966049, + "p95": 0.10990548133850098 + }, + { + "label": "completed_group_6", + "mean": 0.23554196953773499, + "median": 0.2292930632829666, + "p05": 0.10460641980171204, + "p95": 0.3925905227661133 + }, + { + "label": "completed_group_7", + "mean": 0.14986534416675568, + "median": 0.12543796002864838, + "p05": 0.04750998318195343, + "p95": 0.3270293176174164 + }, + { + "label": "current_group_partial", + "mean": 0.3258843719959259, + "median": 0.31322309374809265, + "p05": 0.1389845609664917, + "p95": 0.5583901405334473 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9736928939819336, + "group": 8, + "latest_source_mass": 0.20976582169532776, + "layer": 31, + "max_source_mass": 0.22560220956802368, + "mixer_index": 60, + "normalized_entropy": 0.8982663467084998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09849467873573303, + "median": 0.09779177606105804, + "p05": 0.017307927832007408, + "p95": 0.19556622207164764 + }, + { + "label": "completed_group_1", + "mean": 0.068942591547966, + "median": 0.0661388635635376, + "p05": 0.044743385165929794, + "p95": 0.10368303209543228 + }, + { + "label": "completed_group_2", + "mean": 0.0599900484085083, + "median": 0.05786054581403732, + "p05": 0.040265996009111404, + "p95": 0.08643331378698349 + }, + { + "label": "completed_group_3", + "mean": 0.06335455179214478, + "median": 0.05964011698961258, + "p05": 0.04099462926387787, + "p95": 0.09641734510660172 + }, + { + "label": "completed_group_4", + "mean": 0.04671281576156616, + "median": 0.04506188631057739, + "p05": 0.03403949737548828, + "p95": 0.06450185924768448 + }, + { + "label": "completed_group_5", + "mean": 0.0713595524430275, + "median": 0.06962447613477707, + "p05": 0.048271458595991135, + "p95": 0.10105480253696442 + }, + { + "label": "completed_group_6", + "mean": 0.22560220956802368, + "median": 0.22400957345962524, + "p05": 0.13957585394382477, + "p95": 0.31992682814598083 + }, + { + "label": "completed_group_7", + "mean": 0.1557777225971222, + "median": 0.1399117112159729, + "p05": 0.06281191110610962, + "p95": 0.2967374622821808 + }, + { + "label": "current_group_partial", + "mean": 0.20976582169532776, + "median": 0.2105630338191986, + "p05": 0.08139262348413467, + "p95": 0.34620192646980286 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.809161901473999, + "group": 8, + "latest_source_mass": 0.288146048784256, + "layer": 31, + "max_source_mass": 0.288146048784256, + "mixer_index": 61, + "normalized_entropy": 0.8233850650202157, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08854076266288757, + "median": 0.08406365662813187, + "p05": 0.020090684294700623, + "p95": 0.1807844340801239 + }, + { + "label": "completed_group_1", + "mean": 0.05356752499938011, + "median": 0.05036265030503273, + "p05": 0.031541675329208374, + "p95": 0.08521421998739243 + }, + { + "label": "completed_group_2", + "mean": 0.04243364930152893, + "median": 0.03939494490623474, + "p05": 0.023794323205947876, + "p95": 0.0721740797162056 + }, + { + "label": "completed_group_3", + "mean": 0.04702243581414223, + "median": 0.043501369655132294, + "p05": 0.02430840954184532, + "p95": 0.0788305327296257 + }, + { + "label": "completed_group_4", + "mean": 0.02648795023560524, + "median": 0.025087328627705574, + "p05": 0.01698113977909088, + "p95": 0.040197182446718216 + }, + { + "label": "completed_group_5", + "mean": 0.05820769816637039, + "median": 0.05441046878695488, + "p05": 0.02834305912256241, + "p95": 0.10125020891427994 + }, + { + "label": "completed_group_6", + "mean": 0.18489578366279602, + "median": 0.17386877536773682, + "p05": 0.07876831293106079, + "p95": 0.32456493377685547 + }, + { + "label": "completed_group_7", + "mean": 0.21069815754890442, + "median": 0.19004324078559875, + "p05": 0.07121473550796509, + "p95": 0.42417171597480774 + }, + { + "label": "current_group_partial", + "mean": 0.288146048784256, + "median": 0.28367581963539124, + "p05": 0.0962119847536087, + "p95": 0.5029257535934448 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.8544633388519287, + "group": 8, + "latest_source_mass": 0.12106549739837646, + "layer": 32, + "max_source_mass": 0.3525030016899109, + "mixer_index": 62, + "normalized_entropy": 0.8440026376822011, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07977832853794098, + "median": 0.053830958902835846, + "p05": 0.02079753391444683, + "p95": 0.2620829939842224 + }, + { + "label": "completed_group_1", + "mean": 0.05858254432678223, + "median": 0.056158941239118576, + "p05": 0.03545547276735306, + "p95": 0.09179478138685226 + }, + { + "label": "completed_group_2", + "mean": 0.053744811564683914, + "median": 0.05185101926326752, + "p05": 0.03345105051994324, + "p95": 0.08065605163574219 + }, + { + "label": "completed_group_3", + "mean": 0.07626347988843918, + "median": 0.07124648243188858, + "p05": 0.043040163815021515, + "p95": 0.12173676490783691 + }, + { + "label": "completed_group_4", + "mean": 0.031090255826711655, + "median": 0.02958102896809578, + "p05": 0.01873353309929371, + "p95": 0.04807420074939728 + }, + { + "label": "completed_group_5", + "mean": 0.0994817242026329, + "median": 0.0973888486623764, + "p05": 0.0575672909617424, + "p95": 0.14856761693954468 + }, + { + "label": "completed_group_6", + "mean": 0.3525030016899109, + "median": 0.3572760224342346, + "p05": 0.18325307965278625, + "p95": 0.5024369955062866 + }, + { + "label": "completed_group_7", + "mean": 0.1274903416633606, + "median": 0.1073581799864769, + "p05": 0.039848171174526215, + "p95": 0.2741084098815918 + }, + { + "label": "current_group_partial", + "mean": 0.12106549739837646, + "median": 0.10882791876792908, + "p05": 0.0503288172185421, + "p95": 0.22616314888000488 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9456019401550293, + "group": 8, + "latest_source_mass": 0.18013209104537964, + "layer": 32, + "max_source_mass": 0.2524128556251526, + "mixer_index": 63, + "normalized_entropy": 0.885481602665194, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08823961019515991, + "median": 0.07105386257171631, + "p05": 0.04482517018914223, + "p95": 0.1774362027645111 + }, + { + "label": "completed_group_1", + "mean": 0.06737187504768372, + "median": 0.06628458946943283, + "p05": 0.04582075774669647, + "p95": 0.09290395677089691 + }, + { + "label": "completed_group_2", + "mean": 0.05197303742170334, + "median": 0.051264435052871704, + "p05": 0.035227708518505096, + "p95": 0.07133325934410095 + }, + { + "label": "completed_group_3", + "mean": 0.0631948709487915, + "median": 0.06016676500439644, + "p05": 0.03969947248697281, + "p95": 0.09776238352060318 + }, + { + "label": "completed_group_4", + "mean": 0.02739548496901989, + "median": 0.026907969266176224, + "p05": 0.020028207451105118, + "p95": 0.036658842116594315 + }, + { + "label": "completed_group_5", + "mean": 0.08291903138160706, + "median": 0.08066994696855545, + "p05": 0.05307728424668312, + "p95": 0.12080682814121246 + }, + { + "label": "completed_group_6", + "mean": 0.2524128556251526, + "median": 0.25180506706237793, + "p05": 0.14340050518512726, + "p95": 0.36010295152664185 + }, + { + "label": "completed_group_7", + "mean": 0.186361163854599, + "median": 0.17697101831436157, + "p05": 0.08843781799077988, + "p95": 0.3156980276107788 + }, + { + "label": "current_group_partial", + "mean": 0.18013209104537964, + "median": 0.16958200931549072, + "p05": 0.08391770720481873, + "p95": 0.3105517029762268 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.596132755279541, + "group": null, + "latest_source_mass": 0.29075440764427185, + "layer": null, + "max_source_mass": 0.3670029938220978, + "mixer_index": null, + "normalized_entropy": 0.7264313223797062, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09668220579624176, + "median": 0.1013428270816803, + "p05": 0.02658359333872795, + "p95": 0.15407970547676086 + }, + { + "label": "completed_group_1", + "mean": 0.037084318697452545, + "median": 0.035886794328689575, + "p05": 0.02348068356513977, + "p95": 0.053765345364809036 + }, + { + "label": "completed_group_2", + "mean": 0.03457995876669884, + "median": 0.03355112299323082, + "p05": 0.019709739834070206, + "p95": 0.0531856045126915 + }, + { + "label": "completed_group_3", + "mean": 0.026012206450104713, + "median": 0.023950476199388504, + "p05": 0.013590577989816666, + "p95": 0.04525115340948105 + }, + { + "label": "completed_group_4", + "mean": 0.0258483849465847, + "median": 0.02512776106595993, + "p05": 0.016705092042684555, + "p95": 0.03687155246734619 + }, + { + "label": "completed_group_5", + "mean": 0.037477150559425354, + "median": 0.03506678715348244, + "p05": 0.02071693353354931, + "p95": 0.061963990330696106 + }, + { + "label": "completed_group_6", + "mean": 0.0845583900809288, + "median": 0.07240844517946243, + "p05": 0.03393660485744476, + "p95": 0.1794772744178772 + }, + { + "label": "completed_group_7", + "mean": 0.3670029938220978, + "median": 0.3608853220939636, + "p05": 0.1103469580411911, + "p95": 0.6465575098991394 + }, + { + "label": "completed_group_8", + "mean": 0.29075440764427185, + "median": 0.28802984952926636, + "p05": 0.039541684091091156, + "p95": 0.5658512711524963 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "optimizer_state_before": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.09253507107496262, + 0.19514523446559906, + 0.07917427271604538, + 0.04215213656425476, + 0.12552575767040253, + 0.05027730390429497, + 0.054851483553647995, + 0.036243390291929245, + 0.031880609691143036, + 0.035054463893175125, + 0.050445348024368286, + 0.05600574240088463, + 0.053015924990177155, + 0.032114963978528976, + 0.048427484929561615, + 0.5742782950401306, + 0.04075472429394722, + 0.03505101054906845, + 0.06577442586421967, + 0.08553697913885117, + 0.0493597537279129, + 0.023477228358387947, + 0.03512806445360184, + 0.030729109421372414, + 0.023035654798150063, + 0.019568851217627525, + 0.02376176416873932, + 0.02176014520227909, + 0.03424545377492905, + 0.021422436460852623, + 0.019477492198348045, + 0.017491821199655533 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1261602139711613e-05, + "first_to_last_ratio": 0.5245701598350286, + "imbalance_abs_log_ratio": 0.6451760949151356, + "last_quartile_mean": 2.146824772353284e-05, + "mean": 2.0086123484475138e-05, + "normalized": [ + 0.6085748623432731, + 0.672343922700359, + 0.5609816766389839, + 0.5375859907162287, + 0.641556317168069, + 0.5340725535941322, + 0.4789883309018991, + 0.4512226054976404, + 0.5900913956594765, + 0.540676107431406, + 0.5086908977229898, + 0.473500107930304, + 0.40487760145311497, + 0.38162857664898325, + 0.3654861410934707, + 0.44670015436893523, + 1.1143309981822054, + 1.1164864049647665, + 1.036008527266956, + 0.9253333108915559, + 3.7714016683827163, + 3.048770143020733, + 2.3679155285839633, + 1.872296940712075, + 1.7902480367185682, + 1.5773203624246581, + 1.2887757729919698, + 0.9509073151516803, + 1.1740622366025057, + 0.7631942144418731, + 0.5687018744547615, + 0.43726942333974633 + ], + "peak_layer": 21, + "peak_normalized": 3.7714016683827163, + "population_cv": 0.7926339194277571, + "rest_mean": 1.4245798663776239e-05, + "spike_contrast": 3.6237966529399817, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.1623877516249196e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2223909834574442e-05, + 1.350478305539582e-05, + 1.1267947229498532e-05, + 1.0798018593050074e-05, + 1.2886379408882931e-05, + 1.0727447261160705e-05, + 9.621018762118183e-06, + 9.063312973012216e-06, + 1.1852648640342522e-05, + 1.0860087058972567e-05, + 1.0217628187092487e-05, + 9.510781637800392e-06, + 8.132421498885378e-06, + 7.665438715775963e-06, + 7.341199761867756e-06, + 8.97247446118854e-06, + 2.238259003206622e-05, + 2.2425883798860013e-05, + 2.0809395209653303e-05, + 1.8586359146866016e-05, + 7.575283962069079e-05, + 6.123797356849536e-05, + 4.756224370794371e-05, + 3.760718755074777e-05, + 3.595914313336834e-05, + 3.168225157423876e-05, + 2.5886509320116602e-05, + 1.9100041754427366e-05, + 2.3582359062856995e-05, + 1.532961323391646e-05, + 1.1423016076150816e-05, + 8.78304763318738e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.418448133947095e-05, + "first_to_last_ratio": 0.515514370401758, + "imbalance_abs_log_ratio": 0.662590099285622, + "last_quartile_mean": 8.570950467401417e-05, + "mean": 8.019432914352365e-05, + "normalized": [ + 0.5977618675800617, + 0.6560769396425044, + 0.5549086183167843, + 0.5332641078022274, + 0.6187504975834317, + 0.5235180355140238, + 0.4739614950807707, + 0.44949966520899015, + 0.5901273331445138, + 0.5434806557289369, + 0.5105162863240122, + 0.47519844997761185, + 0.40525254510695746, + 0.3803726356397213, + 0.3632907544081655, + 0.4413534187685878, + 1.1120243122062796, + 1.1101422281940985, + 1.048076642833664, + 0.9240879167666287, + 3.7925106998539326, + 3.0713745688935163, + 2.3834694147362985, + 1.8907998104142179, + 1.7924928574135153, + 1.575834720390749, + 1.2856009256033192, + 0.946613406637061, + 1.179458790809137, + 0.7622280566154953, + 0.5700461300235112, + 0.43790621278127495 + ], + "peak_layer": 21, + "peak_normalized": 3.7925106998539326, + "population_cv": 0.799556889668146, + "rest_mean": 5.663903492084636e-05, + "spike_contrast": 3.6616605179769537, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020739291794598104, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.793711195816286e-05, + 5.261365004116669e-05, + 4.450052438187413e-05, + 4.27647573815193e-05, + 4.9620281060924754e-05, + 4.1983177652582526e-05, + 3.800902413786389e-05, + 3.604732410167344e-05, + 4.7324865590780973e-05, + 4.358406658866443e-05, + 4.09405110985972e-05, + 3.810822090599686e-05, + 3.249895598855801e-05, + 3.0503728339681402e-05, + 2.913385833380744e-05, + 3.539404133334756e-05, + 8.917804370867088e-05, + 8.902711124392226e-05, + 8.404980326304212e-05, + 7.410661055473611e-05, + 0.0003041378513444215, + 0.0002463068231008947, + 0.00019114073074888438, + 0.0001516314223408699, + 0.00014374776219483465, + 0.00012637300824280828, + 0.00010309790377505124, + 7.591302710352466e-05, + 9.458590648137033e-05, + 6.11263676546514e-05, + 4.5714466978097335e-05, + 3.511759496177547e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.039183240849524736, + "first_to_last_ratio": 0.5155143831205776, + "imbalance_abs_log_ratio": 0.6625900746135269, + "last_quartile_mean": 0.07600804581306875, + "mean": 0.07111713383346796, + "normalized": [ + 0.5977618468462375, + 0.6560769852418055, + 0.5549086482011255, + 0.5332641116607529, + 0.618750502626057, + 0.5235180409966333, + 0.4739615115435027, + 0.44949963686515554, + 0.590127313076766, + 0.543480629182993, + 0.5105162957833653, + 0.47519841853167755, + 0.40525256719553, + 0.38037265430860584, + 0.36329073465232276, + 0.44135337886041204, + 1.1120242741359478, + 1.1101421724087162, + 1.048076711910138, + 0.9240879569048468, + 3.79251093493639, + 3.0713746221735128, + 2.383469321905541, + 1.8907997296698473, + 1.7924929415619593, + 1.5758345890827283, + 1.2856009880693582, + 0.9466133568642321, + 1.179458727081905, + 0.7622280464822554, + 0.5700461310047829, + 0.4379062202348958 + ], + "peak_layer": 21, + "peak_normalized": 3.79251093493639, + "population_cv": 0.7995569122050559, + "rest_mean": 0.05022806260320875, + "spike_contrast": 3.6616606125102327, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18391811847686768, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04251110926270485, + 0.046658314764499664, + 0.03946351259946823, + 0.03792421519756317, + 0.04400376230478287, + 0.03723110258579254, + 0.03370678424835205, + 0.031967125833034515, + 0.04196816310286522, + 0.03865078464150429, + 0.03630645573139191, + 0.03379474952816963, + 0.028820401057600975, + 0.027051012963056564, + 0.025836195796728134, + 0.03138778731226921, + 0.07908397912979126, + 0.07895012944936752, + 0.07453621178865433, + 0.06571848690509796, + 0.26971250772476196, + 0.2184273600578308, + 0.1695055067539215, + 0.13446825742721558, + 0.12747696042060852, + 0.11206883937120438, + 0.0914282575249672, + 0.06732042878866196, + 0.08387972414493561, + 0.05420747399330139, + 0.040540046989917755, + 0.03114263527095318 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8925995988183786e-06, + "first_to_last_ratio": 0.3261648413799994, + "imbalance_abs_log_ratio": 1.1203523770075996, + "last_quartile_mean": 5.802586173331292e-06, + "mean": 4.670730803013612e-06, + "normalized": [ + 0.5310348802393875, + 0.46146348274830745, + 0.44476201867725484, + 0.43677383492791344, + 0.3556033019654118, + 0.3423979278935454, + 0.3333438108043655, + 0.33625410358276536, + 0.4223966230117983, + 0.41208725398123947, + 0.41437801392793316, + 0.4155995059181246, + 0.382562022554051, + 0.3815658727614762, + 0.38949800087449105, + 0.4096468013886889, + 0.8179581009736591, + 0.8423033340072443, + 0.8553008421936446, + 0.8530642636826707, + 3.896213232744094, + 3.3402464703409684, + 2.714333567686501, + 2.2725775720601913, + 2.07067894660348, + 1.8582090508366296, + 1.5332685227055798, + 1.1680391073619418, + 1.334393737375711, + 0.8510160788430678, + 0.6469677209502473, + 0.4760619963776144 + ], + "peak_layer": 21, + "peak_normalized": 3.896213232744094, + "population_cv": 0.9180343092487918, + "rest_mean": 3.0629528535226e-06, + "spike_contrast": 4.359431035612757, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3352731730265078e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4803209726087516e-06, + 2.15537170333846e-06, + 2.0773636606463697e-06, + 2.040053004748188e-06, + 1.6609272961431998e-06, + 1.599248548700416e-06, + 1.5569592051178915e-06, + 1.570552399243752e-06, + 1.9729009181901347e-06, + 1.9247486306994688e-06, + 1.935448153744801e-06, + 1.9411534140090225e-06, + 1.786844222806394e-06, + 1.7821914752857992e-06, + 1.819240310396708e-06, + 1.9133499336021487e-06, + 3.820462097792188e-06, + 3.934172127628699e-06, + 3.99487998947734e-06, + 3.984433533332776e-06, + 1.8198163161287084e-05, + 1.5601392078679055e-05, + 1.2677921404247172e-05, + 1.0614598068059422e-05, + 9.671583939052653e-06, + 8.679194252181333e-06, + 7.161484518292127e-06, + 5.455596237879945e-06, + 6.2325939325091895e-06, + 3.974867013312178e-06, + 3.021812062797835e-06, + 2.2235574306250783e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.4072450716848834e-05, + "first_to_last_ratio": 0.634285224932807, + "imbalance_abs_log_ratio": 0.4552565441138088, + "last_quartile_mean": 5.371786915020493e-05, + "mean": 5.588273802459298e-05, + "normalized": [ + 0.6676093152090673, + 0.7197771199409104, + 0.6218041460469672, + 0.5988669281463308, + 0.6666598935307795, + 0.5782912860000531, + 0.5252896450390306, + 0.4994083671227573, + 0.6402001675671648, + 0.5894906078705415, + 0.5499130564119773, + 0.5108819562402718, + 0.4646654187577452, + 0.4356758329586985, + 0.4154501696208735, + 0.4880908883442812, + 1.1821250357964774, + 1.1664741605550812, + 1.078174429062179, + 0.962548943315326, + 3.8682876465312988, + 3.0459843127744204, + 2.2760577554709505, + 1.7581888525052323, + 1.7091478115698895, + 1.4520832989438006, + 1.1543347914386743, + 0.8578423927708055, + 0.9984708567512235, + 0.639080834302184, + 0.48627901888802055, + 0.39284506051698587 + ], + "peak_layer": 21, + "peak_normalized": 3.8682876465312988, + "population_cv": 0.7787255777921933, + "rest_mean": 4.0033428242033415e-05, + "spike_contrast": 3.533772076553717, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014146901085041464, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.7307836464606225e-05, + 4.022311622975394e-05, + 3.4748118196148425e-05, + 3.346632365719415e-05, + 3.72547801816836e-05, + 3.231650043744594e-05, + 2.935462362074759e-05, + 2.7908306947210804e-05, + 3.57761382474564e-05, + 3.294234920758754e-05, + 3.073064726777375e-05, + 2.8549482522066683e-05, + 2.596677586552687e-05, + 2.434675843687728e-05, + 2.321649299119599e-05, + 2.727585524553433e-05, + 6.606038368772715e-05, + 6.518576992675662e-05, + 6.025133916409686e-05, + 5.3789870435139164e-05, + 0.0002161705051548779, + 0.00017021794337779284, + 0.00012719233927782625, + 9.825240704230964e-05, + 9.551185939926654e-05, + 8.114639058476314e-05, + 6.450738874264061e-05, + 4.7938581701600924e-05, + 5.579728531301953e-05, + 3.571358683984727e-05, + 2.7174603019375354e-05, + 2.1953257601126097e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.604143833195849e-05, + "first_to_last_ratio": 0.864500772440069, + "imbalance_abs_log_ratio": 0.14560308034841146, + "last_quartile_mean": 3.012309434780036e-05, + "mean": 3.750934104118642e-05, + "normalized": [ + 0.7613938466965254, + 0.8106407342403823, + 0.7210553143827715, + 0.6926649057839166, + 0.7441900072557481, + 0.6549135446116822, + 0.6010420947141472, + 0.5682228952686117, + 0.7146883031764985, + 0.6605877661848203, + 0.6100617443729247, + 0.5720177685677394, + 0.5632312795246776, + 0.5253333685644455, + 0.49762299559665957, + 0.5644048416919647, + 1.293995056666398, + 1.2838583891993025, + 1.1578542801495904, + 1.0246076437213654, + 3.9830246051038323, + 3.000750743357827, + 2.06181294387121, + 1.5073651467427196, + 1.5060057544008438, + 1.2137998314477398, + 0.951522556430117, + 0.7372102220186553, + 0.7702443482214394, + 0.5110189167704164, + 0.405035438834493, + 0.3298227124305357 + ], + "peak_layer": 21, + "peak_normalized": 3.9830246051038323, + "population_cv": 0.7527493617080463, + "rest_mean": 2.770278890104102e-05, + "spike_contrast": 3.2655456792139823, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.046472259797156e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.855938146240078e-05, + 3.0406599762500264e-05, + 2.7046309696743265e-05, + 2.5981404178310186e-05, + 2.791407678159885e-05, + 2.4565375497331843e-05, + 2.2544692910742015e-05, + 2.1313666366040707e-05, + 2.6807487301994115e-05, + 2.4778211809461936e-05, + 2.2883014025865123e-05, + 2.1456009562825784e-05, + 2.1126434148754925e-05, + 1.9704908481799066e-05, + 1.866551065177191e-05, + 2.1170453692320734e-05, + 4.853690188610926e-05, + 4.815668216906488e-05, + 4.343035107012838e-05, + 3.8432357541751117e-05, + 0.0001494006282882765, + 0.00011255618301220238, + 7.733724487479776e-05, + 5.654027336277068e-05, + 5.648928345181048e-05, + 4.5528831833507866e-05, + 3.5690984077518806e-05, + 2.7652269636746496e-05, + 2.889135794248432e-05, + 1.9167982827639207e-05, + 1.5192612409009598e-05, + 1.2371432603686117e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.932713899412192e-05, + "first_to_last_ratio": 0.48428070567546516, + "imbalance_abs_log_ratio": 0.7250905699794873, + "last_quartile_mean": 0.00018445322712068446, + "mean": 0.00016839920272104791, + "normalized": [ + 0.580113378070693, + 0.6316011322538936, + 0.5371709395526841, + 0.5193916381660382, + 0.5958096458866964, + 0.49825289372245124, + 0.4493748616089108, + 0.43187515015476213, + 0.5789448986804199, + 0.5298739922879939, + 0.5012758869564349, + 0.4661125381937263, + 0.3655735366227801, + 0.35379219756409647, + 0.33290305767004397, + 0.4094948207474536, + 1.0315615517644154, + 1.032222526387711, + 0.9835380166521113, + 0.8863325773824939, + 3.853305598042274, + 3.162866144945307, + 2.502991043300759, + 2.002956870927857, + 1.8436650111759416, + 1.637369378572084, + 1.333674654700857, + 1.0052835546379626, + 1.1909970302670063, + 0.7575881320943505, + 0.5664246618886806, + 0.4276626791211106 + ], + "peak_layer": 21, + "peak_normalized": 3.853305598042274, + "population_cv": 0.8344872340072493, + "rest_mean": 0.00011622174093241079, + "spike_contrast": 3.873264096443705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004501574963796884, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.769063035491854e-05, + 0.00010636112710926682, + 9.045915794558823e-05, + 8.746513776713982e-05, + 0.00010033386934082955, + 8.390539005631581e-05, + 7.567436841782182e-05, + 7.27274309610948e-05, + 9.749385935720056e-05, + 8.923035784391686e-05, + 8.441445970674977e-05, + 7.84929798101075e-05, + 6.156229210318998e-05, + 5.9578323998721316e-05, + 5.606060949503444e-05, + 6.895860133226961e-05, + 0.00017371414287481457, + 0.00017382545047439635, + 0.0001656270178500563, + 0.00014925769937690347, + 0.0006488935905508697, + 0.000532624137122184, + 0.00042150169610977173, + 0.000337296340148896, + 0.00031047171796672046, + 0.0002757316979113966, + 0.0002245897485408932, + 0.0001692889491096139, + 0.00020056295034009963, + 0.00012757723743561655, + 9.538546146359295e-05, + 7.201805419754237e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.116500623524189, + 0.0468219518661499, + 0.09375449270009995, + 0.041188232600688934, + 0.024237047880887985, + 0.022429833188652992, + 0.03027583472430706, + 0.04546776041388512, + 0.030174262821674347, + 0.03663193807005882, + 0.05748353153467178, + 0.10596779733896255, + 0.05394246056675911, + 0.06537146866321564, + 0.08060523867607117, + 0.06690707802772522, + 0.033449217677116394, + 0.022565271705389023, + 0.036697570234537125, + 0.07367803901433945, + 0.042231619358062744, + 0.052135858684778214, + 0.06269875913858414, + 0.0728035643696785, + 0.051585447043180466, + 0.04862172156572342, + 0.05972515419125557, + 0.06787826865911484, + 0.06439156830310822, + 0.07232685387134552, + 0.09627991914749146, + 0.1271906942129135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1220664760003274e-05, + "first_to_last_ratio": 0.6555362798323586, + "imbalance_abs_log_ratio": 0.42230163050239594, + "last_quartile_mean": 1.711677157345548e-05, + "mean": 1.7638031224009865e-05, + "normalized": [ + 0.6920470508787427, + 0.765871148622978, + 0.6390872031764239, + 0.6123477040772916, + 0.7313980706371319, + 0.6060462235938255, + 0.5358729377549785, + 0.5066345241410947, + 0.6361374094466793, + 0.5935131324046057, + 0.553200745756426, + 0.5203044559127653, + 0.44270176965120756, + 0.41555808291178226, + 0.42011495887552464, + 0.5249791307747981, + 1.2555818719274776, + 1.2709584825941431, + 1.1358536396128, + 1.0637844334884017, + 3.6056946953031805, + 2.7844654381945673, + 2.1779734918269082, + 1.746298720031033, + 1.816337422755615, + 1.4912504148810115, + 1.091129247215256, + 0.8818465230617194, + 0.8715920116565027, + 0.6503144256419822, + 0.49863208841232076, + 0.4624725447808252 + ], + "peak_layer": 21, + "peak_normalized": 3.6056946953031805, + "population_cv": 0.7234178171751035, + "rest_mean": 1.2979781601003466e-05, + "spike_contrast": 3.2968643466956427, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2792579188244416e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2206347491883207e-05, + 1.3508459232980385e-05, + 1.12722400444909e-05, + 1.0800607924466021e-05, + 1.2900422007078305e-05, + 1.0689462214941159e-05, + 9.451743608224206e-06, + 8.936035555962007e-06, + 1.1220211490581278e-05, + 1.0468403161212336e-05, + 9.757372026797384e-06, + 9.177146239380818e-06, + 7.808387636032421e-06, + 7.3296264417876955e-06, + 7.410000762320124e-06, + 9.259598300559446e-06, + 2.2145992261357605e-05, + 2.2417205400415696e-05, + 2.0034221961395815e-05, + 1.8763063053484075e-05, + 6.359735562000424e-05, + 4.9112488341052085e-05, + 3.84151644539088e-05, + 3.080127135035582e-05, + 3.2036616175901145e-05, + 2.6302721380488947e-05, + 1.9245371731813066e-05, + 1.5554036508547142e-05, + 1.5373167116194963e-05, + 1.1470266144897323e-05, + 8.794888344709761e-06, + 8.157105185091496e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.3983808154735016e-05, + "first_to_last_ratio": 0.6452579224313538, + "imbalance_abs_log_ratio": 0.43810516237842645, + "last_quartile_mean": 6.816469294790295e-05, + "mean": 7.033422900803998e-05, + "normalized": [ + 0.6809924600708206, + 0.7475303658027369, + 0.6323190441913445, + 0.6077859654839745, + 0.7056348319250383, + 0.5942536325242065, + 0.530225745030374, + 0.5040918553425093, + 0.6368908391102093, + 0.5978963063844708, + 0.5559498242106791, + 0.5233025181432952, + 0.44344812436490716, + 0.41411157823288425, + 0.41744328645242573, + 0.5192458958427938, + 1.2557243607052013, + 1.2676253654309775, + 1.1574430802993265, + 1.060137110625566, + 3.6189237818679025, + 2.8123811167698722, + 2.2018107363871295, + 1.7616009064115667, + 1.8179427908533603, + 1.4871270732826793, + 1.0868453970932423, + 0.8727044844737096, + 0.8717769668640218, + 0.6523346900704022, + 0.4999162674775265, + 0.4645835982748448 + ], + "peak_layer": 21, + "peak_normalized": 3.6189237818679025, + "population_cv": 0.7299683580686596, + "rest_mean": 5.154545740677147e-05, + "spike_contrast": 3.3328561680843984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179359565488994, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789707963936962e-05, + 5.2576971938833594e-05, + 4.4473672460298985e-05, + 4.274815728422254e-05, + 4.963028186466545e-05, + 4.1796371078817174e-05, + 3.729301897692494e-05, + 3.545491199474782e-05, + 4.479522613110021e-05, + 4.205257573630661e-05, + 3.910230225301348e-05, + 3.680607915157452e-05, + 3.1189581932267174e-05, + 2.9126218578312546e-05, + 2.9360551707213745e-05, + 3.6520759749691933e-05, + 8.832040475681424e-05, + 8.915745274862275e-05, + 8.140786667354405e-05, + 7.456392631866038e-05, + 0.0002545342140365392, + 0.00019780665752477944, + 0.00015486266056541353, + 0.00012390084157232195, + 0.00012786360457539558, + 0.00010459593613632023, + 7.644243305549026e-05, + 6.138099706731737e-05, + 6.13157608313486e-05, + 4.588145748130046e-05, + 3.516122524160892e-05, + 3.267612919444218e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.039005281403660774, + "first_to_last_ratio": 0.6452579411239904, + "imbalance_abs_log_ratio": 0.4381051334091792, + "last_quartile_mean": 0.06044913036748767, + "mean": 0.06237309757852927, + "normalized": [ + 0.6809924890770891, + 0.7475303831256814, + 0.6323190938885936, + 0.6077859577612714, + 0.7056348041762752, + 0.5942536183897815, + 0.5302257656275956, + 0.5040918555375298, + 0.6368908140206065, + 0.5978963019204037, + 0.5559498364917699, + 0.5233025167864885, + 0.4434481328217215, + 0.4141115698767486, + 0.41744329043307676, + 0.5192458729418684, + 1.2557243339288315, + 1.2676253196693095, + 1.1574430718922224, + 1.0601371340408086, + 3.6189238450886454, + 2.8123810331522527, + 2.2018108921566237, + 1.761600919246031, + 1.8179427635119274, + 1.487127014661369, + 1.086845368525537, + 0.8727044924490052, + 0.8717770087213936, + 0.6523346812884329, + 0.4999162376710536, + 0.46458358112005493 + ], + "peak_layer": 21, + "peak_normalized": 3.6189238450886454, + "population_cv": 0.7299683644727474, + "rest_mean": 0.045711026768441555, + "spike_contrast": 3.3328562214267015, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15234827995300293, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.042475610971450806, + 0.046625785529613495, + 0.03943970054388046, + 0.03790949285030365, + 0.04401262849569321, + 0.03706543892621994, + 0.03307182341814041, + 0.03144177049398422, + 0.03972485288977623, + 0.03729264438152313, + 0.034676313400268555, + 0.0326399989426136, + 0.027659233659505844, + 0.02582942135632038, + 0.02603723108768463, + 0.03238697350025177, + 0.07832341641187668, + 0.0790657177567482, + 0.07219330966472626, + 0.06612403690814972, + 0.2257234901189804, + 0.17541691660881042, + 0.13733376562595367, + 0.10987650603055954, + 0.1133907213807106, + 0.0927567183971405, + 0.06778991222381592, + 0.05443328246474266, + 0.054375432431697845, + 0.04068813472986221, + 0.031181324273347855, + 0.028977517038583755 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8962810770517535e-06, + "first_to_last_ratio": 0.40446687944391657, + "imbalance_abs_log_ratio": 0.9051854260734966, + "last_quartile_mean": 4.688346990633363e-06, + "mean": 3.997706922831412e-06, + "normalized": [ + 0.617154002791989, + 0.5425132643975126, + 0.522616779881192, + 0.5130533608023556, + 0.4179497664431836, + 0.4022508178223506, + 0.3871440054888278, + 0.39205556299135513, + 0.47572519958981047, + 0.4709737480952342, + 0.46924028063152856, + 0.46856760789403773, + 0.43552024907631937, + 0.4334760110167188, + 0.4573112739567068, + 0.48343767377708236, + 0.9432261829651593, + 0.9556187861127372, + 0.9329616531367625, + 0.91445072544449, + 3.5932868350685228, + 3.016174008278517, + 2.572613821305946, + 2.200605947741084, + 2.1897185116963183, + 1.8429631378409383, + 1.3739246533430545, + 1.1483111158298067, + 1.0092131774108768, + 0.7452469424242577, + 0.5556201735322511, + 0.5170747232130729 + ], + "peak_layer": 21, + "peak_normalized": 3.5932868350685228, + "population_cv": 0.8307963422495312, + "rest_mean": 2.7284499108443724e-06, + "spike_contrast": 3.9772380443675277, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0851694787561427e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.467200829414651e-06, + 2.168809032809804e-06, + 2.089268718918902e-06, + 2.0510369722614996e-06, + 1.670840674705687e-06, + 1.6080808791230083e-06, + 1.5476882708753692e-06, + 1.5673232383051072e-06, + 1.9018099237655406e-06, + 1.8828150132321753e-06, + 1.8758851183520164e-06, + 1.8731959698925493e-06, + 1.7410823147656629e-06, + 1.7329100501228822e-06, + 1.8281964457855793e-06, + 1.932642135216156e-06, + 3.7707418414356653e-06, + 3.82028383683064e-06, + 3.7297072594810743e-06, + 3.6557059956976445e-06, + 1.4364907656272408e-05, + 1.2057779713359196e-05, + 1.0284556083206553e-05, + 8.797377631708514e-06, + 8.753852853260469e-06, + 7.367626494669821e-06, + 5.492548098118277e-06, + 4.590611297317082e-06, + 4.034538505948149e-06, + 2.9792788609483978e-06, + 2.2212066141946707e-06, + 2.067113200610038e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.395081762391783e-05, + "first_to_last_ratio": 0.7815758779298154, + "imbalance_abs_log_ratio": 0.246443041153289, + "last_quartile_mean": 4.343892715041875e-05, + "mean": 4.974787287892468e-05, + "normalized": [ + 0.7500101371289282, + 0.8083785365909619, + 0.6989343871323576, + 0.6730384758360606, + 0.7487834825409285, + 0.6472056011592257, + 0.5806038659736669, + 0.5527069076510954, + 0.6849789424977496, + 0.6414689031465389, + 0.5923280921669694, + 0.5570919747617912, + 0.5025384608304704, + 0.4685796759451795, + 0.4679593283316108, + 0.5622406113180932, + 1.3163192795863237, + 1.3145042344644593, + 1.1718309380215874, + 1.0894177246512402, + 3.678688405943946, + 2.7746227673728385, + 2.0966738705116796, + 1.635642610362089, + 1.6803690663352613, + 1.3632931153614756, + 0.986286743821586, + 0.8095332329033809, + 0.728391794299929, + 0.5544865952259644, + 0.4452050831707052, + 0.41788715495590656 + ], + "peak_layer": 21, + "peak_normalized": 3.678688405943946, + "population_cv": 0.7109227971967927, + "rest_mean": 3.7097179099671645e-05, + "spike_contrast": 3.182495870364873, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011806161928689108, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.731140895979479e-05, + 4.021511267637834e-05, + 3.4770499041769654e-05, + 3.3482232538517565e-05, + 3.725038550328463e-05, + 3.219710197299719e-05, + 2.88838073174702e-05, + 2.7495992981130257e-05, + 3.4076245356118307e-05, + 3.1911713449517265e-05, + 2.9467062631738372e-05, + 2.7714140742318705e-05, + 2.5000219466164708e-05, + 2.331084215256851e-05, + 2.3279981178347953e-05, + 2.79702744592214e-05, + 6.548408418893814e-05, + 6.539378955494612e-05, + 5.8296096540289e-05, + 5.419621447799727e-05, + 0.0001830069231800735, + 0.00013803158071823418, + 0.00010430506517877802, + 8.136974065564573e-05, + 8.359478670172393e-05, + 6.782093259971589e-05, + 4.9065667553804815e-05, + 4.027255636174232e-05, + 3.623594238888472e-05, + 2.7584528652369045e-05, + 2.214800588262733e-05, + 2.078899706248194e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6004547407865175e-05, + "first_to_last_ratio": 1.0323157315408855, + "imbalance_abs_log_ratio": 0.031804561684426046, + "last_quartile_mean": 2.519049803595408e-05, + "mean": 3.413751699099521e-05, + "normalized": [ + 0.838119937847341, + 0.8931761838723624, + 0.7929921852930517, + 0.7615904360235652, + 0.8189280439687685, + 0.7189183912075784, + 0.6527804223286999, + 0.6175632597625974, + 0.7513453279894857, + 0.7058548167769552, + 0.6474515253469071, + 0.6094592164179198, + 0.5931149886633756, + 0.5496093129413688, + 0.5439605510615998, + 0.633537428927029, + 1.4126123740339225, + 1.4259512145086812, + 1.2377137858897012, + 1.131469831457994, + 3.7563109168199906, + 2.718723537642896, + 1.8748185902822487, + 1.4106982999267925, + 1.4327179849324012, + 1.1438362357528318, + 0.8490271550305547, + 0.7287425009809699, + 0.5778680951931764, + 0.464037753169555, + 0.3681285307530028, + 0.3389411651966756 + ], + "peak_layer": 21, + "peak_normalized": 3.7563109168199906, + "population_cv": 0.6884877712720907, + "rest_mean": 2.6307041547692347e-05, + "spike_contrast": 2.9050048917999454, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.642208438483067e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8611333618755452e-05, + 3.0490817152895033e-05, + 2.7070784199167974e-05, + 2.599880644993391e-05, + 2.7956170015386306e-05, + 2.454208879498765e-05, + 2.2284302758635022e-05, + 2.1082076273160055e-05, + 2.5649063900345936e-05, + 2.4096130800899118e-05, + 2.2102387447375804e-05, + 2.0805424355785362e-05, + 2.0247473003109917e-05, + 1.8762297258945182e-05, + 1.856946255429648e-05, + 2.1627394744427875e-05, + 4.822307892027311e-05, + 4.867843381362036e-05, + 4.225247539579868e-05, + 3.862557059619576e-05, + 0.00012823112774640322, + 9.281047096010298e-05, + 6.400165148079395e-05, + 4.8157737182918936e-05, + 4.890943455393426e-05, + 3.9047728932928294e-05, + 2.8983678930671886e-05, + 2.4877459509298205e-05, + 1.9726981918211095e-05, + 1.5841096683288924e-05, + 1.2566993973450735e-05, + 1.1570609785849229e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.908438576327171e-05, + "first_to_last_ratio": 0.6062325309661494, + "imbalance_abs_log_ratio": 0.5004916520467982, + "last_quartile_mean": 0.00014694755100208567, + "mean": 0.00014725192045261792, + "normalized": [ + 0.6637009362863341, + 0.7248732277041845, + 0.6138698495621964, + 0.598513900406796, + 0.680481175716416, + 0.565076258862256, + 0.5041321503581428, + 0.4891880965619703, + 0.6294201840754665, + 0.5850404355839129, + 0.5479525097707834, + 0.5179728424254814, + 0.4076252060436077, + 0.3853916272098706, + 0.3845475777225467, + 0.4800015848780767, + 1.1784027781257396, + 1.1715870365157128, + 1.1011549065529929, + 1.0246566094283616, + 3.66881165389385, + 2.9055737038215397, + 2.326629675317946, + 1.8619320585590522, + 1.8930003135554039, + 1.5415479300660813, + 1.1509068852552828, + 0.9338288403787282, + 0.8717742548708616, + 0.6455051030286945, + 0.49036587220767053, + 0.45653481525404077 + ], + "peak_layer": 21, + "peak_normalized": 3.66881165389385, + "population_cv": 0.7616800575204435, + "rest_mean": 0.00010549810718253462, + "spike_contrast": 3.5329781933070796, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037272251211106777, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.77312374743633e-05, + 0.00010673897486412898, + 9.039351425599307e-05, + 8.81323212524876e-05, + 0.0001002021599560976, + 8.320856431964785e-05, + 7.423442730214447e-05, + 7.203388668131083e-05, + 9.268333087675273e-05, + 8.614832768216729e-05, + 8.068705938057974e-05, + 7.627249578945339e-05, + 6.002359441481531e-05, + 5.674965723301284e-05, + 5.662536932504736e-05, + 7.068115519359708e-05, + 0.00017352207214571536, + 0.00017251844110433012, + 0.00016214717470575124, + 0.0001508826535427943, + 0.0005402395618148148, + 0.0004278513079043478, + 0.00034260068787261844, + 0.0002741730713751167, + 0.00027874793158844113, + 0.00022699589317198843, + 0.00016947324911598116, + 0.0001375080901198089, + 0.00012837043323088437, + 9.505186608294025e-05, + 7.220731640700251e-05, + 6.722562829963863e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.09253507107496262, + 0.30484533309936523, + 0.33420246839523315, + 0.38682517409324646, + 0.12552575767040253, + 0.14749252796173096, + 0.1726136952638626, + 0.1892337054014206, + 0.031880609691143036, + 0.06123264133930206, + 0.10403046011924744, + 0.15778414905071259, + 0.053015924990177155, + 0.09510950744152069, + 0.1597633957862854, + 0.7120755910873413, + 0.04075472429394722, + 0.07218439877033234, + 0.11449843645095825, + 0.16539782285690308, + 0.0493597537279129, + 0.07179203629493713, + 0.10295005142688751, + 0.1323672980070114, + 0.023035654798150063, + 0.06294231116771698, + 0.09175154566764832, + 0.1252463310956955, + 0.03424545377492905, + 0.07851024717092514, + 0.12469635903835297, + 0.19299884140491486 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1261542454121809e-05, + "first_to_last_ratio": 0.5245675018823146, + "imbalance_abs_log_ratio": 0.6451811618435493, + "last_quartile_mean": 2.146824272131198e-05, + "mean": 2.008613361681455e-05, + "normalized": [ + 0.6085515532483275, + 0.6723602464807182, + 0.5609760959264884, + 0.53758893439475, + 0.6415342139885134, + 0.5340779441503937, + 0.4789823840329274, + 0.45122885288223713, + 0.5900862077803115, + 0.5406841208808664, + 0.5086899166409938, + 0.47350367257308246, + 0.40488038567700324, + 0.38162938029260524, + 0.36548145139289145, + 0.4467012874253964, + 1.1143303455046147, + 1.1164970711323288, + 1.0360116270365431, + 0.9253438018068647, + 3.771407372914525, + 3.0487664316584073, + 2.367876661366877, + 1.872337110236809, + 1.790253835037814, + 1.5773117786405777, + 1.2887586410545064, + 0.9509019452609131, + 1.1740685268722015, + 0.7631964556770826, + 0.5687161676492125, + 0.4372655803832159 + ], + "peak_layer": 21, + "peak_normalized": 3.771407372914525, + "population_cv": 0.7926334965261557, + "rest_mean": 1.42457990848386e-05, + "spike_contrast": 3.6238009382307355, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.1623940089484675e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2223447811265942e-05, + 1.3505117749446072e-05, + 1.1267840818618424e-05, + 1.07980831671739e-05, + 1.2885941941931378e-05, + 1.0727560947998427e-05, + 9.62090416578576e-06, + 9.06344303075457e-06, + 1.1852550414914731e-05, + 1.0860253496502992e-05, + 1.0217613635177258e-05, + 9.51085803535534e-06, + 8.132481525535695e-06, + 7.665458724659402e-06, + 7.34110926714493e-06, + 8.972501746029593e-06, + 2.2382588213076815e-05, + 2.2426109353546053e-05, + 2.0809467969229445e-05, + 1.8586579244583845e-05, + 7.575299241580069e-05, + 6.123792991274968e-05, + 4.756148700835183e-05, + 3.760801337193698e-05, + 3.5959277738584206e-05, + 3.168209514115006e-05, + 2.5886178264045157e-05, + 1.9099943528999574e-05, + 2.3582497306051664e-05, + 1.5329665984609164e-05, + 1.1423308933444787e-05, + 8.782974873611238e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.4184340367792174e-05, + "first_to_last_ratio": 0.5155127283755759, + "imbalance_abs_log_ratio": 0.6625932845097232, + "last_quartile_mean": 8.570950421926682e-05, + "mean": 8.019429117211985e-05, + "normalized": [ + 0.5977553912963086, + 0.6560764337279869, + 0.5549115575710053, + 0.5332632715493767, + 0.618731238431461, + 0.5235220032899371, + 0.4739610390296197, + 0.44950831585208795, + 0.5901295178770004, + 0.5434863568103182, + 0.5105118554999238, + 0.4751981306058326, + 0.4052536442822589, + 0.3803760366271764, + 0.36329074496540736, + 0.44135367311072854, + 1.112029465926781, + 1.1101484697729092, + 1.04807895367264, + 0.9240971550402413, + 3.7925041485004933, + 3.0713698535863756, + 2.383447135178006, + 1.890810504439103, + 1.7925006015588865, + 1.5758247605004587, + 1.2855904653729862, + 0.9466104071448787, + 1.1794713255182339, + 0.7622315023149855, + 0.5700532953493974, + 0.43790274559719394 + ], + "peak_layer": 21, + "peak_normalized": 3.7925041485004933, + "population_cv": 0.7995548542275265, + "rest_mean": 5.6639052976000435e-05, + "spike_contrast": 3.6616533387138865, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002073925774311647, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.79365698993206e-05, + 5.2613584557548165e-05, + 4.450073902262375e-05, + 4.276467007002793e-05, + 4.96187130920589e-05, + 4.198347596684471e-05, + 3.800896956818178e-05, + 3.604800076573156e-05, + 4.732501838589087e-05, + 4.358450314612128e-05, + 4.094013638678007e-05, + 3.8108177250251174e-05, + 3.249902874813415e-05, + 3.0503986636176705e-05, + 2.913384378189221e-05, + 3.5394044971326366e-05, + 8.917841478250921e-05, + 8.902756962925196e-05, + 8.40499487821944e-05, + 7.410731632262468e-05, + 0.000304137181956321, + 0.0002463063283357769, + 0.00019113885355181992, + 0.00015163220814429224, + 0.00014374831516761333, + 0.0001263721496798098, + 0.0001030970161082223, + 7.591275061713532e-05, + 9.45868669077754e-05, + 6.112661503721029e-05, + 4.571501995087601e-05, + 3.511730028549209e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03918311558663845, + "first_to_last_ratio": 0.5155127429948628, + "imbalance_abs_log_ratio": 0.6625932561509924, + "last_quartile_mean": 0.07600804464891553, + "mean": 0.07111709873424843, + "normalized": [ + 0.5977553845258899, + 0.656076418540609, + 0.5549115935781082, + 0.5332632748164047, + 0.6187312693383394, + 0.5235220185307409, + 0.4739610644907502, + 0.4495083446741889, + 0.5901295424808332, + 0.5434863451914381, + 0.5105118857031089, + 0.4751981292368421, + 0.4052536577066065, + 0.38037603736972314, + 0.36329075680382195, + 0.4413536490689804, + 1.1120294326243692, + 1.1101484823828944, + 1.0480789054186883, + 0.9240971084727143, + 3.792504006439516, + 3.0713698521253847, + 2.383447240422923, + 1.8908105107644724, + 1.7925005311890823, + 1.575824785560249, + 1.2855905174797924, + 0.9466103668129862, + 1.1794713571647604, + 0.7622314608572955, + 0.5700533268339548, + 0.4379027433945313 + ], + "peak_layer": 21, + "peak_normalized": 3.792504006439516, + "population_cv": 0.7995548405481961, + "rest_mean": 0.05022807840119909, + "spike_contrast": 3.6616532900913086, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18391780853271483, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04251062870025635, + 0.04665825143456459, + 0.03946370258927345, + 0.037924136966466904, + 0.04400237277150154, + 0.03723136708140373, + 0.03370673581957817, + 0.031967729330062866, + 0.04196830093860626, + 0.03865117207169533, + 0.036306124180555344, + 0.03379471227526665, + 0.02882046438753605, + 0.02705124020576477, + 0.02583618462085724, + 0.03138779103755951, + 0.07908430695533752, + 0.07895053923130035, + 0.07453633099794388, + 0.0657191053032875, + 0.2697118818759918, + 0.218426913022995, + 0.169503852725029, + 0.1344689577817917, + 0.12747743725776672, + 0.11206808686256409, + 0.09142746776342392, + 0.06732018291950226, + 0.08388058096170425, + 0.0542076900601387, + 0.04054053872823715, + 0.031142372637987137 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8926682940900719e-06, + "first_to_last_ratio": 0.32617819149698246, + "imbalance_abs_log_ratio": 1.1203114472626277, + "last_quartile_mean": 5.802559286394171e-06, + "mean": 4.670736920786567e-06, + "normalized": [ + 0.5310214304016295, + 0.4614624888755425, + 0.44477058805262604, + 0.4368101139545488, + 0.3555922725307191, + 0.3424096008550773, + 0.333381953461308, + 0.3362983274867149, + 0.4224036639064773, + 0.41210428787587466, + 0.41438837559751834, + 0.41562174402282875, + 0.3825706733995329, + 0.3815673445421813, + 0.3895113889811354, + 0.40964670295389133, + 0.8179200324465078, + 0.8423754461799106, + 0.8552890122092341, + 0.8530595439765862, + 3.8961508812147314, + 3.3402234019552477, + 2.714242582301369, + 2.272602051203693, + 2.0707052479706927, + 1.8581861711397907, + 1.5332291278146746, + 1.1680063245911945, + 1.3343957866516452, + 0.8510283999847308, + 0.6469666788242595, + 0.4760583546381262 + ], + "peak_layer": 21, + "peak_normalized": 3.8961508812147314, + "population_cv": 0.9180173062016742, + "rest_mean": 3.0629785972754084e-06, + "spike_contrast": 4.359361792349546, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3352631867746823e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4802614007057855e-06, + 2.1553698843490565e-06, + 2.077406406897353e-06, + 2.0402251266204985e-06, + 1.6608779560556286e-06, + 1.5993051647456014e-06, + 1.5571393987556803e-06, + 1.5707610145909712e-06, + 1.972936388483504e-06, + 1.9248307125963038e-06, + 1.9354990854481002e-06, + 1.94125982488913e-06, + 1.7868869690573774e-06, + 1.7822006839196547e-06, + 1.8193052255810471e-06, + 1.9133519799652277e-06, + 3.820289293798851e-06, + 3.9345140976365656e-06, + 3.994829967268743e-06, + 3.9844167076807935e-06, + 1.8197895769844763e-05, + 1.5601304767187685e-05, + 1.2677513041126076e-05, + 1.0614726306812372e-05, + 9.671719453763217e-06, + 8.679098755237646e-06, + 7.161309895309387e-06, + 5.45545026398031e-06, + 6.232611667655874e-06, + 3.9749297684466e-06, + 3.0218111533031333e-06, + 2.2235433334572008e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.407248482290015e-05, + "first_to_last_ratio": 0.6342850678385603, + "imbalance_abs_log_ratio": 0.4552567917851402, + "last_quartile_mean": 5.371793622543919e-05, + "mean": 5.588281288737562e-05, + "normalized": [ + 0.667605556447693, + 0.7197799315075553, + 0.6218050056576038, + 0.5988650842787754, + 0.6666530112346837, + 0.5782950032081196, + 0.5252885832899008, + 0.4994128735568847, + 0.6402020441359789, + 0.5894915107696465, + 0.5499104969232754, + 0.5108819879445103, + 0.46466609827668737, + 0.43567603051211307, + 0.41544997111847654, + 0.48809065762969883, + 1.1821280091855806, + 1.1664813213195084, + 1.07816907868966, + 0.9625539685607898, + 3.8682944428485135, + 3.0459807530568472, + 2.2760455923564815, + 1.7581896219738866, + 1.7091546359484804, + 1.4520780986698394, + 1.1543285578371303, + 0.8578398764689898, + 0.9984751177727861, + 0.6390807593663731, + 0.4862828593592872, + 0.39284346009424276 + ], + "peak_layer": 21, + "peak_normalized": 3.8682944428485135, + "population_cv": 0.7787255378412773, + "rest_mean": 4.0033484630704925e-05, + "spike_contrast": 3.5337714610257334, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014146918547339738, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.7307676393538713e-05, + 4.022332723252475e-05, + 3.474821278359741e-05, + 3.346626544953324e-05, + 3.725444548763335e-05, + 3.231675145798363e-05, + 2.935460361186415e-05, + 2.7908596166525967e-05, + 3.57762910425663e-05, + 3.2942443795036525e-05, + 3.073054540436715e-05, + 2.854952253983356e-05, + 2.5966848625103012e-05, + 2.4346802092622966e-05, + 2.321651300007943e-05, + 2.7275878892396577e-05, + 6.606063834624365e-05, + 6.518625741591677e-05, + 6.0251120885368437e-05, + 5.379022331908345e-05, + 0.0002161711745429784, + 0.0001702179724816233, + 0.00012719182996079326, + 9.825258166529238e-05, + 9.551236871629953e-05, + 8.114620868582278e-05, + 6.45071268081665e-05, + 4.7938505304045975e-05, + 5.579759817919694e-05, + 3.571363049559295e-05, + 2.7174854039913043e-05, + 2.195319757447578e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.60414058175229e-05, + "first_to_last_ratio": 0.8644500280195812, + "imbalance_abs_log_ratio": 0.14566178001816363, + "last_quartile_mean": 3.0124825002530997e-05, + "mean": 3.7509687871306596e-05, + "normalized": [ + 0.7612877820794312, + 0.810711701779243, + 0.7209397300058344, + 0.6925594766630951, + 0.7442562064075515, + 0.6550198977679047, + 0.601099191291831, + 0.5681910666111114, + 0.7147935701981128, + 0.6606027044500686, + 0.6100202180357266, + 0.5720600519271453, + 0.5632479423839041, + 0.5253282201512559, + 0.49767906018209906, + 0.5644858935407635, + 1.2937489635148631, + 1.2840324435565709, + 1.1578011905188617, + 1.0245733409201465, + 3.982499346289561, + 3.0010314180971562, + 2.0616055293573603, + 1.5074555677980233, + 1.505937419140994, + 1.2140551303841993, + 0.9515353865179874, + 0.7371695567558888, + 0.7704610253685644, + 0.5109788396566676, + 0.4050543403397244, + 0.3297777883083533 + ], + "peak_layer": 21, + "peak_normalized": 3.982499346289561, + "population_cv": 0.7526960469489694, + "rest_mean": 2.7703642310236183e-05, + "spike_contrast": 3.2653588610499065, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.046233390108682e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.855566708603874e-05, + 3.04095428873552e-05, + 2.70422242465429e-05, + 2.5977689801948145e-05, + 2.7916817998629995e-05, + 2.4569591914769262e-05, + 2.2547043045051396e-05, + 2.1312669559847564e-05, + 2.6811683710548095e-05, + 2.4779001250863075e-05, + 2.28816679737065e-05, + 2.1457793991430663e-05, + 2.1127254512975924e-05, + 1.9704897567862645e-05, + 1.8667786207515746e-05, + 2.1173689674469642e-05, + 4.852811980526894e-05, + 4.8163656174438074e-05, + 4.342876127338968e-05, + 3.84314262191765e-05, + 0.00014938230742700398, + 0.00011256775178480893, + 7.733017991995439e-05, + 5.654418782796711e-05, + 5.64872425456997e-05, + 4.5538828999269754e-05, + 3.569179534679279e-05, + 2.765099998214282e-05, + 2.8899752578581683e-05, + 1.9166656784364022e-05, + 1.5193461877061054e-05, + 1.2369861906336155e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.932642231229693e-05, + "first_to_last_ratio": 0.48429290534855984, + "imbalance_abs_log_ratio": 0.7250653789708609, + "last_quartile_mean": 0.00018444710076437332, + "mean": 0.00016840073044477322, + "normalized": [ + 0.5799699418435614, + 0.631638262959337, + 0.5372322150726369, + 0.519448365501922, + 0.5960027732717111, + 0.4981229459623488, + 0.4492943531121762, + 0.43180823754645586, + 0.5789679033812432, + 0.5298804189104165, + 0.5013705840728442, + 0.4662175781515477, + 0.3655512526294271, + 0.35395025516272205, + 0.3328677409454649, + 0.4094748170873465, + 1.0315739694159025, + 1.0322187789196813, + 0.9834616923499129, + 0.8864297869456468, + 3.853800867660393, + 3.163217666231461, + 2.5024524541124884, + 2.002752567834613, + 1.843634113893829, + 1.6377082968935632, + 1.3336197815157738, + 1.0052459186517901, + 1.1908891844448808, + 0.7572668908929535, + 0.5664028889185376, + 0.4275274957074122 + ], + "peak_layer": 21, + "peak_normalized": 3.853800867660393, + "population_cv": 0.8345380635674979, + "rest_mean": 0.00011622233998625436, + "spike_contrast": 3.873300425494927, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00045016403892077505, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.766736184246838e-05, + 0.00010636834485922009, + 9.047029743669555e-05, + 8.747548417886719e-05, + 0.00010036730236606672, + 8.388426795136184e-05, + 7.566149724880233e-05, + 7.271682261489332e-05, + 9.749861783348024e-05, + 8.923224959289655e-05, + 8.443117258138955e-05, + 7.851138070691377e-05, + 6.155909795779735e-05, + 5.9605481510516256e-05, + 5.605517071671784e-05, + 6.895585829624906e-05, + 0.00017371780995745212, + 0.0001738263963488862, + 0.00016561566735617816, + 0.00014927542360965163, + 0.000648982881102711, + 0.0005326881655491889, + 0.0004214148211758584, + 0.0003372649953234941, + 0.000310469331452623, + 0.00027579127345234156, + 0.00022458254534285516, + 0.00016928414697758853, + 0.00020054660853929818, + 0.00012752429756801575, + 9.538266021991149e-05, + 7.199594256235287e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.15345558524131775, + 0.3249167501926422, + 0.3818626403808594, + 0.40600064396858215, + 0.1351035088300705, + 0.15324510633945465, + 0.18001793324947357, + 0.20391276478767395, + 0.04374876990914345, + 0.08211024105548859, + 0.1380448192358017, + 0.2295975536108017, + 0.08199761807918549, + 0.13892096281051636, + 0.21687114238739014, + 0.7360114455223083, + 0.057135771960020065, + 0.08251863718032837, + 0.12661415338516235, + 0.19619159400463104, + 0.06753352284431458, + 0.095451220870018, + 0.1274663656949997, + 0.15628865361213684, + 0.058436937630176544, + 0.08794140070676804, + 0.12247690558433533, + 0.1640285700559616, + 0.07537907361984253, + 0.12290703505277634, + 0.19157102704048157, + 0.29249706864356995 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1220713531656656e-05, + "first_to_last_ratio": 0.6555413366591524, + "imbalance_abs_log_ratio": 0.4222939165017446, + "last_quartile_mean": 1.7116713934228756e-05, + "mean": 1.7638015336274293e-05, + "normalized": [ + 0.6920608231884311, + 0.7659177824262653, + 0.6390754033749174, + 0.6123424804403663, + 0.7313911494807722, + 0.6060429021656846, + 0.5358717188243214, + 0.5066293084101811, + 0.6361394778273108, + 0.5935071698997143, + 0.5532047504443424, + 0.5203003869133469, + 0.4427001574083234, + 0.41555518288954896, + 0.42010731902783316, + 0.5249828006548773, + 1.2555950689969066, + 1.2709301325611273, + 1.1358461030518159, + 1.0637808540376092, + 3.605678554956403, + 2.784492491030135, + 2.1780228929741425, + 1.7462695606216458, + 1.8163019324419567, + 1.491254439501435, + 1.0911345614825427, + 0.8818559802285882, + 0.8715835151550508, + 0.6503123816361532, + 0.4986363533333601, + 0.46247636461489006 + ], + "peak_layer": 21, + "peak_normalized": 3.605678554956403, + "population_cv": 0.7234183193184713, + "rest_mean": 1.2979772741851371e-05, + "spike_contrast": 3.2968624487684486, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2792525346158075e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.220657941303216e-05, + 1.3509269592759665e-05, + 1.1272021765762474e-05, + 1.0800506061059423e-05, + 1.2900288311357144e-05, + 1.0689394002838526e-05, + 9.451713594899047e-06, + 8.935935511544812e-06, + 1.122023786592763e-05, + 1.0468288564879913e-05, + 9.757433872437105e-06, + 9.177066203847062e-06, + 7.808352165739052e-06, + 7.329568688874133e-06, + 7.409859335893998e-06, + 9.259654689230956e-06, + 2.214620508311782e-05, + 2.2416685169446282e-05, + 2.003407098527532e-05, + 1.876298301795032e-05, + 6.359701364999637e-05, + 4.911292126053013e-05, + 3.841600118903443e-05, + 3.080072929151356e-05, + 3.203596133971587e-05, + 2.630276867421344e-05, + 1.9245448129368015e-05, + 1.555418930365704e-05, + 1.5373003407148644e-05, + 1.1470219760667533e-05, + 8.794955647317693e-06, + 8.157165211741813e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "rest_mean": 5.15456125598306e-05, + "spike_contrast": 3.3328477735441013, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179368005599828, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.039005559869110584, + "first_to_last_ratio": 0.6452634722813873, + "imbalance_abs_log_ratio": 0.43809656143611553, + "last_quartile_mean": 0.06044904375448823, + "mean": 0.06237322464585304, + "normalized": [ + 0.6809932518812878, + 0.7475633220322179, + 0.6323153569646397, + 0.6077851973803072, + 0.7056447145510248, + 0.5942445836918125, + 0.530223252027616, + 0.5040898132583496, + 0.6368974600710677, + 0.5978889321237267, + 0.5559539597752757, + 0.523297269904143, + 0.4434509324241475, + 0.4141123687040429, + 0.4174418427557242, + 0.5192489959347639, + 1.2557404102027183, + 1.2676085225122937, + 1.157434741358947, + 1.0601387967701486, + 3.61892435638739, + 2.812397521737949, + 2.201832924858681, + 1.7615672286963322, + 1.8179181559541986, + 1.487118729202359, + 1.086854143939758, + 0.8727033118250799, + 0.8717668711180101, + 0.6523365178130685, + 0.4999226849610752, + 0.46458382918184327 + ], + "peak_layer": 21, + "peak_normalized": 3.61892435638739, + "population_cv": 0.7299683256836244, + "rest_mean": 0.045711164397222025, + "spike_contrast": 3.3328477188762906, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15234834998846053, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04247574508190155, + 0.04662793502211571, + 0.039439547806978226, + 0.03790952265262604, + 0.044013336300849915, + 0.03706495091319084, + 0.03307173401117325, + 0.03144170716404915, + 0.039725348353385925, + 0.03729226067662239, + 0.03467664122581482, + 0.03263973817229271, + 0.02765946462750435, + 0.02582952380180359, + 0.026037193834781647, + 0.03238723427057266, + 0.0783245787024498, + 0.07906483113765717, + 0.07219293713569641, + 0.06612427532672882, + 0.2257239818572998, + 0.17541830241680145, + 0.1373354196548462, + 0.10987462848424911, + 0.11338941752910614, + 0.09275639057159424, + 0.06779059767723083, + 0.054433319717645645, + 0.05437491089105606, + 0.04068833217024803, + 0.031181789934635162, + 0.028977591544389725 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.896312141980161e-06, + "first_to_last_ratio": 0.40447227453095713, + "imbalance_abs_log_ratio": 0.905172087401449, + "last_quartile_mean": 4.688361258331497e-06, + "mean": 3.997668517996544e-06, + "normalized": [ + 0.6172058310645722, + 0.5425329228673286, + 0.5226213455478163, + 0.5130242205418598, + 0.4179738021592733, + 0.4022397236307717, + 0.38715722309433015, + 0.3920811131204843, + 0.47573503087335345, + 0.47091681751525505, + 0.4692356598433531, + 0.46858709630953144, + 0.435518404119361, + 0.43348810961922396, + 0.457344361488258, + 0.4834240606909693, + 0.943243320831244, + 0.9556851843891272, + 0.9328847891755226, + 0.9144352241016374, + 3.593133434897055, + 3.0161686306352355, + 2.572714523011844, + 2.200458051372808, + 2.189693136620607, + 1.8429744726585007, + 1.3739587829516955, + 1.1483183935808434, + 1.0092761091983693, + 0.7452413615169265, + 0.5556410385724699, + 0.5170878240003723 + ], + "peak_layer": 21, + "peak_normalized": 3.593133434897055, + "population_cv": 0.8307777929041875, + "rest_mean": 2.728457953135485e-06, + "spike_contrast": 3.9771203202072667, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0851405568246264e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4673843199707335e-06, + 2.1688667857233668e-06, + 2.0892668999294983e-06, + 2.050900775429909e-06, + 1.670920710239443e-06, + 1.6080210798463668e-06, + 1.5477262422791682e-06, + 1.567410322422802e-06, + 1.901830955830519e-06, + 1.8825693359758588e-06, + 1.8758486248771078e-06, + 1.8732558828560286e-06, + 1.741058213156066e-06, + 1.7329417687506066e-06, + 1.8283111558048404e-06, + 1.9325691482663387e-06, + 3.770774128497578e-06, + 3.820512574748136e-06, + 3.7293641526048305e-06, + 3.6556089071382303e-06, + 1.4364156413648743e-05, + 1.2057642379659228e-05, + 1.0284859854436945e-05, + 8.796701877145097e-06, + 8.753667316341307e-06, + 7.3676010288181715e-06, + 5.49263177163084e-06, + 4.590596290654503e-06, + 4.0347513277083635e-06, + 2.9792279292450985e-06, + 2.2212686872080667e-06, + 2.0671457150456263e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.3950954957617796e-05, + "first_to_last_ratio": 0.7815786467251918, + "imbalance_abs_log_ratio": 0.24643949857917652, + "last_quartile_mean": 4.343894897829159e-05, + "mean": 4.974799179535694e-05, + "normalized": [ + 0.7500094412446059, + 0.8083859646655934, + 0.6989360803110847, + 0.6730394265060721, + 0.7487882742003255, + 0.6472039809659893, + 0.5806034653418889, + 0.5527037948336986, + 0.6849833016487895, + 0.641463932771085, + 0.5923270784846439, + 0.5570934219720602, + 0.5025383930607131, + 0.4685801281186291, + 0.467958282860767, + 0.5622422290415673, + 1.3163199357516149, + 1.3145043099431595, + 1.1718295263393368, + 1.089421848322846, + 3.6786851702332073, + 2.7746307606104894, + 2.0966820217393844, + 1.6356296326692725, + 1.680361539469081, + 1.3632870777077486, + 0.9862895783238333, + 0.8095320290956769, + 0.7283886637343489, + 0.5544846482049088, + 0.4452099057793211, + 0.417886156048257 + ], + "peak_layer": 21, + "peak_normalized": 3.6786851702332073, + "population_cv": 0.710922698103497, + "rest_mean": 3.709728177151798e-05, + "spike_contrast": 3.18249263251226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011806182592408732, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.7311463529476896e-05, + 4.021557833766565e-05, + 3.477066638879478e-05, + 3.348235986777581e-05, + 3.725071292137727e-05, + 3.219709833501838e-05, + 2.8883856430184096e-05, + 2.7495903850649484e-05, + 3.407654367038049e-05, + 3.191154246451333e-05, + 2.946708264062181e-05, + 2.7714278985513374e-05, + 2.5000275854836218e-05, + 2.3310920369112864e-05, + 2.327998481632676e-05, + 2.7970421797363088e-05, + 6.548427336383611e-05, + 6.539394962601364e-05, + 5.8296165661886334e-05, + 5.4196549172047526e-05, + 0.00018300719966646284, + 0.0001380323083139956, + 0.0001043057200149633, + 8.136928954627365e-05, + 8.359461207874119e-05, + 6.782079435652122e-05, + 4.906592585030012e-05, + 4.027259274153039e-05, + 3.623587326728739e-05, + 2.7584497729549184e-05, + 2.21482987399213e-05, + 2.078899706248194e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.600453626655508e-05, + "first_to_last_ratio": 1.032180910721196, + "imbalance_abs_log_ratio": 0.031673952782361334, + "last_quartile_mean": 2.5193777560161834e-05, + "mean": 3.414015603198095e-05, + "normalized": [ + 0.8380570158080917, + 0.8932999084125006, + 0.7929620023886231, + 0.7615443521416788, + 0.8187638282400099, + 0.7186815599217726, + 0.6527123798513835, + 0.6175741300582618, + 0.7513411682592698, + 0.705872928051971, + 0.6475562024934578, + 0.6094685286755085, + 0.5930459106503145, + 0.5494388493370179, + 0.5439464215496852, + 0.6333820560847124, + 1.4124654565316987, + 1.425994860990475, + 1.2380543674563398, + 1.1313103339756791, + 3.756835524951107, + 2.7186060870583466, + 1.87468091251249, + 1.410793634954463, + 1.4329939421834126, + 1.1438971076949418, + 0.8490265267268121, + 0.7287288463621843, + 0.5779352074283672, + 0.4639524923457935, + 0.36814597511060676, + 0.33893148179302335 + ], + "peak_layer": 21, + "peak_normalized": 3.756835524951107, + "population_cv": 0.6885594503102112, + "rest_mean": 2.6308265020176163e-05, + "spike_contrast": 2.9052606637917697, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.643236749572679e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8611397283384576e-05, + 3.0497398256557062e-05, + 2.7071846488979645e-05, + 2.599924300739076e-05, + 2.795272484945599e-05, + 2.4535900593036786e-05, + 2.228370249213185e-05, + 2.1084077161503956e-05, + 2.5650904717622325e-05, + 2.4098611902445555e-05, + 2.21076697926037e-05, + 2.0807350665563717e-05, + 2.024667992372997e-05, + 1.8757928046397865e-05, + 1.8570415704743937e-05, + 2.1623762222588994e-05, + 4.82217910757754e-05, + 4.86836870550178e-05, + 4.226736928103492e-05, + 3.8623111322522163e-05, + 0.00012825895100831985, + 9.281363600166515e-05, + 6.400189886335284e-05, + 4.816471482627094e-05, + 4.89226367790252e-05, + 3.905282574123703e-05, + 2.898589809774421e-05, + 2.4878916519810446e-05, + 1.9730798157979734e-05, + 1.5839410480111837e-05, + 1.256856103282189e-05, + 1.1571173672564328e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.909050666261464e-05, + "first_to_last_ratio": 0.6062699294291527, + "imbalance_abs_log_ratio": 0.5004299639855584, + "last_quartile_mean": 0.00014694858236907749, + "mean": 0.00014725223013556388, + "normalized": [ + 0.6636255714147841, + 0.7248917149077715, + 0.6138660879695628, + 0.5985975801010156, + 0.6806345015153643, + 0.5652332367621573, + 0.5042378190281714, + 0.4890714447869022, + 0.6294477660992072, + 0.5850399957826026, + 0.5478904329704705, + 0.518038310415621, + 0.40778029155663986, + 0.3853440981019431, + 0.3845833582250337, + 0.48007518680371525, + 1.178544482689773, + 1.1716070054101007, + 1.1012637666749874, + 1.0246953672401915, + 3.668461219741745, + 2.9054304267727518, + 2.326469432391579, + 1.8616676451976302, + 1.8927854440285927, + 1.5414183922061209, + 1.1508159193509495, + 0.9341585266360799, + 0.8717798331906053, + 0.6455697098701952, + 0.49047863567768574, + 0.45649679648005004 + ], + "peak_layer": 21, + "peak_normalized": 3.668461219741745, + "population_cv": 0.7615887971484168, + "rest_mean": 0.00010550450948627528, + "spike_contrast": 3.5324549012779802, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003726899216417223, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.772034536581486e-05, + 0.00010674192162696272, + 9.039315045811236e-05, + 8.814482862362638e-05, + 0.00010022494825534523, + 8.323185465997085e-05, + 7.42501433705911e-05, + 7.201686094049364e-05, + 9.268758731195703e-05, + 8.614844409748912e-05, + 8.067808812484145e-05, + 7.628229650435969e-05, + 6.004655733704567e-05, + 5.674277781508863e-05, + 5.663075717166066e-05, + 7.06921418895945e-05, + 0.00017354330339003354, + 0.00017252174438908696, + 0.00016216354561038315, + 0.00015088867803569883, + 0.0005401890957728028, + 0.0004278311098460108, + 0.00034257781226187944, + 0.00027413471252657473, + 0.0002787168778013438, + 0.00022697729582432657, + 0.00016946021059993654, + 0.0001375569263473153, + 0.0001283715246245265, + 9.506157948635519e-05, + 7.222407293738797e-05, + 6.722017133142799e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.06279198825359344, + 0.07690861076116562, + 0.18058517575263977, + 0.15882407128810883, + 0.19892723858356476, + 0.15647411346435547, + 0.1533493995666504, + 0.1527898609638214, + 0.16355258226394653, + 0.11789151281118393, + 0.11456070095300674, + 0.12148918211460114, + 0.15094918012619019, + 0.09854138642549515, + 0.11213528364896774, + 0.16254977881908417, + 0.2323070764541626, + 0.1390199512243271, + 0.14812250435352325, + 0.15333011746406555, + 0.1781008094549179, + 0.07593096792697906, + 0.09691466391086578, + 0.09368696808815002, + 0.10724716633558273, + 0.06538715958595276, + 0.07412076741456985, + 0.07655765861272812, + 0.08420819044113159, + 0.07230556756258011, + 0.07723049819469452, + 0.08422473073005676 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.792817086738069e-06, + "first_to_last_ratio": 0.9644943347481165, + "imbalance_abs_log_ratio": 0.03615132042434886, + "last_quartile_mean": 8.079691923512655e-06, + "mean": 1.231628780118399e-05, + "normalized": [ + 0.899719870687867, + 0.727426048108731, + 0.7370807482678542, + 0.3965794568165141, + 0.5003083522454715, + 0.7486231449014935, + 0.6212780315806493, + 0.4307803771911971, + 0.40860273125956736, + 0.5996948368722662, + 0.5787702645812915, + 0.5364690416901848, + 0.2710899543731052, + 0.3344315013927567, + 0.25145386433912675, + 1.0500832430459097, + 0.7256359007107253, + 1.3787305863467312, + 2.2756402165685268, + 2.51490055711116, + 3.4405522362788585, + 3.2818395195720442, + 2.448485364965397, + 1.5936896131932947, + 0.9036087632058596, + 1.0199413906977872, + 1.0227020812544427, + 0.5747747031450406, + 0.6830971927098096, + 0.5175337025733492, + 0.3003192317874535, + 0.22615747252553492 + ], + "peak_layer": 21, + "peak_normalized": 3.4405522362788585, + "population_cv": 0.8497149496404188, + "rest_mean": 9.274540818498483e-06, + "spike_contrast": 3.098991321528187, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.8741721507685725e-05, + "top_five_layers": [ + 21, + 22, + 20, + 23, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1081208867835812e-05, + 8.959188562585041e-06, + 9.078098628378939e-06, + 4.884386726189405e-06, + 6.161941655591363e-06, + 9.220258107234258e-06, + 7.651839041500352e-06, + 5.3056151045893785e-06, + 5.032468834542669e-06, + 7.386014203802915e-06, + 7.128301149350591e-06, + 6.6073071138816886e-06, + 3.338821898069e-06, + 4.118954620935256e-06, + 3.0969781619205605e-06, + 1.293312743655406e-05, + 8.937140592024662e-06, + 1.6980842701741494e-05, + 2.8027439839206636e-05, + 3.0974239052738994e-05, + 4.23748315370176e-05, + 4.0420080040348694e-05, + 3.0156250431900844e-05, + 1.9628339941846207e-05, + 1.112910558731528e-05, + 1.2561891708173789e-05, + 1.259589316759957e-05, + 7.079090664774412e-06, + 8.413221621594857e-06, + 6.374094027705723e-06, + 3.6988180909247603e-06, + 2.785420520012849e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.7120979211758822e-05, + "first_to_last_ratio": 0.8594772224120714, + "imbalance_abs_log_ratio": 0.15143095548529256, + "last_quartile_mean": 3.1555204145661264e-05, + "mean": 4.800072150601409e-05, + "normalized": [ + 0.509760479963835, + 0.6173326836717695, + 0.7324351629463104, + 0.39842116793036353, + 0.4759789060227602, + 0.7458666061827044, + 0.6096831903000134, + 0.4306170616938511, + 0.4047810550951731, + 0.5961537125874838, + 0.6210341956779442, + 0.5605157741769637, + 0.27837672031341976, + 0.3443893546178484, + 0.25310408037089555, + 1.0527443019909122, + 0.7465088513707779, + 1.4090569908205048, + 2.369608868078584, + 2.622524932193837, + 3.51885122958336, + 3.2980052855840127, + 2.541370727154349, + 1.6037570225598534, + 0.918310576031736, + 1.0270328185033981, + 1.0005888996255676, + 0.5708475165844191, + 0.6850494580580762, + 0.5307877571911549, + 0.29761830732641337, + 0.22888630579170677 + ], + "peak_layer": 21, + "peak_normalized": 3.51885122958336, + "population_cv": 0.8786708090610466, + "rest_mean": 3.5768902374979075e-05, + "spike_contrast": 3.1885950432011234, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011405254481360317, + "top_five_layers": [ + 21, + 22, + 20, + 23, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.446887083351612e-05, + 2.96324142254889e-05, + 3.515741627779789e-05, + 1.912450352392625e-05, + 2.2847330910735764e-05, + 3.580213524401188e-05, + 2.9265233024489135e-05, + 2.0669929654104635e-05, + 1.9429782696533948e-05, + 2.8615808332688175e-05, + 2.9810089472448453e-05, + 2.6905161575996317e-05, + 1.3362283425522037e-05, + 1.653093750064727e-05, + 1.2149178473919164e-05, + 5.0532486056908965e-05, + 3.5832963476423174e-05, + 6.76357522024773e-05, + 0.00011374293535482138, + 0.00012588308891281486, + 0.00016890739789232612, + 0.00015830663323868066, + 0.00012198762851767242, + 7.698149420320988e-05, + 4.407957021612674e-05, + 4.929831629851833e-05, + 4.802898911293596e-05, + 2.7401092665968463e-05, + 3.288286825409159e-05, + 2.5478195311734453e-05, + 1.4285893485066481e-05, + 1.0986707820848096e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.024051155894994736, + "first_to_last_ratio": 0.8594772374722103, + "imbalance_abs_log_ratio": 0.15143093796285173, + "last_quartile_mean": 0.027983470470644534, + "mean": 0.042567519849399105, + "normalized": [ + 0.5097604616222596, + 0.6173327141338325, + 0.7324351822855276, + 0.3984211759638306, + 0.4759788964629159, + 0.7458666120674253, + 0.6096832157472697, + 0.430617059477717, + 0.40478105545710685, + 0.5961537255711045, + 0.6210341986548017, + 0.5605157515644588, + 0.278376722109868, + 0.3443893736332492, + 0.25310409496395464, + 1.0527443624204256, + 0.7465088835832933, + 1.4090569785936171, + 2.3696089052435614, + 2.6225247338771562, + 3.5188511717225395, + 3.298005336484889, + 2.541370809226851, + 1.6037569634689308, + 0.9183106068252992, + 1.0270328482800881, + 1.000588834809077, + 0.5708474927211314, + 0.6850494767792821, + 0.5307877431850209, + 0.29761830682761936, + 0.2288863062358959 + ], + "peak_layer": 21, + "peak_normalized": 3.5188511717225395, + "population_cv": 0.8786708002138178, + "rest_mean": 0.0317202202492842, + "spike_contrast": 3.1885950631853515, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10114293769001961, + "top_five_layers": [ + 21, + 22, + 20, + 23, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.021699238568544388, + 0.02627832256257534, + 0.031177949160337448, + 0.01695980131626129, + 0.020261241123080254, + 0.03174969181418419, + 0.02595270238816738, + 0.018330300226807594, + 0.017230525612831116, + 0.025376785546541214, + 0.026435885578393936, + 0.023859765380620956, + 0.011849806644022465, + 0.014659801498055458, + 0.010774013586342335, + 0.04481271654367447, + 0.0317770317196846, + 0.059980060905218124, + 0.10086837410926819, + 0.11163437366485596, + 0.1497887670993805, + 0.1403879076242447, + 0.10817985236644745, + 0.06826795637607574, + 0.03909020498394966, + 0.04371824115514755, + 0.04259258508682251, + 0.024299561977386475, + 0.02916085720062256, + 0.02259431779384613, + 0.012668873183429241, + 0.00974312238395214 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.670534108517586e-06, + "first_to_last_ratio": 0.516077911622693, + "imbalance_abs_log_ratio": 0.6614975333817992, + "last_quartile_mean": 5.174672367047606e-06, + "mean": 3.915999473136367e-06, + "normalized": [ + 2.91792462644838, + 1.0330499050224828, + 0.21324233730395303, + 0.31813747476101534, + 0.1980657666006576, + 0.22572388219114395, + 0.26853595304608496, + 0.28095737396172366, + 0.2790646446283966, + 0.3792913632687372, + 0.3694120454168363, + 0.4225628265552778, + 0.22944152357264785, + 0.35050475469892045, + 0.38580405805880497, + 0.330583746531144, + 0.275347554842962, + 0.6571160489647494, + 0.9522439259464477, + 1.1118382618025555, + 1.9569752952109616, + 3.466111019148444, + 2.5633468283111025, + 2.243374425678547, + 1.2633072391816056, + 1.9652520232619801, + 1.881284353579538, + 1.3019726140072538, + 1.7404895149568589, + 1.1347055768273686, + 0.7207866671170219, + 0.5635463690963973 + ], + "peak_layer": 22, + "peak_normalized": 3.466111019148444, + "population_cv": 0.881830620778404, + "rest_mean": 2.974257467013996e-06, + "spike_contrast": 3.026438163483582, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.001406306197169e-06, + "top_five_layers": [ + 22, + 1, + 23, + 24, + 26 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1426591299823485e-05, + 4.045422883791616e-06, + 8.350568805326475e-07, + 1.2458261835490703e-06, + 7.756254376545257e-07, + 8.83934603734815e-07, + 1.051586650646641e-06, + 1.100228928407887e-06, + 1.0928170013357885e-06, + 1.4853047787255491e-06, + 1.4466173752225586e-06, + 1.6547558061574819e-06, + 8.984928854260943e-07, + 1.372576434732764e-06, + 1.5108084880921524e-06, + 1.294565777243406e-06, + 1.078260879694426e-06, + 2.5732661015354097e-06, + 3.7289867123035947e-06, + 4.353958047431661e-06, + 7.663514224987011e-06, + 1.3573288924817462e-05, + 1.0038064829132054e-05, + 8.785053069004789e-06, + 4.947110483044526e-06, + 7.695925887674093e-06, + 7.3671085374371614e-06, + 5.0985240704903845e-06, + 6.8157560235704295e-06, + 4.4435064410208724e-06, + 2.8226002086739754e-06, + 2.2068472844694043e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.836107992403413e-05, + "first_to_last_ratio": 0.9160653111165196, + "imbalance_abs_log_ratio": 0.08766761650421023, + "last_quartile_mean": 2.004341797601228e-05, + "mean": 3.121470177802621e-05, + "normalized": [ + 0.5128089259228045, + 0.6437171011700566, + 0.7320107521642394, + 0.42714084499329735, + 0.4777939631770697, + 0.7761341517210342, + 0.6673269525799402, + 0.4688191183265314, + 0.43572863736759326, + 0.6442483222440443, + 0.6589268299586913, + 0.5254485027294052, + 0.30608252008544473, + 0.35836487745274936, + 0.2721971888139584, + 1.0581434296571979, + 0.7638096596387699, + 1.3818036797996578, + 2.4488923222411234, + 2.174595772916911, + 3.2150744119330064, + 3.5033815383059985, + 2.7401154537196812, + 1.6705176716332228, + 0.9292103758537513, + 1.0277197763891293, + 1.0216310132104693, + 0.5115354754917107, + 0.6598856618533152, + 0.50319316016592, + 0.28185438701748666, + 0.20188752146578876 + ], + "peak_layer": 22, + "peak_normalized": 3.5033815383059985, + "population_cv": 0.8621259030079774, + "rest_mean": 2.3054601317400942e-05, + "spike_contrast": 3.2652589922942665, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.527924426540267e-05, + "top_five_layers": [ + 22, + 21, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.6007177691790275e-05, + 2.0093437342438847e-05, + 2.2849497327115387e-05, + 1.3333074093679897e-05, + 1.4914196071913466e-05, + 2.422679608571343e-05, + 2.0830411813221872e-05, + 1.463404896639986e-05, + 1.3601139471575152e-05, + 2.011001924984157e-05, + 2.0568204490700737e-05, + 1.6401718312408775e-05, + 9.554274583933875e-06, + 1.118625277740648e-05, + 8.496554073644802e-06, + 3.3029631595127285e-05, + 2.3842090740799904e-05, + 4.313258978072554e-05, + 7.644144352525473e-05, + 6.787935853935778e-05, + 0.00010035758896265179, + 0.00010935700993286446, + 8.553188672522083e-05, + 5.2144710934953764e-05, + 2.9005024771322496e-05, + 3.207996633136645e-05, + 3.188990740454756e-05, + 1.5967427316354588e-05, + 2.059813414234668e-05, + 1.570702443132177e-05, + 8.798000635579228e-06, + 6.301858775259461e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.2228491186760948e-05, + "first_to_last_ratio": 0.9571529484026068, + "imbalance_abs_log_ratio": 0.04379207960676997, + "last_quartile_mean": 1.2775900870565238e-05, + "mean": 2.0228415721135207e-05, + "normalized": [ + 0.4968956893086432, + 0.680742973654979, + 0.7082959423523417, + 0.4553628765634101, + 0.48416833058916253, + 0.8025982815331242, + 0.71544927601163, + 0.49265031392139635, + 0.46201741019996967, + 0.6900348438458223, + 0.6974262922446428, + 0.46524054678094645, + 0.3363013254017149, + 0.39144273721666156, + 0.29629939799755733, + 1.053294975127128, + 0.7915013073840591, + 1.3752427386344863, + 2.531667923112097, + 1.9941217404782703, + 2.7172330416452364, + 3.752050564767614, + 2.871442549447273, + 1.6858638659668104, + 0.9381697357727052, + 1.0373218643217643, + 1.0308781989118676, + 0.4725668463395891, + 0.6400815997091446, + 0.4765607085370807, + 0.2663935850069364, + 0.19068251721593543 + ], + "peak_layer": 22, + "peak_normalized": 3.752050564767614, + "population_cv": 0.8541640127467388, + "rest_mean": 1.5010413655410823e-05, + "spike_contrast": 3.224802992594281, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.840562687604688e-05, + "top_five_layers": [ + 22, + 23, + 21, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.0051412573375273e-05, + 1.3770351870334707e-05, + 1.4327704775496386e-05, + 9.211269571096636e-06, + 9.793958270165604e-06, + 1.623529169592075e-05, + 1.4472405382548459e-05, + 9.965535355149768e-06, + 9.34588024392724e-06, + 1.3958311683381908e-05, + 1.410782897437457e-05, + 9.411079190613236e-06, + 6.8028430177946575e-06, + 7.918266419437714e-06, + 5.993667400616687e-06, + 2.1306488633854315e-05, + 1.601081748958677e-05, + 2.7818981834570877e-05, + 5.121163121657446e-05, + 4.033792356494814e-05, + 5.496531957760453e-05, + 7.589803863083944e-05, + 5.808473360957578e-05, + 3.4102355130016804e-05, + 1.8977687432197854e-05, + 2.0983377908123657e-05, + 2.0853032765444368e-05, + 9.55927862378303e-06, + 1.2947836694365833e-05, + 9.640068128646817e-06, + 5.38872018296388e-06, + 3.857205228996463e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 5.4042374358687084e-05, + "first_to_last_ratio": 0.8965146957898991, + "imbalance_abs_log_ratio": 0.10924059366762352, + "last_quartile_mean": 6.028052257534e-05, + "mean": 9.390137262244025e-05, + "normalized": [ + 0.5389360174242739, + 0.5974650105374263, + 0.7267651782696123, + 0.4014182444828728, + 0.4819177901244493, + 0.7732899466309664, + 0.6299309978744704, + 0.45445865774623606, + 0.41201557220312385, + 0.5831302663155696, + 0.6244048764918407, + 0.5807706511889962, + 0.27596931886992426, + 0.3314258726431575, + 0.23879880407819865, + 1.0945434973487749, + 0.734405519454758, + 1.357347632155138, + 2.3148429937772614, + 2.252423783537609, + 3.7724800608232885, + 3.3168691995820305, + 2.6904907774938076, + 1.6802536301987592, + 0.9186543261041, + 0.9881029074846317, + 1.0077452263535627, + 0.5222119037959454, + 0.6630699433389028, + 0.5491195746896886, + 0.2929408036924596, + 0.19380101528816393 + ], + "peak_layer": 21, + "peak_normalized": 3.7724800608232885, + "population_cv": 0.8932088546517, + "rest_mean": 6.823935170056454e-05, + "spike_contrast": 3.406777465012843, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002324762856005691, + "top_five_layers": [ + 21, + 22, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.060683179181069e-05, + 5.6102784583345056e-05, + 6.824424781370908e-05, + 3.769372415263206e-05, + 4.525274198385887e-05, + 7.26129874237813e-05, + 5.915138535783626e-05, + 4.267429176252335e-05, + 3.868882777169347e-05, + 5.475673242472112e-05, + 5.863247497472912e-05, + 5.45351613254752e-05, + 2.5913897843565792e-05, + 3.1121344363782555e-05, + 2.2423535483540036e-05, + 0.00010277913679601625, + 6.896168633829802e-05, + 0.0001274568057851866, + 0.00021736693452112377, + 0.0002115056850016117, + 0.00035424105590209365, + 0.0003114585706498474, + 0.000252640777034685, + 0.0001577781222295016, + 8.626290218671784e-05, + 9.2784219305031e-05, + 9.462866000831127e-05, + 4.903641456621699e-05, + 6.226317782420665e-05, + 5.156308179721236e-05, + 2.7507543563842773e-05, + 1.819818135118112e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0596221424639225, + 0.07634982466697693, + 0.06968431174755096, + 0.11288930475711823, + 0.11885104328393936, + 0.1495160460472107, + 0.11554427444934845, + 0.13194940984249115, + 0.08884616196155548, + 0.10897023230791092, + 0.09843935817480087, + 0.09431816637516022, + 0.09025302529335022, + 0.0895693451166153, + 0.10807885229587555, + 0.2204425185918808, + 0.15380074083805084, + 0.08548793941736221, + 0.09071414917707443, + 0.10860678553581238, + 0.04808543995022774, + 0.05631518363952637, + 0.07238366454839706, + 0.08572373539209366, + 0.06546958535909653, + 0.06323372572660446, + 0.06869969516992569, + 0.07729499042034149, + 0.04753003641963005, + 0.06484207510948181, + 0.07835857570171356, + 0.08203105628490448 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.6929306190813804e-06, + "first_to_last_ratio": 0.13688476606561775, + "imbalance_abs_log_ratio": 1.9886158307055912, + "last_quartile_mean": 2.6978390110343753e-05, + "mean": 1.631737328722238e-05, + "normalized": [ + 0.4314194517233057, + 0.19806398425123214, + 0.34668833324353665, + 0.22046602853167355, + 0.10956900273172894, + 0.12688527812920286, + 0.16715726937306416, + 0.21030216864948645, + 0.2743958044688482, + 0.24925629504156072, + 0.32761471427281874, + 0.40984521150785025, + 0.28786819126149943, + 0.29273992694234957, + 0.30660983581768325, + 0.24937431986069702, + 0.3349491137477184, + 0.5177030633789016, + 0.6154108922798558, + 0.7890850685582481, + 3.8103077986911464, + 3.428556172833401, + 2.788926643404586, + 2.2799755314310137, + 2.2723621920706982, + 2.1349169811597655, + 2.003931559830236, + 1.5323918909280079, + 1.9997634861702531, + 1.35547405261909, + 1.065904631827636, + 0.8620851052629044 + ], + "peak_layer": 21, + "peak_normalized": 3.8103077986911464, + "population_cv": 1.0256784776243844, + "rest_mean": 1.0527649945086321e-05, + "spike_contrast": 4.519705687684409, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.7581879334757106e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.039632237137994e-06, + 3.23188396578189e-06, + 5.657042947859736e-06, + 3.597426484702737e-06, + 1.78787831828231e-06, + 2.070434447887237e-06, + 2.727567562033073e-06, + 3.4315789889660664e-06, + 4.47741876996588e-06, + 4.067208010383183e-06, + 5.345811587176286e-06, + 6.687597306154203e-06, + 4.697252734331414e-06, + 4.776746663992526e-06, + 5.003067144571105e-06, + 4.069133865414187e-06, + 5.465489721245831e-06, + 8.447554137092084e-06, + 1.004188925435301e-05, + 1.28757956190384e-05, + 6.217421469045803e-05, + 5.594503090833314e-05, + 4.5507957111112773e-05, + 3.7203211832093075e-05, + 3.707898213178851e-05, + 3.48362373188138e-05, + 3.269889930379577e-05, + 2.5004610506584868e-05, + 3.263088728999719e-05, + 2.2117776097729802e-05, + 1.7392763766110875e-05, + 1.406696446792921e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.4786839983571554e-05, + "first_to_last_ratio": 0.1368590004352419, + "imbalance_abs_log_ratio": 1.9888040770410698, + "last_quartile_mean": 0.00010804433713929029, + "mean": 6.53327875852483e-05, + "normalized": [ + 0.4303813973935663, + 0.1954661274427069, + 0.34854762874244455, + 0.22114030543306437, + 0.10883074349044422, + 0.12627252893821267, + 0.16836982659974983, + 0.21164003420329566, + 0.27785076214706256, + 0.25390299121217014, + 0.3323483997673342, + 0.40930824768453605, + 0.28788144954928885, + 0.2920379121787492, + 0.30710829102986115, + 0.24768611608853355, + 0.33445308089382414, + 0.5081445956986791, + 0.6162106751311494, + 0.7807238268363051, + 3.828274701557073, + 3.4515650116091967, + 2.7664213382241543, + 2.2654046605337452, + 2.2812415827715613, + 2.1356021257971247, + 2.0106696974531495, + 1.537539473929636, + 2.0088419319348305, + 1.3422522168191928, + 1.0598743605751482, + 0.8540079583342102 + ], + "peak_layer": 21, + "peak_normalized": 3.828274701557073, + "population_cv": 1.0279395854576847, + "rest_mean": 4.2120514451569225e-05, + "spike_contrast": 4.52698797699541, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001906790625071153, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.8118016416556202e-05, + 1.2770346984325442e-05, + 2.2771588191972114e-05, + 1.4447712601395324e-05, + 7.110215847205836e-06, + 8.249736310972366e-06, + 1.1000070117006544e-05, + 1.38270333991386e-05, + 1.8152764823753387e-05, + 1.6588190192123875e-05, + 2.1713247406296432e-05, + 2.6741248802863993e-05, + 1.8808097593137063e-05, + 1.9079650883213617e-05, + 2.0064240743522532e-05, + 1.6182024410227314e-05, + 2.1850752091268077e-05, + 3.3198502933373675e-05, + 4.0258761146105826e-05, + 5.1006863941438496e-05, + 0.00025011185789480805, + 0.0002255003637401387, + 0.000180738017661497, + 0.0001480052014812827, + 0.00014903987175785005, + 0.00013952484005130827, + 0.00013136265624780208, + 0.00010045173985417932, + 0.0001312432432314381, + 8.769307896727696e-05, + 6.924454646650702e-05, + 5.579472053796053e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.013113117485772818, + "first_to_last_ratio": 0.136858997118483, + "imbalance_abs_log_ratio": 1.9888041012759314, + "last_quartile_mean": 0.09581480035558343, + "mean": 0.05793777071812656, + "normalized": [ + 0.4303813960583082, + 0.19546612151423692, + 0.3485475910344393, + 0.22114030042116126, + 0.10883074113033038, + 0.12627253310783348, + 0.1683698287227952, + 0.21164002670866952, + 0.27785076861558566, + 0.25390299449217174, + 0.3323483845259541, + 0.40930823057023097, + 0.28788143760030005, + 0.2920379258991919, + 0.3071082822929024, + 0.2476861047748367, + 0.3344530552067231, + 0.5081445507864877, + 0.6162106424834214, + 0.7807238372353658, + 3.828274894940391, + 3.4515650124215562, + 2.7664213738142998, + 2.265404688647299, + 2.281241575194701, + 2.135602067657764, + 2.0106696442072307, + 1.5375394436971075, + 2.00884203416751, + 1.3422522638779362, + 1.0598743603945742, + 0.8540078877986846 + ], + "peak_layer": 21, + "peak_normalized": 3.828274894940391, + "population_cv": 1.0279396080658147, + "rest_mean": 0.03735289376022087, + "spike_contrast": 4.526988119750361, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.16909610629081726, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.024935338646173477, + 0.011324871331453323, + 0.020194070413708687, + 0.012812376022338867, + 0.006305410526692867, + 0.007315949071198702, + 0.009754972532391548, + 0.012261951342225075, + 0.016098054125905037, + 0.014710573479533195, + 0.01925552450120449, + 0.023714406415820122, + 0.01667920872569084, + 0.016920026391744614, + 0.017793169245123863, + 0.01435038074851036, + 0.019377464428544044, + 0.029440762475132942, + 0.035701870918273926, + 0.04523339867591858, + 0.22180171310901642, + 0.19997598230838776, + 0.1602802872657776, + 0.1312524974346161, + 0.13217005133628845, + 0.12373202294111252, + 0.11649371683597565, + 0.08908160775899887, + 0.11638782918453217, + 0.07776710391044617, + 0.061406757682561874, + 0.04947931319475174 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 9.471203732402955e-07, + "first_to_last_ratio": 0.13038144854942058, + "imbalance_abs_log_ratio": 2.037290905335615, + "last_quartile_mean": 7.264226496772608e-06, + "mean": 4.254466777098287e-06, + "normalized": [ + 0.436364407529321, + 0.1911843551159761, + 0.3504684074329156, + 0.22599177156694852, + 0.09379418393340402, + 0.12911388991485367, + 0.1538375030619483, + 0.20018851917359348, + 0.21440564422614256, + 0.20760264501895292, + 0.28595251377841346, + 0.3787305694072829, + 0.27369502895368186, + 0.29556005516396594, + 0.2777983154447535, + 0.10238349267586677, + 0.302592474036744, + 0.3867848818041926, + 0.5416819159884301, + 0.6344790509535327, + 4.041575473617487, + 3.5020644333651707, + 2.77570587266537, + 2.338562686179214, + 2.3623514652944007, + 2.22406080211689, + 2.0998230053367104, + 1.589088734701869, + 1.995381756812838, + 1.3883763621564047, + 1.1160049955561004, + 0.8843947870166255 + ], + "peak_layer": 21, + "peak_normalized": 4.041575473617487, + "population_cv": 1.0776419744012766, + "rest_mean": 2.6755459261780573e-06, + "spike_contrast": 4.776834232976263, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2780639372067526e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.8564978745416738e-06, + 8.133874871418811e-07, + 1.4910561958458857e-06, + 9.614744840291678e-07, + 3.9904423942971334e-07, + 5.493107551046705e-07, + 6.54496545848815e-07, + 8.516954039805569e-07, + 9.121816901824786e-07, + 8.832385560708644e-07, + 1.2165754696980002e-06, + 1.611296625014802e-06, + 1.1644264077403932e-06, + 1.25745043533243e-06, + 1.1818837037935737e-06, + 4.35587168112761e-07, + 1.2873696277893032e-06, + 1.6455634295198251e-06, + 2.304567715327721e-06, + 2.699370043046656e-06, + 1.7194748579640873e-05, + 1.4899416783009656e-05, + 1.1809148418251425e-05, + 9.949337254511192e-06, + 1.0050545824924484e-05, + 9.462192792852875e-06, + 8.933627213991713e-06, + 6.760725227650255e-06, + 8.489285391988233e-06, + 5.9068011069030035e-06, + 4.748006176669151e-06, + 3.762628239201149e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 9.853123231096106e-06, + "first_to_last_ratio": 0.15080465707059237, + "imbalance_abs_log_ratio": 1.89176994145415, + "last_quartile_mean": 6.533699570354656e-05, + "mean": 4.1971269041596315e-05, + "normalized": [ + 0.4681313077158978, + 0.20820887504839763, + 0.36906491110676237, + 0.20482622912946075, + 0.11627096124793933, + 0.12407290917887392, + 0.1787682927302214, + 0.20872662343689463, + 0.3073254147861055, + 0.27783033377805905, + 0.35982154325450194, + 0.4577819740053479, + 0.34235028174668086, + 0.3396230936294121, + 0.3348757051705037, + 0.17757659165663509, + 0.3534791092433193, + 0.5341192315280864, + 0.6428744767315707, + 0.8246883036040644, + 4.08439765980215, + 3.623760084464356, + 2.7887442899646078, + 2.2190205767814533, + 2.346653344716544, + 2.0729802100068144, + 1.8735227194091606, + 1.4202102880569494, + 1.8401836496959596, + 1.1967241409050977, + 0.93540866538392, + 0.7679782020842529 + ], + "peak_layer": 21, + "peak_normalized": 4.08439765980215, + "population_cv": 1.0467961731297342, + "rest_mean": 2.6329080793840586e-05, + "spike_contrast": 4.8022597738792445, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012643908557947726, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.9648065062938258e-05, + 8.738790711504407e-06, + 1.549012267787475e-05, + 8.59681676956825e-06, + 4.880039796262281e-06, + 5.207497451920062e-06, + 7.50313211028697e-06, + 8.760521268413868e-06, + 1.2898837667307816e-05, + 1.166089168691542e-05, + 1.5102166798897088e-05, + 1.9213690393371508e-05, + 1.4368875781656243e-05, + 1.4254412235459313e-05, + 1.4055158317205496e-05, + 7.453114903910318e-06, + 1.483596679463517e-05, + 2.2417661966755986e-05, + 2.6982257622876205e-05, + 3.461321466602385e-05, + 0.0001714273530524224, + 0.00015209380944725126, + 0.00011704713688232005, + 9.313510963693261e-05, + 9.849201887845993e-05, + 8.700561011210084e-05, + 7.863412611186504e-05, + 5.960802809568122e-05, + 7.723484304733574e-05, + 5.0228030886501074e-05, + 3.926028875866905e-05, + 3.2233019737759605e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 6.539388465398588e-06, + "first_to_last_ratio": 0.18539523353460505, + "imbalance_abs_log_ratio": 1.6852653352714542, + "last_quartile_mean": 3.52726892742794e-05, + "mean": 2.5340588074129755e-05, + "normalized": [ + 0.5326820585081189, + 0.22955400672130744, + 0.4075699321599326, + 0.20399783458406082, + 0.13335091090399903, + 0.12913807236762478, + 0.20439277726135907, + 0.22379324161336578, + 0.3549539057327596, + 0.3176654165435907, + 0.4067751656526541, + 0.5369739555670685, + 0.4337606182550087, + 0.42213995757068684, + 0.3963582115760412, + 0.12453444551735621, + 0.40318005313905786, + 0.6070024892440241, + 0.7294977435014097, + 0.8929234728836388, + 4.460419429548373, + 3.910252651443441, + 2.738748156484065, + 2.0647804157793166, + 2.3165049051088284, + 1.8671237368915825, + 1.5983578514794328, + 1.197615503415084, + 1.590133109210288, + 1.0414283798969246, + 0.8387268861490448, + 0.6856647052905551 + ], + "peak_layer": 21, + "peak_normalized": 4.460419429548373, + "population_cv": 1.0654839953495887, + "rest_mean": 1.5494638142222862e-05, + "spike_contrast": 5.066831311954996, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.850871770642698e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.3498476619133726e-05, + 5.817033525090665e-06, + 1.0328061762265861e-05, + 5.169425094209146e-06, + 3.3791905025282176e-06, + 3.2724346965551376e-06, + 5.179433173907455e-06, + 5.6710523494984955e-06, + 8.994740710477345e-06, + 8.049828466027975e-06, + 1.0307921911589801e-05, + 1.3607235814561136e-05, + 1.0991749149980024e-05, + 1.0697274774429388e-05, + 1.0043950169347227e-05, + 3.1557760848954786e-06, + 1.021681964630261e-05, + 1.5381800039904192e-05, + 1.848590181907639e-05, + 2.2627205908065662e-05, + 0.00011302965140203014, + 9.908810170600191e-05, + 6.940148887224495e-05, + 5.2322749979794025e-05, + 5.870159657206386e-05, + 4.731401349999942e-05, + 4.0503327909391373e-05, + 3.034828114323318e-05, + 4.029490810353309e-05, + 2.639040758367628e-05, + 2.125383252860047e-05, + 1.7375146853737533e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.8892148407066998e-05, + "first_to_last_ratio": 0.12323172113154097, + "imbalance_abs_log_ratio": 2.0936887843071057, + "last_quartile_mean": 0.00023445382521458669, + "mean": 0.00013867109626630736, + "normalized": [ + 0.40760164765243684, + 0.18549021085330977, + 0.32629733370345226, + 0.19134578357607018, + 0.0997358093357467, + 0.1135317976584037, + 0.15336948879180487, + 0.18942939953458193, + 0.2684956409479822, + 0.25415697053702413, + 0.3183782664979506, + 0.3953176437860155, + 0.26214994057141733, + 0.2686491657455234, + 0.2734202925021933, + 0.2068336337941873, + 0.30327711407815156, + 0.45518103395508264, + 0.5758332130814036, + 0.7564185139065018, + 3.831959637972952, + 3.474608896989398, + 2.829513018719538, + 2.3332553915396175, + 2.3723457642549124, + 2.19075436705495, + 2.160872753795178, + 1.6201515958627097, + 2.023759263900917, + 1.3235642487571164, + 1.0060145229556705, + 0.8282876376878003 + ], + "peak_layer": 21, + "peak_normalized": 3.831959637972952, + "population_cv": 1.0570572109274479, + "rest_mean": 8.812454328748302e-05, + "spike_contrast": 4.670917624039755, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004116224823519588, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.652256731991656e-05, + 2.572213088569697e-05, + 4.524800897343084e-05, + 2.6534129574429244e-05, + 1.3830474017595407e-05, + 1.5743578842375427e-05, + 2.126791514456272e-05, + 2.626838249852881e-05, + 3.7232584872981533e-05, + 3.5244225728092715e-05, + 4.4149863242637366e-05, + 5.481913103722036e-05, + 3.635261964518577e-05, + 3.7253874324960634e-05, + 3.791549170273356e-05, + 2.8681846742983907e-05, + 4.2055769881699234e-05, + 6.312045297818258e-05, + 7.985142292454839e-05, + 0.00010489338455954567, + 0.0005313820438459516, + 0.0004818278248421848, + 0.00039237167220562696, + 0.000323555083014071, + 0.00032897578785195947, + 0.0003037943097297102, + 0.00029965059366077185, + 0.00022466819791588932, + 0.00028063691570423543, + 0.00018354010535404086, + 0.00013950513675808907, + 0.00011485955474199727 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 1.877417995231997, + "forward": { + "activation_sha256": { + "attention_branch_output": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", + "mlp_branch_output": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", + "post_attention_state": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", + "post_mlp_state": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", + "pre_attention_input": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", + "pre_mlp_input": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a" + }, + "logits_sha256": "5f6747d51014b7e09e13dda43380fde8b7ba2972518bba8957bd86ff4ed7f868", + "loss_nats": 1.3013269901275635, + "mixer_summary_sha256": "9abaf96810a98e69acfbc813a9b903aa15d6279ddc462a3c26f1a1846711526f" + }, + "loss_nats": 1.3013269901275635, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.2200722098350525, + "group": 1, + "latest_source_mass": 0.06159520894289017, + "layer": 1, + "max_source_mass": 0.9384047985076904, + "mixer_index": 1, + "normalized_entropy": 0.3174970857665056, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9384047985076904, + "median": 0.949082612991333, + "p05": 0.8598991632461548, + "p95": 0.9761404991149902 + }, + { + "label": "current_group_partial", + "mean": 0.06159520894289017, + "median": 0.05091731622815132, + "p05": 0.02385946735739708, + "p95": 0.14010077714920044 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5651201605796814, + "group": 1, + "latest_source_mass": 0.3442652225494385, + "layer": 2, + "max_source_mass": 0.6557347774505615, + "mixer_index": 2, + "normalized_entropy": 0.815296053174681, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6557347774505615, + "median": 0.6636689305305481, + "p05": 0.3472050428390503, + "p95": 0.9190996289253235 + }, + { + "label": "current_group_partial", + "mean": 0.3442652225494385, + "median": 0.3363310396671295, + "p05": 0.08090042322874069, + "p95": 0.6527949571609497 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4018177092075348, + "group": 1, + "latest_source_mass": 0.14357808232307434, + "layer": 2, + "max_source_mass": 0.856421947479248, + "mixer_index": 3, + "normalized_entropy": 0.5797004164150741, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.856421947479248, + "median": 0.8660546541213989, + "p05": 0.7540444135665894, + "p95": 0.9155591726303101 + }, + { + "label": "current_group_partial", + "mean": 0.14357808232307434, + "median": 0.13394534587860107, + "p05": 0.08444082736968994, + "p95": 0.24595558643341064 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.687358021736145, + "group": 1, + "latest_source_mass": 0.5312775373458862, + "layer": 3, + "max_source_mass": 0.5312775373458862, + "mixer_index": 4, + "normalized_entropy": 0.9916480092739848, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46872249245643616, + "median": 0.4674533009529114, + "p05": 0.3992259204387665, + "p95": 0.5431661605834961 + }, + { + "label": "current_group_partial", + "mean": 0.5312775373458862, + "median": 0.5325466990470886, + "p05": 0.45683377981185913, + "p95": 0.6007740497589111 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.2678314447402954, + "group": 1, + "latest_source_mass": 0.07946884632110596, + "layer": 3, + "max_source_mass": 0.920531153678894, + "mixer_index": 5, + "normalized_entropy": 0.38639909712095066, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.920531153678894, + "median": 0.9314625263214111, + "p05": 0.8444089889526367, + "p95": 0.9650533199310303 + }, + { + "label": "current_group_partial", + "mean": 0.07946884632110596, + "median": 0.06853749603033066, + "p05": 0.03494669869542122, + "p95": 0.1555909812450409 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.657238245010376, + "group": 1, + "latest_source_mass": 0.3777551054954529, + "layer": 4, + "max_source_mass": 0.6222448348999023, + "mixer_index": 6, + "normalized_entropy": 0.948194356759035, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6222448348999023, + "median": 0.6208515167236328, + "p05": 0.5314525961875916, + "p95": 0.7065955400466919 + }, + { + "label": "current_group_partial", + "mean": 0.3777551054954529, + "median": 0.3791484832763672, + "p05": 0.2934044897556305, + "p95": 0.4685474634170532 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5044373869895935, + "group": 1, + "latest_source_mass": 0.2207653522491455, + "layer": 4, + "max_source_mass": 0.7792346477508545, + "mixer_index": 7, + "normalized_entropy": 0.7277493166488734, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7792346477508545, + "median": 0.7866228222846985, + "p05": 0.6320486664772034, + "p95": 0.9385759830474854 + }, + { + "label": "current_group_partial", + "mean": 0.2207653522491455, + "median": 0.21337716281414032, + "p05": 0.06142403930425644, + "p95": 0.367951363325119 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.650418758392334, + "group": 2, + "latest_source_mass": 0.4602207541465759, + "layer": 5, + "max_source_mass": 0.5397792458534241, + "mixer_index": 8, + "normalized_entropy": 0.9383559172337771, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5397792458534241, + "median": 0.5213949680328369, + "p05": 0.34510529041290283, + "p95": 0.7807042598724365 + }, + { + "label": "completed_group_1", + "mean": 0.4602207541465759, + "median": 0.4786049723625183, + "p05": 0.2192957103252411, + "p95": 0.6548947095870972 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0032464265823364, + "group": 2, + "latest_source_mass": 0.37668097019195557, + "layer": 5, + "max_source_mass": 0.4337552785873413, + "mixer_index": 9, + "normalized_entropy": 0.913194251448444, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4337552785873413, + "median": 0.42766237258911133, + "p05": 0.2500072717666626, + "p95": 0.6592159271240234 + }, + { + "label": "completed_group_1", + "mean": 0.1895637810230255, + "median": 0.18978442251682281, + "p05": 0.1284591555595398, + "p95": 0.25696656107902527 + }, + { + "label": "current_group_partial", + "mean": 0.37668097019195557, + "median": 0.3803442120552063, + "p05": 0.1759825050830841, + "p95": 0.568291425704956 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0385558605194092, + "group": 2, + "latest_source_mass": 0.4949595034122467, + "layer": 6, + "max_source_mass": 0.4949595034122467, + "mixer_index": 10, + "normalized_entropy": 0.9453342832879565, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24478834867477417, + "median": 0.23865866661071777, + "p05": 0.19890648126602173, + "p95": 0.32154381275177 + }, + { + "label": "completed_group_1", + "mean": 0.26025211811065674, + "median": 0.26050686836242676, + "p05": 0.23401768505573273, + "p95": 0.2860541343688965 + }, + { + "label": "current_group_partial", + "mean": 0.4949595034122467, + "median": 0.5017017722129822, + "p05": 0.42678141593933105, + "p95": 0.5358227491378784 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.031773328781128, + "group": 2, + "latest_source_mass": 0.33116668462753296, + "layer": 6, + "max_source_mass": 0.40370824933052063, + "mixer_index": 11, + "normalized_entropy": 0.9391605568439314, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40370824933052063, + "median": 0.38690465688705444, + "p05": 0.18056227266788483, + "p95": 0.7100913524627686 + }, + { + "label": "completed_group_1", + "mean": 0.2651250660419464, + "median": 0.2730395793914795, + "p05": 0.12547528743743896, + "p95": 0.35344114899635315 + }, + { + "label": "current_group_partial", + "mean": 0.33116668462753296, + "median": 0.33928292989730835, + "p05": 0.1502782255411148, + "p95": 0.47667479515075684 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0838522911071777, + "group": 2, + "latest_source_mass": 0.38127952814102173, + "layer": 7, + "max_source_mass": 0.38127952814102173, + "mixer_index": 12, + "normalized_entropy": 0.9865648712351232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35612422227859497, + "median": 0.36401283740997314, + "p05": 0.2946957051753998, + "p95": 0.39828816056251526 + }, + { + "label": "completed_group_1", + "mean": 0.2625962495803833, + "median": 0.2590169608592987, + "p05": 0.24180665612220764, + "p95": 0.2929879426956177 + }, + { + "label": "current_group_partial", + "mean": 0.38127952814102173, + "median": 0.37846946716308594, + "p05": 0.3489896059036255, + "p95": 0.41648316383361816 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.96053147315979, + "group": 2, + "latest_source_mass": 0.3146248161792755, + "layer": 7, + "max_source_mass": 0.5271762013435364, + "mixer_index": 13, + "normalized_entropy": 0.874313425279704, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5271762013435364, + "median": 0.5114221572875977, + "p05": 0.3548204302787781, + "p95": 0.7728348970413208 + }, + { + "label": "completed_group_1", + "mean": 0.1581989824771881, + "median": 0.1618250608444214, + "p05": 0.07541970163583755, + "p95": 0.21691158413887024 + }, + { + "label": "current_group_partial", + "mean": 0.3146248161792755, + "median": 0.32365772128105164, + "p05": 0.14578065276145935, + "p95": 0.4483411908149719 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0704245567321777, + "group": 2, + "latest_source_mass": 0.38260507583618164, + "layer": 8, + "max_source_mass": 0.38260507583618164, + "mixer_index": 14, + "normalized_entropy": 0.9743424206822726, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3788670599460602, + "median": 0.39538437128067017, + "p05": 0.27489906549453735, + "p95": 0.43731769919395447 + }, + { + "label": "completed_group_1", + "mean": 0.23852786421775818, + "median": 0.2294245958328247, + "p05": 0.20573674142360687, + "p95": 0.29735344648361206 + }, + { + "label": "current_group_partial", + "mean": 0.38260507583618164, + "median": 0.37669745087623596, + "p05": 0.34335845708847046, + "p95": 0.435106486082077 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9674104452133179, + "group": 2, + "latest_source_mass": 0.3281455934047699, + "layer": 8, + "max_source_mass": 0.48829177021980286, + "mixer_index": 15, + "normalized_entropy": 0.8805749354816949, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48829177021980286, + "median": 0.44956350326538086, + "p05": 0.2823859453201294, + "p95": 0.8209846615791321 + }, + { + "label": "completed_group_1", + "mean": 0.18356263637542725, + "median": 0.18210050463676453, + "p05": 0.05149933695793152, + "p95": 0.2906108498573303 + }, + { + "label": "current_group_partial", + "mean": 0.3281455934047699, + "median": 0.34843379259109497, + "p05": 0.12376503646373749, + "p95": 0.46937939524650574 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0832805633544922, + "group": 3, + "latest_source_mass": 0.31498202681541443, + "layer": 9, + "max_source_mass": 0.40388280153274536, + "mixer_index": 16, + "normalized_entropy": 0.9860444622076776, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40388280153274536, + "median": 0.403384804725647, + "p05": 0.3489384353160858, + "p95": 0.4601682126522064 + }, + { + "label": "completed_group_1", + "mean": 0.2811351716518402, + "median": 0.28186845779418945, + "p05": 0.25417307019233704, + "p95": 0.30605652928352356 + }, + { + "label": "completed_group_2", + "mean": 0.31498202681541443, + "median": 0.3153840899467468, + "p05": 0.2613564431667328, + "p95": 0.3658811151981354 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2917464971542358, + "group": 3, + "latest_source_mass": 0.33769795298576355, + "layer": 9, + "max_source_mass": 0.33769795298576355, + "mixer_index": 17, + "normalized_entropy": 0.9317981327650527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3349480926990509, + "median": 0.3352771997451782, + "p05": 0.18474003672599792, + "p95": 0.48752525448799133 + }, + { + "label": "completed_group_1", + "mean": 0.1244347020983696, + "median": 0.11896269023418427, + "p05": 0.07986846566200256, + "p95": 0.1843445897102356 + }, + { + "label": "completed_group_2", + "mean": 0.20291924476623535, + "median": 0.20060715079307556, + "p05": 0.1253921389579773, + "p95": 0.2836313247680664 + }, + { + "label": "current_group_partial", + "mean": 0.33769795298576355, + "median": 0.33908897638320923, + "p05": 0.26979348063468933, + "p95": 0.4016563892364502 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3533791303634644, + "group": 3, + "latest_source_mass": 0.28473395109176636, + "layer": 10, + "max_source_mass": 0.32115638256073, + "mixer_index": 18, + "normalized_entropy": 0.976256679908994, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32115638256073, + "median": 0.3355404734611511, + "p05": 0.22348493337631226, + "p95": 0.3863494396209717 + }, + { + "label": "completed_group_1", + "mean": 0.20602071285247803, + "median": 0.2052205502986908, + "p05": 0.1802767962217331, + "p95": 0.23218519985675812 + }, + { + "label": "completed_group_2", + "mean": 0.18808895349502563, + "median": 0.18706050515174866, + "p05": 0.16633892059326172, + "p95": 0.2130376398563385 + }, + { + "label": "current_group_partial", + "mean": 0.28473395109176636, + "median": 0.275795578956604, + "p05": 0.22748424112796783, + "p95": 0.3660258948802948 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.308367371559143, + "group": 3, + "latest_source_mass": 0.29719752073287964, + "layer": 10, + "max_source_mass": 0.3113970160484314, + "mixer_index": 19, + "normalized_entropy": 0.9437875593046517, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3113970160484314, + "median": 0.33913683891296387, + "p05": 0.0915825292468071, + "p95": 0.47554588317871094 + }, + { + "label": "completed_group_1", + "mean": 0.16179001331329346, + "median": 0.16501447558403015, + "p05": 0.10682816803455353, + "p95": 0.20702867209911346 + }, + { + "label": "completed_group_2", + "mean": 0.2296154499053955, + "median": 0.2261994481086731, + "p05": 0.14510810375213623, + "p95": 0.32595163583755493 + }, + { + "label": "current_group_partial", + "mean": 0.29719752073287964, + "median": 0.2761377692222595, + "p05": 0.194716677069664, + "p95": 0.4515363574028015 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3437637090682983, + "group": 3, + "latest_source_mass": 0.3098427951335907, + "layer": 11, + "max_source_mass": 0.3217787742614746, + "mixer_index": 20, + "normalized_entropy": 0.969320619599697, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3217787742614746, + "median": 0.31754863262176514, + "p05": 0.25967735052108765, + "p95": 0.40468457341194153 + }, + { + "label": "completed_group_1", + "mean": 0.1920534074306488, + "median": 0.1915990114212036, + "p05": 0.16391108930110931, + "p95": 0.22057634592056274 + }, + { + "label": "completed_group_2", + "mean": 0.1763250231742859, + "median": 0.1779440939426422, + "p05": 0.14714840054512024, + "p95": 0.1996835470199585 + }, + { + "label": "current_group_partial", + "mean": 0.3098427951335907, + "median": 0.31089526414871216, + "p05": 0.2459525167942047, + "p95": 0.36900240182876587 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2280659675598145, + "group": 3, + "latest_source_mass": 0.29383569955825806, + "layer": 11, + "max_source_mass": 0.42534634470939636, + "mixer_index": 21, + "normalized_entropy": 0.8858623406415255, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42534634470939636, + "median": 0.4574331045150757, + "p05": 0.17354995012283325, + "p95": 0.6023476123809814 + }, + { + "label": "completed_group_1", + "mean": 0.14848899841308594, + "median": 0.14150214195251465, + "p05": 0.07509882748126984, + "p95": 0.23531809449195862 + }, + { + "label": "completed_group_2", + "mean": 0.13232898712158203, + "median": 0.13064970076084137, + "p05": 0.07000484317541122, + "p95": 0.19124652445316315 + }, + { + "label": "current_group_partial", + "mean": 0.29383569955825806, + "median": 0.27858495712280273, + "p05": 0.187642902135849, + "p95": 0.4577033817768097 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3462648391723633, + "group": 3, + "latest_source_mass": 0.3101534843444824, + "layer": 12, + "max_source_mass": 0.3221249580383301, + "mixer_index": 22, + "normalized_entropy": 0.9711248035985732, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3221249580383301, + "median": 0.3120178282260895, + "p05": 0.2763444483280182, + "p95": 0.3882659375667572 + }, + { + "label": "completed_group_1", + "mean": 0.18557845056056976, + "median": 0.1851869523525238, + "p05": 0.16533124446868896, + "p95": 0.20681798458099365 + }, + { + "label": "completed_group_2", + "mean": 0.18214309215545654, + "median": 0.18278372287750244, + "p05": 0.15564095973968506, + "p95": 0.2072875201702118 + }, + { + "label": "current_group_partial", + "mean": 0.3101534843444824, + "median": 0.31112557649612427, + "p05": 0.2608441412448883, + "p95": 0.36039745807647705 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.147221326828003, + "group": 3, + "latest_source_mass": 0.2585829496383667, + "layer": 12, + "max_source_mass": 0.5053804516792297, + "mixer_index": 23, + "normalized_entropy": 0.8275452595084083, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5053804516792297, + "median": 0.48097139596939087, + "p05": 0.30125147104263306, + "p95": 0.7477225065231323 + }, + { + "label": "completed_group_1", + "mean": 0.12089012563228607, + "median": 0.11902710795402527, + "p05": 0.05662901699542999, + "p95": 0.1844676434993744 + }, + { + "label": "completed_group_2", + "mean": 0.11514648050069809, + "median": 0.11994394659996033, + "p05": 0.05982710421085358, + "p95": 0.16130289435386658 + }, + { + "label": "current_group_partial", + "mean": 0.2585829496383667, + "median": 0.25007814168930054, + "p05": 0.12433226406574249, + "p95": 0.42273372411727905 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3729398250579834, + "group": 4, + "latest_source_mass": 0.22271886467933655, + "layer": 13, + "max_source_mass": 0.30785465240478516, + "mixer_index": 24, + "normalized_entropy": 0.9903667385250569, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30785465240478516, + "median": 0.3129041790962219, + "p05": 0.25217515230178833, + "p95": 0.3537779450416565 + }, + { + "label": "completed_group_1", + "mean": 0.23079943656921387, + "median": 0.22906282544136047, + "p05": 0.19741898775100708, + "p95": 0.26931890845298767 + }, + { + "label": "completed_group_2", + "mean": 0.23862707614898682, + "median": 0.23815912008285522, + "p05": 0.21215641498565674, + "p95": 0.2663886845111847 + }, + { + "label": "completed_group_3", + "mean": 0.22271886467933655, + "median": 0.22075486183166504, + "p05": 0.186451256275177, + "p95": 0.2638319730758667 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5232064723968506, + "group": 4, + "latest_source_mass": 0.24461600184440613, + "layer": 13, + "max_source_mass": 0.3136816620826721, + "mixer_index": 25, + "normalized_entropy": 0.9464213938474744, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3136816620826721, + "median": 0.326117604970932, + "p05": 0.20238417387008667, + "p95": 0.4362232983112335 + }, + { + "label": "completed_group_1", + "mean": 0.11285123229026794, + "median": 0.11373943835496902, + "p05": 0.08906605839729309, + "p95": 0.13395145535469055 + }, + { + "label": "completed_group_2", + "mean": 0.13196846842765808, + "median": 0.1334267258644104, + "p05": 0.10183953493833542, + "p95": 0.16085752844810486 + }, + { + "label": "completed_group_3", + "mean": 0.19688265025615692, + "median": 0.1940557211637497, + "p05": 0.13830327987670898, + "p95": 0.2671412229537964 + }, + { + "label": "current_group_partial", + "mean": 0.24461600184440613, + "median": 0.2419893592596054, + "p05": 0.18994617462158203, + "p95": 0.3077915906906128 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4876129627227783, + "group": 4, + "latest_source_mass": 0.396677166223526, + "layer": 14, + "max_source_mass": 0.396677166223526, + "mixer_index": 26, + "normalized_entropy": 0.9243059028433878, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2035508155822754, + "median": 0.20008960366249084, + "p05": 0.16096048057079315, + "p95": 0.255715548992157 + }, + { + "label": "completed_group_1", + "mean": 0.15372203290462494, + "median": 0.15584711730480194, + "p05": 0.12048184871673584, + "p95": 0.18010932207107544 + }, + { + "label": "completed_group_2", + "mean": 0.13167491555213928, + "median": 0.13113251328468323, + "p05": 0.11609883606433868, + "p95": 0.14900363981723785 + }, + { + "label": "completed_group_3", + "mean": 0.11437506973743439, + "median": 0.11201842874288559, + "p05": 0.09515875577926636, + "p95": 0.1399364471435547 + }, + { + "label": "current_group_partial", + "mean": 0.396677166223526, + "median": 0.39533013105392456, + "p05": 0.3409242630004883, + "p95": 0.4550648033618927 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4587602615356445, + "group": 4, + "latest_source_mass": 0.33461427688598633, + "layer": 14, + "max_source_mass": 0.33461427688598633, + "mixer_index": 27, + "normalized_entropy": 0.906378711639412, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28519001603126526, + "median": 0.28675830364227295, + "p05": 0.1500462144613266, + "p95": 0.45491817593574524 + }, + { + "label": "completed_group_1", + "mean": 0.10532153397798538, + "median": 0.09640030562877655, + "p05": 0.06345808506011963, + "p95": 0.1692449301481247 + }, + { + "label": "completed_group_2", + "mean": 0.09675674885511398, + "median": 0.09718051552772522, + "p05": 0.058796193450689316, + "p95": 0.13363884389400482 + }, + { + "label": "completed_group_3", + "mean": 0.17811740934848785, + "median": 0.1759425848722458, + "p05": 0.1284305453300476, + "p95": 0.23726706206798553 + }, + { + "label": "current_group_partial", + "mean": 0.33461427688598633, + "median": 0.33546411991119385, + "p05": 0.23162710666656494, + "p95": 0.4484652876853943 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4565696716308594, + "group": 4, + "latest_source_mass": 0.4221859574317932, + "layer": 15, + "max_source_mass": 0.4221859574317932, + "mixer_index": 28, + "normalized_entropy": 0.9050176216042753, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20232918858528137, + "median": 0.1998670995235443, + "p05": 0.15170496702194214, + "p95": 0.26285088062286377 + }, + { + "label": "completed_group_1", + "mean": 0.14928826689720154, + "median": 0.14749380946159363, + "p05": 0.12863489985466003, + "p95": 0.17874351143836975 + }, + { + "label": "completed_group_2", + "mean": 0.12029382586479187, + "median": 0.11950792372226715, + "p05": 0.10211217403411865, + "p95": 0.14088989794254303 + }, + { + "label": "completed_group_3", + "mean": 0.1059027910232544, + "median": 0.10411787033081055, + "p05": 0.08704904466867447, + "p95": 0.13138693571090698 + }, + { + "label": "current_group_partial", + "mean": 0.4221859574317932, + "median": 0.4195709228515625, + "p05": 0.35523390769958496, + "p95": 0.4969034492969513 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4788098335266113, + "group": 4, + "latest_source_mass": 0.34753501415252686, + "layer": 15, + "max_source_mass": 0.34753501415252686, + "mixer_index": 29, + "normalized_entropy": 0.9188362111403675, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2697942852973938, + "median": 0.25651901960372925, + "p05": 0.146310955286026, + "p95": 0.42084938287734985 + }, + { + "label": "completed_group_1", + "mean": 0.11948063224554062, + "median": 0.11549928039312363, + "p05": 0.0849025622010231, + "p95": 0.16021209955215454 + }, + { + "label": "completed_group_2", + "mean": 0.11090938746929169, + "median": 0.11184529960155487, + "p05": 0.08139486610889435, + "p95": 0.13886672258377075 + }, + { + "label": "completed_group_3", + "mean": 0.15228068828582764, + "median": 0.15164361894130707, + "p05": 0.11261995136737823, + "p95": 0.19522227346897125 + }, + { + "label": "current_group_partial", + "mean": 0.34753501415252686, + "median": 0.3453425168991089, + "p05": 0.25332537293434143, + "p95": 0.44237542152404785 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5018762350082397, + "group": 4, + "latest_source_mass": 0.31088152527809143, + "layer": 16, + "max_source_mass": 0.31088152527809143, + "mixer_index": 30, + "normalized_entropy": 0.9331681721954809, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.208971306681633, + "median": 0.20881521701812744, + "p05": 0.16508029401302338, + "p95": 0.2516583502292633 + }, + { + "label": "completed_group_1", + "mean": 0.27331358194351196, + "median": 0.26948875188827515, + "p05": 0.23055149614810944, + "p95": 0.33120542764663696 + }, + { + "label": "completed_group_2", + "mean": 0.12766750156879425, + "median": 0.1261082887649536, + "p05": 0.10924770683050156, + "p95": 0.1494429111480713 + }, + { + "label": "completed_group_3", + "mean": 0.07916608452796936, + "median": 0.07811431586742401, + "p05": 0.0561223030090332, + "p95": 0.10760308802127838 + }, + { + "label": "current_group_partial", + "mean": 0.31088152527809143, + "median": 0.3123162090778351, + "p05": 0.2707694172859192, + "p95": 0.3467172384262085 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5627843141555786, + "group": 4, + "latest_source_mass": 0.22652018070220947, + "layer": 16, + "max_source_mass": 0.27452588081359863, + "mixer_index": 31, + "normalized_entropy": 0.9710124895666443, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27452588081359863, + "median": 0.2678954601287842, + "p05": 0.19414913654327393, + "p95": 0.3916982412338257 + }, + { + "label": "completed_group_1", + "mean": 0.21448183059692383, + "median": 0.21617943048477173, + "p05": 0.17793960869312286, + "p95": 0.24279263615608215 + }, + { + "label": "completed_group_2", + "mean": 0.143874391913414, + "median": 0.1446641981601715, + "p05": 0.11340779811143875, + "p95": 0.17275942862033844 + }, + { + "label": "completed_group_3", + "mean": 0.14059773087501526, + "median": 0.13659776747226715, + "p05": 0.11039599776268005, + "p95": 0.18301524221897125 + }, + { + "label": "current_group_partial", + "mean": 0.22652018070220947, + "median": 0.22071748971939087, + "p05": 0.16668984293937683, + "p95": 0.293569415807724 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.580754280090332, + "group": 5, + "latest_source_mass": 0.24107450246810913, + "layer": 17, + "max_source_mass": 0.2518634498119354, + "mixer_index": 32, + "normalized_entropy": 0.9821778571747528, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2518634498119354, + "median": 0.24593469500541687, + "p05": 0.21242231130599976, + "p95": 0.3016672730445862 + }, + { + "label": "completed_group_1", + "mean": 0.18758200109004974, + "median": 0.1801876276731491, + "p05": 0.15723729133605957, + "p95": 0.24892441928386688 + }, + { + "label": "completed_group_2", + "mean": 0.13903990387916565, + "median": 0.13839681446552277, + "p05": 0.12513971328735352, + "p95": 0.15491701662540436 + }, + { + "label": "completed_group_3", + "mean": 0.18044015765190125, + "median": 0.1819951832294464, + "p05": 0.14810611307621002, + "p95": 0.20801132917404175 + }, + { + "label": "completed_group_4", + "mean": 0.24107450246810913, + "median": 0.24384509027004242, + "p05": 0.1871698796749115, + "p95": 0.2886185050010681 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7564777135849, + "group": 5, + "latest_source_mass": 0.2205006182193756, + "layer": 17, + "max_source_mass": 0.2205006182193756, + "mixer_index": 33, + "normalized_entropy": 0.9803088772521708, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15440437197685242, + "median": 0.15525281429290771, + "p05": 0.11778812110424042, + "p95": 0.18865147233009338 + }, + { + "label": "completed_group_1", + "mean": 0.19411899149417877, + "median": 0.2002902328968048, + "p05": 0.14482440054416656, + "p95": 0.2256385236978531 + }, + { + "label": "completed_group_2", + "mean": 0.17806550860404968, + "median": 0.1781877875328064, + "p05": 0.15101563930511475, + "p95": 0.20418882369995117 + }, + { + "label": "completed_group_3", + "mean": 0.14696575701236725, + "median": 0.14620032906532288, + "p05": 0.10836094617843628, + "p95": 0.1874053031206131 + }, + { + "label": "completed_group_4", + "mean": 0.10594475269317627, + "median": 0.10578415542840958, + "p05": 0.08951808512210846, + "p95": 0.1216079369187355 + }, + { + "label": "current_group_partial", + "mean": 0.2205006182193756, + "median": 0.21638520061969757, + "p05": 0.16007332503795624, + "p95": 0.2944811284542084 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5899518728256226, + "group": 5, + "latest_source_mass": 0.4157395362854004, + "layer": 18, + "max_source_mass": 0.4157395362854004, + "mixer_index": 34, + "normalized_entropy": 0.8873690359290373, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1636897772550583, + "median": 0.16218292713165283, + "p05": 0.1178273856639862, + "p95": 0.21766462922096252 + }, + { + "label": "completed_group_1", + "mean": 0.11789833009243011, + "median": 0.11790023744106293, + "p05": 0.1032494530081749, + "p95": 0.13267967104911804 + }, + { + "label": "completed_group_2", + "mean": 0.07030131667852402, + "median": 0.0698043555021286, + "p05": 0.06258689612150192, + "p95": 0.07938572019338608 + }, + { + "label": "completed_group_3", + "mean": 0.09238425642251968, + "median": 0.09013497084379196, + "p05": 0.0768275037407875, + "p95": 0.1164383590221405 + }, + { + "label": "completed_group_4", + "mean": 0.1399867981672287, + "median": 0.1400289535522461, + "p05": 0.12679682672023773, + "p95": 0.15317322313785553 + }, + { + "label": "current_group_partial", + "mean": 0.4157395362854004, + "median": 0.4171099066734314, + "p05": 0.367803692817688, + "p95": 0.4605288803577423 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.2894535064697266, + "group": 5, + "latest_source_mass": 0.5977407693862915, + "layer": 18, + "max_source_mass": 0.5977407693862915, + "mixer_index": 35, + "normalized_entropy": 0.7196577044045218, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1212274506688118, + "median": 0.10892187058925629, + "p05": 0.07249757647514343, + "p95": 0.2145335078239441 + }, + { + "label": "completed_group_1", + "mean": 0.06944876164197922, + "median": 0.0678764283657074, + "p05": 0.05350388586521149, + "p95": 0.08999023586511612 + }, + { + "label": "completed_group_2", + "mean": 0.05726535990834236, + "median": 0.0559837631881237, + "p05": 0.0409373864531517, + "p95": 0.07825973629951477 + }, + { + "label": "completed_group_3", + "mean": 0.09096579253673553, + "median": 0.08796361088752747, + "p05": 0.06407731771469116, + "p95": 0.12661314010620117 + }, + { + "label": "completed_group_4", + "mean": 0.06335186958312988, + "median": 0.06283852458000183, + "p05": 0.05032140761613846, + "p95": 0.0781375840306282 + }, + { + "label": "current_group_partial", + "mean": 0.5977407693862915, + "median": 0.6085352301597595, + "p05": 0.4878399074077606, + "p95": 0.6737192869186401 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6581614017486572, + "group": 5, + "latest_source_mass": 0.35773903131484985, + "layer": 19, + "max_source_mass": 0.35773903131484985, + "mixer_index": 36, + "normalized_entropy": 0.9254374988530375, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18132120370864868, + "median": 0.1806161105632782, + "p05": 0.1410265862941742, + "p95": 0.22331592440605164 + }, + { + "label": "completed_group_1", + "mean": 0.12711825966835022, + "median": 0.12722507119178772, + "p05": 0.11291128396987915, + "p95": 0.14178316295146942 + }, + { + "label": "completed_group_2", + "mean": 0.08560527861118317, + "median": 0.08513904362916946, + "p05": 0.07705061137676239, + "p95": 0.09564155340194702 + }, + { + "label": "completed_group_3", + "mean": 0.09748885035514832, + "median": 0.0953599065542221, + "p05": 0.08346174657344818, + "p95": 0.1194010004401207 + }, + { + "label": "completed_group_4", + "mean": 0.15072739124298096, + "median": 0.150638610124588, + "p05": 0.13762864470481873, + "p95": 0.1634974181652069 + }, + { + "label": "current_group_partial", + "mean": 0.35773903131484985, + "median": 0.35794761776924133, + "p05": 0.32225728034973145, + "p95": 0.39409786462783813 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3699336051940918, + "group": 5, + "latest_source_mass": 0.5385624766349792, + "layer": 19, + "max_source_mass": 0.5385624766349792, + "mixer_index": 37, + "normalized_entropy": 0.7645745027284836, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13604649901390076, + "median": 0.11998516321182251, + "p05": 0.06740576773881912, + "p95": 0.26897257566452026 + }, + { + "label": "completed_group_1", + "mean": 0.10151474922895432, + "median": 0.0991508960723877, + "p05": 0.07438431680202484, + "p95": 0.13600951433181763 + }, + { + "label": "completed_group_2", + "mean": 0.07715044170618057, + "median": 0.0767412930727005, + "p05": 0.054624155163764954, + "p95": 0.09943743050098419 + }, + { + "label": "completed_group_3", + "mean": 0.11300846189260483, + "median": 0.11003986746072769, + "p05": 0.08064281940460205, + "p95": 0.1516837179660797 + }, + { + "label": "completed_group_4", + "mean": 0.033717282116413116, + "median": 0.03286907076835632, + "p05": 0.024589763954281807, + "p95": 0.04592139273881912 + }, + { + "label": "current_group_partial", + "mean": 0.5385624766349792, + "median": 0.5415897369384766, + "p05": 0.4144754409790039, + "p95": 0.6506978273391724 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6926933526992798, + "group": 5, + "latest_source_mass": 0.3198680281639099, + "layer": 20, + "max_source_mass": 0.3198680281639099, + "mixer_index": 38, + "normalized_entropy": 0.9447101476341264, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19425281882286072, + "median": 0.19403350353240967, + "p05": 0.139950692653656, + "p95": 0.2494640350341797 + }, + { + "label": "completed_group_1", + "mean": 0.14001017808914185, + "median": 0.14058321714401245, + "p05": 0.12015820294618607, + "p95": 0.158815398812294 + }, + { + "label": "completed_group_2", + "mean": 0.09434760361909866, + "median": 0.09424005448818207, + "p05": 0.08333250135183334, + "p95": 0.10548526048660278 + }, + { + "label": "completed_group_3", + "mean": 0.10331346094608307, + "median": 0.10153411328792572, + "p05": 0.08690286427736282, + "p95": 0.12658309936523438 + }, + { + "label": "completed_group_4", + "mean": 0.14820793271064758, + "median": 0.14894580841064453, + "p05": 0.13278263807296753, + "p95": 0.16199614107608795 + }, + { + "label": "current_group_partial", + "mean": 0.3198680281639099, + "median": 0.3201481103897095, + "p05": 0.2800636887550354, + "p95": 0.3604968786239624 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4148362874984741, + "group": 5, + "latest_source_mass": 0.4996800422668457, + "layer": 20, + "max_source_mass": 0.4996800422668457, + "mixer_index": 39, + "normalized_entropy": 0.789635166883214, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16652248799800873, + "median": 0.14292314648628235, + "p05": 0.08186839520931244, + "p95": 0.350705623626709 + }, + { + "label": "completed_group_1", + "mean": 0.10329679399728775, + "median": 0.10057851672172546, + "p05": 0.07478320598602295, + "p95": 0.13858266174793243 + }, + { + "label": "completed_group_2", + "mean": 0.08063912391662598, + "median": 0.0807541012763977, + "p05": 0.056308042258024216, + "p95": 0.10469165444374084 + }, + { + "label": "completed_group_3", + "mean": 0.1165856122970581, + "median": 0.11373114585876465, + "p05": 0.08339665830135345, + "p95": 0.15619920194149017 + }, + { + "label": "completed_group_4", + "mean": 0.03327595070004463, + "median": 0.032142095267772675, + "p05": 0.022759035229682922, + "p95": 0.04896518588066101 + }, + { + "label": "current_group_partial", + "mean": 0.4996800422668457, + "median": 0.5001581907272339, + "p05": 0.3463003933429718, + "p95": 0.6266770362854004 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.740808129310608, + "group": 6, + "latest_source_mass": 0.24231041967868805, + "layer": 21, + "max_source_mass": 0.24231041967868805, + "mixer_index": 40, + "normalized_entropy": 0.9715635157550481, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21459883451461792, + "median": 0.21599377691745758, + "p05": 0.1570315659046173, + "p95": 0.27208006381988525 + }, + { + "label": "completed_group_1", + "mean": 0.15640269219875336, + "median": 0.1569681316614151, + "p05": 0.13785089552402496, + "p95": 0.1749057024717331 + }, + { + "label": "completed_group_2", + "mean": 0.10570268332958221, + "median": 0.10564640164375305, + "p05": 0.09223169088363647, + "p95": 0.11907710134983063 + }, + { + "label": "completed_group_3", + "mean": 0.11097013205289841, + "median": 0.10862545669078827, + "p05": 0.09454305469989777, + "p95": 0.13635265827178955 + }, + { + "label": "completed_group_4", + "mean": 0.17001524567604065, + "median": 0.17010903358459473, + "p05": 0.15088173747062683, + "p95": 0.18767674267292023 + }, + { + "label": "completed_group_5", + "mean": 0.24231041967868805, + "median": 0.24274763464927673, + "p05": 0.2087050974369049, + "p95": 0.27555277943611145 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3179612159729004, + "group": 6, + "latest_source_mass": 0.603651762008667, + "layer": 21, + "max_source_mass": 0.603651762008667, + "mixer_index": 41, + "normalized_entropy": 0.6772980841960945, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08068332821130753, + "median": 0.06917878985404968, + "p05": 0.039396293461322784, + "p95": 0.17595769464969635 + }, + { + "label": "completed_group_1", + "mean": 0.049232251942157745, + "median": 0.04861311614513397, + "p05": 0.03742668777704239, + "p95": 0.06363335251808167 + }, + { + "label": "completed_group_2", + "mean": 0.045073024928569794, + "median": 0.04482679069042206, + "p05": 0.03333880379796028, + "p95": 0.057495348155498505 + }, + { + "label": "completed_group_3", + "mean": 0.06111945956945419, + "median": 0.059528060257434845, + "p05": 0.042710162699222565, + "p95": 0.08362975716590881 + }, + { + "label": "completed_group_4", + "mean": 0.028317855671048164, + "median": 0.027924545109272003, + "p05": 0.022549759596586227, + "p95": 0.03542562201619148 + }, + { + "label": "completed_group_5", + "mean": 0.13192236423492432, + "median": 0.12939827144145966, + "p05": 0.10056710243225098, + "p95": 0.17195004224777222 + }, + { + "label": "current_group_partial", + "mean": 0.603651762008667, + "median": 0.6094026565551758, + "p05": 0.5065152645111084, + "p95": 0.6787958145141602 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.5319744348526, + "group": 6, + "latest_source_mass": 0.5132461190223694, + "layer": 22, + "max_source_mass": 0.5132461190223694, + "mixer_index": 42, + "normalized_entropy": 0.7872791226235868, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12406684458255768, + "median": 0.1289612352848053, + "p05": 0.06980825215578079, + "p95": 0.16972985863685608 + }, + { + "label": "completed_group_1", + "mean": 0.07462044060230255, + "median": 0.07392798364162445, + "p05": 0.06016036123037338, + "p95": 0.0917147696018219 + }, + { + "label": "completed_group_2", + "mean": 0.05378449708223343, + "median": 0.053309448063373566, + "p05": 0.04370884597301483, + "p95": 0.06546851992607117 + }, + { + "label": "completed_group_3", + "mean": 0.05575074627995491, + "median": 0.053552836179733276, + "p05": 0.0437602773308754, + "p95": 0.07444392144680023 + }, + { + "label": "completed_group_4", + "mean": 0.07357081025838852, + "median": 0.07329098880290985, + "p05": 0.06283499300479889, + "p95": 0.08542977273464203 + }, + { + "label": "completed_group_5", + "mean": 0.10496053099632263, + "median": 0.1038457602262497, + "p05": 0.08522139489650726, + "p95": 0.1291121542453766 + }, + { + "label": "current_group_partial", + "mean": 0.5132461190223694, + "median": 0.5090781450271606, + "p05": 0.4398295283317566, + "p95": 0.59932541847229 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.2461028099060059, + "group": 6, + "latest_source_mass": 0.6375080347061157, + "layer": 22, + "max_source_mass": 0.6375080347061157, + "mixer_index": 43, + "normalized_entropy": 0.640370168432985, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08556011319160461, + "median": 0.07947102189064026, + "p05": 0.043556369841098785, + "p95": 0.16172736883163452 + }, + { + "label": "completed_group_1", + "mean": 0.0446893647313118, + "median": 0.04398339241743088, + "p05": 0.03128381073474884, + "p95": 0.05990619957447052 + }, + { + "label": "completed_group_2", + "mean": 0.04084867238998413, + "median": 0.04042873531579971, + "p05": 0.02942952886223793, + "p95": 0.05392487719655037 + }, + { + "label": "completed_group_3", + "mean": 0.05386577919125557, + "median": 0.052273523062467575, + "p05": 0.033789411187171936, + "p95": 0.076903335750103 + }, + { + "label": "completed_group_4", + "mean": 0.025119207799434662, + "median": 0.024625632911920547, + "p05": 0.019138969480991364, + "p95": 0.03265690058469772 + }, + { + "label": "completed_group_5", + "mean": 0.11240880191326141, + "median": 0.11003861576318741, + "p05": 0.07852104306221008, + "p95": 0.155371755361557 + }, + { + "label": "current_group_partial", + "mean": 0.6375080347061157, + "median": 0.6392797231674194, + "p05": 0.5317062735557556, + "p95": 0.7386638522148132 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.6323832273483276, + "group": 6, + "latest_source_mass": 0.4532663822174072, + "layer": 23, + "max_source_mass": 0.4532663822174072, + "mixer_index": 44, + "normalized_entropy": 0.8388790346464895, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1418324112892151, + "median": 0.14852997660636902, + "p05": 0.07787702977657318, + "p95": 0.1923113763332367 + }, + { + "label": "completed_group_1", + "mean": 0.08991365134716034, + "median": 0.08918912708759308, + "p05": 0.0752246081829071, + "p95": 0.10654127597808838 + }, + { + "label": "completed_group_2", + "mean": 0.06188817322254181, + "median": 0.06161077320575714, + "p05": 0.04994015768170357, + "p95": 0.07481888681650162 + }, + { + "label": "completed_group_3", + "mean": 0.05739111453294754, + "median": 0.0548475980758667, + "p05": 0.04428064078092575, + "p95": 0.07816634327173233 + }, + { + "label": "completed_group_4", + "mean": 0.09848874807357788, + "median": 0.09768162667751312, + "p05": 0.08320866525173187, + "p95": 0.11606857925653458 + }, + { + "label": "completed_group_5", + "mean": 0.0972195416688919, + "median": 0.09613433480262756, + "p05": 0.07767169177532196, + "p95": 0.12004032731056213 + }, + { + "label": "current_group_partial", + "mean": 0.4532663822174072, + "median": 0.45108917355537415, + "p05": 0.37653055787086487, + "p95": 0.5389904975891113 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.301480770111084, + "group": 6, + "latest_source_mass": 0.6134276390075684, + "layer": 23, + "max_source_mass": 0.6134276390075684, + "mixer_index": 45, + "normalized_entropy": 0.6688288103861927, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09513165056705475, + "median": 0.08380547165870667, + "p05": 0.055099427700042725, + "p95": 0.1882638931274414 + }, + { + "label": "completed_group_1", + "mean": 0.050291214138269424, + "median": 0.04947858303785324, + "p05": 0.03757611662149429, + "p95": 0.06516233086585999 + }, + { + "label": "completed_group_2", + "mean": 0.045042067766189575, + "median": 0.0443364642560482, + "p05": 0.031873252242803574, + "p95": 0.06115761026740074 + }, + { + "label": "completed_group_3", + "mean": 0.05902485549449921, + "median": 0.05624900758266449, + "p05": 0.038259170949459076, + "p95": 0.08772744238376617 + }, + { + "label": "completed_group_4", + "mean": 0.02615261636674404, + "median": 0.02565642073750496, + "p05": 0.01966264471411705, + "p95": 0.03415588289499283 + }, + { + "label": "completed_group_5", + "mean": 0.1109299436211586, + "median": 0.10745325684547424, + "p05": 0.07642066478729248, + "p95": 0.15984633564949036 + }, + { + "label": "current_group_partial", + "mean": 0.6134276390075684, + "median": 0.6200529336929321, + "p05": 0.49638131260871887, + "p95": 0.7104787826538086 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.629331111907959, + "group": 6, + "latest_source_mass": 0.45539987087249756, + "layer": 24, + "max_source_mass": 0.45539987087249756, + "mixer_index": 46, + "normalized_entropy": 0.837310557580963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14790503680706024, + "median": 0.15420395135879517, + "p05": 0.08257082104682922, + "p95": 0.20396333932876587 + }, + { + "label": "completed_group_1", + "mean": 0.08658000826835632, + "median": 0.08578156679868698, + "p05": 0.07299856841564178, + "p95": 0.10278668999671936 + }, + { + "label": "completed_group_2", + "mean": 0.06608415395021439, + "median": 0.06551258265972137, + "p05": 0.054121918976306915, + "p95": 0.07912646979093552 + }, + { + "label": "completed_group_3", + "mean": 0.06427294760942459, + "median": 0.06133077293634415, + "p05": 0.04914242774248123, + "p95": 0.09066928923130035 + }, + { + "label": "completed_group_4", + "mean": 0.0819322019815445, + "median": 0.08114765584468842, + "p05": 0.06886082887649536, + "p95": 0.09821749478578568 + }, + { + "label": "completed_group_5", + "mean": 0.0978257954120636, + "median": 0.09643188118934631, + "p05": 0.07740232348442078, + "p95": 0.12209523469209671 + }, + { + "label": "current_group_partial", + "mean": 0.45539987087249756, + "median": 0.45274412631988525, + "p05": 0.3780696392059326, + "p95": 0.5462272763252258 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.338958501815796, + "group": 6, + "latest_source_mass": 0.5946891903877258, + "layer": 24, + "max_source_mass": 0.5946891903877258, + "mixer_index": 47, + "normalized_entropy": 0.6880885545850224, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10803166776895523, + "median": 0.09507323801517487, + "p05": 0.04510519653558731, + "p95": 0.2335781455039978 + }, + { + "label": "completed_group_1", + "mean": 0.0559723861515522, + "median": 0.05525296553969383, + "p05": 0.042058296501636505, + "p95": 0.07207933813333511 + }, + { + "label": "completed_group_2", + "mean": 0.047874726355075836, + "median": 0.04723185673356056, + "p05": 0.03294869884848595, + "p95": 0.06497423350811005 + }, + { + "label": "completed_group_3", + "mean": 0.060293328016996384, + "median": 0.05662515014410019, + "p05": 0.040346790105104446, + "p95": 0.08978720009326935 + }, + { + "label": "completed_group_4", + "mean": 0.029215168207883835, + "median": 0.028496116399765015, + "p05": 0.021991699934005737, + "p95": 0.039112430065870285 + }, + { + "label": "completed_group_5", + "mean": 0.10392352938652039, + "median": 0.10063231736421585, + "p05": 0.06900570541620255, + "p95": 0.1502695530653 + }, + { + "label": "current_group_partial", + "mean": 0.5946891903877258, + "median": 0.600748598575592, + "p05": 0.46449366211891174, + "p95": 0.7016042470932007 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7611865997314453, + "group": 7, + "latest_source_mass": 0.3679775297641754, + "layer": 25, + "max_source_mass": 0.3679775297641754, + "mixer_index": 48, + "normalized_entropy": 0.9050708742058073, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1573912650346756, + "median": 0.15898014605045319, + "p05": 0.09474922716617584, + "p95": 0.2298184633255005 + }, + { + "label": "completed_group_1", + "mean": 0.10317619889974594, + "median": 0.10270150750875473, + "p05": 0.08812867850065231, + "p95": 0.11994096636772156 + }, + { + "label": "completed_group_2", + "mean": 0.08533225208520889, + "median": 0.08495655655860901, + "p05": 0.07036984711885452, + "p95": 0.1013721376657486 + }, + { + "label": "completed_group_3", + "mean": 0.08218386024236679, + "median": 0.07942138612270355, + "p05": 0.06530377268791199, + "p95": 0.1088358536362648 + }, + { + "label": "completed_group_4", + "mean": 0.09052872657775879, + "median": 0.08931052684783936, + "p05": 0.07639768719673157, + "p95": 0.10804671049118042 + }, + { + "label": "completed_group_5", + "mean": 0.11341015249490738, + "median": 0.1123356968164444, + "p05": 0.09177497029304504, + "p95": 0.13803066313266754 + }, + { + "label": "completed_group_6", + "mean": 0.3679775297641754, + "median": 0.3671194612979889, + "p05": 0.2961871325969696, + "p95": 0.4462817311286926 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.476489782333374, + "group": 7, + "latest_source_mass": 0.3396714925765991, + "layer": 25, + "max_source_mass": 0.3836256265640259, + "mixer_index": 49, + "normalized_entropy": 0.7100414956318613, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09070298075675964, + "median": 0.0818760022521019, + "p05": 0.04576021432876587, + "p95": 0.1613437533378601 + }, + { + "label": "completed_group_1", + "mean": 0.03713828697800636, + "median": 0.035560622811317444, + "p05": 0.02337636426091194, + "p95": 0.054472144693136215 + }, + { + "label": "completed_group_2", + "mean": 0.030013620853424072, + "median": 0.028701743111014366, + "p05": 0.01836019568145275, + "p95": 0.04582716524600983 + }, + { + "label": "completed_group_3", + "mean": 0.03533655032515526, + "median": 0.031870774924755096, + "p05": 0.018502909690141678, + "p95": 0.061826568096876144 + }, + { + "label": "completed_group_4", + "mean": 0.020730752497911453, + "median": 0.020491186529397964, + "p05": 0.014567261561751366, + "p95": 0.027856238186359406 + }, + { + "label": "completed_group_5", + "mean": 0.06278066337108612, + "median": 0.059403590857982635, + "p05": 0.03608490154147148, + "p95": 0.10122650861740112 + }, + { + "label": "completed_group_6", + "mean": 0.3836256265640259, + "median": 0.3773127794265747, + "p05": 0.1874689906835556, + "p95": 0.5895293951034546 + }, + { + "label": "current_group_partial", + "mean": 0.3396714925765991, + "median": 0.34821513295173645, + "p05": 0.09212851524353027, + "p95": 0.587702751159668 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7213938236236572, + "group": 7, + "latest_source_mass": 0.37924307584762573, + "layer": 26, + "max_source_mass": 0.37924307584762573, + "mixer_index": 50, + "normalized_entropy": 0.8278154442529139, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10584281384944916, + "median": 0.10982948541641235, + "p05": 0.03868259862065315, + "p95": 0.16580678522586823 + }, + { + "label": "completed_group_1", + "mean": 0.06887699663639069, + "median": 0.06925956159830093, + "p05": 0.04082746431231499, + "p95": 0.09546782076358795 + }, + { + "label": "completed_group_2", + "mean": 0.05207539349794388, + "median": 0.051871635019779205, + "p05": 0.035383015871047974, + "p95": 0.07048650085926056 + }, + { + "label": "completed_group_3", + "mean": 0.050678279250860214, + "median": 0.04831986129283905, + "p05": 0.03378130868077278, + "p95": 0.07452918589115143 + }, + { + "label": "completed_group_4", + "mean": 0.04768219590187073, + "median": 0.04721469432115555, + "p05": 0.03680850565433502, + "p95": 0.060343313962221146 + }, + { + "label": "completed_group_5", + "mean": 0.06809806823730469, + "median": 0.06714263558387756, + "p05": 0.04759174585342407, + "p95": 0.09223967790603638 + }, + { + "label": "completed_group_6", + "mean": 0.2275032103061676, + "median": 0.22425106167793274, + "p05": 0.14320722222328186, + "p95": 0.3203037679195404 + }, + { + "label": "current_group_partial", + "mean": 0.37924307584762573, + "median": 0.3716055750846863, + "p05": 0.20950253307819366, + "p95": 0.5737906098365784 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5405172109603882, + "group": 7, + "latest_source_mass": 0.4017985463142395, + "layer": 26, + "max_source_mass": 0.4017985463142395, + "mixer_index": 51, + "normalized_entropy": 0.7408321802188831, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.074287548661232, + "median": 0.06164507195353508, + "p05": 0.028203874826431274, + "p95": 0.19368377327919006 + }, + { + "label": "completed_group_1", + "mean": 0.04491765797138214, + "median": 0.04365704953670502, + "p05": 0.02776508405804634, + "p95": 0.06526201963424683 + }, + { + "label": "completed_group_2", + "mean": 0.035157643258571625, + "median": 0.03418130800127983, + "p05": 0.022541508078575134, + "p95": 0.050758615136146545 + }, + { + "label": "completed_group_3", + "mean": 0.046252522617578506, + "median": 0.04353209584951401, + "p05": 0.026215802878141403, + "p95": 0.07329559326171875 + }, + { + "label": "completed_group_4", + "mean": 0.023427214473485947, + "median": 0.023055266588926315, + "p05": 0.016644639894366264, + "p95": 0.03150089830160141 + }, + { + "label": "completed_group_5", + "mean": 0.07749318331480026, + "median": 0.07363636791706085, + "p05": 0.04229677468538284, + "p95": 0.12579958140850067 + }, + { + "label": "completed_group_6", + "mean": 0.29666566848754883, + "median": 0.2920765280723572, + "p05": 0.14379237592220306, + "p95": 0.45965293049812317 + }, + { + "label": "current_group_partial", + "mean": 0.4017985463142395, + "median": 0.4004032611846924, + "p05": 0.18456703424453735, + "p95": 0.6275562047958374 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8063910007476807, + "group": 7, + "latest_source_mass": 0.3437207341194153, + "layer": 27, + "max_source_mass": 0.3437207341194153, + "mixer_index": 52, + "normalized_entropy": 0.868690446228377, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11433833092451096, + "median": 0.11714513599872589, + "p05": 0.06115472689270973, + "p95": 0.16258974373340607 + }, + { + "label": "completed_group_1", + "mean": 0.07374332845211029, + "median": 0.0731307864189148, + "p05": 0.05447638779878616, + "p95": 0.09509895741939545 + }, + { + "label": "completed_group_2", + "mean": 0.05932536721229553, + "median": 0.05929289758205414, + "p05": 0.04452420026063919, + "p95": 0.07498160749673843 + }, + { + "label": "completed_group_3", + "mean": 0.05389238893985748, + "median": 0.051720209419727325, + "p05": 0.03926049917936325, + "p95": 0.07536239922046661 + }, + { + "label": "completed_group_4", + "mean": 0.06126017868518829, + "median": 0.06108957529067993, + "p05": 0.0506296306848526, + "p95": 0.07238088548183441 + }, + { + "label": "completed_group_5", + "mean": 0.07349519431591034, + "median": 0.07249070703983307, + "p05": 0.05706139653921127, + "p95": 0.09332700818777084 + }, + { + "label": "completed_group_6", + "mean": 0.22022445499897003, + "median": 0.21959853172302246, + "p05": 0.15368635952472687, + "p95": 0.2904376685619354 + }, + { + "label": "current_group_partial", + "mean": 0.3437207341194153, + "median": 0.33986932039260864, + "p05": 0.22015926241874695, + "p95": 0.4802429676055908 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.4859801530838013, + "group": 7, + "latest_source_mass": 0.4424910843372345, + "layer": 27, + "max_source_mass": 0.4424910843372345, + "mixer_index": 53, + "normalized_entropy": 0.7146053992378076, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08387120068073273, + "median": 0.07644708454608917, + "p05": 0.0386628657579422, + "p95": 0.15876422822475433 + }, + { + "label": "completed_group_1", + "mean": 0.041406162083148956, + "median": 0.0394824743270874, + "p05": 0.025755178183317184, + "p95": 0.06136181950569153 + }, + { + "label": "completed_group_2", + "mean": 0.032444946467876434, + "median": 0.0307333804666996, + "p05": 0.018246563151478767, + "p95": 0.052875272929668427 + }, + { + "label": "completed_group_3", + "mean": 0.03686721250414848, + "median": 0.03247762471437454, + "p05": 0.017809681594371796, + "p95": 0.06780916452407837 + }, + { + "label": "completed_group_4", + "mean": 0.025052767246961594, + "median": 0.024561133235692978, + "p05": 0.017057543620467186, + "p95": 0.034822143614292145 + }, + { + "label": "completed_group_5", + "mean": 0.06012478470802307, + "median": 0.056938059628009796, + "p05": 0.03254516050219536, + "p95": 0.09905251860618591 + }, + { + "label": "completed_group_6", + "mean": 0.27774184942245483, + "median": 0.2747848629951477, + "p05": 0.11654777824878693, + "p95": 0.4562506675720215 + }, + { + "label": "current_group_partial", + "mean": 0.4424910843372345, + "median": 0.4387660026550293, + "p05": 0.21194897592067719, + "p95": 0.6782112717628479 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7777256965637207, + "group": 7, + "latest_source_mass": 0.3409850299358368, + "layer": 28, + "max_source_mass": 0.3409850299358368, + "mixer_index": 54, + "normalized_entropy": 0.8549053488311195, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10586117208003998, + "median": 0.10652337223291397, + "p05": 0.05095779523253441, + "p95": 0.1629783660173416 + }, + { + "label": "completed_group_1", + "mean": 0.07087038457393646, + "median": 0.06906798481941223, + "p05": 0.04985840618610382, + "p95": 0.09773125499486923 + }, + { + "label": "completed_group_2", + "mean": 0.05850498005747795, + "median": 0.05767729878425598, + "p05": 0.04179685190320015, + "p95": 0.07779045403003693 + }, + { + "label": "completed_group_3", + "mean": 0.06360965222120285, + "median": 0.06068425625562668, + "p05": 0.04235687851905823, + "p95": 0.09371945261955261 + }, + { + "label": "completed_group_4", + "mean": 0.04672512412071228, + "median": 0.04617916792631149, + "p05": 0.0356140062212944, + "p95": 0.05914818495512009 + }, + { + "label": "completed_group_5", + "mean": 0.07485505938529968, + "median": 0.07274073362350464, + "p05": 0.052147433161735535, + "p95": 0.10333258658647537 + }, + { + "label": "completed_group_6", + "mean": 0.2385886013507843, + "median": 0.2345724105834961, + "p05": 0.15389031171798706, + "p95": 0.33735349774360657 + }, + { + "label": "current_group_partial", + "mean": 0.3409850299358368, + "median": 0.33667099475860596, + "p05": 0.18655306100845337, + "p95": 0.5089476704597473 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6318418979644775, + "group": 7, + "latest_source_mass": 0.3292362689971924, + "layer": 28, + "max_source_mass": 0.3292362689971924, + "mixer_index": 55, + "normalized_entropy": 0.7847500712360619, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09244748950004578, + "median": 0.06782519817352295, + "p05": 0.025312144309282303, + "p95": 0.24087309837341309 + }, + { + "label": "completed_group_1", + "mean": 0.05762210115790367, + "median": 0.056058838963508606, + "p05": 0.03864594176411629, + "p95": 0.08028930425643921 + }, + { + "label": "completed_group_2", + "mean": 0.045099519193172455, + "median": 0.04426075518131256, + "p05": 0.03003283590078354, + "p95": 0.06244165822863579 + }, + { + "label": "completed_group_3", + "mean": 0.056236449629068375, + "median": 0.052902162075042725, + "p05": 0.03282861411571503, + "p95": 0.08620619773864746 + }, + { + "label": "completed_group_4", + "mean": 0.029373466968536377, + "median": 0.02887408435344696, + "p05": 0.020401213318109512, + "p95": 0.04018516466021538 + }, + { + "label": "completed_group_5", + "mean": 0.07237439602613449, + "median": 0.06993190199136734, + "p05": 0.04555661231279373, + "p95": 0.10748817771673203 + }, + { + "label": "completed_group_6", + "mean": 0.31761035323143005, + "median": 0.3173876404762268, + "p05": 0.16091161966323853, + "p95": 0.4724525213241577 + }, + { + "label": "current_group_partial", + "mean": 0.3292362689971924, + "median": 0.3128150701522827, + "p05": 0.12605780363082886, + "p95": 0.5847846865653992 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8334553241729736, + "group": 8, + "latest_source_mass": 0.270294725894928, + "layer": 29, + "max_source_mass": 0.27155569195747375, + "mixer_index": 56, + "normalized_entropy": 0.8817056346252721, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1157165914773941, + "median": 0.113271564245224, + "p05": 0.06316040456295013, + "p95": 0.17676270008087158 + }, + { + "label": "completed_group_1", + "mean": 0.07161639630794525, + "median": 0.07118864357471466, + "p05": 0.04294680058956146, + "p95": 0.102086141705513 + }, + { + "label": "completed_group_2", + "mean": 0.06577154994010925, + "median": 0.06490545719861984, + "p05": 0.04486069828271866, + "p95": 0.09022088348865509 + }, + { + "label": "completed_group_3", + "mean": 0.07425849884748459, + "median": 0.07179903984069824, + "p05": 0.045271825045347214, + "p95": 0.11117851734161377 + }, + { + "label": "completed_group_4", + "mean": 0.04881400242447853, + "median": 0.04833249747753143, + "p05": 0.0363147035241127, + "p95": 0.0629924014210701 + }, + { + "label": "completed_group_5", + "mean": 0.08197253197431564, + "median": 0.08029600977897644, + "p05": 0.05680408701300621, + "p95": 0.11223416775465012 + }, + { + "label": "completed_group_6", + "mean": 0.27155569195747375, + "median": 0.2687028646469116, + "p05": 0.18294592201709747, + "p95": 0.3740781545639038 + }, + { + "label": "completed_group_7", + "mean": 0.270294725894928, + "median": 0.2569413185119629, + "p05": 0.13936233520507812, + "p95": 0.43686625361442566 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6227548122406006, + "group": 8, + "latest_source_mass": 0.45173269510269165, + "layer": 29, + "max_source_mass": 0.45173269510269165, + "mixer_index": 57, + "normalized_entropy": 0.7385475426494315, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0605507455766201, + "median": 0.05293343961238861, + "p05": 0.02206447720527649, + "p95": 0.1254502236843109 + }, + { + "label": "completed_group_1", + "mean": 0.040063515305519104, + "median": 0.03803131729364395, + "p05": 0.022060411050915718, + "p95": 0.06445677578449249 + }, + { + "label": "completed_group_2", + "mean": 0.032619886100292206, + "median": 0.030926726758480072, + "p05": 0.020269516855478287, + "p95": 0.050711989402770996 + }, + { + "label": "completed_group_3", + "mean": 0.03868011012673378, + "median": 0.0344192236661911, + "p05": 0.020861119031906128, + "p95": 0.06798171997070312 + }, + { + "label": "completed_group_4", + "mean": 0.020290488377213478, + "median": 0.01974661275744438, + "p05": 0.014590941369533539, + "p95": 0.027847949415445328 + }, + { + "label": "completed_group_5", + "mean": 0.04760073497891426, + "median": 0.044880062341690063, + "p05": 0.027415042743086815, + "p95": 0.07675685733556747 + }, + { + "label": "completed_group_6", + "mean": 0.19551873207092285, + "median": 0.18908299505710602, + "p05": 0.10506269335746765, + "p95": 0.30520933866500854 + }, + { + "label": "completed_group_7", + "mean": 0.11294309794902802, + "median": 0.09412655979394913, + "p05": 0.03902517631649971, + "p95": 0.24146248400211334 + }, + { + "label": "current_group_partial", + "mean": 0.45173269510269165, + "median": 0.4618796110153198, + "p05": 0.24265648424625397, + "p95": 0.6204620599746704 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9787883758544922, + "group": 8, + "latest_source_mass": 0.1984800100326538, + "layer": 30, + "max_source_mass": 0.24502354860305786, + "mixer_index": 58, + "normalized_entropy": 0.9005854004479842, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0788881778717041, + "median": 0.055554650723934174, + "p05": 0.03367501497268677, + "p95": 0.21530622243881226 + }, + { + "label": "completed_group_1", + "mean": 0.08108963817358017, + "median": 0.07848536968231201, + "p05": 0.05139587074518204, + "p95": 0.12043403089046478 + }, + { + "label": "completed_group_2", + "mean": 0.057054273784160614, + "median": 0.056182146072387695, + "p05": 0.04127398133277893, + "p95": 0.07570968568325043 + }, + { + "label": "completed_group_3", + "mean": 0.08211587369441986, + "median": 0.0802752673625946, + "p05": 0.055154237896203995, + "p95": 0.113825723528862 + }, + { + "label": "completed_group_4", + "mean": 0.026040805503726006, + "median": 0.025504354387521744, + "p05": 0.01917124353349209, + "p95": 0.03502596169710159 + }, + { + "label": "completed_group_5", + "mean": 0.10537257045507431, + "median": 0.10299044847488403, + "p05": 0.07248030602931976, + "p95": 0.14682361483573914 + }, + { + "label": "completed_group_6", + "mean": 0.24502354860305786, + "median": 0.24406442046165466, + "p05": 0.1495005488395691, + "p95": 0.3413054645061493 + }, + { + "label": "completed_group_7", + "mean": 0.12593510746955872, + "median": 0.11572076380252838, + "p05": 0.051278628408908844, + "p95": 0.2299422323703766 + }, + { + "label": "current_group_partial", + "mean": 0.1984800100326538, + "median": 0.19106052815914154, + "p05": 0.11614266037940979, + "p95": 0.3049595355987549 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7590525150299072, + "group": 8, + "latest_source_mass": 0.3258843719959259, + "layer": 30, + "max_source_mass": 0.3258843719959259, + "mixer_index": 59, + "normalized_entropy": 0.800579300438408, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07131639868021011, + "median": 0.052984386682510376, + "p05": 0.03260878100991249, + "p95": 0.19837424159049988 + }, + { + "label": "completed_group_1", + "mean": 0.044687941670417786, + "median": 0.04335416108369827, + "p05": 0.027018578723073006, + "p95": 0.06556333601474762 + }, + { + "label": "completed_group_2", + "mean": 0.033290669322013855, + "median": 0.03222261369228363, + "p05": 0.01970716379582882, + "p95": 0.05042369291186333 + }, + { + "label": "completed_group_3", + "mean": 0.043597716838121414, + "median": 0.03997292369604111, + "p05": 0.024005703628063202, + "p95": 0.07244924455881119 + }, + { + "label": "completed_group_4", + "mean": 0.02658817172050476, + "median": 0.02605675905942917, + "p05": 0.017710059881210327, + "p95": 0.03680335357785225 + }, + { + "label": "completed_group_5", + "mean": 0.06922741234302521, + "median": 0.06634850800037384, + "p05": 0.03838749974966049, + "p95": 0.10990548133850098 + }, + { + "label": "completed_group_6", + "mean": 0.23554196953773499, + "median": 0.2292930632829666, + "p05": 0.10460641980171204, + "p95": 0.3925905227661133 + }, + { + "label": "completed_group_7", + "mean": 0.14986534416675568, + "median": 0.12543796002864838, + "p05": 0.04750998318195343, + "p95": 0.3270293176174164 + }, + { + "label": "current_group_partial", + "mean": 0.3258843719959259, + "median": 0.31322309374809265, + "p05": 0.1389845609664917, + "p95": 0.5583901405334473 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9736928939819336, + "group": 8, + "latest_source_mass": 0.20976582169532776, + "layer": 31, + "max_source_mass": 0.22560220956802368, + "mixer_index": 60, + "normalized_entropy": 0.8982663467084998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09849467873573303, + "median": 0.09779177606105804, + "p05": 0.017307927832007408, + "p95": 0.19556622207164764 + }, + { + "label": "completed_group_1", + "mean": 0.068942591547966, + "median": 0.0661388635635376, + "p05": 0.044743385165929794, + "p95": 0.10368303209543228 + }, + { + "label": "completed_group_2", + "mean": 0.0599900484085083, + "median": 0.05786054581403732, + "p05": 0.040265996009111404, + "p95": 0.08643331378698349 + }, + { + "label": "completed_group_3", + "mean": 0.06335455179214478, + "median": 0.05964011698961258, + "p05": 0.04099462926387787, + "p95": 0.09641734510660172 + }, + { + "label": "completed_group_4", + "mean": 0.04671281576156616, + "median": 0.04506188631057739, + "p05": 0.03403949737548828, + "p95": 0.06450185924768448 + }, + { + "label": "completed_group_5", + "mean": 0.0713595524430275, + "median": 0.06962447613477707, + "p05": 0.048271458595991135, + "p95": 0.10105480253696442 + }, + { + "label": "completed_group_6", + "mean": 0.22560220956802368, + "median": 0.22400957345962524, + "p05": 0.13957585394382477, + "p95": 0.31992682814598083 + }, + { + "label": "completed_group_7", + "mean": 0.1557777225971222, + "median": 0.1399117112159729, + "p05": 0.06281191110610962, + "p95": 0.2967374622821808 + }, + { + "label": "current_group_partial", + "mean": 0.20976582169532776, + "median": 0.2105630338191986, + "p05": 0.08139262348413467, + "p95": 0.34620192646980286 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.809161901473999, + "group": 8, + "latest_source_mass": 0.288146048784256, + "layer": 31, + "max_source_mass": 0.288146048784256, + "mixer_index": 61, + "normalized_entropy": 0.8233850650202157, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08854076266288757, + "median": 0.08406365662813187, + "p05": 0.020090684294700623, + "p95": 0.1807844340801239 + }, + { + "label": "completed_group_1", + "mean": 0.05356752499938011, + "median": 0.05036265030503273, + "p05": 0.031541675329208374, + "p95": 0.08521421998739243 + }, + { + "label": "completed_group_2", + "mean": 0.04243364930152893, + "median": 0.03939494490623474, + "p05": 0.023794323205947876, + "p95": 0.0721740797162056 + }, + { + "label": "completed_group_3", + "mean": 0.04702243581414223, + "median": 0.043501369655132294, + "p05": 0.02430840954184532, + "p95": 0.0788305327296257 + }, + { + "label": "completed_group_4", + "mean": 0.02648795023560524, + "median": 0.025087328627705574, + "p05": 0.01698113977909088, + "p95": 0.040197182446718216 + }, + { + "label": "completed_group_5", + "mean": 0.05820769816637039, + "median": 0.05441046878695488, + "p05": 0.02834305912256241, + "p95": 0.10125020891427994 + }, + { + "label": "completed_group_6", + "mean": 0.18489578366279602, + "median": 0.17386877536773682, + "p05": 0.07876831293106079, + "p95": 0.32456493377685547 + }, + { + "label": "completed_group_7", + "mean": 0.21069815754890442, + "median": 0.19004324078559875, + "p05": 0.07121473550796509, + "p95": 0.42417171597480774 + }, + { + "label": "current_group_partial", + "mean": 0.288146048784256, + "median": 0.28367581963539124, + "p05": 0.0962119847536087, + "p95": 0.5029257535934448 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.8544633388519287, + "group": 8, + "latest_source_mass": 0.12106549739837646, + "layer": 32, + "max_source_mass": 0.3525030016899109, + "mixer_index": 62, + "normalized_entropy": 0.8440026376822011, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07977832853794098, + "median": 0.053830958902835846, + "p05": 0.02079753391444683, + "p95": 0.2620829939842224 + }, + { + "label": "completed_group_1", + "mean": 0.05858254432678223, + "median": 0.056158941239118576, + "p05": 0.03545547276735306, + "p95": 0.09179478138685226 + }, + { + "label": "completed_group_2", + "mean": 0.053744811564683914, + "median": 0.05185101926326752, + "p05": 0.03345105051994324, + "p95": 0.08065605163574219 + }, + { + "label": "completed_group_3", + "mean": 0.07626347988843918, + "median": 0.07124648243188858, + "p05": 0.043040163815021515, + "p95": 0.12173676490783691 + }, + { + "label": "completed_group_4", + "mean": 0.031090255826711655, + "median": 0.02958102896809578, + "p05": 0.01873353309929371, + "p95": 0.04807420074939728 + }, + { + "label": "completed_group_5", + "mean": 0.0994817242026329, + "median": 0.0973888486623764, + "p05": 0.0575672909617424, + "p95": 0.14856761693954468 + }, + { + "label": "completed_group_6", + "mean": 0.3525030016899109, + "median": 0.3572760224342346, + "p05": 0.18325307965278625, + "p95": 0.5024369955062866 + }, + { + "label": "completed_group_7", + "mean": 0.1274903416633606, + "median": 0.1073581799864769, + "p05": 0.039848171174526215, + "p95": 0.2741084098815918 + }, + { + "label": "current_group_partial", + "mean": 0.12106549739837646, + "median": 0.10882791876792908, + "p05": 0.0503288172185421, + "p95": 0.22616314888000488 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9456019401550293, + "group": 8, + "latest_source_mass": 0.18013209104537964, + "layer": 32, + "max_source_mass": 0.2524128556251526, + "mixer_index": 63, + "normalized_entropy": 0.885481602665194, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08823961019515991, + "median": 0.07105386257171631, + "p05": 0.04482517018914223, + "p95": 0.1774362027645111 + }, + { + "label": "completed_group_1", + "mean": 0.06737187504768372, + "median": 0.06628458946943283, + "p05": 0.04582075774669647, + "p95": 0.09290395677089691 + }, + { + "label": "completed_group_2", + "mean": 0.05197303742170334, + "median": 0.051264435052871704, + "p05": 0.035227708518505096, + "p95": 0.07133325934410095 + }, + { + "label": "completed_group_3", + "mean": 0.0631948709487915, + "median": 0.06016676500439644, + "p05": 0.03969947248697281, + "p95": 0.09776238352060318 + }, + { + "label": "completed_group_4", + "mean": 0.02739548496901989, + "median": 0.026907969266176224, + "p05": 0.020028207451105118, + "p95": 0.036658842116594315 + }, + { + "label": "completed_group_5", + "mean": 0.08291903138160706, + "median": 0.08066994696855545, + "p05": 0.05307728424668312, + "p95": 0.12080682814121246 + }, + { + "label": "completed_group_6", + "mean": 0.2524128556251526, + "median": 0.25180506706237793, + "p05": 0.14340050518512726, + "p95": 0.36010295152664185 + }, + { + "label": "completed_group_7", + "mean": 0.186361163854599, + "median": 0.17697101831436157, + "p05": 0.08843781799077988, + "p95": 0.3156980276107788 + }, + { + "label": "current_group_partial", + "mean": 0.18013209104537964, + "median": 0.16958200931549072, + "p05": 0.08391770720481873, + "p95": 0.3105517029762268 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.596132755279541, + "group": null, + "latest_source_mass": 0.29075440764427185, + "layer": null, + "max_source_mass": 0.3670029938220978, + "mixer_index": null, + "normalized_entropy": 0.7264313223797062, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09668220579624176, + "median": 0.1013428270816803, + "p05": 0.02658359333872795, + "p95": 0.15407970547676086 + }, + { + "label": "completed_group_1", + "mean": 0.037084318697452545, + "median": 0.035886794328689575, + "p05": 0.02348068356513977, + "p95": 0.053765345364809036 + }, + { + "label": "completed_group_2", + "mean": 0.03457995876669884, + "median": 0.03355112299323082, + "p05": 0.019709739834070206, + "p95": 0.0531856045126915 + }, + { + "label": "completed_group_3", + "mean": 0.026012206450104713, + "median": 0.023950476199388504, + "p05": 0.013590577989816666, + "p95": 0.04525115340948105 + }, + { + "label": "completed_group_4", + "mean": 0.0258483849465847, + "median": 0.02512776106595993, + "p05": 0.016705092042684555, + "p95": 0.03687155246734619 + }, + { + "label": "completed_group_5", + "mean": 0.037477150559425354, + "median": 0.03506678715348244, + "p05": 0.02071693353354931, + "p95": 0.061963990330696106 + }, + { + "label": "completed_group_6", + "mean": 0.0845583900809288, + "median": 0.07240844517946243, + "p05": 0.03393660485744476, + "p95": 0.1794772744178772 + }, + { + "label": "completed_group_7", + "mean": 0.3670029938220978, + "median": 0.3608853220939636, + "p05": 0.1103469580411911, + "p95": 0.6465575098991394 + }, + { + "label": "completed_group_8", + "mean": 0.29075440764427185, + "median": 0.28802984952926636, + "p05": 0.039541684091091156, + "p95": 0.5658512711524963 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "optimizer_state_before": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "33fdf3c665794af7b295af7e48427b302bf8844450f6b4ebc3f0b6525419497e", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.09253507107496262, + 0.19514523446559906, + 0.07917427271604538, + 0.04215213656425476, + 0.12552575767040253, + 0.05027730390429497, + 0.054851483553647995, + 0.036243390291929245, + 0.031880609691143036, + 0.035054463893175125, + 0.050445348024368286, + 0.05600574240088463, + 0.053015924990177155, + 0.032114963978528976, + 0.048427484929561615, + 0.5742782950401306, + 0.04075472429394722, + 0.03505101054906845, + 0.06577442586421967, + 0.08553697913885117, + 0.0493597537279129, + 0.023477228358387947, + 0.03512806445360184, + 0.030729109421372414, + 0.023035654798150063, + 0.019568851217627525, + 0.02376176416873932, + 0.02176014520227909, + 0.03424545377492905, + 0.021422436460852623, + 0.019477492198348045, + 0.017491821199655533 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.125071417453e-06, + "first_to_last_ratio": 2.282823561454729, + "imbalance_abs_log_ratio": 0.8254130809232482, + "last_quartile_mean": 3.5592200617884373e-06, + "mean": 5.20536254811077e-06, + "normalized": [ + 1.9367384702517454, + 2.0575431033491354, + 1.6801075995543924, + 1.4400668185251226, + 1.5661035211648109, + 1.3925792134893262, + 1.2466333024952874, + 1.1674602703335213, + 1.0945737588640874, + 1.02847032983817, + 0.9743308590636915, + 0.9218384112827168, + 0.879353639681705, + 0.854229397242741, + 0.8405216196536197, + 0.8631772058707392, + 0.8729986277133472, + 0.864352128283068, + 0.8430011960406467, + 0.8174196170838346, + 0.8115146906436438, + 0.7998541382616392, + 0.7909603189303038, + 0.7860896631748897, + 0.7834851600981735, + 0.7653029103464635, + 0.7413896705455917, + 0.7072726720787379, + 0.6729563587569708, + 0.6325907107306379, + 0.6002222927934864, + 0.5668623238577527 + ], + "peak_layer": 2, + "peak_normalized": 2.0575431033491354, + "population_cv": 0.3713778073179401, + "rest_mean": 5.4035704208756755e-06, + "spike_contrast": 0.7652421849089509, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1350400351802815e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.008142589853378e-05, + 1.0710257811297197e-05, + 8.745569175516721e-06, + 7.496069883927703e-06, + 8.15213661553571e-06, + 7.248879683174891e-06, + 6.4891783040366136e-06, + 6.077053967601387e-06, + 5.69765325053595e-06, + 5.353560936782742e-06, + 5.071745363238733e-06, + 4.798503141500987e-06, + 4.57735450254404e-06, + 4.446573711902602e-06, + 4.3752197598223574e-06, + 4.493150299822446e-06, + 4.544274361251155e-06, + 4.499266196944518e-06, + 4.388126853882568e-06, + 4.254965460859239e-06, + 4.224228177918121e-06, + 4.16353077525855e-06, + 4.117235221201554e-06, + 4.091881692147581e-06, + 4.078324309375603e-06, + 3.983679107477656e-06, + 3.859202024614206e-06, + 3.681610678540892e-06, + 3.502981826386531e-06, + 3.2928639939200366e-06, + 3.124374643448391e-06, + 2.950723910544184e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.1829306408326374e-05, + "first_to_last_ratio": 2.242285131401237, + "imbalance_abs_log_ratio": 0.8074954938166318, + "last_quartile_mean": 1.419503075794637e-05, + "mean": 2.0616567582010248e-05, + "normalized": [ + 1.9245546228372667, + 2.0377975361457814, + 1.6531866271625524, + 1.423879750583757, + 1.538930309842627, + 1.3703305332714228, + 1.2383647911586764, + 1.1639182408400186, + 1.0938180520406116, + 1.0314014172841877, + 0.9796503171364493, + 0.9244131813022366, + 0.8827206850480096, + 0.8588358130176241, + 0.8455923007135348, + 0.8694241468157592, + 0.8793999914403602, + 0.8731192864119536, + 0.8549795663584978, + 0.8252911344236574, + 0.8191203633494749, + 0.8077330232195155, + 0.8000322644244665, + 0.7953027221963918, + 0.7913068082422922, + 0.7709379699276236, + 0.7460244301827129, + 0.7109724829635561, + 0.6762339269204858, + 0.6361542658855333, + 0.6048340735041462, + 0.5717393653488173 + ], + "peak_layer": 2, + "peak_normalized": 2.0377975361457814, + "population_cv": 0.36204807030480873, + "rest_mean": 2.1369839554676286e-05, + "spike_contrast": 0.7744044538692995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6548898929613643e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.967771044699475e-05, + 4.201239062240347e-05, + 3.408303382457234e-05, + 2.9355513106565922e-05, + 3.172746073687449e-05, + 2.8251512048882432e-05, + 2.553083140810486e-05, + 2.3995999072212726e-05, + 2.255077379231807e-05, + 2.1263957023620605e-05, + 2.019702696998138e-05, + 1.9058226826018654e-05, + 1.8198670659330674e-05, + 1.7706246580928564e-05, + 1.743321081448812e-05, + 1.79245416802587e-05, + 1.8130209355149418e-05, + 1.8000722775468603e-05, + 1.7626744011067785e-05, + 1.7014670447679237e-05, + 1.688745032879524e-05, + 1.6652682461426593e-05, + 1.6493919247295707e-05, + 1.6396412320318632e-05, + 1.631403029023204e-05, + 1.5894094758550636e-05, + 1.5380463082692586e-05, + 1.4657812243967783e-05, + 1.3941622455604374e-05, + 1.3115317415213212e-05, + 1.2469602552300785e-05, + 1.1787303265009541e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.028226546943187714, + "first_to_last_ratio": 2.242285122589081, + "imbalance_abs_log_ratio": 0.8074954898866428, + "last_quartile_mean": 0.01258829515427351, + "mean": 0.018282978096976876, + "normalized": [ + 1.924554554381315, + 2.0377975654135123, + 1.6531866056622397, + 1.4238797675022206, + 1.5389303095647766, + 1.3703305145930151, + 1.2383648563072303, + 1.1639182544549975, + 1.0938181015007578, + 1.0314014326659862, + 0.9796503379512251, + 0.9244131595482207, + 0.8827206459493627, + 0.8588358074372765, + 0.8455922449868795, + 0.8694241575396449, + 0.8794000131907383, + 0.8731192962202396, + 0.854979598609066, + 0.8252910899109279, + 0.8191203509444464, + 0.8077330175197024, + 0.8000322649688487, + 0.795302701402855, + 0.79130681694379, + 0.7709379635040872, + 0.7460244053110716, + 0.7109724950182295, + 0.6762339125172862, + 0.6361542907855866, + 0.6048340936091356, + 0.5717393740853287 + ], + "peak_layer": 2, + "peak_normalized": 2.0377975654135123, + "population_cv": 0.36204806883319857, + "rest_mean": 0.018950987227812962, + "spike_contrast": 0.7744044473273423, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.014675728790462018, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.035186588764190674, + 0.037257008254528046, + 0.030225174501538277, + 0.026032762601971626, + 0.028136229142546654, + 0.02505372278392315, + 0.022640997543931007, + 0.021279891952872276, + 0.019998252391815186, + 0.018857089802622795, + 0.017910925671458244, + 0.01690102554857731, + 0.01613876223564148, + 0.015702076256275177, + 0.015459944494068623, + 0.0158956628292799, + 0.016078051179647446, + 0.015963220968842506, + 0.015631573274731636, + 0.015088778920471668, + 0.014975959435105324, + 0.014767765067517757, + 0.014626972377300262, + 0.014540501870214939, + 0.014467445202171803, + 0.014095041900873184, + 0.013639547862112522, + 0.01299869455397129, + 0.012363569810986519, + 0.01163079496473074, + 0.011058168485760689, + 0.010453098453581333 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3494145605363883e-06, + "first_to_last_ratio": 1.5177010929884052, + "imbalance_abs_log_ratio": 0.4171967511446403, + "last_quartile_mean": 8.891174729797058e-07, + "mean": 1.0250130149103143e-06, + "normalized": [ + 2.738596187346452, + 1.4794765430196648, + 1.3169609982966792, + 1.1024070457743096, + 1.0148842654352301, + 0.9797975144938003, + 0.96217162006378, + 0.9375881795084541, + 0.9109044983437103, + 0.8889313853518959, + 0.900009111988326, + 0.9067663499946724, + 0.8764724089040279, + 0.8712282957663963, + 0.8670476678993289, + 0.8512105160986065, + 0.8567132773721255, + 0.8738651868927485, + 0.8997173009911049, + 0.9374637355940177, + 0.949227789273703, + 0.9688228256439222, + 0.9778524405833988, + 0.9925195201244542, + 0.9855597557475544, + 0.9661836608096859, + 0.9353656023236144, + 0.8926044217979134, + 0.8523421571494035, + 0.800136382267235, + 0.7757454859502885, + 0.7314278691934953 + ], + "peak_layer": 1, + "peak_normalized": 2.738596187346452, + "population_cv": 0.3416815162600759, + "rest_mean": 1.0297970797418437e-06, + "spike_contrast": 0.9702679144036179, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.99179064820055e-07, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.8070967346138787e-06, + 1.516482711849676e-06, + 1.3499021633833763e-06, + 1.129981569647498e-06, + 1.040269580698805e-06, + 1.0043052043329226e-06, + 9.862384331427165e-07, + 9.610400866222335e-07, + 9.33688966142654e-07, + 9.111662393479492e-07, + 9.225210533259087e-07, + 9.294473102272605e-07, + 8.983956263364234e-07, + 8.930203421186889e-07, + 8.88735144144448e-07, + 8.725018574295973e-07, + 8.781422593528987e-07, + 8.957231898421014e-07, + 9.222219432558632e-07, + 9.609125299903098e-07, + 9.729708381200908e-07, + 9.930560054272064e-07, + 1.0023114782597986e-06, + 1.0173454256801051e-06, + 1.0102115766130737e-06, + 9.903508271236205e-07, + 9.5876191608113e-07, + 9.149311495093571e-07, + 8.736618042348709e-07, + 8.201502055271703e-07, + 7.95149219356972e-07, + 7.497230853914516e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.385878633504035e-05, + "first_to_last_ratio": 2.4650377133645693, + "imbalance_abs_log_ratio": 0.9022071069169957, + "last_quartile_mean": 9.678872743279499e-06, + "mean": 1.5052104913593212e-05, + "normalized": [ + 1.9687527575116004, + 2.0294475105176777, + 1.6968710846545252, + 1.4810263904268979, + 1.5821534963273902, + 1.4269282679347157, + 1.2874882094723543, + 1.207970092684867, + 1.1337626821930848, + 1.062868269769312, + 1.0013238472605475, + 0.9383693915051101, + 0.893007722864056, + 0.8652298987199482, + 0.8502729480950922, + 0.8889049949994228, + 0.902010159160841, + 0.8913150303986681, + 0.8591568947108383, + 0.8064459655370104, + 0.7966210501283933, + 0.7772027605950662, + 0.7602554733595914, + 0.748418831566216, + 0.7418869141825569, + 0.7197374405224362, + 0.6919515801072532, + 0.6600392455527566, + 0.631139243766499, + 0.594431491034554, + 0.565796746616864, + 0.5392136078238537 + ], + "peak_layer": 2, + "peak_normalized": 2.0294475105176777, + "population_cv": 0.38729311327642874, + "rest_mean": 1.5707493056888104e-05, + "spike_contrast": 0.7329628539770106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1513008939800783e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.9633873054990545e-05, + 3.0547456844942644e-05, + 2.554148159106262e-05, + 2.2292564608505927e-05, + 2.381474041612819e-05, + 2.1478273993125185e-05, + 1.937940760399215e-05, + 1.8182492567575537e-05, + 1.706551483948715e-05, + 1.5998404705896974e-05, + 1.5072031601448543e-05, + 1.412443452863954e-05, + 1.3441645933198743e-05, + 1.3023531209910288e-05, + 1.2798397619917523e-05, + 1.337989124294836e-05, + 1.3577151548815891e-05, + 1.3416167348623276e-05, + 1.2932119716424495e-05, + 1.2138709280407056e-05, + 1.1990823622909375e-05, + 1.1698537491611205e-05, + 1.144344514614204e-05, + 1.126527877204353e-05, + 1.1166959666297771e-05, + 1.0833563464984763e-05, + 1.0415327778900973e-05, + 9.934979971149005e-06, + 9.499974112259224e-06, + 8.947445166995749e-06, + 8.516431989846751e-06, + 8.116299795801751e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.751610693645489e-05, + "first_to_last_ratio": 2.3594114654247895, + "imbalance_abs_log_ratio": 0.8584122088800984, + "last_quartile_mean": 7.423930583172478e-06, + "mean": 1.1241273455198098e-05, + "normalized": [ + 1.9313123081075616, + 1.9532131132782136, + 1.6584498227621727, + 1.458759062451573, + 1.5411339282816632, + 1.4256617364160402, + 1.287861390019934, + 1.2091763566215845, + 1.1379116841730463, + 1.0621022378529084, + 0.9972026598452313, + 0.9362662172108906, + 0.8866971704599771, + 0.8635601104202946, + 0.8515524995644437, + 0.9050254579606748, + 0.9262405785753693, + 0.9113277687772972, + 0.8653429711725167, + 0.8112155582189827, + 0.8037618636954775, + 0.7801120976934229, + 0.763059385733637, + 0.7497162474482649, + 0.7450633821823573, + 0.7253917192991086, + 0.6970996868130734, + 0.6708244568956084, + 0.6493864786868861, + 0.6191431780854164, + 0.5992363193314693, + 0.5771925519649024 + ], + "peak_layer": 2, + "peak_normalized": 1.9532131132782136, + "population_cv": 0.367876005198269, + "rest_mean": 1.172351868363977e-05, + "spike_contrast": 0.7367369349328763, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.637149221613072e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.17104097828269e-05, + 2.1956602722639218e-05, + 1.86430879693944e-05, + 1.639830952626653e-05, + 1.732430791889783e-05, + 1.602625343366526e-05, + 1.447720205760561e-05, + 1.3592682080343366e-05, + 1.2791576409654226e-05, + 1.1939381693082396e-05, + 1.1209827789571136e-05, + 1.0524824574531522e-05, + 9.967605365091003e-06, + 9.707515346235596e-06, + 9.57253450906137e-06, + 1.0173638656851836e-05, + 1.0412123629066627e-05, + 1.024448465614114e-05, + 9.727556971483864e-06, + 9.119095921050757e-06, + 9.035306902660523e-06, + 8.76945341587998e-06, + 8.5777592175873e-06, + 8.427765351370908e-06, + 8.375461220566649e-06, + 8.15432667877758e-06, + 7.83628820499871e-06, + 7.540921160398284e-06, + 7.2999309850274585e-06, + 6.95995777277858e-06, + 6.736179329891456e-06, + 6.488379312941106e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.572680194949498e-05, + "first_to_last_ratio": 2.1392798428041573, + "imbalance_abs_log_ratio": 0.760469250343375, + "last_quartile_mean": 3.072379809054837e-05, + "mean": 4.3481031354986044e-05, + "normalized": [ + 1.9052658835371106, + 1.9135764766323078, + 1.63425226196895, + 1.412889347119181, + 1.4908784667860193, + 1.3457175192541695, + 1.2381709412398325, + 1.1522100846113699, + 1.1016927428728427, + 1.0254980546866814, + 0.9643692448719049, + 0.9205305483556507, + 0.884025033242643, + 0.8683067103948131, + 0.8527072800323677, + 0.8631028842013038, + 0.8693948150903978, + 0.864259931478139, + 0.8658847675556672, + 0.8471741358779341, + 0.8431650913109492, + 0.8356030774323144, + 0.8293172543204428, + 0.8191888026538522, + 0.8215172881346767, + 0.7971178058174285, + 0.7748791379460219, + 0.7321907952521126, + 0.7000071600353317, + 0.6525819851008854, + 0.6088054122091195, + 0.5657190599775788 + ], + "peak_layer": 2, + "peak_normalized": 1.9135764766323078, + "population_cv": 0.33981080750369574, + "rest_mean": 4.485182479483127e-05, + "spike_contrast": 0.8043986380678882, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6078746779821814e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.284292562166229e-05, + 8.32042787806131e-05, + 7.105897384462878e-05, + 6.143388600321487e-05, + 6.482493336079642e-05, + 5.8513185649644583e-05, + 5.383694951888174e-05, + 5.00992828165181e-05, + 4.790273669641465e-05, + 4.458971307030879e-05, + 4.193176937405951e-05, + 4.0025617636274546e-05, + 3.843832018901594e-05, + 3.7754871300421655e-05, + 3.707659197971225e-05, + 3.752860357053578e-05, + 3.780218321480788e-05, + 3.757891317945905e-05, + 3.764956272789277e-05, + 3.683600516524166e-05, + 3.666168777272105e-05, + 3.6332883610157296e-05, + 3.605956953833811e-05, + 3.561917401384562e-05, + 3.5720418964046985e-05, + 3.4659504308365285e-05, + 3.3692544093355536e-05, + 3.1836410926189274e-05, + 3.043703327421099e-05, + 2.8374937755870633e-05, + 2.647148721734993e-05, + 2.4598048184998333e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e035a313fa700b2413cb60fbc5eb6e2a9798711f435f32fde650dae79dcc0abb", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.116500623524189, + 0.0468219518661499, + 0.09375449270009995, + 0.041188232600688934, + 0.024237047880887985, + 0.022429833188652992, + 0.03027583472430706, + 0.04546776041388512, + 0.030174262821674347, + 0.03663193807005882, + 0.05748353153467178, + 0.10596779733896255, + 0.05394246056675911, + 0.06537146866321564, + 0.08060523867607117, + 0.06690707802772522, + 0.033449217677116394, + 0.022565271705389023, + 0.036697570234537125, + 0.07367803901433945, + 0.042231619358062744, + 0.052135858684778214, + 0.06269875913858414, + 0.0728035643696785, + 0.051585447043180466, + 0.04862172156572342, + 0.05972515419125557, + 0.06787826865911484, + 0.06439156830310822, + 0.07232685387134552, + 0.09627991914749146, + 0.1271906942129135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.901735841642221e-06, + "first_to_last_ratio": 2.319841868234872, + "imbalance_abs_log_ratio": 0.8414990231116138, + "last_quartile_mean": 3.40615278560108e-06, + "mean": 5.0787282575015524e-06, + "normalized": [ + 1.8471312527953234, + 2.083119522653031, + 1.6411120697088142, + 1.4465389141337504, + 1.6030351711912503, + 1.4179806134683028, + 1.2455229358727704, + 1.1623539789603097, + 1.087561721245063, + 1.0281126130561575, + 0.9635841018330411, + 0.9122839040717997, + 0.8813528037235394, + 0.8604108318814777, + 0.8561651321622351, + 0.8887436718072528, + 0.8917236395900334, + 0.8870579095571277, + 0.8597154672649976, + 0.8329428550551852, + 0.8190933157744664, + 0.8104435207647023, + 0.8055448086648276, + 0.8031059287493388, + 0.7839217090043826, + 0.7606298596198403, + 0.7253003945161415, + 0.6892581493416868, + 0.6485348180910961, + 0.6152321716145727, + 0.5812511282777914, + 0.5612350855496906 + ], + "peak_layer": 2, + "peak_normalized": 2.083119522653031, + "population_cv": 0.3684028416393891, + "rest_mean": 5.262670524812334e-06, + "spike_contrast": 0.7763054887744506, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.085440014023334e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.381077688885853e-06, + 1.0579597983451094e-05, + 8.334762242157012e-06, + 7.34657805878669e-06, + 8.141380021697842e-06, + 7.201538210210856e-06, + 6.325672529783333e-06, + 5.90327999816509e-06, + 5.523430445464328e-06, + 5.221504579822067e-06, + 4.893781806458719e-06, + 4.6332420424732845e-06, + 4.476151389098959e-06, + 4.369792804936878e-06, + 4.348230049799895e-06, + 4.5136875996831805e-06, + 4.528822046268033e-06, + 4.505126071308041e-06, + 4.366261237009894e-06, + 4.230290414852789e-06, + 4.159952368354425e-06, + 4.11602241001674e-06, + 4.091143182449741e-06, + 4.078756774106296e-06, + 3.981325335189467e-06, + 3.863032361550722e-06, + 3.683603608806152e-06, + 3.50055483977485e-06, + 3.293732106612879e-06, + 3.1245970149029745e-06, + 2.952016529889079e-06, + 2.850360488082515e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.0931253377275425e-05, + "first_to_last_ratio": 2.2774504398256985, + "imbalance_abs_log_ratio": 0.8230565891361505, + "last_quartile_mean": 1.3581526445705094e-05, + "mean": 2.0111286517021654e-05, + "normalized": [ + 1.8305147349095139, + 2.0638827488953453, + 1.6144583041246288, + 1.4292024216268573, + 1.5742414021575613, + 1.3951965505267199, + 1.2383780380531688, + 1.1581634757031298, + 1.0870733067057088, + 1.0303290805210676, + 0.9686529136024508, + 0.9150769272896286, + 0.8858327771787193, + 0.8660099552886569, + 0.8620450653043313, + 0.8950473453908027, + 0.8985730286373522, + 0.8961938413974023, + 0.8726197634428304, + 0.8416466438098196, + 0.8273537929153006, + 0.8203765923046102, + 0.8150971572776908, + 0.8114850979119946, + 0.7901082294468773, + 0.7656064448388112, + 0.7288917101067979, + 0.6928620148788858, + 0.6522549280133905, + 0.6198972579804878, + 0.5863035049818077, + 0.5666249447776498 + ], + "peak_layer": 2, + "peak_normalized": 2.0638827488953453, + "population_cv": 0.3585662940639587, + "rest_mean": 2.0808164292786926e-05, + "spike_contrast": 0.7856602003837604, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6348146527889184e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.6814006307395175e-05, + 4.1507337300572544e-05, + 3.246883352403529e-05, + 2.874309939215891e-05, + 3.1660019885748625e-05, + 2.8059197575203143e-05, + 2.4905375539674424e-05, + 2.329215749341529e-05, + 2.1862442736164667e-05, + 2.0721243345178664e-05, + 1.948085628100671e-05, + 1.8403374269837514e-05, + 1.7815236788010225e-05, + 1.741657433740329e-05, + 1.733683529892005e-05, + 1.8000553609454073e-05, + 1.8071459635393694e-05, + 1.802361111913342e-05, + 1.754950608301442e-05, + 1.692659679974895e-05, + 1.6639149180264212e-05, + 1.6498828699695878e-05, + 1.6392652469221503e-05, + 1.6320009308401495e-05, + 1.5890092981862836e-05, + 1.5397330571431667e-05, + 1.46589500218397e-05, + 1.3934346497990191e-05, + 1.3117685739416629e-05, + 1.2466931366361678e-05, + 1.1791317774623167e-05, + 1.1395556612114888e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.027430144604295492, + "first_to_last_ratio": 2.2774504162405003, + "imbalance_abs_log_ratio": 0.8230565787801849, + "last_quartile_mean": 0.012044233502820134, + "mean": 0.017834889993537217, + "normalized": [ + 1.8305147145419467, + 2.0638826670965, + 1.6144583460455657, + 1.429202391478539, + 1.5742413594714617, + 1.3951965580068768, + 1.2383779969546256, + 1.1581635530608791, + 1.087073254641334, + 1.0303291198940532, + 0.9686528876006529, + 0.9150769923697103, + 0.8858327087407725, + 0.866009958574656, + 0.8620451161142623, + 0.8950472977612934, + 0.8985730294936267, + 0.8961938211463923, + 0.8726198047926574, + 0.8416466499561293, + 0.8273538542424902, + 0.8203766463914877, + 0.8150971349229298, + 0.8114850849561472, + 0.7901082410359804, + 0.7656064994836816, + 0.7288916938108543, + 0.6928620554424487, + 0.6522549396913511, + 0.6198972466405528, + 0.5863034743836752, + 0.5666249012564666 + ], + "peak_layer": 2, + "peak_normalized": 2.0638826670965, + "population_cv": 0.3585662833971213, + "rest_mean": 0.01845288811320508, + "spike_contrast": 0.7856602206868666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.014497700147330761, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0326470285654068, + 0.03680912032723427, + 0.028793687000870705, + 0.025489667430520058, + 0.028076421469449997, + 0.024883177131414413, + 0.022086335346102715, + 0.020655719563364983, + 0.019387831911444664, + 0.018375806510448456, + 0.017275817692279816, + 0.016320297494530678, + 0.01579872891306877, + 0.015445192344486713, + 0.015374479815363884, + 0.01596307009458542, + 0.016025951132178307, + 0.015983518213033676, + 0.015563078224658966, + 0.01501067541539669, + 0.014755764976143837, + 0.014631327241659164, + 0.014537167735397816, + 0.014472747221589088, + 0.0140914935618639, + 0.01365450769662857, + 0.0129997031763196, + 0.012357118539512157, + 0.011632895097136497, + 0.011055799201130867, + 0.010456657968461514, + 0.010105692781507969 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.291231697564399e-06, + "first_to_last_ratio": 1.5202661658451317, + "imbalance_abs_log_ratio": 0.41888542863701395, + "last_quartile_mean": 8.493458096836548e-07, + "mean": 9.93008823613195e-07, + "normalized": [ + 2.660433833539824, + 1.4760353050092347, + 1.1979770584425742, + 1.0900493665572268, + 1.0508641630009479, + 1.017485294217113, + 0.969866960695177, + 0.9398678696701823, + 0.9117101913843627, + 0.9035984270963272, + 0.9003870600648762, + 0.9006127144113643, + 0.88474134857999, + 0.8793888984633811, + 0.8796998458023265, + 0.8766255199949985, + 0.8797862836671659, + 0.8997429548627749, + 0.9211914520157214, + 0.9587231163907084, + 0.9600152766046289, + 0.9836978204801501, + 1.0020995832501742, + 1.0127953245043364, + 0.9873250051607358, + 0.9639498022761569, + 0.915477107739169, + 0.8777702206461201, + 0.8263790746766105, + 0.7874102520005254, + 0.7530783342023123, + 0.731214534592803 + ], + "peak_layer": 1, + "peak_normalized": 2.660433833539824, + "population_cv": 0.3259281809459305, + "rest_mean": 9.949973050256463e-07, + "spike_contrast": 0.9872097331566548, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.82271023985959e-07, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.6418342713441234e-06, + 1.4657160818387638e-06, + 1.1896017895196564e-06, + 1.0824286391653004e-06, + 1.043517386278836e-06, + 1.0103718750542612e-06, + 9.630864497012226e-07, + 9.332970876130275e-07, + 9.05336264622747e-07, + 8.972812111096573e-07, + 8.94092295311566e-07, + 8.943163720687153e-07, + 8.785559657553677e-07, + 8.732409355616255e-07, + 8.735497090128774e-07, + 8.704968763595389e-07, + 8.736355425753572e-07, + 8.934526931625442e-07, + 9.147512400886626e-07, + 9.520205139779137e-07, + 9.533036404718587e-07, + 9.768206155058579e-07, + 9.950937283065286e-07, + 1.0057146937469952e-06, + 9.80422441898554e-07, + 9.572106591804186e-07, + 9.090768458008824e-07, + 8.716335742064985e-07, + 8.206017128031817e-07, + 7.819053280400112e-07, + 7.478134307348228e-07, + 7.261024848048692e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.325468449271284e-05, + "first_to_last_ratio": 2.49752403690097, + "imbalance_abs_log_ratio": 0.9152998558790303, + "last_quartile_mean": 9.31109536850272e-06, + "mean": 1.4726752056049008e-05, + "normalized": [ + 1.8853975232156222, + 2.0473116740268433, + 1.6582395075614986, + 1.4857492998515238, + 1.6139665585544047, + 1.450824782288974, + 1.2878281373152924, + 1.2033034259685012, + 1.1275524261509264, + 1.0620148402767218, + 0.9899925412551716, + 0.9287541765485573, + 0.8955670967012345, + 0.8710703550878681, + 0.8665017455395924, + 0.9137558765532069, + 0.9194683059544567, + 0.9140179774903064, + 0.8758167041500393, + 0.8223153812435762, + 0.8027710133686686, + 0.7865201376885268, + 0.7720207805468117, + 0.7611819433729968, + 0.737807344555837, + 0.7143323889943822, + 0.6774046890577128, + 0.6461902213762475, + 0.6100577174023712, + 0.5819822851664526, + 0.5536782248275314, + 0.5366049179081446 + ], + "peak_layer": 2, + "peak_normalized": 2.0473116740268433, + "population_cv": 0.38353466933491126, + "rest_mean": 1.5348383958216464e-05, + "spike_contrast": 0.7407906796766099, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1369939784344751e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.776578185148537e-05, + 3.015025140484795e-05, + 2.4420482077402994e-05, + 2.1880261556361802e-05, + 2.376848533458542e-05, + 2.1365936845541e-05, + 1.8965525669045746e-05, + 1.772075120243244e-05, + 1.6605185010121204e-05, + 1.5640029232599773e-05, + 1.457937469240278e-05, + 1.367753247905057e-05, + 1.3188794582674745e-05, + 1.2828037142753601e-05, + 1.2760756362695247e-05, + 1.3456656233756803e-05, + 1.3540781765186694e-05, + 1.3460516129271127e-05, + 1.2897935448563658e-05, + 1.211003473144956e-05, + 1.1822209671663586e-05, + 1.1582887054828461e-05, + 1.1369358617230318e-05, + 1.120973774959566e-05, + 1.086550582840573e-05, + 1.0519795978325419e-05, + 9.975970897357911e-06, + 9.516283171251416e-06, + 8.984168744063936e-06, + 8.570708814659156e-06, + 8.153881935868412e-06, + 7.902447578089777e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.711517734293011e-05, + "first_to_last_ratio": 2.384061847169584, + "imbalance_abs_log_ratio": 0.8688056914712909, + "last_quartile_mean": 7.178998885137844e-06, + "mean": 1.102206732639388e-05, + "normalized": [ + 1.8531196998586799, + 1.963921090780072, + 1.6275918064471853, + 1.4630803862888466, + 1.5667065333158392, + 1.442954782698019, + 1.2970011659327807, + 1.2081052444056364, + 1.1324273638297324, + 1.0585415613956308, + 0.9863531089650689, + 0.9268820741278728, + 0.8902461316863771, + 0.8668001748241623, + 0.8666116262214356, + 0.9296576580884496, + 0.9423272990336848, + 0.9350798538143901, + 0.8819279614478344, + 0.8237566329745545, + 0.8069480820231186, + 0.7863655059325534, + 0.7701915014821978, + 0.7627658221351614, + 0.7427791751513323, + 0.714459505084982, + 0.683554697524179, + 0.6634891654356642, + 0.6327577648283097, + 0.612919811315865, + 0.5879751612383226, + 0.5727016517120618 + ], + "peak_layer": 2, + "peak_normalized": 1.963921090780072, + "population_cv": 0.36382703407118977, + "rest_mean": 1.1483749033316229e-05, + "spike_contrast": 0.7427004965250654, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.528986109013204e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.042521009570919e-05, + 2.164647048630286e-05, + 1.7939426470547915e-05, + 1.6126170521602035e-05, + 1.7268344890908338e-05, + 1.5904344763839617e-05, + 1.4295634173322469e-05, + 1.3315817341208458e-05, + 1.2481690646382049e-05, + 1.1667316357488744e-05, + 1.0871650374610908e-05, + 1.0216156624665018e-05, + 9.812352800508961e-06, + 9.553929885441903e-06, + 9.551851690048352e-06, + 1.0246749297948554e-05, + 1.0386394933448173e-05, + 1.0306513104296755e-05, + 9.720669368107338e-06, + 9.079501069209073e-06, + 8.894236088963225e-06, + 8.66737354954239e-06, + 8.489102583553176e-06, + 8.407256245845929e-06, + 8.186962077161297e-06, + 7.874820767028723e-06, + 7.5341858973843046e-06, + 7.313022251764778e-06, + 6.9742986852361355e-06, + 6.755643426004099e-06, + 6.48070181341609e-06, + 6.312356163107324e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.3685856275697e-05, + "first_to_last_ratio": 2.1822227741405626, + "imbalance_abs_log_ratio": 0.780343978505805, + "last_quartile_mean": 2.918393897743954e-05, + "mean": 4.2313938195093215e-05, + "normalized": [ + 1.7787137281139571, + 1.9558551999471387, + 1.6050111202233173, + 1.4144048771289366, + 1.5329849841630987, + 1.3719415533754096, + 1.229370474014663, + 1.15235696212458, + 1.0910654327049436, + 1.0293903648071914, + 0.9556505571608248, + 0.9144304448831003, + 0.8945689808825378, + 0.8756421608621586, + 0.8706776546209917, + 0.8937649342904419, + 0.8888623306953729, + 0.8872279288865894, + 0.8842406103569739, + 0.8655423142717734, + 0.8568957182744374, + 0.8484542607681987, + 0.8405793844255519, + 0.8447650350125491, + 0.8186152079032946, + 0.7955630923758639, + 0.7545060763754283, + 0.7184499344207425, + 0.6699909954890287, + 0.6249915977031716, + 0.5822893469308854, + 0.5531967368068471 + ], + "peak_layer": 2, + "peak_normalized": 1.9558551999471387, + "population_cv": 0.3361774415143179, + "rest_mean": 4.355309465609025e-05, + "spike_contrast": 0.8179095788025235, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5622493305709214e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.526438275817782e-05, + 8.275993604911491e-05, + 6.791434134356678e-05, + 5.984904055367224e-05, + 6.486663187388331e-05, + 5.8052250096807256e-05, + 5.2019506256328896e-05, + 4.8760761274024844e-05, + 4.616727528627962e-05, + 4.355756027507596e-05, + 4.043733861180954e-05, + 3.86931533284951e-05, + 3.785273656831123e-05, + 3.705186827573925e-05, + 3.6841800465481356e-05, + 3.781871419050731e-05, + 3.761126572499052e-05, + 3.7542107747867703e-05, + 3.7415702536236495e-05, + 3.662450399133377e-05, + 3.6258632462704554e-05, + 3.590144115150906e-05, + 3.55682241206523e-05, + 3.5745335480896756e-05, + 3.463883331278339e-05, + 3.366340752108954e-05, + 3.1926123483572155e-05, + 3.0400446121348068e-05, + 2.8349957574391738e-05, + 2.6445855837664567e-05, + 2.4638955437694676e-05, + 2.3407932530972175e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "52ffcbeb09bc61876e4e47d2e0a4cedd5afcca30e802adc931a3e85275d4df39", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.09253507107496262, + 0.30484533309936523, + 0.33420246839523315, + 0.38682517409324646, + 0.12552575767040253, + 0.14749252796173096, + 0.1726136952638626, + 0.1892337054014206, + 0.031880609691143036, + 0.06123264133930206, + 0.10403046011924744, + 0.15778414905071259, + 0.053015924990177155, + 0.09510950744152069, + 0.1597633957862854, + 0.7120755910873413, + 0.04075472429394722, + 0.07218439877033234, + 0.11449843645095825, + 0.16539782285690308, + 0.0493597537279129, + 0.07179203629493713, + 0.10295005142688751, + 0.1323672980070114, + 0.023035654798150063, + 0.06294231116771698, + 0.09175154566764832, + 0.1252463310956955, + 0.03424545377492905, + 0.07851024717092514, + 0.12469635903835297, + 0.19299884140491486 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.125052374907682e-06, + "first_to_last_ratio": 2.282830133186621, + "imbalance_abs_log_ratio": 0.8254159596925686, + "last_quartile_mean": 3.55920147399047e-06, + "mean": 5.2053669250540224e-06, + "normalized": [ + 1.936708187251961, + 2.057532287690085, + 1.6800784059556717, + 1.440095135743866, + 1.5661357510227474, + 1.3925489512407077, + 1.246644135390895, + 1.1674496789349236, + 1.0945761582169642, + 1.0284891213276248, + 0.9743321364655805, + 0.9218510024239784, + 0.8793642572376682, + 0.8542321734121537, + 0.8405229222080287, + 0.8631893221699054, + 0.8729930887820545, + 0.864356118998234, + 0.8429990020601789, + 0.8174384986744575, + 0.8115121736936753, + 0.7998527668126937, + 0.7909597412109762, + 0.7860900505247909, + 0.7834756778170519, + 0.7652930939089312, + 0.7413802236605472, + 0.7072803330050248, + 0.6729535215078144, + 0.6325835393606178, + 0.6002196914254118, + 0.5668628518647791 + ], + "peak_layer": 2, + "peak_normalized": 2.057532287690085, + "population_cv": 0.3713746971880558, + "rest_mean": 5.403577578935828e-06, + "spike_contrast": 0.765239201933808, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1350293940922714e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.008127674140269e-05, + 1.0710210517572705e-05, + 8.745424565859139e-06, + 7.4962235885323025e-06, + 8.15231123851845e-06, + 7.248728252307046e-06, + 6.489240149676334e-06, + 6.077003945392789e-06, + 5.697670530935284e-06, + 5.353663254936691e-06, + 5.0717562771751545e-06, + 4.798572717845673e-06, + 4.577413619699655e-06, + 4.446591901796637e-06, + 4.375230219011428e-06, + 4.493217147683026e-06, + 4.544249350146856e-06, + 4.499290753301466e-06, + 4.388119123177603e-06, + 4.255067324265838e-06, + 4.224218628223753e-06, + 4.163527137279743e-06, + 4.117235675948905e-06, + 4.0918871491157915e-06, + 4.078278379893163e-06, + 3.9836313590058126e-06, + 3.859156095131766e-06, + 3.6816536521655507e-06, + 3.502970002955408e-06, + 3.2928294331213692e-06, + 3.1243637295119697e-06, + 2.9507291401387192e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.1829198633204214e-05, + "first_to_last_ratio": 2.242284345131851, + "imbalance_abs_log_ratio": 0.8074951431611719, + "last_quartile_mean": 1.4194987670634873e-05, + "mean": 2.061655351326408e-05, + "normalized": [ + 1.9245211737084353, + 2.037756753011099, + 1.6531757560780274, + 1.423897927003799, + 1.5389403594243347, + 1.3703405560301187, + 1.238377282521067, + 1.1639192115575097, + 1.0938268273523595, + 1.0314185318108018, + 0.9796456918768245, + 0.9244239585214099, + 0.8827240225333328, + 0.8588384283674352, + 0.8455928777461329, + 0.869423769586083, + 0.8794014738389228, + 0.8731224408861616, + 0.8549747677939339, + 0.8252991088856002, + 0.8191238338930235, + 0.807730839300627, + 0.8000289282663227, + 0.7953051177320747, + 0.7913018779978491, + 0.7709359373588888, + 0.7460162927734657, + 0.7109784824788987, + 0.6762365500070177, + 0.6361514355038691, + 0.6048344862432129, + 0.571735299911361 + ], + "peak_layer": 2, + "peak_normalized": 2.037756753011099, + "population_cv": 0.3620430414304777, + "rest_mean": 2.136982847231492e-05, + "spike_contrast": 0.7744034424903768, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6548868734389545e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.9676993765169755e-05, + 4.201152114546858e-05, + 3.408278644201346e-05, + 2.9355867809499614e-05, + 3.172764627379365e-05, + 2.8251699404790998e-05, + 2.5531071514706127e-05, + 2.3996002710191533e-05, + 2.255093932035379e-05, + 2.1264295355649665e-05, + 2.0196917830617167e-05, + 1.905843600980006e-05, + 1.8198727048002183e-05, + 1.7706288417684846e-05, + 1.743321081448812e-05, + 1.792452167137526e-05, + 1.8130227545043454e-05, + 1.8000775526161306e-05, + 1.762663305271417e-05, + 1.7014823242789134e-05, + 1.6887510355445556e-05, + 1.6652626072755083e-05, + 1.649383921176195e-05, + 1.6396450519096106e-05, + 1.631391751288902e-05, + 1.5894042007857934e-05, + 1.538028482173104e-05, + 1.4657925930805504e-05, + 1.3941667020844761e-05, + 1.3115250112605281e-05, + 1.2469602552300785e-05, + 1.1787211406044662e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.028226451249793172, + "first_to_last_ratio": 2.2422842808882666, + "imbalance_abs_log_ratio": 0.8074951145102182, + "last_quartile_mean": 0.012588257202878594, + "mean": 0.018282965727848932, + "normalized": [ + 1.9245212176514708, + 2.0377567662716185, + 1.6531757024174414, + 1.4238979483152348, + 1.5389402141588437, + 1.3703406107590115, + 1.2383773082826783, + 1.1639191437690835, + 1.0938267880508532, + 1.0314185329225836, + 0.9796457030288253, + 0.9244239728212152, + 0.8827239938691996, + 0.8588384260470624, + 0.8455928680018439, + 0.869423828829971, + 0.8794014231683566, + 0.8731224338869547, + 0.8549747774634064, + 0.825299136337954, + 0.8191238086543232, + 0.8077308132566419, + 0.8000289348300117, + 0.7953051751513364, + 0.7913019017816899, + 0.770935989044561, + 0.7460162503339971, + 0.7109785284086558, + 0.6762365604079628, + 0.6361514610493322, + 0.6048345028026112, + 0.571735278225269 + ], + "peak_layer": 2, + "peak_normalized": 2.0377567662716185, + "population_cv": 0.3620430350402813, + "rest_mean": 0.018950977466172643, + "spike_contrast": 0.774403450539526, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.014675702340900898, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.035185955464839935, + 0.037256237119436264, + 0.030224954709410667, + 0.026033077389001846, + 0.028136391192674637, + 0.02505389042198658, + 0.02264120988547802, + 0.021279893815517426, + 0.019998397678136826, + 0.01885738968849182, + 0.017910828813910484, + 0.016901211813092232, + 0.01613881252706051, + 0.01570211350917816, + 0.015459945425391197, + 0.015895646065473557, + 0.01607806608080864, + 0.015963267534971237, + 0.015631474554538727, + 0.015088915824890137, + 0.014976012520492077, + 0.014767714776098728, + 0.014626901596784592, + 0.014540537260472775, + 0.014467345550656319, + 0.014094996266067028, + 0.013639389537274837, + 0.012998796068131924, + 0.012363609857857227, + 0.011630735360085964, + 0.011058168485760689, + 0.010453016497194767 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3494244655021248e-06, + "first_to_last_ratio": 1.5177024210142414, + "imbalance_abs_log_ratio": 0.41719762616888667, + "last_quartile_mean": 8.891232212704381e-07, + "mean": 1.0250225503938282e-06, + "normalized": [ + 2.738505051354672, + 1.4794936132945526, + 1.3168876347326839, + 1.1024576808453903, + 1.0149632207688972, + 0.979817680374155, + 0.9622164613647413, + 0.9375203415461504, + 0.9109530884583005, + 0.8889289387407567, + 0.8999297560774143, + 0.9067763813865627, + 0.876468303610318, + 0.8712274556920194, + 0.8670923959065272, + 0.8512620461270487, + 0.8567121286805068, + 0.8739411285350861, + 0.899701222832916, + 0.9374377124429466, + 0.9492248371986144, + 0.9687717774932662, + 0.9778602024648368, + 0.9925052959874211, + 0.9856180215959583, + 0.9661752272448075, + 0.9353666611085539, + 0.8925798141637848, + 0.8523571867512545, + 0.8001195113520716, + 0.7757205790261442, + 0.7314086428416404 + ], + "peak_layer": 1, + "peak_normalized": 2.738505051354672, + "population_cv": 0.34166636728519467, + "rest_mean": 1.02980674312305e-06, + "spike_contrast": 0.970267398546873, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.991879096560297e-07, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.8070294320059475e-06, + 1.5165143167905626e-06, + 1.3498395219357917e-06, + 1.1300439837214071e-06, + 1.040360189108469e-06, + 1.004335217658081e-06, + 9.862935712590115e-07, + 9.609794915377279e-07, + 9.337474580206617e-07, + 9.111722079069295e-07, + 9.224482937497669e-07, + 9.294662390857411e-07, + 8.983997759060003e-07, + 8.930277886065596e-07, + 8.887892590792035e-07, + 8.725627935746161e-07, + 8.781492510934186e-07, + 8.958093644650944e-07, + 9.222140420206415e-07, + 9.608947948436253e-07, + 9.7297686352249e-07, + 9.9301291811571e-07, + 1.0023287586591323e-06, + 1.0173403097724076e-06, + 1.0102806982104084e-06, + 9.903513955578092e-07, + 9.587719205228495e-07, + 9.14914437544212e-07, + 8.736853374102793e-07, + 8.20140542145964e-07, + 7.951310863063554e-07, + 7.497103524656268e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.3858743134042015e-05, + "first_to_last_ratio": 2.465036348010912, + "imbalance_abs_log_ratio": 0.9022065530293095, + "last_quartile_mean": 9.678860578787862e-06, + "mean": 1.5052110796887064e-05, + "normalized": [ + 1.9687217764679332, + 2.0294404332853553, + 1.6968677627976259, + 1.4810317330109646, + 1.5821503401553387, + 1.4269352026634141, + 1.2874921775494743, + 1.2079704664585447, + 1.13377287350848, + 1.0628809057173056, + 1.0013232141891217, + 0.9383772422695752, + 0.8930109387824885, + 0.8652307689960489, + 0.8502741867556953, + 0.8889045267143569, + 0.9020099274457369, + 0.8913174614788361, + 0.8591545649380836, + 0.8064478255585749, + 0.7966230348362755, + 0.7772037861237395, + 0.760253544782019, + 0.7484175118458054, + 0.7418822737465284, + 0.7197362528586176, + 0.691945931996893, + 0.6600449694523348, + 0.6311394200398015, + 0.5944298085404294, + 0.5657980360452561, + 0.5392111009893452 + ], + "peak_layer": 2, + "peak_normalized": 2.0294404332853553, + "population_cv": 0.3872905960483728, + "rest_mean": 1.57075019160402e-05, + "spike_contrast": 0.73296179207868, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1512998753460124e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.963341830763966e-05, + 3.054736225749366e-05, + 2.5541441573295742e-05, + 2.22926537389867e-05, + 2.3814702217350714e-05, + 2.1478386770468205e-05, + 1.937947490660008e-05, + 1.818250530050136e-05, + 1.7065674910554662e-05, + 1.5998601156752557e-05, + 1.5072027963469736e-05, + 1.4124558219918981e-05, + 1.3441699593386147e-05, + 1.3023549399804324e-05, + 1.279842126677977e-05, + 1.3379889423958957e-05, + 1.3577153367805295e-05, + 1.3416209185379557e-05, + 1.2932089703099336e-05, + 1.213874202221632e-05, + 1.1990858183708042e-05, + 1.1698557500494644e-05, + 1.1443420589785092e-05, + 1.12652633106336e-05, + 1.1166894182679243e-05, + 1.0833549822564237e-05, + 1.0415246833872516e-05, + 9.93507001112448e-06, + 9.499980478722136e-06, + 8.947423339122906e-06, + 8.516454727214295e-06, + 8.116265235003084e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7515224044473143e-05, + "first_to_last_ratio": 2.359437553652934, + "imbalance_abs_log_ratio": 0.8584232659103358, + "last_quartile_mean": 7.423474301049282e-06, + "mean": 1.1241153700325413e-05, + "normalized": [ + 1.9309435555261214, + 1.9526313215933524, + 1.6582244442374867, + 1.4588157040411531, + 1.5412217068718899, + 1.4258707789492027, + 1.2881056965786792, + 1.2092589806313907, + 1.137794759023235, + 1.0622568400477936, + 0.9971593988413544, + 0.9361807205321625, + 0.8867849352218846, + 0.8639024876188973, + 0.8515161822097123, + 0.9048685915984879, + 0.9265194638027697, + 0.9115225939595661, + 0.8653717695293225, + 0.8111735521509221, + 0.8039114483119388, + 0.7800093222994203, + 0.76323256629171, + 0.7496538447594057, + 0.745119944996388, + 0.7253817283186728, + 0.6970650412445614, + 0.6707070461181087, + 0.6492809756642738, + 0.6191651059644422, + 0.5991054029612273, + 0.5772440901044666 + ], + "peak_layer": 2, + "peak_normalized": 1.9526313215933524, + "population_cv": 0.3678078470590104, + "rest_mean": 1.1723304632377457e-05, + "spike_contrast": 0.7367836065087988, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.63753866724437e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.170603329432197e-05, + 2.1949828806100413e-05, + 1.8640355847310275e-05, + 1.639877154957503e-05, + 1.7325110093224794e-05, + 1.602843258297071e-05, + 1.4479794117505662e-05, + 1.3593466064776294e-05, + 1.2790125765604898e-05, + 1.1940992408199236e-05, + 1.1209222066099755e-05, + 1.052375137078343e-05, + 9.96848575596232e-06, + 9.711260645417497e-06, + 9.572024282533675e-06, + 1.0171766916755587e-05, + 1.0415147698950022e-05, + 1.0246565580018796e-05, + 9.727777069201693e-06, + 9.118526577367447e-06, + 9.036892151925713e-06, + 8.768204679654445e-06, + 8.579614586778916e-06, + 8.426974090980366e-06, + 8.376007826882415e-06, + 8.154127499437891e-06, + 7.835815267753787e-06, + 7.5395209933049046e-06, + 7.298667242139345e-06, + 6.960130122024566e-06, + 6.734635917382548e-06, + 6.488889539468801e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.572684515049332e-05, + "first_to_last_ratio": 2.139075374807702, + "imbalance_abs_log_ratio": 0.7603736678186844, + "last_quartile_mean": 3.072675508519751e-05, + "mean": 4.3482417027007614e-05, + "normalized": [ + 1.9051243467602497, + 1.9135350736361767, + 1.6339952020703241, + 1.4129520831778495, + 1.4905720952473844, + 1.3455550767012106, + 1.2384699946115487, + 1.15237968581952, + 1.1017299217830712, + 1.0257825506862357, + 0.9642904052505235, + 0.9205031377378704, + 0.884021208319066, + 0.8684631875068248, + 0.8525398830567881, + 0.8631133634913525, + 0.8692805159213056, + 0.8642712942167508, + 0.865837261658028, + 0.84731405133431, + 0.8430444327809488, + 0.8356094968046114, + 0.8292110092261231, + 0.819222183409267, + 0.8216396147950802, + 0.7972044318616739, + 0.7748498429513314, + 0.7322001754439701, + 0.700092530137593, + 0.6526455238420115, + 0.6087378616535022, + 0.5658125581074975 + ], + "peak_layer": 2, + "peak_normalized": 1.9135350736361767, + "population_cv": 0.3397683108879149, + "rest_mean": 4.4853358472378374e-05, + "spike_contrast": 0.8043842078007138, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6079333222005515e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.283941133413464e-05, + 8.320513006765395e-05, + 7.10500607965514e-05, + 6.14385717199184e-05, + 6.481367745436728e-05, + 5.850798697792925e-05, + 5.385166878113523e-05, + 5.0108254072256386e-05, + 4.790587991010398e-05, + 4.4603504647966474e-05, + 4.1929677536245435e-05, + 4.002570130978711e-05, + 3.8439378840848804e-05, + 3.7762878491776064e-05, + 3.707049472723156e-05, + 3.75302552129142e-05, + 3.779841790674254e-05, + 3.758060483960435e-05, + 3.7648696888936684e-05, + 3.684326293296181e-05, + 3.66576095984783e-05, + 3.63343206117861e-05, + 3.6056098906556144e-05, + 3.5621760616777465e-05, + 3.572687637642957e-05, + 3.466437556198798e-05, + 3.3692344004521146e-05, + 3.1837833375902846e-05, + 3.0441715352935717e-05, + 2.837860483850818e-05, + 2.646939356054645e-05, + 2.4602897610748187e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "a10595ef96f6b35550f566eb8338085abe3f571b2ae6319226145e91aa34ec7b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.15345558524131775, + 0.3249167501926422, + 0.3818626403808594, + 0.40600064396858215, + 0.1351035088300705, + 0.15324510633945465, + 0.18001793324947357, + 0.20391276478767395, + 0.04374876990914345, + 0.08211024105548859, + 0.1380448192358017, + 0.2295975536108017, + 0.08199761807918549, + 0.13892096281051636, + 0.21687114238739014, + 0.7360114455223083, + 0.057135771960020065, + 0.08251863718032837, + 0.12661415338516235, + 0.19619159400463104, + 0.06753352284431458, + 0.095451220870018, + 0.1274663656949997, + 0.15628865361213684, + 0.058436937630176544, + 0.08794140070676804, + 0.12247690558433533, + 0.1640285700559616, + 0.07537907361984253, + 0.12290703505277634, + 0.19157102704048157, + 0.29249706864356995 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.9018017800081e-06, + "first_to_last_ratio": 2.3198553421908015, + "imbalance_abs_log_ratio": 0.8415048312302555, + "last_quartile_mean": 3.4061614258007467e-06, + "mean": 5.078743377850969e-06, + "normalized": [ + 1.847125037232453, + 2.083137317368821, + 1.641111481706062, + 1.4465372937027328, + 1.6030646026913287, + 1.4179987767162792, + 1.2455258536342846, + 1.1623609049834964, + 1.0875552599581997, + 1.028118506110977, + 0.9635771142568214, + 0.9122601462896323, + 0.8813638792944334, + 0.8604039723922948, + 0.8561688509558816, + 0.8887307288299607, + 0.8917013756490454, + 0.8870568803314235, + 0.8597145194427529, + 0.8329319585333728, + 0.8191043976245732, + 0.810440302071778, + 0.8055455442874069, + 0.803104343609964, + 0.7839286872176334, + 0.7606233867362351, + 0.7253002050295685, + 0.689251306939001, + 0.6485362450095404, + 0.6152354884679782, + 0.5812522182760415, + 0.5612334146500276 + ], + "peak_layer": 2, + "peak_normalized": 2.083137317368821, + "population_cv": 0.36840679032180734, + "rest_mean": 5.262683560903059e-06, + "spike_contrast": 0.7763085775707618, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.085466389369685e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.381074050907046e-06, + 1.0579719855741132e-05, + 8.334784070029855e-06, + 7.346591701207217e-06, + 8.14155373518588e-06, + 7.201651897048578e-06, + 6.325706181087298e-06, + 5.9033327488577925e-06, + 5.523414074559696e-06, + 5.221550054557156e-06, + 4.893760888080578e-06, + 4.633135176845826e-06, + 4.476220965443645e-06, + 4.369770977064036e-06, + 4.348261882114457e-06, + 4.5136353037378285e-06, + 4.528722456598189e-06, + 4.505134256760357e-06, + 4.36626942246221e-06, + 4.230247668601805e-06, + 4.160021035204409e-06, + 4.116018317290582e-06, + 4.091159098607022e-06, + 4.078760866832454e-06, + 3.9813726289139595e-06, + 3.863010988425231e-06, + 3.6836136132478714e-06, + 3.5005305107915774e-06, + 3.293749159638537e-06, + 3.1246231628756505e-06, + 2.9520308544306317e-06, + 2.850360488082515e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.093152440669655e-05, + "first_to_last_ratio": 2.2774685844866087, + "imbalance_abs_log_ratio": 0.8230645561980515, + "last_quartile_mean": 1.3581537245954678e-05, + "mean": 2.01113496416383e-05, + "normalized": [ + 1.8305028390484899, + 2.063895445405066, + 1.6144389462797186, + 1.4292254312672816, + 1.5742789705777873, + 1.395222470726611, + 1.2383859090556808, + 1.15815685579556, + 1.0870603073810894, + 1.0303295548521039, + 0.9686496018996943, + 0.9150613022126579, + 0.8858369611012855, + 0.866010221809998, + 0.8620422691094124, + 0.8950413704487017, + 0.898565866832108, + 0.8961986259205167, + 0.8726168436106015, + 0.8416436403028231, + 0.8273619591175169, + 0.8203659676564082, + 0.8150936944275661, + 0.8114785712381606, + 0.790107287070579, + 0.7656015997474095, + 0.7288914121037795, + 0.6928588452491169, + 0.6522528807444096, + 0.6198966689630461, + 0.586304513764539, + 0.566623166280281 + ], + "peak_layer": 2, + "peak_normalized": 2.063895445405066, + "population_cv": 0.3585699713300157, + "rest_mean": 2.0808239578737238e-05, + "spike_contrast": 0.7856572354159773, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.634814398130402e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.6813882616115734e-05, + 4.1507722926326096e-05, + 3.246854612370953e-05, + 2.874365236493759e-05, + 3.166087481076829e-05, + 2.805980693665333e-05, + 2.4905612008296885e-05, + 2.329209746676497e-05, + 2.186224992328789e-05, + 2.072131792374421e-05, + 1.9480850824038498e-05, + 1.8403117792331614e-05, + 1.7815376850194298e-05, + 1.7416634364053607e-05, + 1.7336833479930647e-05, + 1.800048994482495e-05, + 1.8071372323902324e-05, + 1.8023763914243318e-05, + 1.7549502445035614e-05, + 1.6926589523791336e-05, + 1.6639365640003234e-05, + 1.6498666809638962e-05, + 1.6392634279327467e-05, + 1.631992927286774e-05, + 1.5890123904682696e-05, + 1.539728145871777e-05, + 1.4658990039606579e-05, + 1.3934326489106752e-05, + 1.3117685739416629e-05, + 1.246695865120273e-05, + 1.1791375072789378e-05, + 1.1395556612114888e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.027430385118350387, + "first_to_last_ratio": 2.2774685803861345, + "imbalance_abs_log_ratio": 0.8230645543975988, + "last_quartile_mean": 0.012044243048876524, + "mean": 0.017834946047514677, + "normalized": [ + 1.8305029039623888, + 2.0638953970410903, + 1.6144389639181254, + 1.4292254712142256, + 1.5742789179863164, + 1.3952224600071201, + 1.2383859063079345, + 1.158156779896113, + 1.0870603341935974, + 1.0303295369704457, + 0.9686496343170561, + 0.9150613749177388, + 0.8858370264114429, + 0.866010161028954, + 0.8620423023293187, + 0.8950413515523591, + 0.8985659233861498, + 0.8961986284425522, + 0.8726168533342663, + 0.8416436914062322, + 0.827361906598272, + 0.82036592184569, + 0.8150936331886041, + 0.8114785658737781, + 0.790107272131739, + 0.7656015867257281, + 0.7288914394956378, + 0.6928588334464092, + 0.6522528897000817, + 0.6198967082595142, + 0.5863045037167507, + 0.5666231203943681 + ], + "peak_layer": 2, + "peak_normalized": 2.0638953970410903, + "population_cv": 0.35856997121771, + "rest_mean": 0.018452955133937025, + "spike_contrast": 0.7856571955876671, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.014497696980834007, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.032646920531988144, + 0.03680946305394173, + 0.02879343181848526, + 0.025490159168839455, + 0.028077179566025734, + 0.02488371729850769, + 0.022086545825004578, + 0.020655663684010506, + 0.019387662410736084, + 0.01837587170302868, + 0.017275813966989517, + 0.01632007025182247, + 0.01579885557293892, + 0.015445244498550892, + 0.015374477952718735, + 0.015963014215230942, + 0.016025874763727188, + 0.01598365418612957, + 0.015563074499368668, + 0.015010669827461243, + 0.014755954965949059, + 0.014631181955337524, + 0.014537150971591473, + 0.014472676441073418, + 0.014091520570218563, + 0.013654462993144989, + 0.01299973949790001, + 0.012357099913060665, + 0.011632895097136497, + 0.011055824346840382, + 0.010456709191203117, + 0.010105692781507969 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2912549820498498e-06, + "first_to_last_ratio": 1.5202665797334127, + "imbalance_abs_log_ratio": 0.4188857008842256, + "last_quartile_mean": 8.49360894505935e-07, + "mean": 9.93022506889929e-07, + "normalized": [ + 2.660532038406061, + 1.475963218576449, + 1.19801710692682, + 1.0899716081643689, + 1.0508156804583857, + 1.0174876453067656, + 0.9698761501636428, + 0.9399606463412474, + 0.9117614543001846, + 0.9036711533632844, + 0.9003299466006321, + 0.9006091198840781, + 0.8847274400701823, + 0.8793430077222689, + 0.8796797672930117, + 0.8766142420011561, + 0.8797972295808179, + 0.8997445814178318, + 0.9212125890421617, + 0.9587869545893736, + 0.9600087455596155, + 0.9836840367208892, + 1.0020615039365162, + 1.0127125628875548, + 0.9873575381135197, + 0.9639449915433914, + 0.9154718200755378, + 0.8777908111354492, + 0.8263667145220306, + 0.7874398153801374, + 0.7530554210378857, + 0.7312044588787489 + ], + "peak_layer": 1, + "peak_normalized": 2.660532038406061, + "population_cv": 0.3259399130744097, + "rest_mean": 9.950150106979656e-07, + "spike_contrast": 0.9871840884465768, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.82262986326532e-07, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.6419681944389595e-06, + 1.4656646953881136e-06, + 1.189657950817491e-06, + 1.082366338778229e-06, + 1.0434836212880327e-06, + 1.0103881322720554e-06, + 9.631088460082537e-07, + 9.334020774076635e-07, + 9.053996450347768e-07, + 8.973657941169222e-07, + 8.940479006014357e-07, + 8.943251259552198e-07, + 8.785542604528018e-07, + 8.732073979444976e-07, + 8.735418077776558e-07, + 8.704976721674029e-07, + 8.736584504731582e-07, + 8.934666198001651e-07, + 9.147848345492093e-07, + 9.520970252197003e-07, + 9.533102911518654e-07, + 9.768203881321824e-07, + 9.950696266969317e-07, + 1.0056463679575245e-06, + 9.80468257694156e-07, + 9.5721907200641e-07, + 9.090841217584966e-07, + 8.71666031798668e-07, + 8.206007464650611e-07, + 7.81945459493727e-07, + 7.478009820260922e-07, + 7.261024848048692e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.32548172789393e-05, + "first_to_last_ratio": 2.4975346081461725, + "imbalance_abs_log_ratio": 0.9153040885601471, + "last_quartile_mean": 9.311109124610084e-06, + "mean": 1.4726773656548175e-05, + "normalized": [ + 1.8853895701442087, + 2.0473234930293196, + 1.6582347285378365, + 1.4857598427569607, + 1.6139858065355936, + 1.4508395759575576, + 1.287840205679765, + 1.2033012904754463, + 1.1275415086272205, + 1.0620110592815606, + 0.9899911509412549, + 0.9287416361171535, + 0.8955665859807346, + 0.8710690774451937, + 0.8664988688923737, + 0.9137534246592263, + 0.9194604727439549, + 0.9140176249806901, + 0.8758139373559145, + 0.8223144839018022, + 0.8027749000523561, + 0.7865160814383286, + 0.7720189070902561, + 0.7611779242876019, + 0.7378071887453072, + 0.7143310324576317, + 0.6774044365690762, + 0.6461920526823346, + 0.6100565755684288, + 0.5819809992384402, + 0.553681426983732, + 0.5366041308427388 + ], + "peak_layer": 2, + "peak_normalized": 2.0473234930293196, + "population_cv": 0.3835372248130478, + "rest_mean": 1.534840986197297e-05, + "spike_contrast": 0.7407893227704523, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1369938147254288e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.7765705453930423e-05, + 3.0150469683576375e-05, + 2.4420447516604327e-05, + 2.1880448912270367e-05, + 2.376880365773104e-05, + 2.1366186047089286e-05, + 1.8965731214848347e-05, + 1.7720745745464228e-05, + 1.6605048585915938e-05, + 1.563999649079051e-05, + 1.4579375601897482e-05, + 1.3677367860509548e-05, + 1.3188806406105869e-05, + 1.2828037142753601e-05, + 1.2760732715833e-05, + 1.3456639862852171e-05, + 1.3540686268243007e-05, + 1.3460530681186356e-05, + 1.2897913620690815e-05, + 1.2110039278923068e-05, + 1.1822284250229131e-05, + 1.1582844308577478e-05, + 1.1369347703293897e-05, + 1.1209695003344677e-05, + 1.0865519470826257e-05, + 1.051979143085191e-05, + 9.975981811294332e-06, + 9.516324098512996e-06, + 8.984165106085129e-06, + 8.570702448196243e-06, + 8.153941053024027e-06, + 7.902447578089777e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7115447462856537e-05, + "first_to_last_ratio": 2.384273554189065, + "imbalance_abs_log_ratio": 0.8688944885049729, + "last_quartile_mean": 7.178474731972528e-06, + "mean": 1.1022389131198906e-05, + "normalized": [ + 1.8533266693628003, + 1.9634876569720725, + 1.62746111449772, + 1.4631152334273978, + 1.5665541851429352, + 1.44292255643436, + 1.297243679761451, + 1.2082029846844278, + 1.1325776467348811, + 1.0586573654925269, + 0.9865546892223965, + 0.9267660638529281, + 0.890239943667283, + 0.8665873151718257, + 0.8665599207260849, + 0.9296718554228318, + 0.9425328050776204, + 0.9351613887984032, + 0.8819628604108036, + 0.823795705683662, + 0.8070268393278794, + 0.7865168158338803, + 0.7700336933503881, + 0.7629366340955328, + 0.7426802567648084, + 0.7143746981629545, + 0.6833977272777232, + 0.6635906353598262, + 0.6326960953519897, + 0.6128141637886501, + 0.5878658687743828, + 0.5726849313675726 + ], + "peak_layer": 2, + "peak_normalized": 1.9634876569720725, + "population_cv": 0.363814433471477, + "rest_mean": 1.1484025469178084e-05, + "spike_contrast": 0.7427319739932458, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.529552906111348e-06, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.04280877369456e-05, + 2.164232500945218e-05, + 1.7938509699888527e-05, + 1.61270254466217e-05, + 1.7267169823753648e-05, + 1.590445390320383e-05, + 1.4298724636319093e-05, + 1.3317283446667716e-05, + 1.2483711543609388e-05, + 1.1668933439068496e-05, + 1.0874189683818258e-05, + 1.0215176189376507e-05, + 9.812571079237387e-06, + 9.551862603984773e-06, + 9.551560651743785e-06, + 1.0247204954794142e-05, + 1.038896334648598e-05, + 1.0307712727808394e-05, + 9.72133784671314e-06, + 9.080196832655929e-06, + 8.895363862393424e-06, + 8.669294402352534e-06, + 8.487611012242269e-06, + 8.409384463448077e-06, + 8.186110790120438e-06, + 7.874115908634849e-06, + 7.532675681432011e-06, + 7.3143542067555245e-06, + 6.973822564759757e-06, + 6.754676178388763e-06, + 6.47968636258156e-06, + 6.312356163107324e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.367657533701276e-05, + "first_to_last_ratio": 2.18189651420632, + "imbalance_abs_log_ratio": 0.7801944592496646, + "last_quartile_mean": 2.918404925367213e-05, + "mean": 4.231288824030344e-05, + "normalized": [ + 1.7779407300288732, + 1.9563222737204058, + 1.603774689185993, + 1.4143174555706712, + 1.5327227884698011, + 1.3724537206523966, + 1.2293267806538501, + 1.152324512384129, + 1.0911327441927006, + 1.0293532301829285, + 0.9557045350033907, + 0.9145594044662916, + 0.8946717401908558, + 0.8760144215357385, + 0.8707713092633232, + 0.8938212455104791, + 0.8890009731933908, + 0.8873877674020535, + 0.8842233459795642, + 0.8657151992078328, + 0.8569039986584633, + 0.8485896651489135, + 0.8407244808721062, + 0.844482236684939, + 0.8186411956248972, + 0.7956332166567571, + 0.7546037265710437, + 0.7184850006798041, + 0.6700028489089177, + 0.624855570006522, + 0.582328729525934, + 0.5532104638670327 + ], + "peak_layer": 2, + "peak_normalized": 1.9563222737204058, + "population_cv": 0.33607795885441427, + "rest_mean": 4.3551963851011046e-05, + "spike_contrast": 0.8179167319375746, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.562187994248234e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.522980740759522e-05, + 8.277764572994784e-05, + 6.78603391861543e-05, + 5.9843856433872133e-05, + 6.485392805188894e-05, + 5.807248089695349e-05, + 5.2016366680618376e-05, + 4.875817830907181e-05, + 4.616897786036134e-05, + 4.355490818852559e-05, + 4.043861918034963e-05, + 3.8697649870300665e-05, + 3.7856145354453474e-05, + 3.7066700315335765e-05, + 3.68448490917217e-05, + 3.782015846809372e-05, + 3.761619882425293e-05, + 3.754793942789547e-05, + 3.741404361790046e-05, + 3.6630910472013056e-05, + 3.625808312790468e-05, + 3.5906279663322493e-05, + 3.557348100002855e-05, + 3.57324825017713e-05, + 3.463907341938466e-05, + 3.3665539376670495e-05, + 3.1929463148117065e-05, + 3.040117553609889e-05, + 2.8349755666567944e-05, + 2.6439443900017068e-05, + 2.4640010451548733e-05, + 2.3407932530972175e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "8c720ab728e5d91b12380e66f13ddaee2a9b90e8aeb102317b6f7b3ab51f7764", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.06279198825359344, + 0.07690861076116562, + 0.18058517575263977, + 0.15882407128810883, + 0.19892723858356476, + 0.15647411346435547, + 0.1533493995666504, + 0.1527898609638214, + 0.16355258226394653, + 0.11789151281118393, + 0.11456070095300674, + 0.12148918211460114, + 0.15094918012619019, + 0.09854138642549515, + 0.11213528364896774, + 0.16254977881908417, + 0.2323070764541626, + 0.1390199512243271, + 0.14812250435352325, + 0.15333011746406555, + 0.1781008094549179, + 0.07593096792697906, + 0.09691466391086578, + 0.09368696808815002, + 0.10724716633558273, + 0.06538715958595276, + 0.07412076741456985, + 0.07655765861272812, + 0.08420819044113159, + 0.07230556756258011, + 0.07723049819469452, + 0.08422473073005676 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 6.718497104429844e-06, + "first_to_last_ratio": 5.4600950128558265, + "imbalance_abs_log_ratio": 1.6974661912273534, + "last_quartile_mean": 1.2304725629519453e-06, + "mean": 3.460733235982616e-06, + "normalized": [ + 4.1554392029645175, + 2.1310249348511077, + 2.186628621017466, + 1.156264880926304, + 1.051728395133589, + 1.9487043640861321, + 1.6569575754204475, + 1.2440581208520776, + 0.7492436301896301, + 1.1025523326858364, + 1.1770939413199029, + 1.0278305718772012, + 0.5042914345692446, + 0.5528163951643698, + 0.4411680047894231, + 0.9973694803382674, + 0.4275370158551335, + 0.7739052934101094, + 1.5627780657381267, + 1.739415630377695, + 0.5961441757664185, + 0.7016049433942307, + 0.7230338540262741, + 0.5479886519610823, + 0.2846566311183848, + 0.3910340264134103, + 0.5107042475345903, + 0.37580476181367506, + 0.29294599638498453, + 0.45911022842212285, + 0.29896279650123675, + 0.231201795097008 + ], + "peak_layer": 1, + "peak_normalized": 4.1554392029645175, + "population_cv": 0.7923058521157783, + "rest_mean": 3.7358707240181737e-06, + "spike_contrast": 0.5286560875603242, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9749908005906037e-06, + "top_five_layers": [ + 1, + 3, + 2, + 6, + 20 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.4380866559804417e-05, + 7.374908818746917e-06, + 7.5673383435059804e-06, + 4.001524303021142e-06, + 3.6397514122654684e-06, + 6.743945959897246e-06, + 5.7342881518707145e-06, + 4.305353286326863e-06, + 2.592932332845521e-06, + 3.815639502136037e-06, + 4.073608124599559e-06, + 3.557047421054449e-06, + 1.7452181282351376e-06, + 1.913150072141434e-06, + 1.5267647768268944e-06, + 3.4516297091613524e-06, + 1.479591560382687e-06, + 2.6782797704072436e-06, + 5.408357992564561e-06, + 6.019653483235743e-06, + 2.063095962512307e-06, + 2.428067546134116e-06, + 2.50222728936933e-06, + 1.8964425407830277e-06, + 9.851206641542376e-07, + 1.3532644516089931e-06, + 1.7674111632004497e-06, + 1.300560029449116e-06, + 1.0138079460375593e-06, + 1.5888580264800112e-06, + 1.0346304861741373e-06, + 8.001277365110582e-07 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.6456544674147153e-05, + "first_to_last_ratio": 5.448763030808658, + "imbalance_abs_log_ratio": 1.6953886160901857, + "last_quartile_mean": 4.855513907386921e-06, + "mean": 1.3720428839292254e-05, + "normalized": [ + 4.173598008637582, + 2.103709111147759, + 2.234357653663474, + 1.1576980683143674, + 1.0174050734502822, + 1.9214205167195906, + 1.5972559788378484, + 1.2206300853156902, + 0.7360677134431396, + 1.1059605072991032, + 1.1320345474510969, + 1.045897352609526, + 0.49038680071526464, + 0.5622676476822898, + 0.4266762355069258, + 1.004384658425444, + 0.4110778614318569, + 0.7749237983047368, + 1.5517430250838387, + 1.9055106902179004, + 0.6175511328557414, + 0.6917605316674716, + 0.7266963661277974, + 0.5598716813185096, + 0.2878797501636235, + 0.3879341985116693, + 0.5111757613209584, + 0.3605598270780299, + 0.2888857643708419, + 0.4593768179608925, + 0.3088935915519739, + 0.2264092428147734 + ], + "peak_layer": 1, + "peak_normalized": 4.173598008637582, + "population_cv": 0.7986066807104257, + "rest_mean": 1.4795826161759948e-05, + "spike_contrast": 0.5348321351881458, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.9132832979667e-06, + "top_five_layers": [ + 1, + 3, + 2, + 6, + 20 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.726355448132381e-05, + 2.8863791158073582e-05, + 3.0656345188617706e-05, + 1.588411396369338e-05, + 1.3959233911009505e-05, + 2.6362713470007293e-05, + 2.191503699577879e-05, + 1.674756822467316e-05, + 1.009916468319716e-05, + 1.5174252439464908e-05, + 1.5531999451923184e-05, + 1.435016019968316e-05, + 6.72831720294198e-06, + 7.714553248661105e-06, + 5.854180926689878e-06, + 1.3780588233203162e-05, + 5.640164545184234e-06, + 1.0632286830514204e-05, + 2.1290579752530903e-05, + 2.614442382764537e-05, + 8.473066372971516e-06, + 9.49125114857452e-06, + 9.970585779228713e-06, + 7.681679562665522e-06, + 3.949833626393229e-06, + 5.322623565007234e-06, + 7.0135506575752515e-06, + 4.947035449731629e-06, + 3.963636572734686e-06, + 6.3028469412529375e-06, + 4.238152541802265e-06, + 3.1064319045981392e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.023461928823962808, + "first_to_last_ratio": 5.448763126593669, + "imbalance_abs_log_ratio": 1.6953886336694068, + "last_quartile_mean": 0.004305918293539435, + "mean": 0.012167413689894602, + "normalized": [ + 4.173598084973268, + 2.1037090871122417, + 2.234357712750731, + 1.157698009629034, + 1.0174050464680184, + 1.9214205450906374, + 1.59725599079182, + 1.2206300674885242, + 0.7360677803275659, + 1.1059605017352738, + 1.1320345805663141, + 1.0458974046616514, + 0.4903867781433125, + 0.5622676330882879, + 0.42667620792714267, + 1.0043846550160789, + 0.41107786974695965, + 0.7749237446127663, + 1.551743137345238, + 1.9055106029576432, + 0.6175510806823439, + 0.691760545613661, + 0.7266963928653994, + 0.5598716275528304, + 0.2878797436072962, + 0.3879341769467224, + 0.5111757600120082, + 0.3605598347532703, + 0.28888575899731955, + 0.4593768268733131, + 0.30889356866228096, + 0.226409243001046 + ], + "peak_layer": 1, + "peak_normalized": 4.173598084973268, + "population_cv": 0.7986066956152146, + "rest_mean": 0.013121086836758035, + "spike_contrast": 0.534832120550615, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.007017578696832061, + "top_five_layers": [ + 1, + 3, + 2, + 6, + 20 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.050781894475221634, + 0.025596698746085167, + 0.027186354622244835, + 0.014086190611124039, + 0.01237918809056282, + 0.02337871864438057, + 0.019434474408626556, + 0.01485191099345684, + 0.008956041187047958, + 0.013456678949296474, + 0.013773933053016663, + 0.01272586639970541, + 0.005966738797724247, + 0.006841342896223068, + 0.005191545933485031, + 0.012220763601362705, + 0.005001754499971867, + 0.00942881777882576, + 0.018880700692534447, + 0.023185135796666145, + 0.0075139994733035564, + 0.008416936732828617, + 0.008842015638947487, + 0.00681218970566988, + 0.0035027519334107637, + 0.004720155615359545, + 0.0062196869403123856, + 0.004387080669403076, + 0.0035149925388395786, + 0.005589427892118692, + 0.0037584358360618353, + 0.002754814922809601 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.9513348138721085e-06, + "first_to_last_ratio": 3.6583742473448075, + "imbalance_abs_log_ratio": 1.297018854015045, + "last_quartile_mean": 8.067339791750783e-07, + "mean": 1.2634741715089604e-06, + "normalized": [ + 12.309526402394347, + 3.186602737808338, + 0.716414074901559, + 0.5881410276372666, + 0.41150136269951676, + 0.4806569456453473, + 0.5688709171733375, + 0.42539470014768993, + 0.37578688206301336, + 0.4971639190900172, + 0.5295259641376998, + 0.5986589622164143, + 0.3870590927171303, + 0.5372409907363719, + 0.5606252361769879, + 0.5642027331159046, + 0.2035736261311389, + 0.4495004466120918, + 0.6101764796194364, + 0.688524328430386, + 0.29283475717020535, + 0.6520152158906203, + 0.6017552935102983, + 0.656211682155798, + 0.4004572277104296, + 0.6993974572841979, + 0.7821063899475319, + 0.6457741443048106, + 0.6234421647147482, + 0.8431613839218123, + 0.5292562954255609, + 0.5844411585099919 + ], + "peak_layer": 1, + "peak_normalized": 12.309526402394347, + "population_cv": 2.086670622186628, + "rest_mean": 1.375629770555625e-06, + "spike_contrast": 0.47820565586572594, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.578339366569708e-07, + "top_five_layers": [ + 1, + 2, + 30, + 27, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.555276867293287e-05, + 4.026190254080575e-06, + 9.051706797436054e-07, + 7.431009976244241e-07, + 5.199213433115801e-07, + 6.072976361792826e-07, + 7.18753710771125e-07, + 5.374752163334051e-07, + 4.7479701947850117e-07, + 6.281537707764073e-07, + 6.690423788313637e-07, + 7.563901363027981e-07, + 4.89039166495786e-07, + 6.787901156712906e-07, + 7.08335505805735e-07, + 7.128555807867087e-07, + 2.5721001861711557e-07, + 5.679322043761204e-07, + 7.709422220614215e-07, + 8.699327054273454e-07, + 3.699891522046528e-07, + 8.238043847086374e-07, + 7.603022709190554e-07, + 8.291065114462981e-07, + 5.059673640062101e-07, + 8.836706228976254e-07, + 9.881712230708217e-07, + 8.159189519574284e-07, + 7.877030725467193e-07, + 1.0653126309989602e-06, + 6.687016593787121e-07, + 7.384263085441489e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.5771814105391968e-05, + "first_to_last_ratio": 4.993572550529696, + "imbalance_abs_log_ratio": 1.6081515955891383, + "last_quartile_mean": 3.1584229418513132e-06, + "mean": 8.800217244697706e-06, + "normalized": [ + 2.6265759069342214, + 2.1186852630402058, + 2.2538421407700397, + 1.2139819550758302, + 1.0751886352347675, + 2.0201548997891736, + 1.7460426507992353, + 1.2831874191256807, + 0.8200277209467072, + 1.2322326860980624, + 1.2530395508876175, + 1.027820633392001, + 0.5423322546419725, + 0.6355820979921598, + 0.4727811911372265, + 1.062578905841453, + 0.460994689153592, + 0.8637231601610539, + 1.753316568512886, + 1.8155319115306476, + 0.5982235617014492, + 0.804229051128676, + 0.8286878603973532, + 0.62001658378724, + 0.31698601103139457, + 0.41847205574491303, + 0.5364769069396538, + 0.33379863905874946, + 0.2942071158594462, + 0.4522629804378397, + 0.2919398687178099, + 0.2270791241309413 + ], + "peak_layer": 1, + "peak_normalized": 2.6265759069342214, + "population_cv": 0.6434515693524889, + "rest_mean": 9.397281650712506e-06, + "spike_contrast": 0.5933704723849582, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.57606945221778e-06, + "top_five_layers": [ + 1, + 3, + 2, + 6, + 20 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.311443859071005e-05, + 1.8644890587893315e-05, + 1.9834300474030897e-05, + 1.0683304935810156e-05, + 9.461893569095992e-06, + 1.777780198608525e-05, + 1.5365554645541124e-05, + 1.1292328053968959e-05, + 7.21642209100537e-06, + 1.0843915333680343e-05, + 1.102702026400948e-05, + 9.045044862432405e-06, + 4.772641659656074e-06, + 5.593260539171752e-06, + 4.160577191214543e-06, + 9.350925211037975e-06, + 4.056853413203498e-06, + 7.600951448694104e-06, + 1.5429566701641306e-05, + 1.5977075236150995e-05, + 5.264497303869575e-06, + 7.077390364429448e-06, + 7.2926331995404325e-06, + 5.456280632643029e-06, + 2.7895457606064156e-06, + 3.6826450013904832e-06, + 4.721113327832427e-06, + 2.937500539701432e-06, + 2.5890865344990743e-06, + 3.980012479587458e-06, + 2.5691342671052553e-06, + 1.9983456240879605e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.604969477550185e-06, + "first_to_last_ratio": 4.6554299483226265, + "imbalance_abs_log_ratio": 1.538034269154005, + "last_quartile_mean": 2.063175600142131e-06, + "mean": 5.666794919534368e-06, + "normalized": [ + 1.5974352749631073, + 2.1598435301941388, + 2.226941855679588, + 1.2194393211432404, + 1.1026312268209173, + 2.0507201176776584, + 1.8513987535553562, + 1.3512399358016263, + 0.8771842556929774, + 1.317420097511909, + 1.3400408855092742, + 0.9362708096810504, + 0.5836862581919815, + 0.7178415815889991, + 0.5251745185955801, + 1.0976134976177288, + 0.4952393518791189, + 0.9200124618684795, + 1.9599358762567376, + 1.6538913088117144, + 0.5562244464626633, + 0.9341508255125205, + 0.947277666692926, + 0.665733569627415, + 0.34870485527422174, + 0.4467667219601853, + 0.5556686106558906, + 0.325474706243329, + 0.2993237409593487, + 0.43817195080407734, + 0.2727026444295764, + 0.22583934233666234 + ], + "peak_layer": 3, + "peak_normalized": 2.226941855679588, + "population_cv": 0.5912253593182908, + "rest_mean": 5.991671986083492e-06, + "spike_contrast": 0.6529828016714437, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.912458760169102e-06, + "top_five_layers": [ + 3, + 2, + 6, + 19, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.052338100445922e-06, + 1.223939034389332e-05, + 1.2619622793863527e-05, + 6.910312549734954e-06, + 6.248385034268722e-06, + 1.1621010344242677e-05, + 1.0491497050679754e-05, + 7.657199603272602e-06, + 4.9708232836565e-06, + 7.465549515472958e-06, + 7.5937368819722906e-06, + 5.305654667608906e-06, + 3.3076303225243464e-06, + 4.067861027579056e-06, + 2.976056293846341e-06, + 6.219950591912493e-06, + 2.806419843182084e-06, + 5.213521944824606e-06, + 1.110655466618482e-05, + 9.37226286623627e-06, + 3.152009867335437e-06, + 5.293641152093187e-06, + 5.368028269003844e-06, + 3.772575610128115e-06, + 1.9760389022849267e-06, + 2.5317353902210016e-06, + 3.1488600598095218e-06, + 1.8443984117766377e-06, + 1.6962062545644585e-06, + 2.4830305846990086e-06, + 1.545349959997111e-06, + 1.279785237784381e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 4.9967670520345564e-05, + "first_to_last_ratio": 5.362751833258153, + "imbalance_abs_log_ratio": 1.6794772450644015, + "last_quartile_mean": 9.317542946973845e-06, + "mean": 2.6670801545947143e-05, + "normalized": [ + 3.476508751938897, + 2.0142464518918617, + 2.234943048215937, + 1.2211413922997942, + 1.0839213513178267, + 2.0139857856539773, + 1.6922839548152497, + 1.2509464146134932, + 0.7654291525620647, + 1.1423734022107805, + 1.2080765712998793, + 1.1341630295311584, + 0.5240637082381794, + 0.5667171140044547, + 0.4265924381702442, + 1.0426236214132871, + 0.4450052137430249, + 0.8079063596821926, + 1.5538794579061908, + 1.911286561600711, + 0.633864177631036, + 0.7039911705961566, + 0.7649962092658604, + 0.5862251379064324, + 0.29627943867847506, + 0.39112771210174047, + 0.5128146165033627, + 0.3335988370842466, + 0.2814936032846977, + 0.4583551171276488, + 0.3049685860807425, + 0.21619161263039666 + ], + "peak_layer": 1, + "peak_normalized": 3.476508751938897, + "population_cv": 0.7197953943382049, + "rest_mean": 2.86608817953459e-05, + "spike_contrast": 0.5556133378206015, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.592436819919385e-05, + "top_five_layers": [ + 1, + 3, + 2, + 6, + 20 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.27212749957107e-05, + 5.372156738303602e-05, + 5.9607722505461425e-05, + 3.25688197335694e-05, + 2.890905125241261e-05, + 5.3714615205535665e-05, + 4.513456951826811e-05, + 3.336374356877059e-05, + 2.0414609025465325e-05, + 3.0468014301732183e-05, + 3.222037048544735e-05, + 3.0249037081375718e-05, + 1.3977199159853626e-05, + 1.5114799680304714e-05, + 1.137756225944031e-05, + 2.7807607693830505e-05, + 1.1868645742652006e-05, + 2.154751018679235e-05, + 4.144321064813994e-05, + 5.0975544581888244e-05, + 1.6905665688682348e-05, + 1.877600880106911e-05, + 2.0403062080731615e-05, + 1.5635094314347953e-05, + 7.902010111138225e-06, + 1.0431689588585868e-05, + 1.3677176866622176e-05, + 8.897348379832692e-06, + 7.507660029659746e-06, + 1.222469836648088e-05, + 8.133756637107581e-06, + 5.766003596363589e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bfe704b6792741d68aab4d63062eaa658667914c6d7baa7bccfcf34794944a5a", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0596221424639225, + 0.07634982466697693, + 0.06968431174755096, + 0.11288930475711823, + 0.11885104328393936, + 0.1495160460472107, + 0.11554427444934845, + 0.13194940984249115, + 0.08884616196155548, + 0.10897023230791092, + 0.09843935817480087, + 0.09431816637516022, + 0.09025302529335022, + 0.0895693451166153, + 0.10807885229587555, + 0.2204425185918808, + 0.15380074083805084, + 0.08548793941736221, + 0.09071414917707443, + 0.10860678553581238, + 0.04808543995022774, + 0.05631518363952637, + 0.07238366454839706, + 0.08572373539209366, + 0.06546958535909653, + 0.06323372572660446, + 0.06869969516992569, + 0.07729499042034149, + 0.04753003641963005, + 0.06484207510948181, + 0.07835857570171356, + 0.08203105628490448 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.5725815504529237e-06, + "first_to_last_ratio": 0.39907105360304757, + "imbalance_abs_log_ratio": 0.9186157987408602, + "last_quartile_mean": 6.446424833939091e-06, + "mean": 3.6914688479328106e-06, + "normalized": [ + 1.5294867291228127, + 0.63348823892386, + 1.1487447413186276, + 0.5666523794690017, + 0.30417708299540647, + 0.347931809665085, + 0.480405579791847, + 0.5643067429040126, + 0.6040226655402136, + 0.5371983242313995, + 0.7139156338995314, + 0.9519107124097672, + 0.7033642113963686, + 0.7309137073086721, + 0.64465215578137, + 0.26016492268192315, + 0.27317744038515385, + 0.38409968136196165, + 0.5564685521294294, + 0.6459697208189646, + 1.1764134223250242, + 1.3443189191088418, + 1.4240704391297414, + 1.5037184806193353, + 1.5204890244483313, + 1.618553411169592, + 1.865181309342729, + 1.5801984775975157, + 2.316915113115335, + 1.8792787809683902, + 1.727277195313884, + 1.4625343947258718 + ], + "peak_layer": 29, + "peak_normalized": 2.316915113115335, + "population_cv": 0.5613822274727525, + "rest_mean": 3.422263657908102e-06, + "spike_contrast": 1.5034425714619797, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.145176874066237e-06, + "top_five_layers": [ + 29, + 30, + 27, + 31, + 26 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.646052613883512e-06, + 2.3385020995192463e-06, + 4.240555426804349e-06, + 2.0917796064168215e-06, + 1.122860226132616e-06, + 1.2843794365835492e-06, + 1.7734022321747034e-06, + 2.0831207621085923e-06, + 2.2297308532870375e-06, + 1.9830508790619206e-06, + 2.6353973225923255e-06, + 3.5139487408741843e-06, + 2.5964470751205226e-06, + 2.6981451810570434e-06, + 2.3797133508196566e-06, + 9.603907074051676e-07, + 1.008426011139818e-06, + 1.4178920082486002e-06, + 2.054186325040064e-06, + 2.384577101111063e-06, + 4.342693500802852e-06, + 4.9625114115769975e-06, + 5.256911663309438e-06, + 5.550929927267134e-06, + 5.612837867374765e-06, + 5.9748394960479345e-06, + 6.885258699185215e-06, + 5.833253453602083e-06, + 8.552819963369984e-06, + 6.93729907652596e-06, + 6.376189958245959e-06, + 5.398900157160824e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.0313041968856851e-05, + "first_to_last_ratio": 0.39957890355691367, + "imbalance_abs_log_ratio": 0.9173440275030031, + "last_quartile_mean": 2.5809775934249046e-05, + "mean": 1.4741760296033135e-05, + "normalized": [ + 1.5558178418705222, + 0.6371812985554408, + 1.1430115844766933, + 0.5651347220620626, + 0.30571071286989043, + 0.3485567104343656, + 0.4774182631957655, + 0.563809572533981, + 0.6002213549646791, + 0.5340330387924979, + 0.7130055907666595, + 0.9511132257714032, + 0.6924732073436101, + 0.7238195117723928, + 0.6406610268334267, + 0.24972491904320285, + 0.2665104960045955, + 0.3806326830724979, + 0.5562541410983002, + 0.6421888448376827, + 1.1712220492382042, + 1.3430540694729625, + 1.426395171973413, + 1.5057031409504333, + 1.5356562461314327, + 1.6199050072685577, + 1.8779968028411653, + 1.5872014101530674, + 2.3447342214096167, + 1.8683154808873674, + 1.7248296546827906, + 1.4477079986913193 + ], + "peak_layer": 29, + "peak_normalized": 2.3447342214096167, + "population_cv": 0.5659781309990658, + "rest_mean": 1.3659589142291764e-05, + "spike_contrast": 1.5070354102014207, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0585484526236543e-05, + "top_five_layers": [ + 29, + 27, + 30, + 31, + 26 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.2935493689146824e-05, + 9.393173968419433e-06, + 1.685000279394444e-05, + 8.331080607604235e-06, + 4.5067140490573365e-06, + 5.138339474797249e-06, + 7.037985596980434e-06, + 8.311545570904855e-06, + 8.848319339449517e-06, + 7.872587048041169e-06, + 1.051095750881359e-05, + 1.4021083188708872e-05, + 1.0208274034084752e-05, + 1.0670373740140349e-05, + 9.44447128858883e-06, + 3.681384896481177e-06, + 3.928833848476643e-06, + 5.611195774690714e-06, + 8.200165211746935e-06, + 9.466994015383534e-06, + 1.7265874703298323e-05, + 1.9798981156782247e-05, + 2.1027575712651014e-05, + 2.219671478087548e-05, + 2.2638276277575642e-05, + 2.3880251319496892e-05, + 2.7684978704201058e-05, + 2.339814273000229e-05, + 3.456550984992646e-05, + 2.7542258976609446e-05, + 2.5427025320823304e-05, + 2.134176429535728e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.009145708754658699, + "first_to_last_ratio": 0.39957889743135716, + "imbalance_abs_log_ratio": 0.9173440428330331, + "last_quartile_mean": 0.02288836776278913, + "mean": 0.013073140537017025, + "normalized": [ + 1.5558178691361477, + 0.6371812873883458, + 1.143011598762468, + 0.5651346825162038, + 0.3057107042206736, + 0.3485567100234092, + 0.47741826112733665, + 0.5638095585980932, + 0.6002213650547769, + 0.5340330246694877, + 0.7130055747038815, + 0.9511132667062547, + 0.692473171053213, + 0.7238195005894487, + 0.6406609754578104, + 0.24972491326456617, + 0.2665104811282862, + 0.3806326734056363, + 0.5562541580916099, + 0.6421888466836433, + 1.1712220415317927, + 1.3430540180354118, + 1.4263952722058089, + 1.505703094517941, + 1.5356562665173468, + 1.6199051105332753, + 1.877996734112475, + 1.5872013856346572, + 2.34473428701603, + 1.8683154433096476, + 1.7248296206158347, + 1.4477081033884864 + ], + "peak_layer": 29, + "peak_normalized": 2.34473428701603, + "population_cv": 0.5659781409949523, + "rest_mean": 0.012113460322359094, + "spike_contrast": 1.507035414354221, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.018255413696169853, + "top_five_layers": [ + 29, + 27, + 30, + 31, + 26 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.020339425653219223, + 0.008329960517585278, + 0.01494275126606226, + 0.007388085126876831, + 0.0039965989999473095, + 0.004556730855256319, + 0.006241356022655964, + 0.0073707615956664085, + 0.007846778258681297, + 0.006981488782912493, + 0.009321222081780434, + 0.012434037402272224, + 0.009052799083292484, + 0.00946259405463934, + 0.00837545096874237, + 0.0032646888867020607, + 0.003484128974378109, + 0.004976064432412386, + 0.007271988783031702, + 0.008395425044000149, + 0.015311550348997116, + 0.017557933926582336, + 0.018647465854883194, + 0.019684268161654472, + 0.020075850188732147, + 0.021177247166633606, + 0.02455131523311138, + 0.020749706774950027, + 0.03065304085612297, + 0.024424750357866287, + 0.022548940032720566, + 0.018926091492176056 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.574768711686829e-07, + "first_to_last_ratio": 0.3822604187682079, + "imbalance_abs_log_ratio": 0.9616531781200647, + "last_quartile_mean": 1.7199710953264002e-06, + "mean": 9.503018119083606e-07, + "normalized": [ + 1.710462287653802, + 0.6106596152780777, + 1.1204318353188867, + 0.578372698668433, + 0.26243500094691574, + 0.3526857345218465, + 0.40843249489314853, + 0.4914092526788544, + 0.4957557052633209, + 0.40719059148141235, + 0.6062004989292337, + 0.8649403856141179, + 0.6653960206485303, + 0.7108456658711597, + 0.6350704165794606, + 0.1774114383534533, + 0.2713061284073076, + 0.34448335189062385, + 0.5523846644819398, + 0.6344543697998888, + 1.205861295517406, + 1.3960194260372356, + 1.458471938449579, + 1.5599521119745998, + 1.5844376236777387, + 1.6451437430639262, + 1.8959829898090674, + 1.626547492622029, + 2.4008070962757677, + 1.9696132310887802, + 1.813011017317309, + 1.543823876886148 + ], + "peak_layer": 29, + "peak_normalized": 2.4008070962757677, + "population_cv": 0.6132533733917195, + "rest_mean": 8.727028973385973e-07, + "spike_contrast": 1.569074601174147, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3693359505850822e-06, + "top_five_layers": [ + 29, + 30, + 27, + 31, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.6254554111583275e-06, + 5.803109388580197e-07, + 1.0647484032233479e-06, + 5.496286235029402e-07, + 2.4939245690802636e-07, + 3.351578925503418e-07, + 3.881341399392113e-07, + 4.6698710320924874e-07, + 4.7111754497564107e-07, + 3.869539568768232e-07, + 5.76073432512203e-07, + 8.219544156418124e-07, + 6.323270440589113e-07, + 6.755179242645681e-07, + 6.035085675648588e-07, + 1.6859441132055508e-07, + 2.578227054073068e-07, + 3.273631534739252e-07, + 5.249321475275792e-07, + 6.029231371940114e-07, + 1.1459321740403539e-06, + 1.3266397900224547e-06, + 1.385988525726134e-06, + 1.482425318499736e-06, + 1.5056939446367323e-06, + 1.5633830798833515e-06, + 1.8017560705629876e-06, + 1.5457110293937149e-06, + 2.281491333633312e-06, + 1.8717270222623483e-06, + 1.722907654766459e-06, + 1.4670986274722964e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.837243347490585e-06, + "first_to_last_ratio": 0.40636222975912695, + "imbalance_abs_log_ratio": 0.9005103256217483, + "last_quartile_mean": 1.6825489296934393e-05, + "mean": 9.834350791493307e-06, + "normalized": [ + 1.5251245589223423, + 0.6268534751803682, + 1.1358978319631863, + 0.5342766168184842, + 0.32156135683620435, + 0.34603480684096316, + 0.5056955121723139, + 0.5664834091985615, + 0.6366995092986562, + 0.5714340782015497, + 0.7569982736970645, + 0.9928803137356949, + 0.7186055380199308, + 0.7479928031827885, + 0.6849140027219738, + 0.21832989932248908, + 0.2793170013221158, + 0.4130909093851681, + 0.5875122963256634, + 0.6732858979284184, + 1.1886843754660874, + 1.3704808317703552, + 1.427662092234709, + 1.4830671545938088, + 1.595028118071864, + 1.627225338625226, + 1.8396645878168607, + 1.5251907756173002, + 2.3393243002734927, + 1.8118900136130478, + 1.6024649187818438, + 1.3463294020614671 + ], + "peak_layer": 29, + "peak_normalized": 2.3393243002734927, + "population_cv": 0.5475043977949907, + "rest_mean": 9.082233275505672e-06, + "spike_contrast": 1.5299965279798284, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3895785377826541e-05, + "top_five_layers": [ + 29, + 27, + 30, + 26, + 31 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.4998609913163818e-05, + 6.164696969790384e-06, + 1.1170817742822692e-05, + 5.254263669485226e-06, + 3.162347184115788e-06, + 3.4030276765406597e-06, + 4.973187060386408e-06, + 5.570996563619701e-06, + 6.261526323214639e-06, + 5.619683179247659e-06, + 7.444586572091794e-06, + 9.764333299244754e-06, + 7.06701894159778e-06, + 7.356023616011953e-06, + 6.735684564773692e-06, + 2.1471328182087746e-06, + 2.7469013730296865e-06, + 4.062480911670718e-06, + 5.7778020163823385e-06, + 6.6213297031936236e-06, + 1.1689939128700644e-05, + 1.3477789252647199e-05, + 1.40401298267534e-05, + 1.4585002645617351e-05, + 1.5686066035414115e-05, + 1.6002704796846956e-05, + 1.8091906895278953e-05, + 1.4999261111370288e-05, + 2.300573578395415e-05, + 1.7818761989474297e-05, + 1.5759202142362483e-05, + 1.32402756207739e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.331668606027961e-06, + "first_to_last_ratio": 0.37652711875536754, + "imbalance_abs_log_ratio": 0.9767652057694082, + "last_quartile_mean": 1.1504267263262591e-05, + "mean": 6.6139987815461154e-06, + "normalized": [ + 1.43041458747986, + 0.5915824778695964, + 1.0628524255756664, + 0.4720208928952709, + 0.3249571951503614, + 0.32530495936634546, + 0.5021667837664909, + 0.5300951792850189, + 0.6375122437915461, + 0.5808919253486969, + 0.7210798198798327, + 0.97702603221025, + 0.717856778528966, + 0.720609258730328, + 0.7009724751910035, + 0.1867600754297654, + 0.2849012445298506, + 0.44885820098819357, + 0.6203057532024762, + 0.7121274696413161, + 1.1940331031533082, + 1.4444483655844333, + 1.424043860567118, + 1.4741267205385535, + 1.7485546276719308, + 1.7169348985773352, + 1.9028313045331813, + 1.5326281147396288, + 2.428156586134552, + 1.8387406559537405, + 1.4841764050766164, + 1.263029578608766 + ], + "peak_layer": 29, + "peak_normalized": 2.428156586134552, + "population_cv": 0.5648545726504486, + "rest_mean": 6.054207885965836e-06, + "spike_contrast": 1.5917639101919008, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.636869617679621e-06, + "top_five_layers": [ + 29, + 27, + 30, + 25, + 26 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 9.460760338697582e-06, + 3.912725787813542e-06, + 7.029704647720791e-06, + 3.121945610473631e-06, + 2.149266492779134e-06, + 2.1515666048799176e-06, + 3.321330495964503e-06, + 3.5060488698945846e-06, + 4.216505203658016e-06, + 3.842018486466259e-06, + 4.769221050082706e-06, + 6.462048986577429e-06, + 4.747903858515201e-06, + 4.766108759213239e-06, + 4.636231096810661e-06, + 1.2352309113339288e-06, + 1.8843364841814036e-06, + 2.9687475944228936e-06, + 4.1027014958672225e-06, + 4.710010216513183e-06, + 7.897333489381708e-06, + 9.55357972998172e-06, + 9.418624358659144e-06, + 9.749872333486564e-06, + 1.1564938176888973e-05, + 1.1355805327184498e-05, + 1.2585323929670267e-05, + 1.0136800483451225e-05, + 1.6059824702097103e-05, + 1.2161428458057344e-05, + 9.816340934776235e-06, + 8.353676093975082e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.1292764813551912e-05, + "first_to_last_ratio": 0.37783048366494065, + "imbalance_abs_log_ratio": 0.9733096398618312, + "last_quartile_mean": 5.635533852910157e-05, + "mean": 3.159250317708029e-05, + "normalized": [ + 1.482066067786081, + 0.6128089902703769, + 1.1079572882309123, + 0.5339964494498499, + 0.29044186792909177, + 0.3352417717243252, + 0.469628384430032, + 0.5597117204362547, + 0.5936182437594482, + 0.5198266141043868, + 0.7058453157249553, + 0.9273089878310177, + 0.6699840087338256, + 0.7439837778533973, + 0.6502527329579387, + 0.22627068614548218, + 0.262454163087834, + 0.36907877128447675, + 0.5613139624883691, + 0.6571387234246169, + 1.1835844028068934, + 1.3363166236750326, + 1.4226757890468276, + 1.5079347804524084, + 1.5296526797645231, + 1.6193459917783275, + 1.941711012573152, + 1.6212855176431946, + 2.3766952020840555, + 1.9054387812782299, + 1.7892038011178224, + 1.4872268901268606 + ], + "peak_layer": 29, + "peak_normalized": 2.3766952020840555, + "population_cv": 0.5799347827395998, + "rest_mean": 2.9275527392908567e-05, + "spike_contrast": 1.5065201668165664, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.410417241160758e-05, + "top_five_layers": [ + 29, + 27, + 30, + 31, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.6822176955174655e-05, + 1.9360169972060248e-05, + 3.500314414850436e-05, + 1.6870284525793977e-05, + 9.175785635306966e-06, + 1.0591126738290768e-05, + 1.4836736227152869e-05, + 1.7682694306131452e-05, + 1.8753886251943186e-05, + 1.6422623957623728e-05, + 2.229942037956789e-05, + 2.9296012144186534e-05, + 2.1166471924516372e-05, + 2.350430986552965e-05, + 2.054311153187882e-05, + 7.148457370931283e-06, + 8.291583981190342e-06, + 1.166012225439772e-05, + 1.7733313143253326e-05, + 2.0760657207574695e-05, + 3.739239400601946e-05, + 4.2217587179038674e-05, + 4.4945889385417104e-05, + 4.763943434227258e-05, + 4.832555714529008e-05, + 5.115919339004904e-05, + 6.13435113336891e-05, + 5.1220467867096886e-05, + 7.5085750722792e-05, + 6.019758075126447e-05, + 5.6525426771258935e-05, + 4.6985220251372084e-05 + ] + } + } + } + } + } + }, + "step": 8000 + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.034013546791773, + "cross_entropy_nats": 5.5687538385391235, + "step": 0 + }, + { + "bits_per_byte": 5.132431212443787, + "cross_entropy_nats": 3.5575302243232727, + "step": 100 + }, + { + "bits_per_byte": 3.0911456537212687, + "cross_entropy_nats": 2.1426188945770264, + "step": 500 + }, + { + "bits_per_byte": 1.947918907793682, + "cross_entropy_nats": 1.3501944988965988, + "step": 2000 + }, + { + "bits_per_byte": 1.7911587423283797, + "cross_entropy_nats": 1.241536632180214, + "step": 4000 + }, + { + "bits_per_byte": 1.7093240655856785, + "cross_entropy_nats": 1.1848131567239761, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "91dda38f7c99ebfb81b2efc51836d81bcd8f26bf6deadbc08f763cb6225c959d", + "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1", + "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "final_public_parameters": "f4d3594a7bf1e78aff69c12541a837933d83661dec81d23a744ee66756f670a9", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "bbf2cfc5c0019fb54bf6927274ab9b185b564a77b4b8ef2dc6f5d81575f835c2" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "add6249175ac3bb3fa911069fbe8f25d17d9ab05df34b66ee89996983522ce83", + "1": "619b5ee4f70e975312ead9adf5df4f9d8584ce2839369c749442e429923405cf", + "7999": "fe22994f7750c2f8323fdb5ac51c9be1e208733abaf6a337f5b1ae1df1cd2730" + }, + "parent_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "parent_path": "experiments/k3/attnres_gradient/manifest.json", + "path": "experiments/k3/attnres_spike/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "aggregation_groups": 8, + "blocks_per_group": 4, + "d_ff": 768, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + } + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", + "round05_equivalence": { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true + }, + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073002, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 223.82424076982844, + "measured_steps": 7980, + "median_ms": 227.36876149429008, + "p95_ms": 229.48265075683594, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.018870646365954, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.558257579803467, + "step": 1, + "unclipped_grad_norm": 26.834381103515625 + }, + { + "bits_per_byte": 7.214299345307752, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.000571250915527, + "step": 10, + "unclipped_grad_norm": 12.740906715393066 + }, + { + "bits_per_byte": 6.684460697974439, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.633315086364746, + "step": 20, + "unclipped_grad_norm": 3.454355239868164 + }, + { + "bits_per_byte": 6.541223915307444, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.534030914306641, + "step": 30, + "unclipped_grad_norm": 3.060823440551758 + }, + { + "bits_per_byte": 6.411011034824987, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.443774223327637, + "step": 40, + "unclipped_grad_norm": 3.0002496242523193 + }, + { + "bits_per_byte": 6.085588902515397, + "learning_rate": 3.75e-05, + "loss_nats": 4.2182087898254395, + "step": 50, + "unclipped_grad_norm": 2.976402997970581 + }, + { + "bits_per_byte": 5.879418164452873, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.0753021240234375, + "step": 60, + "unclipped_grad_norm": 2.708517551422119 + }, + { + "bits_per_byte": 5.710178980836628, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9579944610595703, + "step": 70, + "unclipped_grad_norm": 2.559852361679077 + }, + { + "bits_per_byte": 5.512089019607543, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8206889629364014, + "step": 80, + "unclipped_grad_norm": 2.5563650131225586 + }, + { + "bits_per_byte": 5.341444737733749, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.7024073600769043, + "step": 90, + "unclipped_grad_norm": 2.3861565589904785 + }, + { + "bits_per_byte": 5.169528816139475, + "learning_rate": 7.5e-05, + "loss_nats": 3.5832443237304688, + "step": 100, + "unclipped_grad_norm": 2.241894006729126 + }, + { + "bits_per_byte": 5.000991665704052, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.466423273086548, + "step": 110, + "unclipped_grad_norm": 2.1930899620056152 + }, + { + "bits_per_byte": 4.733823840860015, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.2812366485595703, + "step": 120, + "unclipped_grad_norm": 1.8853988647460938 + }, + { + "bits_per_byte": 4.585053344418857, + "learning_rate": 9.75e-05, + "loss_nats": 3.178116798400879, + "step": 130, + "unclipped_grad_norm": 1.6627076864242554 + }, + { + "bits_per_byte": 4.391709416422983, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0441009998321533, + "step": 140, + "unclipped_grad_norm": 1.4586857557296753 + }, + { + "bits_per_byte": 4.260228335873064, + "learning_rate": 0.0001125, + "loss_nats": 2.952965259552002, + "step": 150, + "unclipped_grad_norm": 1.235063076019287 + }, + { + "bits_per_byte": 4.153197684831009, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.878777265548706, + "step": 160, + "unclipped_grad_norm": 1.0407519340515137 + }, + { + "bits_per_byte": 3.997309514401244, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.770723819732666, + "step": 170, + "unclipped_grad_norm": 0.9049285054206848 + }, + { + "bits_per_byte": 3.860879805055398, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.6761579513549805, + "step": 180, + "unclipped_grad_norm": 1.0858614444732666 + }, + { + "bits_per_byte": 3.870312021559318, + "learning_rate": 0.0001425, + "loss_nats": 2.6826958656311035, + "step": 190, + "unclipped_grad_norm": 0.6896197199821472 + }, + { + "bits_per_byte": 3.747811875037455, + "learning_rate": 0.00015, + "loss_nats": 2.597785234451294, + "step": 200, + "unclipped_grad_norm": 0.712465226650238 + }, + { + "bits_per_byte": 3.69472358243852, + "learning_rate": 0.0001575, + "loss_nats": 2.5609872341156006, + "step": 210, + "unclipped_grad_norm": 0.8807040452957153 + }, + { + "bits_per_byte": 3.662194783854328, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.538439989089966, + "step": 220, + "unclipped_grad_norm": 0.7631296515464783 + }, + { + "bits_per_byte": 3.6752317567203328, + "learning_rate": 0.0001725, + "loss_nats": 2.5474765300750732, + "step": 230, + "unclipped_grad_norm": 0.6869579553604126 + }, + { + "bits_per_byte": 3.6565805821995627, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.53454852104187, + "step": 240, + "unclipped_grad_norm": 0.781292736530304 + }, + { + "bits_per_byte": 3.5245367744160205, + "learning_rate": 0.0001875, + "loss_nats": 2.4430227279663086, + "step": 250, + "unclipped_grad_norm": 1.0127525329589844 + }, + { + "bits_per_byte": 3.524651658826797, + "learning_rate": 0.000195, + "loss_nats": 2.4431023597717285, + "step": 260, + "unclipped_grad_norm": 0.8615073561668396 + }, + { + "bits_per_byte": 3.485441678221887, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.415924072265625, + "step": 270, + "unclipped_grad_norm": 0.7731815576553345 + }, + { + "bits_per_byte": 3.4956767097400343, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.423018455505371, + "step": 280, + "unclipped_grad_norm": 0.9061474800109863 + }, + { + "bits_per_byte": 3.3938416545805445, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3524317741394043, + "step": 290, + "unclipped_grad_norm": 1.0149996280670166 + }, + { + "bits_per_byte": 3.483576010425327, + "learning_rate": 0.000225, + "loss_nats": 2.414630889892578, + "step": 300, + "unclipped_grad_norm": 1.1414116621017456 + }, + { + "bits_per_byte": 3.381058528107214, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.343571186065674, + "step": 310, + "unclipped_grad_norm": 1.1472890377044678 + }, + { + "bits_per_byte": 3.3707543595634197, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3364288806915283, + "step": 320, + "unclipped_grad_norm": 1.2828246355056763 + }, + { + "bits_per_byte": 3.3462052120139605, + "learning_rate": 0.0002475, + "loss_nats": 2.3194127082824707, + "step": 330, + "unclipped_grad_norm": 0.9300209879875183 + }, + { + "bits_per_byte": 3.3510090314178607, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.322742462158203, + "step": 340, + "unclipped_grad_norm": 0.7300398945808411 + }, + { + "bits_per_byte": 3.324897249502662, + "learning_rate": 0.0002625, + "loss_nats": 2.304643154144287, + "step": 350, + "unclipped_grad_norm": 0.96440589427948 + }, + { + "bits_per_byte": 3.301325651340707, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.288304567337036, + "step": 360, + "unclipped_grad_norm": 1.186806321144104 + }, + { + "bits_per_byte": 3.330423052074884, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3084733486175537, + "step": 370, + "unclipped_grad_norm": 1.40411376953125 + }, + { + "bits_per_byte": 3.351273196769586, + "learning_rate": 0.000285, + "loss_nats": 2.322925567626953, + "step": 380, + "unclipped_grad_norm": 0.8571750521659851 + }, + { + "bits_per_byte": 3.2969108566930587, + "learning_rate": 0.0002925, + "loss_nats": 2.2852444648742676, + "step": 390, + "unclipped_grad_norm": 1.172652006149292 + }, + { + "bits_per_byte": 3.2316042286462263, + "learning_rate": 0.0003, + "loss_nats": 2.2399773597717285, + "step": 400, + "unclipped_grad_norm": 1.0324530601501465 + }, + { + "bits_per_byte": 3.191360632309631, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.212082624435425, + "step": 410, + "unclipped_grad_norm": 0.9553669691085815 + }, + { + "bits_per_byte": 3.2040835648551598, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2209014892578125, + "step": 420, + "unclipped_grad_norm": 1.0212959051132202 + }, + { + "bits_per_byte": 3.1886670400318176, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2102155685424805, + "step": 430, + "unclipped_grad_norm": 0.8366522789001465 + }, + { + "bits_per_byte": 3.2100664973133752, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.225048542022705, + "step": 440, + "unclipped_grad_norm": 1.1677532196044922 + }, + { + "bits_per_byte": 3.1886219805772913, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.210184335708618, + "step": 450, + "unclipped_grad_norm": 1.3463479280471802 + }, + { + "bits_per_byte": 3.136037597145384, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.1737356185913086, + "step": 460, + "unclipped_grad_norm": 1.704139232635498 + }, + { + "bits_per_byte": 3.1589835224235, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.189640522003174, + "step": 470, + "unclipped_grad_norm": 1.1849488019943237 + }, + { + "bits_per_byte": 3.1210654754918714, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.163357734680176, + "step": 480, + "unclipped_grad_norm": 0.8924318552017212 + }, + { + "bits_per_byte": 3.0823876092087312, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1365482807159424, + "step": 490, + "unclipped_grad_norm": 1.3279240131378174 + }, + { + "bits_per_byte": 3.12389562199447, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1653194427490234, + "step": 500, + "unclipped_grad_norm": 1.3776742219924927 + }, + { + "bits_per_byte": 3.102860080002138, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.1507387161254883, + "step": 510, + "unclipped_grad_norm": 1.061196208000183 + }, + { + "bits_per_byte": 2.961672642602824, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.05287504196167, + "step": 520, + "unclipped_grad_norm": 1.0864008665084839 + }, + { + "bits_per_byte": 3.0521231342057993, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.115570545196533, + "step": 530, + "unclipped_grad_norm": 0.9000556468963623 + }, + { + "bits_per_byte": 2.992112883874938, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.073974609375, + "step": 540, + "unclipped_grad_norm": 0.8655318021774292 + }, + { + "bits_per_byte": 2.9456885750316344, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0417957305908203, + "step": 550, + "unclipped_grad_norm": 0.8011268973350525 + }, + { + "bits_per_byte": 2.9526635034200264, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.046630382537842, + "step": 560, + "unclipped_grad_norm": 1.2049078941345215 + }, + { + "bits_per_byte": 2.943776471919342, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0404703617095947, + "step": 570, + "unclipped_grad_norm": 0.9911091923713684 + }, + { + "bits_per_byte": 2.836173462615215, + "learning_rate": 0.000299626473885547, + "loss_nats": 1.9658856391906738, + "step": 580, + "unclipped_grad_norm": 0.9621384143829346 + }, + { + "bits_per_byte": 2.898327304706431, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.008967399597168, + "step": 590, + "unclipped_grad_norm": 1.0092018842697144 + }, + { + "bits_per_byte": 2.807946982025008, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.9463205337524414, + "step": 600, + "unclipped_grad_norm": 1.041255235671997 + }, + { + "bits_per_byte": 2.8662821213364653, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.98675537109375, + "step": 610, + "unclipped_grad_norm": 1.0585267543792725 + }, + { + "bits_per_byte": 2.7525066546928727, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9078922271728516, + "step": 620, + "unclipped_grad_norm": 0.9587937593460083 + }, + { + "bits_per_byte": 2.7676626258001575, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9183975458145142, + "step": 630, + "unclipped_grad_norm": 0.9836012721061707 + }, + { + "bits_per_byte": 2.7750938241612886, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9235484600067139, + "step": 640, + "unclipped_grad_norm": 0.9689151644706726 + }, + { + "bits_per_byte": 2.6921524390602856, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8660578727722168, + "step": 650, + "unclipped_grad_norm": 1.0390667915344238 + }, + { + "bits_per_byte": 2.6851900654054064, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8612319231033325, + "step": 660, + "unclipped_grad_norm": 1.6513670682907104 + }, + { + "bits_per_byte": 2.6846809967588525, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8608790636062622, + "step": 670, + "unclipped_grad_norm": 0.9175130724906921 + }, + { + "bits_per_byte": 2.7078205745018398, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8769181966781616, + "step": 680, + "unclipped_grad_norm": 1.0189234018325806 + }, + { + "bits_per_byte": 2.5875252455640383, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.793535828590393, + "step": 690, + "unclipped_grad_norm": 0.9444974064826965 + }, + { + "bits_per_byte": 2.5413828162854992, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.761552333831787, + "step": 700, + "unclipped_grad_norm": 1.1904406547546387 + }, + { + "bits_per_byte": 2.623998466245768, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.818817138671875, + "step": 710, + "unclipped_grad_norm": 1.1858620643615723 + }, + { + "bits_per_byte": 2.683836561943116, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.8602937459945679, + "step": 720, + "unclipped_grad_norm": 0.9852609634399414 + }, + { + "bits_per_byte": 2.4849188482154285, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.722414493560791, + "step": 730, + "unclipped_grad_norm": 1.1975579261779785 + }, + { + "bits_per_byte": 2.5244201674309004, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7497947216033936, + "step": 740, + "unclipped_grad_norm": 0.9646193385124207 + }, + { + "bits_per_byte": 2.5296962511940695, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.7534518241882324, + "step": 750, + "unclipped_grad_norm": 1.191988468170166 + }, + { + "bits_per_byte": 2.4988650933565446, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7320812940597534, + "step": 760, + "unclipped_grad_norm": 1.092863917350769 + }, + { + "bits_per_byte": 2.4838112799438723, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.721646785736084, + "step": 770, + "unclipped_grad_norm": 1.0348618030548096 + }, + { + "bits_per_byte": 2.4223032327066463, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.679012656211853, + "step": 780, + "unclipped_grad_norm": 1.034103274345398 + }, + { + "bits_per_byte": 2.485370818621896, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7227277755737305, + "step": 790, + "unclipped_grad_norm": 1.060265302658081 + }, + { + "bits_per_byte": 2.4233009040643014, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.679704189300537, + "step": 800, + "unclipped_grad_norm": 1.3169445991516113 + }, + { + "bits_per_byte": 2.4599308009409677, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7050940990447998, + "step": 810, + "unclipped_grad_norm": 0.9821087121963501 + }, + { + "bits_per_byte": 2.425854846429614, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6814744472503662, + "step": 820, + "unclipped_grad_norm": 1.0924170017242432 + }, + { + "bits_per_byte": 2.4277851797028984, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6828124523162842, + "step": 830, + "unclipped_grad_norm": 1.055789828300476 + }, + { + "bits_per_byte": 2.3982261775338056, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.6623237133026123, + "step": 840, + "unclipped_grad_norm": 1.230089545249939 + }, + { + "bits_per_byte": 2.339956047664374, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.621933937072754, + "step": 850, + "unclipped_grad_norm": 0.94926518201828 + }, + { + "bits_per_byte": 2.3803994878412724, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6499671936035156, + "step": 860, + "unclipped_grad_norm": 1.1705771684646606 + }, + { + "bits_per_byte": 2.3436123988217177, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6244683265686035, + "step": 870, + "unclipped_grad_norm": 1.1496235132217407 + }, + { + "bits_per_byte": 2.3796778486382517, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6494669914245605, + "step": 880, + "unclipped_grad_norm": 1.1764546632766724 + }, + { + "bits_per_byte": 2.357840007579645, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.634330153465271, + "step": 890, + "unclipped_grad_norm": 0.9926145672798157 + }, + { + "bits_per_byte": 2.2929997964819524, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.5893863439559937, + "step": 900, + "unclipped_grad_norm": 1.009945034980774 + }, + { + "bits_per_byte": 2.3411444477818364, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6227576732635498, + "step": 910, + "unclipped_grad_norm": 1.2830723524093628 + }, + { + "bits_per_byte": 2.2736926801307917, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.5760036706924438, + "step": 920, + "unclipped_grad_norm": 1.0900828838348389 + }, + { + "bits_per_byte": 2.3038844064724087, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.596930980682373, + "step": 930, + "unclipped_grad_norm": 1.1090145111083984 + }, + { + "bits_per_byte": 2.2389674911124313, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.551934003829956, + "step": 940, + "unclipped_grad_norm": 0.9897664189338684 + }, + { + "bits_per_byte": 2.2434025797128814, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5550081729888916, + "step": 950, + "unclipped_grad_norm": 1.0138952732086182 + }, + { + "bits_per_byte": 2.2236732459538526, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.5413328409194946, + "step": 960, + "unclipped_grad_norm": 1.0875126123428345 + }, + { + "bits_per_byte": 2.2667958318658914, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5712231397628784, + "step": 970, + "unclipped_grad_norm": 1.245374321937561 + }, + { + "bits_per_byte": 2.2344890628839638, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.5488297939300537, + "step": 980, + "unclipped_grad_norm": 1.1757959127426147 + }, + { + "bits_per_byte": 2.1613678871472266, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.4981460571289062, + "step": 990, + "unclipped_grad_norm": 1.066755771636963 + }, + { + "bits_per_byte": 2.207448574654106, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5300867557525635, + "step": 1000, + "unclipped_grad_norm": 0.9658553004264832 + }, + { + "bits_per_byte": 2.213320750285166, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5341570377349854, + "step": 1010, + "unclipped_grad_norm": 1.1267611980438232 + }, + { + "bits_per_byte": 2.2186436133293697, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.537846565246582, + "step": 1020, + "unclipped_grad_norm": 0.9853240251541138 + }, + { + "bits_per_byte": 2.1280228589020402, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.4750330448150635, + "step": 1030, + "unclipped_grad_norm": 0.972481369972229 + }, + { + "bits_per_byte": 2.2322469250646653, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5472756624221802, + "step": 1040, + "unclipped_grad_norm": 1.0531573295593262 + }, + { + "bits_per_byte": 2.179971250491057, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5110409259796143, + "step": 1050, + "unclipped_grad_norm": 0.9985654950141907 + }, + { + "bits_per_byte": 2.1476089311129005, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.4886090755462646, + "step": 1060, + "unclipped_grad_norm": 1.100019931793213 + }, + { + "bits_per_byte": 2.2640807417567204, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5693411827087402, + "step": 1070, + "unclipped_grad_norm": 1.0959177017211914 + }, + { + "bits_per_byte": 2.1797610876917024, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5108952522277832, + "step": 1080, + "unclipped_grad_norm": 1.088409423828125 + }, + { + "bits_per_byte": 2.1080023585151006, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.461155891418457, + "step": 1090, + "unclipped_grad_norm": 1.0188393592834473 + }, + { + "bits_per_byte": 2.1918825969071642, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5192972421646118, + "step": 1100, + "unclipped_grad_norm": 0.9501848816871643 + }, + { + "bits_per_byte": 2.205643788715952, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.5288357734680176, + "step": 1110, + "unclipped_grad_norm": 0.9797388911247253 + }, + { + "bits_per_byte": 2.0930500148664373, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.4507917165756226, + "step": 1120, + "unclipped_grad_norm": 0.8856902122497559 + }, + { + "bits_per_byte": 2.095190166973774, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4522751569747925, + "step": 1130, + "unclipped_grad_norm": 0.9447507262229919 + }, + { + "bits_per_byte": 2.116739593126884, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4672120809555054, + "step": 1140, + "unclipped_grad_norm": 1.0488460063934326 + }, + { + "bits_per_byte": 2.1023439573190634, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4572337865829468, + "step": 1150, + "unclipped_grad_norm": 0.9838997721672058 + }, + { + "bits_per_byte": 2.0562952585852448, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.425315260887146, + "step": 1160, + "unclipped_grad_norm": 0.9282211065292358 + }, + { + "bits_per_byte": 2.160992449020393, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.497885823249817, + "step": 1170, + "unclipped_grad_norm": 1.0253955125808716 + }, + { + "bits_per_byte": 2.0716469459492117, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4359562397003174, + "step": 1180, + "unclipped_grad_norm": 0.9352066516876221 + }, + { + "bits_per_byte": 2.0893013090255805, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4481933116912842, + "step": 1190, + "unclipped_grad_norm": 1.0826658010482788 + }, + { + "bits_per_byte": 2.0752424152481495, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.438448429107666, + "step": 1200, + "unclipped_grad_norm": 0.9942638874053955 + }, + { + "bits_per_byte": 2.2049758081000004, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.5283727645874023, + "step": 1210, + "unclipped_grad_norm": 1.1482313871383667 + }, + { + "bits_per_byte": 2.0506430487646385, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4213974475860596, + "step": 1220, + "unclipped_grad_norm": 0.9142818450927734 + }, + { + "bits_per_byte": 2.00269480165204, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.3881622552871704, + "step": 1230, + "unclipped_grad_norm": 0.9387761354446411 + }, + { + "bits_per_byte": 2.0166881700394925, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.3978617191314697, + "step": 1240, + "unclipped_grad_norm": 1.0545530319213867 + }, + { + "bits_per_byte": 2.1787518934964396, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.5101957321166992, + "step": 1250, + "unclipped_grad_norm": 1.1504697799682617 + }, + { + "bits_per_byte": 2.1047604855463367, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4589087963104248, + "step": 1260, + "unclipped_grad_norm": 0.9371413588523865 + }, + { + "bits_per_byte": 1.9885089846782722, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.3783293962478638, + "step": 1270, + "unclipped_grad_norm": 0.9821643233299255 + }, + { + "bits_per_byte": 2.1346452228561636, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.4796233177185059, + "step": 1280, + "unclipped_grad_norm": 0.9382128715515137 + }, + { + "bits_per_byte": 2.022505311222276, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4018938541412354, + "step": 1290, + "unclipped_grad_norm": 0.9164487719535828 + }, + { + "bits_per_byte": 2.050535903573151, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4213231801986694, + "step": 1300, + "unclipped_grad_norm": 0.9040083885192871 + }, + { + "bits_per_byte": 2.128253659619498, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4751930236816406, + "step": 1310, + "unclipped_grad_norm": 1.045804738998413 + }, + { + "bits_per_byte": 2.0627867437420613, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4298148155212402, + "step": 1320, + "unclipped_grad_norm": 1.0330690145492554 + }, + { + "bits_per_byte": 1.954123977343318, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.3544955253601074, + "step": 1330, + "unclipped_grad_norm": 0.9727330803871155 + }, + { + "bits_per_byte": 1.9791288789175636, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.3718276023864746, + "step": 1340, + "unclipped_grad_norm": 0.966886043548584 + }, + { + "bits_per_byte": 2.1481475807754027, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4889824390411377, + "step": 1350, + "unclipped_grad_norm": 0.9399362802505493 + }, + { + "bits_per_byte": 1.9898208683390537, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3792387247085571, + "step": 1360, + "unclipped_grad_norm": 0.9223746061325073 + }, + { + "bits_per_byte": 2.0666878262115826, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4325188398361206, + "step": 1370, + "unclipped_grad_norm": 0.9086402654647827 + }, + { + "bits_per_byte": 2.05495087020345, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4243834018707275, + "step": 1380, + "unclipped_grad_norm": 0.954355776309967 + }, + { + "bits_per_byte": 2.067550663170961, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4331169128417969, + "step": 1390, + "unclipped_grad_norm": 0.8977138996124268 + }, + { + "bits_per_byte": 2.0721085473841274, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4362761974334717, + "step": 1400, + "unclipped_grad_norm": 0.8833977580070496 + }, + { + "bits_per_byte": 1.9799110560136877, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.3723697662353516, + "step": 1410, + "unclipped_grad_norm": 0.9529625177383423 + }, + { + "bits_per_byte": 2.038817521691312, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4132006168365479, + "step": 1420, + "unclipped_grad_norm": 1.083083152770996 + }, + { + "bits_per_byte": 2.0292872750764, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4065947532653809, + "step": 1430, + "unclipped_grad_norm": 0.9355389475822449 + }, + { + "bits_per_byte": 1.987968787171912, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3779549598693848, + "step": 1440, + "unclipped_grad_norm": 0.8836045265197754 + }, + { + "bits_per_byte": 2.0423230440638482, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.4156304597854614, + "step": 1450, + "unclipped_grad_norm": 0.924534022808075 + }, + { + "bits_per_byte": 1.9816687187055055, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.3735880851745605, + "step": 1460, + "unclipped_grad_norm": 0.8437807559967041 + }, + { + "bits_per_byte": 1.9797782854072217, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3722777366638184, + "step": 1470, + "unclipped_grad_norm": 0.9143778085708618 + }, + { + "bits_per_byte": 1.8767507025466492, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3008644580841064, + "step": 1480, + "unclipped_grad_norm": 0.8930306434631348 + }, + { + "bits_per_byte": 2.001770738868954, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.387521743774414, + "step": 1490, + "unclipped_grad_norm": 0.8246955871582031 + }, + { + "bits_per_byte": 1.9125255016832254, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3256616592407227, + "step": 1500, + "unclipped_grad_norm": 0.9061107635498047 + }, + { + "bits_per_byte": 1.973825621895558, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3681516647338867, + "step": 1510, + "unclipped_grad_norm": 0.9889123439788818 + }, + { + "bits_per_byte": 1.980869687309597, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3730342388153076, + "step": 1520, + "unclipped_grad_norm": 0.9045401811599731 + }, + { + "bits_per_byte": 1.9378121108395843, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.343189001083374, + "step": 1530, + "unclipped_grad_norm": 0.8896906971931458 + }, + { + "bits_per_byte": 2.017761685746178, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3986058235168457, + "step": 1540, + "unclipped_grad_norm": 0.9600386619567871 + }, + { + "bits_per_byte": 1.9800400430018348, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.3724591732025146, + "step": 1550, + "unclipped_grad_norm": 0.8903042078018188 + }, + { + "bits_per_byte": 2.037820882229562, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.412509799003601, + "step": 1560, + "unclipped_grad_norm": 0.9084417819976807 + }, + { + "bits_per_byte": 2.015308525214271, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3969054222106934, + "step": 1570, + "unclipped_grad_norm": 0.9122006893157959 + }, + { + "bits_per_byte": 1.985575476602505, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.376296043395996, + "step": 1580, + "unclipped_grad_norm": 0.8903481960296631 + }, + { + "bits_per_byte": 1.996281568601366, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3837169408798218, + "step": 1590, + "unclipped_grad_norm": 0.933029294013977 + }, + { + "bits_per_byte": 2.002402775110875, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3879598379135132, + "step": 1600, + "unclipped_grad_norm": 0.9777830243110657 + }, + { + "bits_per_byte": 2.0119596790366705, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3945841789245605, + "step": 1610, + "unclipped_grad_norm": 0.9627588391304016 + }, + { + "bits_per_byte": 2.0159194075901357, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3973288536071777, + "step": 1620, + "unclipped_grad_norm": 0.8731178641319275 + }, + { + "bits_per_byte": 1.8905791395719242, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3104496002197266, + "step": 1630, + "unclipped_grad_norm": 0.8998029828071594 + }, + { + "bits_per_byte": 2.0235616286638756, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.4026260375976562, + "step": 1640, + "unclipped_grad_norm": 0.9714035391807556 + }, + { + "bits_per_byte": 1.9588359580116574, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3577616214752197, + "step": 1650, + "unclipped_grad_norm": 0.9218164682388306 + }, + { + "bits_per_byte": 1.936817019221692, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3424992561340332, + "step": 1660, + "unclipped_grad_norm": 0.8974215984344482 + }, + { + "bits_per_byte": 2.096671109580354, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.4533016681671143, + "step": 1670, + "unclipped_grad_norm": 0.9378093481063843 + }, + { + "bits_per_byte": 2.0150522710644854, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3967278003692627, + "step": 1680, + "unclipped_grad_norm": 0.8720370531082153 + }, + { + "bits_per_byte": 1.8838673446393523, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.3057973384857178, + "step": 1690, + "unclipped_grad_norm": 0.8577803373336792 + }, + { + "bits_per_byte": 1.927341291107104, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3359311819076538, + "step": 1700, + "unclipped_grad_norm": 0.9562329053878784 + }, + { + "bits_per_byte": 1.9962937793695772, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3837254047393799, + "step": 1710, + "unclipped_grad_norm": 0.9462636113166809 + }, + { + "bits_per_byte": 1.995081645646296, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.382885217666626, + "step": 1720, + "unclipped_grad_norm": 0.8408851027488708 + }, + { + "bits_per_byte": 1.894888164889292, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3134363889694214, + "step": 1730, + "unclipped_grad_norm": 0.8187386989593506 + }, + { + "bits_per_byte": 1.9475287866480313, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.349924087524414, + "step": 1740, + "unclipped_grad_norm": 0.9181671142578125 + }, + { + "bits_per_byte": 2.0484275682562236, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.4198617935180664, + "step": 1750, + "unclipped_grad_norm": 0.9259082674980164 + }, + { + "bits_per_byte": 1.9089950418263137, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3232145309448242, + "step": 1760, + "unclipped_grad_norm": 0.9000674486160278 + }, + { + "bits_per_byte": 1.9653874651136254, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3623027801513672, + "step": 1770, + "unclipped_grad_norm": 0.8686579465866089 + }, + { + "bits_per_byte": 1.9032847018697159, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.3192564249038696, + "step": 1780, + "unclipped_grad_norm": 0.8219738006591797 + }, + { + "bits_per_byte": 1.8754542973244455, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.2999658584594727, + "step": 1790, + "unclipped_grad_norm": 0.8401536345481873 + }, + { + "bits_per_byte": 1.9695277754504965, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3651726245880127, + "step": 1800, + "unclipped_grad_norm": 0.9855192303657532 + }, + { + "bits_per_byte": 1.952977025044714, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3537005186080933, + "step": 1810, + "unclipped_grad_norm": 0.9152796268463135 + }, + { + "bits_per_byte": 2.0236675699768067, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.4026994705200195, + "step": 1820, + "unclipped_grad_norm": 0.9160932898521423 + }, + { + "bits_per_byte": 1.863515605666851, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.2916905879974365, + "step": 1830, + "unclipped_grad_norm": 0.8749093413352966 + }, + { + "bits_per_byte": 1.9532654399502112, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.35390043258667, + "step": 1840, + "unclipped_grad_norm": 0.8691527843475342 + }, + { + "bits_per_byte": 1.8425976998077092, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.277191400527954, + "step": 1850, + "unclipped_grad_norm": 0.918012797832489 + }, + { + "bits_per_byte": 1.9936586611930571, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3818988800048828, + "step": 1860, + "unclipped_grad_norm": 0.9178117513656616 + }, + { + "bits_per_byte": 1.995530520365048, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3831963539123535, + "step": 1870, + "unclipped_grad_norm": 0.8769021034240723 + }, + { + "bits_per_byte": 1.8636951555543517, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.2918150424957275, + "step": 1880, + "unclipped_grad_norm": 0.8614026308059692 + }, + { + "bits_per_byte": 1.9426209177403593, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3465222120285034, + "step": 1890, + "unclipped_grad_norm": 0.8975904583930969 + }, + { + "bits_per_byte": 1.861964838104021, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.2906156778335571, + "step": 1900, + "unclipped_grad_norm": 0.8269307613372803 + }, + { + "bits_per_byte": 1.8502344174366214, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.282484769821167, + "step": 1910, + "unclipped_grad_norm": 0.8237605690956116 + }, + { + "bits_per_byte": 1.9149965484108225, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3273744583129883, + "step": 1920, + "unclipped_grad_norm": 0.8641052842140198 + }, + { + "bits_per_byte": 1.8801597426520518, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.303227424621582, + "step": 1930, + "unclipped_grad_norm": 0.8907292485237122 + }, + { + "bits_per_byte": 1.8629336163763313, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2912871837615967, + "step": 1940, + "unclipped_grad_norm": 0.8837904930114746 + }, + { + "bits_per_byte": 1.923144226495448, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.3330219984054565, + "step": 1950, + "unclipped_grad_norm": 1.0997384786605835 + }, + { + "bits_per_byte": 1.88083511852199, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.303695559501648, + "step": 1960, + "unclipped_grad_norm": 0.8385953307151794 + }, + { + "bits_per_byte": 1.8359603733629608, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2725907564163208, + "step": 1970, + "unclipped_grad_norm": 0.839948832988739 + }, + { + "bits_per_byte": 1.7988822896981462, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.2468901872634888, + "step": 1980, + "unclipped_grad_norm": 0.7658324837684631 + }, + { + "bits_per_byte": 1.9198664091526536, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.3307499885559082, + "step": 1990, + "unclipped_grad_norm": 0.8840731382369995 + }, + { + "bits_per_byte": 1.863330208369221, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.2915620803833008, + "step": 2000, + "unclipped_grad_norm": 0.8192090392112732 + }, + { + "bits_per_byte": 1.9331472534175815, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3399555683135986, + "step": 2010, + "unclipped_grad_norm": 0.8343153595924377 + }, + { + "bits_per_byte": 1.9117808168049895, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3251454830169678, + "step": 2020, + "unclipped_grad_norm": 0.913158118724823 + }, + { + "bits_per_byte": 1.8970498148279866, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3149347305297852, + "step": 2030, + "unclipped_grad_norm": 0.8877862691879272 + }, + { + "bits_per_byte": 1.9115138997308505, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.324960470199585, + "step": 2040, + "unclipped_grad_norm": 0.8749900460243225 + }, + { + "bits_per_byte": 1.8657040848990802, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2932075262069702, + "step": 2050, + "unclipped_grad_norm": 0.9185486435890198 + }, + { + "bits_per_byte": 1.7675556498434857, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.225176215171814, + "step": 2060, + "unclipped_grad_norm": 0.7904725670814514 + }, + { + "bits_per_byte": 1.8617049723185672, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.290435552597046, + "step": 2070, + "unclipped_grad_norm": 0.8348630666732788 + }, + { + "bits_per_byte": 1.973646415973359, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3680274486541748, + "step": 2080, + "unclipped_grad_norm": 0.8117388486862183 + }, + { + "bits_per_byte": 1.8279640400110946, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2670481204986572, + "step": 2090, + "unclipped_grad_norm": 0.7758682370185852 + }, + { + "bits_per_byte": 1.8962239541385433, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.3143622875213623, + "step": 2100, + "unclipped_grad_norm": 0.8137099146842957 + }, + { + "bits_per_byte": 1.8350517890184526, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.271960973739624, + "step": 2110, + "unclipped_grad_norm": 0.9251447319984436 + }, + { + "bits_per_byte": 1.870564486595114, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2965764999389648, + "step": 2120, + "unclipped_grad_norm": 0.9138389825820923 + }, + { + "bits_per_byte": 1.8882444750864615, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.3088313341140747, + "step": 2130, + "unclipped_grad_norm": 0.8262670040130615 + }, + { + "bits_per_byte": 1.7726164113277747, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.2286840677261353, + "step": 2140, + "unclipped_grad_norm": 0.7359873056411743 + }, + { + "bits_per_byte": 1.8268258588276844, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2662591934204102, + "step": 2150, + "unclipped_grad_norm": 0.8173519372940063 + }, + { + "bits_per_byte": 1.8637190611428216, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.291831612586975, + "step": 2160, + "unclipped_grad_norm": 0.8174240589141846 + }, + { + "bits_per_byte": 1.8528119214251022, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2842713594436646, + "step": 2170, + "unclipped_grad_norm": 0.8957990407943726 + }, + { + "bits_per_byte": 1.782856774308099, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2357821464538574, + "step": 2180, + "unclipped_grad_norm": 0.8048707246780396 + }, + { + "bits_per_byte": 1.8923490130319534, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.3116763830184937, + "step": 2190, + "unclipped_grad_norm": 0.8952655792236328 + }, + { + "bits_per_byte": 1.9442205283760343, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.3476309776306152, + "step": 2200, + "unclipped_grad_norm": 0.8895230889320374 + }, + { + "bits_per_byte": 1.9876000563684622, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.377699375152588, + "step": 2210, + "unclipped_grad_norm": 0.8686981797218323 + }, + { + "bits_per_byte": 1.852814845130167, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.284273386001587, + "step": 2220, + "unclipped_grad_norm": 0.8053512573242188 + }, + { + "bits_per_byte": 1.859576858996791, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2889604568481445, + "step": 2230, + "unclipped_grad_norm": 0.8349927663803101 + }, + { + "bits_per_byte": 1.8206912376714082, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2620069980621338, + "step": 2240, + "unclipped_grad_norm": 0.8471757769584656 + }, + { + "bits_per_byte": 1.8270817690121683, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2664365768432617, + "step": 2250, + "unclipped_grad_norm": 0.8009039163589478 + }, + { + "bits_per_byte": 1.8453999851208678, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.2791337966918945, + "step": 2260, + "unclipped_grad_norm": 0.8003636002540588 + }, + { + "bits_per_byte": 1.9009572606555896, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.317643165588379, + "step": 2270, + "unclipped_grad_norm": 0.8468067646026611 + }, + { + "bits_per_byte": 1.7920475271702074, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2421526908874512, + "step": 2280, + "unclipped_grad_norm": 0.9071624875068665 + }, + { + "bits_per_byte": 1.8281408381761814, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2671706676483154, + "step": 2290, + "unclipped_grad_norm": 0.822325587272644 + }, + { + "bits_per_byte": 1.8827026781277094, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.3049900531768799, + "step": 2300, + "unclipped_grad_norm": 0.8915430307388306 + }, + { + "bits_per_byte": 1.831042185496237, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.269181728363037, + "step": 2310, + "unclipped_grad_norm": 0.8500391840934753 + }, + { + "bits_per_byte": 1.8431096921593277, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2775462865829468, + "step": 2320, + "unclipped_grad_norm": 0.8417274355888367 + }, + { + "bits_per_byte": 1.7906957435344257, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.241215705871582, + "step": 2330, + "unclipped_grad_norm": 0.777140736579895 + }, + { + "bits_per_byte": 1.825728265549878, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.265498399734497, + "step": 2340, + "unclipped_grad_norm": 0.8567083477973938 + }, + { + "bits_per_byte": 1.7348746466132805, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2025234699249268, + "step": 2350, + "unclipped_grad_norm": 0.7593752145767212 + }, + { + "bits_per_byte": 1.8440781264663364, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2782175540924072, + "step": 2360, + "unclipped_grad_norm": 0.8551052212715149 + }, + { + "bits_per_byte": 1.871989878805465, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.2975645065307617, + "step": 2370, + "unclipped_grad_norm": 0.8494746685028076 + }, + { + "bits_per_byte": 1.7757177744707837, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2308337688446045, + "step": 2380, + "unclipped_grad_norm": 0.7841326594352722 + }, + { + "bits_per_byte": 1.7735628318554726, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2293400764465332, + "step": 2390, + "unclipped_grad_norm": 0.8951802849769592 + }, + { + "bits_per_byte": 1.8680005692360513, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2947993278503418, + "step": 2400, + "unclipped_grad_norm": 0.8339796662330627 + }, + { + "bits_per_byte": 1.8459044102358484, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.279483437538147, + "step": 2410, + "unclipped_grad_norm": 0.9356110692024231 + }, + { + "bits_per_byte": 1.931278833898608, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.338660478591919, + "step": 2420, + "unclipped_grad_norm": 0.8573734760284424 + }, + { + "bits_per_byte": 1.825236739133712, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.265157699584961, + "step": 2430, + "unclipped_grad_norm": 0.8783476948738098 + }, + { + "bits_per_byte": 1.7732133631089195, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.229097843170166, + "step": 2440, + "unclipped_grad_norm": 0.8393950462341309 + }, + { + "bits_per_byte": 1.8696945983470499, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.295973539352417, + "step": 2450, + "unclipped_grad_norm": 0.8328628540039062 + }, + { + "bits_per_byte": 1.8721511985319743, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.2976763248443604, + "step": 2460, + "unclipped_grad_norm": 0.8645251989364624 + }, + { + "bits_per_byte": 1.8203225068679583, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.261751413345337, + "step": 2470, + "unclipped_grad_norm": 0.8000954985618591 + }, + { + "bits_per_byte": 1.8206716316492098, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.261993408203125, + "step": 2480, + "unclipped_grad_norm": 0.7867690920829773 + }, + { + "bits_per_byte": 1.7380601092725614, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2047314643859863, + "step": 2490, + "unclipped_grad_norm": 0.7984139919281006 + }, + { + "bits_per_byte": 1.822108202731867, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2629891633987427, + "step": 2500, + "unclipped_grad_norm": 0.7798662781715393 + }, + { + "bits_per_byte": 1.7878630172920644, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2392522096633911, + "step": 2510, + "unclipped_grad_norm": 0.8921790719032288 + }, + { + "bits_per_byte": 1.7912486677569497, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2415989637374878, + "step": 2520, + "unclipped_grad_norm": 0.8250295519828796 + }, + { + "bits_per_byte": 1.7871405181757898, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2387514114379883, + "step": 2530, + "unclipped_grad_norm": 0.7922425270080566 + }, + { + "bits_per_byte": 1.7956409326773348, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2446434497833252, + "step": 2540, + "unclipped_grad_norm": 0.8244069218635559 + }, + { + "bits_per_byte": 1.833951787983534, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2711985111236572, + "step": 2550, + "unclipped_grad_norm": 0.8060020804405212 + }, + { + "bits_per_byte": 1.7851159384098327, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2373480796813965, + "step": 2560, + "unclipped_grad_norm": 0.7821792960166931 + }, + { + "bits_per_byte": 1.768823849910948, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2260552644729614, + "step": 2570, + "unclipped_grad_norm": 0.7924907207489014 + }, + { + "bits_per_byte": 1.8169237857216074, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2593955993652344, + "step": 2580, + "unclipped_grad_norm": 0.813592255115509 + }, + { + "bits_per_byte": 1.7917042497990852, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2419147491455078, + "step": 2590, + "unclipped_grad_norm": 0.8239418268203735 + }, + { + "bits_per_byte": 1.8340940176357976, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2712970972061157, + "step": 2600, + "unclipped_grad_norm": 0.7961696982383728 + }, + { + "bits_per_byte": 1.7288738279593754, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.198364019393921, + "step": 2610, + "unclipped_grad_norm": 0.7893867492675781 + }, + { + "bits_per_byte": 1.7710386424887592, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2275904417037964, + "step": 2620, + "unclipped_grad_norm": 0.7791327238082886 + }, + { + "bits_per_byte": 1.783225161146247, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.2360374927520752, + "step": 2630, + "unclipped_grad_norm": 0.7614359259605408 + }, + { + "bits_per_byte": 1.8200944578729141, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.2615933418273926, + "step": 2640, + "unclipped_grad_norm": 0.831463098526001 + }, + { + "bits_per_byte": 1.7566091260987187, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2175886631011963, + "step": 2650, + "unclipped_grad_norm": 0.7739477157592773 + }, + { + "bits_per_byte": 1.7470520501902724, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2109642028808594, + "step": 2660, + "unclipped_grad_norm": 0.7929992079734802 + }, + { + "bits_per_byte": 1.775358846678433, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.230584979057312, + "step": 2670, + "unclipped_grad_norm": 0.7438913583755493 + }, + { + "bits_per_byte": 1.7678870604116983, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2254059314727783, + "step": 2680, + "unclipped_grad_norm": 0.8285056352615356 + }, + { + "bits_per_byte": 1.8436208245976922, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2779005765914917, + "step": 2690, + "unclipped_grad_norm": 0.7684076428413391 + }, + { + "bits_per_byte": 1.825501592416041, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2653412818908691, + "step": 2700, + "unclipped_grad_norm": 0.8987715840339661 + }, + { + "bits_per_byte": 1.7152053347994027, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.188889741897583, + "step": 2710, + "unclipped_grad_norm": 0.8146651983261108 + }, + { + "bits_per_byte": 1.8198911743795942, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2614524364471436, + "step": 2720, + "unclipped_grad_norm": 0.8013731241226196 + }, + { + "bits_per_byte": 1.7448410412307802, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.2094316482543945, + "step": 2730, + "unclipped_grad_norm": 0.7481213212013245 + }, + { + "bits_per_byte": 1.7681415947349752, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2255823612213135, + "step": 2740, + "unclipped_grad_norm": 0.8155582547187805 + }, + { + "bits_per_byte": 1.781660806953999, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2349531650543213, + "step": 2750, + "unclipped_grad_norm": 0.816902756690979 + }, + { + "bits_per_byte": 1.869134278870539, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2955851554870605, + "step": 2760, + "unclipped_grad_norm": 0.8422117233276367 + }, + { + "bits_per_byte": 1.7766440730283308, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.231475830078125, + "step": 2770, + "unclipped_grad_norm": 0.8072130084037781 + }, + { + "bits_per_byte": 1.778421169759696, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2327076196670532, + "step": 2780, + "unclipped_grad_norm": 0.8882554173469543 + }, + { + "bits_per_byte": 1.7671284449387423, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2248800992965698, + "step": 2790, + "unclipped_grad_norm": 0.8406802415847778 + }, + { + "bits_per_byte": 1.719355792112675, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.191766619682312, + "step": 2800, + "unclipped_grad_norm": 0.8179776072502136 + }, + { + "bits_per_byte": 1.7902867687906738, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2409322261810303, + "step": 2810, + "unclipped_grad_norm": 0.8046079874038696 + }, + { + "bits_per_byte": 1.7916516231079211, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2418782711029053, + "step": 2820, + "unclipped_grad_norm": 0.7889978885650635 + }, + { + "bits_per_byte": 1.717342391219024, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.190371036529541, + "step": 2830, + "unclipped_grad_norm": 0.7270697355270386 + }, + { + "bits_per_byte": 1.8021411889493455, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2491490840911865, + "step": 2840, + "unclipped_grad_norm": 0.8529788255691528 + }, + { + "bits_per_byte": 1.7264863648000979, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.196709156036377, + "step": 2850, + "unclipped_grad_norm": 0.7487845420837402 + }, + { + "bits_per_byte": 1.698717702026487, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.1774613857269287, + "step": 2860, + "unclipped_grad_norm": 0.8107967972755432 + }, + { + "bits_per_byte": 1.8323354950307253, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.270078182220459, + "step": 2870, + "unclipped_grad_norm": 0.822874128818512 + }, + { + "bits_per_byte": 1.7418315168233318, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2073456048965454, + "step": 2880, + "unclipped_grad_norm": 0.7772940993309021 + }, + { + "bits_per_byte": 1.7973201712803593, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.245807409286499, + "step": 2890, + "unclipped_grad_norm": 0.7877682447433472 + }, + { + "bits_per_byte": 1.7076477291898875, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1836512088775635, + "step": 2900, + "unclipped_grad_norm": 0.8104777932167053 + }, + { + "bits_per_byte": 1.7337485042154308, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2017428874969482, + "step": 2910, + "unclipped_grad_norm": 0.8031578063964844 + }, + { + "bits_per_byte": 1.8229534974608574, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2635750770568848, + "step": 2920, + "unclipped_grad_norm": 0.7966766953468323 + }, + { + "bits_per_byte": 1.766033431400699, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.22412109375, + "step": 2930, + "unclipped_grad_norm": 0.8636581301689148 + }, + { + "bits_per_byte": 1.8135838826418518, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2570805549621582, + "step": 2940, + "unclipped_grad_norm": 0.8673250675201416 + }, + { + "bits_per_byte": 1.7613176671140407, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208523750305176, + "step": 2950, + "unclipped_grad_norm": 0.7721947431564331 + }, + { + "bits_per_byte": 1.7305283010606756, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.1995108127593994, + "step": 2960, + "unclipped_grad_norm": 0.7505601644515991 + }, + { + "bits_per_byte": 1.8081494028572378, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.253313660621643, + "step": 2970, + "unclipped_grad_norm": 0.7939547896385193 + }, + { + "bits_per_byte": 1.72656255311443, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.196761965751648, + "step": 2980, + "unclipped_grad_norm": 0.7610272765159607 + }, + { + "bits_per_byte": 1.8584056571444152, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2881486415863037, + "step": 2990, + "unclipped_grad_norm": 0.833854615688324 + }, + { + "bits_per_byte": 1.7959412143857414, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2448515892028809, + "step": 3000, + "unclipped_grad_norm": 0.7871666550636292 + }, + { + "bits_per_byte": 1.7543190051198296, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.216001272201538, + "step": 3010, + "unclipped_grad_norm": 0.7730440497398376 + }, + { + "bits_per_byte": 1.7304921847039942, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1994857788085938, + "step": 3020, + "unclipped_grad_norm": 0.8199000954627991 + }, + { + "bits_per_byte": 1.7578830015936595, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.218471646308899, + "step": 3030, + "unclipped_grad_norm": 0.9139093160629272 + }, + { + "bits_per_byte": 1.7370483353375354, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.204030156135559, + "step": 3040, + "unclipped_grad_norm": 0.7832452058792114 + }, + { + "bits_per_byte": 1.745498014957076, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2098870277404785, + "step": 3050, + "unclipped_grad_norm": 0.7990915179252625 + }, + { + "bits_per_byte": 1.7370017280391516, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2039978504180908, + "step": 3060, + "unclipped_grad_norm": 0.8140543103218079 + }, + { + "bits_per_byte": 1.7324813360438736, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.200864553451538, + "step": 3070, + "unclipped_grad_norm": 0.8290132284164429 + }, + { + "bits_per_byte": 1.73554830265671, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2029904127120972, + "step": 3080, + "unclipped_grad_norm": 0.8363073468208313 + }, + { + "bits_per_byte": 1.7064868462965634, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.1828465461730957, + "step": 3090, + "unclipped_grad_norm": 0.7568578124046326 + }, + { + "bits_per_byte": 1.8015361539836103, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.2487297058105469, + "step": 3100, + "unclipped_grad_norm": 0.8538148999214172 + }, + { + "bits_per_byte": 1.7730212584879057, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.2289646863937378, + "step": 3110, + "unclipped_grad_norm": 0.8123647570610046 + }, + { + "bits_per_byte": 1.664611306585931, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.1538206338882446, + "step": 3120, + "unclipped_grad_norm": 0.814990758895874 + }, + { + "bits_per_byte": 1.7603762340832176, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2201998233795166, + "step": 3130, + "unclipped_grad_norm": 0.8308927416801453 + }, + { + "bits_per_byte": 1.713619138810495, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1877902746200562, + "step": 3140, + "unclipped_grad_norm": 0.7710813879966736 + }, + { + "bits_per_byte": 1.7093383186478688, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.1848230361938477, + "step": 3150, + "unclipped_grad_norm": 0.7859835624694824 + }, + { + "bits_per_byte": 1.7563532159142348, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.2174112796783447, + "step": 3160, + "unclipped_grad_norm": 0.811743974685669 + }, + { + "bits_per_byte": 1.8161149513145995, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.258834958076477, + "step": 3170, + "unclipped_grad_norm": 0.8695415258407593 + }, + { + "bits_per_byte": 1.6889931150161004, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.1707208156585693, + "step": 3180, + "unclipped_grad_norm": 0.8201394081115723 + }, + { + "bits_per_byte": 1.7704115937437135, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2271558046340942, + "step": 3190, + "unclipped_grad_norm": 0.8031932711601257 + }, + { + "bits_per_byte": 1.6638831320421776, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1533159017562866, + "step": 3200, + "unclipped_grad_norm": 0.7531365156173706 + }, + { + "bits_per_byte": 1.7080410535124106, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1839238405227661, + "step": 3210, + "unclipped_grad_norm": 0.7989531755447388 + }, + { + "bits_per_byte": 1.7142219380017691, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1882081031799316, + "step": 3220, + "unclipped_grad_norm": 0.8516196012496948 + }, + { + "bits_per_byte": 1.687988048404458, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.1700241565704346, + "step": 3230, + "unclipped_grad_norm": 0.8453141450881958 + }, + { + "bits_per_byte": 1.725164850110868, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1957931518554688, + "step": 3240, + "unclipped_grad_norm": 0.8131914734840393 + }, + { + "bits_per_byte": 1.7438406181507111, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2087382078170776, + "step": 3250, + "unclipped_grad_norm": 0.8390238285064697 + }, + { + "bits_per_byte": 1.8351064795014271, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.2719988822937012, + "step": 3260, + "unclipped_grad_norm": 0.8244672417640686 + }, + { + "bits_per_byte": 1.6914478233918653, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1724222898483276, + "step": 3270, + "unclipped_grad_norm": 0.8076629042625427 + }, + { + "bits_per_byte": 1.6199020087370317, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.1228305101394653, + "step": 3280, + "unclipped_grad_norm": 0.7831076979637146 + }, + { + "bits_per_byte": 1.6792207168287738, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.1639471054077148, + "step": 3290, + "unclipped_grad_norm": 0.8022177219390869 + }, + { + "bits_per_byte": 1.7567814527148833, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.2177081108093262, + "step": 3300, + "unclipped_grad_norm": 0.8066262602806091 + }, + { + "bits_per_byte": 1.745768887632185, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.210074782371521, + "step": 3310, + "unclipped_grad_norm": 0.8170367479324341 + }, + { + "bits_per_byte": 1.7488066171943748, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.2121803760528564, + "step": 3320, + "unclipped_grad_norm": 0.8205949664115906 + }, + { + "bits_per_byte": 1.6972405430382262, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.176437497138977, + "step": 3330, + "unclipped_grad_norm": 0.7750418782234192 + }, + { + "bits_per_byte": 1.7083963696690934, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.184170126914978, + "step": 3340, + "unclipped_grad_norm": 0.7907351851463318 + }, + { + "bits_per_byte": 1.7221589373390878, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1937096118927002, + "step": 3350, + "unclipped_grad_norm": 0.7935147285461426 + }, + { + "bits_per_byte": 1.6939794080125659, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.1741770505905151, + "step": 3360, + "unclipped_grad_norm": 0.7715696096420288 + }, + { + "bits_per_byte": 1.6536716621471985, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.146237850189209, + "step": 3370, + "unclipped_grad_norm": 0.7847044467926025 + }, + { + "bits_per_byte": 1.7430156173745224, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.2081663608551025, + "step": 3380, + "unclipped_grad_norm": 0.9182605743408203 + }, + { + "bits_per_byte": 1.7282608817917002, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.197939157485962, + "step": 3390, + "unclipped_grad_norm": 0.8387140035629272 + }, + { + "bits_per_byte": 1.7168453613580306, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1900265216827393, + "step": 3400, + "unclipped_grad_norm": 0.827458918094635 + }, + { + "bits_per_byte": 1.744452876387783, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2091625928878784, + "step": 3410, + "unclipped_grad_norm": 0.8461803197860718 + }, + { + "bits_per_byte": 1.7169175940713928, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1900765895843506, + "step": 3420, + "unclipped_grad_norm": 0.7754979133605957 + }, + { + "bits_per_byte": 1.665198971303929, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1542279720306396, + "step": 3430, + "unclipped_grad_norm": 0.784263551235199 + }, + { + "bits_per_byte": 1.6807204055442975, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1649866104125977, + "step": 3440, + "unclipped_grad_norm": 0.768701434135437 + }, + { + "bits_per_byte": 1.674807642007634, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.1608881950378418, + "step": 3450, + "unclipped_grad_norm": 0.8573304414749146 + }, + { + "bits_per_byte": 1.6607244986764313, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.151126503944397, + "step": 3460, + "unclipped_grad_norm": 0.7546505331993103 + }, + { + "bits_per_byte": 1.7168016777647113, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1899962425231934, + "step": 3470, + "unclipped_grad_norm": 0.8346678614616394 + }, + { + "bits_per_byte": 1.7794081781929978, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.2333917617797852, + "step": 3480, + "unclipped_grad_norm": 0.8080501556396484 + }, + { + "bits_per_byte": 1.7098234817059528, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1851593255996704, + "step": 3490, + "unclipped_grad_norm": 0.7828541398048401 + }, + { + "bits_per_byte": 1.6618220919542377, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.15188729763031, + "step": 3500, + "unclipped_grad_norm": 0.8354499936103821 + }, + { + "bits_per_byte": 1.670962969847596, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.158223271369934, + "step": 3510, + "unclipped_grad_norm": 0.7484581470489502 + }, + { + "bits_per_byte": 1.7077951183216769, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1837533712387085, + "step": 3520, + "unclipped_grad_norm": 0.8215554356575012 + }, + { + "bits_per_byte": 1.6640239858332344, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1534135341644287, + "step": 3530, + "unclipped_grad_norm": 0.7679988741874695 + }, + { + "bits_per_byte": 1.6934656958344385, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.173820972442627, + "step": 3540, + "unclipped_grad_norm": 0.849611759185791 + }, + { + "bits_per_byte": 1.7380399873024104, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.2047175168991089, + "step": 3550, + "unclipped_grad_norm": 0.8348926305770874 + }, + { + "bits_per_byte": 1.6724150193688307, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1592297554016113, + "step": 3560, + "unclipped_grad_norm": 0.82209312915802 + }, + { + "bits_per_byte": 1.707797526078789, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1837550401687622, + "step": 3570, + "unclipped_grad_norm": 0.814924955368042 + }, + { + "bits_per_byte": 1.725196322935976, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1958149671554565, + "step": 3580, + "unclipped_grad_norm": 0.8165888786315918 + }, + { + "bits_per_byte": 1.7086591591596116, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1843522787094116, + "step": 3590, + "unclipped_grad_norm": 0.8172020316123962 + }, + { + "bits_per_byte": 1.7535423314685332, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2154629230499268, + "step": 3600, + "unclipped_grad_norm": 0.8490144610404968 + }, + { + "bits_per_byte": 1.7320164669385913, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.2005423307418823, + "step": 3610, + "unclipped_grad_norm": 0.8558303117752075 + }, + { + "bits_per_byte": 1.671523289324107, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1586116552352905, + "step": 3620, + "unclipped_grad_norm": 0.7700185179710388 + }, + { + "bits_per_byte": 1.6783824733884685, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1633660793304443, + "step": 3630, + "unclipped_grad_norm": 0.8705776333808899 + }, + { + "bits_per_byte": 1.6085100499265301, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1149342060089111, + "step": 3640, + "unclipped_grad_norm": 0.7363900542259216 + }, + { + "bits_per_byte": 1.69346002040696, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1738170385360718, + "step": 3650, + "unclipped_grad_norm": 0.7923496961593628 + }, + { + "bits_per_byte": 1.6919579239343243, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1727758646011353, + "step": 3660, + "unclipped_grad_norm": 0.8016297221183777 + }, + { + "bits_per_byte": 1.7219477426438283, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1935632228851318, + "step": 3670, + "unclipped_grad_norm": 0.8946071863174438 + }, + { + "bits_per_byte": 1.6611216066172736, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1514017581939697, + "step": 3680, + "unclipped_grad_norm": 0.7832621932029724 + }, + { + "bits_per_byte": 1.7299588665036687, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1991161108016968, + "step": 3690, + "unclipped_grad_norm": 0.8317174911499023 + }, + { + "bits_per_byte": 1.6188276331170917, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1220858097076416, + "step": 3700, + "unclipped_grad_norm": 0.7848891615867615 + }, + { + "bits_per_byte": 1.69280218676741, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.17336106300354, + "step": 3710, + "unclipped_grad_norm": 0.7887690663337708 + }, + { + "bits_per_byte": 1.7133850704226707, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1876280307769775, + "step": 3720, + "unclipped_grad_norm": 0.8315525650978088 + }, + { + "bits_per_byte": 1.6847540866376338, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1677825450897217, + "step": 3730, + "unclipped_grad_norm": 0.8175208568572998 + }, + { + "bits_per_byte": 1.6646517225088835, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.153848648071289, + "step": 3740, + "unclipped_grad_norm": 0.795343816280365 + }, + { + "bits_per_byte": 1.6976003307438312, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1766868829727173, + "step": 3750, + "unclipped_grad_norm": 0.8138338327407837 + }, + { + "bits_per_byte": 1.7200370153927425, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1922388076782227, + "step": 3760, + "unclipped_grad_norm": 0.8444334268569946 + }, + { + "bits_per_byte": 1.7215936303657018, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1933177709579468, + "step": 3770, + "unclipped_grad_norm": 0.7583279609680176 + }, + { + "bits_per_byte": 1.7717356881727062, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.2280735969543457, + "step": 3780, + "unclipped_grad_norm": 0.8467698693275452 + }, + { + "bits_per_byte": 1.6571842358084203, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1486725807189941, + "step": 3790, + "unclipped_grad_norm": 0.769040584564209 + }, + { + "bits_per_byte": 1.6766210270783317, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1621451377868652, + "step": 3800, + "unclipped_grad_norm": 0.7948989272117615 + }, + { + "bits_per_byte": 1.6280214816669156, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.1284584999084473, + "step": 3810, + "unclipped_grad_norm": 0.8209450840950012 + }, + { + "bits_per_byte": 1.6241315780697008, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1257622241973877, + "step": 3820, + "unclipped_grad_norm": 0.81658935546875 + }, + { + "bits_per_byte": 1.6981439679032087, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1770637035369873, + "step": 3830, + "unclipped_grad_norm": 0.8478888273239136 + }, + { + "bits_per_byte": 1.649875145129402, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1436063051223755, + "step": 3840, + "unclipped_grad_norm": 0.8229350447654724 + }, + { + "bits_per_byte": 1.621423539249215, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.123885154724121, + "step": 3850, + "unclipped_grad_norm": 0.8153557181358337 + }, + { + "bits_per_byte": 1.6024271955481633, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1107178926467896, + "step": 3860, + "unclipped_grad_norm": 0.794911801815033 + }, + { + "bits_per_byte": 1.5664059452729007, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.085749864578247, + "step": 3870, + "unclipped_grad_norm": 0.75998854637146 + }, + { + "bits_per_byte": 1.6739675067581694, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1603058576583862, + "step": 3880, + "unclipped_grad_norm": 0.8571500182151794 + }, + { + "bits_per_byte": 1.6411153807903556, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.137534499168396, + "step": 3890, + "unclipped_grad_norm": 0.8254691958427429 + }, + { + "bits_per_byte": 1.6427378651185953, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.138659119606018, + "step": 3900, + "unclipped_grad_norm": 0.8312620520591736 + }, + { + "bits_per_byte": 1.6573687731927962, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1488004922866821, + "step": 3910, + "unclipped_grad_norm": 0.8051927089691162 + }, + { + "bits_per_byte": 1.6875147521492837, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1696960926055908, + "step": 3920, + "unclipped_grad_norm": 0.8500935435295105 + }, + { + "bits_per_byte": 1.685968628118027, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1686244010925293, + "step": 3930, + "unclipped_grad_norm": 0.8186569213867188 + }, + { + "bits_per_byte": 1.6142946863883005, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1189438104629517, + "step": 3940, + "unclipped_grad_norm": 0.7923241853713989 + }, + { + "bits_per_byte": 1.7195771337843353, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1919200420379639, + "step": 3950, + "unclipped_grad_norm": 0.8632136583328247 + }, + { + "bits_per_byte": 1.7081789835984027, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1840194463729858, + "step": 3960, + "unclipped_grad_norm": 0.8128528594970703 + }, + { + "bits_per_byte": 1.6418629893736563, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1380527019500732, + "step": 3970, + "unclipped_grad_norm": 0.8211596608161926 + }, + { + "bits_per_byte": 1.6416170541829225, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1378822326660156, + "step": 3980, + "unclipped_grad_norm": 0.8518010973930359 + }, + { + "bits_per_byte": 1.672920992327669, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1595804691314697, + "step": 3990, + "unclipped_grad_norm": 0.8055341243743896 + }, + { + "bits_per_byte": 1.6017095119461129, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.1102204322814941, + "step": 4000, + "unclipped_grad_norm": 0.7798590064048767 + }, + { + "bits_per_byte": 1.6685314791296977, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1565378904342651, + "step": 4010, + "unclipped_grad_norm": 0.8833235502243042 + }, + { + "bits_per_byte": 1.6832612772281446, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.166747808456421, + "step": 4020, + "unclipped_grad_norm": 0.8783014416694641 + }, + { + "bits_per_byte": 1.6355704881438877, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1336910724639893, + "step": 4030, + "unclipped_grad_norm": 0.831271231174469 + }, + { + "bits_per_byte": 1.6221185211413516, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1243668794631958, + "step": 4040, + "unclipped_grad_norm": 0.828468382358551 + }, + { + "bits_per_byte": 1.6444031731269, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1398134231567383, + "step": 4050, + "unclipped_grad_norm": 0.8378818035125732 + }, + { + "bits_per_byte": 1.6213299806871455, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1238203048706055, + "step": 4060, + "unclipped_grad_norm": 0.808568000793457 + }, + { + "bits_per_byte": 1.749336667724334, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.212547779083252, + "step": 4070, + "unclipped_grad_norm": 0.8772934675216675 + }, + { + "bits_per_byte": 1.5885019322904526, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1010656356811523, + "step": 4080, + "unclipped_grad_norm": 0.8058938980102539 + }, + { + "bits_per_byte": 1.6013793052564562, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1099915504455566, + "step": 4090, + "unclipped_grad_norm": 0.8040578365325928 + }, + { + "bits_per_byte": 1.737544161319973, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.204373836517334, + "step": 4100, + "unclipped_grad_norm": 0.8918697834014893 + }, + { + "bits_per_byte": 1.6258001537483717, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1269187927246094, + "step": 4110, + "unclipped_grad_norm": 0.8254836797714233 + }, + { + "bits_per_byte": 1.6466504704257243, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1413711309432983, + "step": 4120, + "unclipped_grad_norm": 0.8797640800476074 + }, + { + "bits_per_byte": 1.7012891866221878, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.179243803024292, + "step": 4130, + "unclipped_grad_norm": 0.8626418709754944 + }, + { + "bits_per_byte": 1.6871897049391529, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1694707870483398, + "step": 4140, + "unclipped_grad_norm": 0.8501971364021301 + }, + { + "bits_per_byte": 1.629858600343432, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1297318935394287, + "step": 4150, + "unclipped_grad_norm": 0.8614740371704102 + }, + { + "bits_per_byte": 1.6422026551091102, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.138288140296936, + "step": 4160, + "unclipped_grad_norm": 0.8621925115585327 + }, + { + "bits_per_byte": 1.6599294228814925, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1505753993988037, + "step": 4170, + "unclipped_grad_norm": 0.8789071440696716 + }, + { + "bits_per_byte": 1.6412854716320588, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1376523971557617, + "step": 4180, + "unclipped_grad_norm": 0.8332215547561646 + }, + { + "bits_per_byte": 1.6753517951149641, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1612653732299805, + "step": 4190, + "unclipped_grad_norm": 0.8483413457870483 + }, + { + "bits_per_byte": 1.6524258198243482, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1453742980957031, + "step": 4200, + "unclipped_grad_norm": 0.8514559268951416 + }, + { + "bits_per_byte": 1.6764456047744516, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1620235443115234, + "step": 4210, + "unclipped_grad_norm": 0.7994921803474426 + }, + { + "bits_per_byte": 1.6553684429806108, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.147413969039917, + "step": 4220, + "unclipped_grad_norm": 0.8137004375457764 + }, + { + "bits_per_byte": 1.654246428166382, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1466362476348877, + "step": 4230, + "unclipped_grad_norm": 0.8624707460403442 + }, + { + "bits_per_byte": 1.670982919835096, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.158237099647522, + "step": 4240, + "unclipped_grad_norm": 0.83209228515625 + }, + { + "bits_per_byte": 1.605450134602379, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1128132343292236, + "step": 4250, + "unclipped_grad_norm": 0.8165753483772278 + }, + { + "bits_per_byte": 1.64798711958823, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.142297625541687, + "step": 4260, + "unclipped_grad_norm": 0.8390073180198669 + }, + { + "bits_per_byte": 1.5630770490828, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.0834424495697021, + "step": 4270, + "unclipped_grad_norm": 0.7599806189537048 + }, + { + "bits_per_byte": 1.634252757072977, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1327776908874512, + "step": 4280, + "unclipped_grad_norm": 0.8349722027778625 + }, + { + "bits_per_byte": 1.622665597953746, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1247460842132568, + "step": 4290, + "unclipped_grad_norm": 0.8544122576713562 + }, + { + "bits_per_byte": 1.5775992641216556, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0935084819793701, + "step": 4300, + "unclipped_grad_norm": 0.8515903353691101 + }, + { + "bits_per_byte": 1.5682213941354084, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.0870082378387451, + "step": 4310, + "unclipped_grad_norm": 0.8175749182701111 + }, + { + "bits_per_byte": 1.709660270170284, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.1850461959838867, + "step": 4320, + "unclipped_grad_norm": 0.9185000061988831 + }, + { + "bits_per_byte": 1.6398645509706304, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1366674900054932, + "step": 4330, + "unclipped_grad_norm": 0.8850803971290588 + }, + { + "bits_per_byte": 1.5869262272432476, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.099973440170288, + "step": 4340, + "unclipped_grad_norm": 0.8476384878158569 + }, + { + "bits_per_byte": 1.5594358323987327, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.080918550491333, + "step": 4350, + "unclipped_grad_norm": 0.8173791170120239 + }, + { + "bits_per_byte": 1.6383724294917448, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1356332302093506, + "step": 4360, + "unclipped_grad_norm": 0.8429195880889893 + }, + { + "bits_per_byte": 1.6201880158854165, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1230287551879883, + "step": 4370, + "unclipped_grad_norm": 0.8339970707893372 + }, + { + "bits_per_byte": 1.5560377991829855, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.0785632133483887, + "step": 4380, + "unclipped_grad_norm": 0.7866214513778687 + }, + { + "bits_per_byte": 1.7055524645544258, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1821988821029663, + "step": 4390, + "unclipped_grad_norm": 0.8178918361663818 + }, + { + "bits_per_byte": 1.6491636529027824, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.143113136291504, + "step": 4400, + "unclipped_grad_norm": 0.8444706797599792 + }, + { + "bits_per_byte": 1.6592194784987309, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.150083303451538, + "step": 4410, + "unclipped_grad_norm": 0.8914594054222107 + }, + { + "bits_per_byte": 1.676016508060549, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1617261171340942, + "step": 4420, + "unclipped_grad_norm": 0.894481897354126 + }, + { + "bits_per_byte": 1.6150391992838857, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.119459867477417, + "step": 4430, + "unclipped_grad_norm": 0.8697190880775452 + }, + { + "bits_per_byte": 1.63267068866769, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1316810846328735, + "step": 4440, + "unclipped_grad_norm": 0.8085072636604309 + }, + { + "bits_per_byte": 1.6597337066248108, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.150439739227295, + "step": 4450, + "unclipped_grad_norm": 0.8258711099624634 + }, + { + "bits_per_byte": 1.6374223973283786, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.134974718093872, + "step": 4460, + "unclipped_grad_norm": 0.8935436010360718 + }, + { + "bits_per_byte": 1.6156965169754833, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.11991548538208, + "step": 4470, + "unclipped_grad_norm": 0.8515243530273438 + }, + { + "bits_per_byte": 1.663532975365021, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1530731916427612, + "step": 4480, + "unclipped_grad_norm": 0.8462327718734741 + }, + { + "bits_per_byte": 1.6896239473794652, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1711580753326416, + "step": 4490, + "unclipped_grad_norm": 0.8624916672706604 + }, + { + "bits_per_byte": 1.6220770733224936, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.124338150024414, + "step": 4500, + "unclipped_grad_norm": 0.830740213394165 + }, + { + "bits_per_byte": 1.6091536090060587, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1153802871704102, + "step": 4510, + "unclipped_grad_norm": 0.8425999879837036 + }, + { + "bits_per_byte": 1.6507618876772505, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.1442209482192993, + "step": 4520, + "unclipped_grad_norm": 0.8468135595321655 + }, + { + "bits_per_byte": 1.6238777316770272, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1255862712860107, + "step": 4530, + "unclipped_grad_norm": 0.8532826900482178 + }, + { + "bits_per_byte": 1.6469891042652731, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1416058540344238, + "step": 4540, + "unclipped_grad_norm": 0.8736191987991333 + }, + { + "bits_per_byte": 1.6559469926181134, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.1478149890899658, + "step": 4550, + "unclipped_grad_norm": 0.8442186713218689 + }, + { + "bits_per_byte": 1.6446332859137545, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1399729251861572, + "step": 4560, + "unclipped_grad_norm": 0.9025750756263733 + }, + { + "bits_per_byte": 1.637351540476223, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1349256038665771, + "step": 4570, + "unclipped_grad_norm": 0.8963979482650757 + }, + { + "bits_per_byte": 1.6074537324849307, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.1142020225524902, + "step": 4580, + "unclipped_grad_norm": 0.8524222373962402 + }, + { + "bits_per_byte": 1.592079171428399, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1035451889038086, + "step": 4590, + "unclipped_grad_norm": 0.8329559564590454 + }, + { + "bits_per_byte": 1.593515914493707, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1045410633087158, + "step": 4600, + "unclipped_grad_norm": 0.856568455696106 + }, + { + "bits_per_byte": 1.6099590037600493, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1159385442733765, + "step": 4610, + "unclipped_grad_norm": 0.8378997445106506 + }, + { + "bits_per_byte": 1.6641679353120065, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1535133123397827, + "step": 4620, + "unclipped_grad_norm": 0.8407614827156067 + }, + { + "bits_per_byte": 1.64286238055782, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1387454271316528, + "step": 4630, + "unclipped_grad_norm": 0.9177886247634888 + }, + { + "bits_per_byte": 1.6574955244064822, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.148888349533081, + "step": 4640, + "unclipped_grad_norm": 0.8678322434425354 + }, + { + "bits_per_byte": 1.6864685816840852, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.1689709424972534, + "step": 4650, + "unclipped_grad_norm": 0.8395454287528992 + }, + { + "bits_per_byte": 1.6055636431519484, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.1128919124603271, + "step": 4660, + "unclipped_grad_norm": 0.8130927681922913 + }, + { + "bits_per_byte": 1.6111651180905502, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.116774559020996, + "step": 4670, + "unclipped_grad_norm": 0.8343866467475891 + }, + { + "bits_per_byte": 1.6120890088909852, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.117414951324463, + "step": 4680, + "unclipped_grad_norm": 0.9091700911521912 + }, + { + "bits_per_byte": 1.598018764258597, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1076622009277344, + "step": 4690, + "unclipped_grad_norm": 0.8087259531021118 + }, + { + "bits_per_byte": 1.6257145063882419, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.126859426498413, + "step": 4700, + "unclipped_grad_norm": 0.8473510146141052 + }, + { + "bits_per_byte": 1.6037934257266175, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.111664891242981, + "step": 4710, + "unclipped_grad_norm": 0.8299785256385803 + }, + { + "bits_per_byte": 1.6360324335441052, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1340112686157227, + "step": 4720, + "unclipped_grad_norm": 0.8852627873420715 + }, + { + "bits_per_byte": 1.6613310814860245, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1515469551086426, + "step": 4730, + "unclipped_grad_norm": 0.8318192958831787 + }, + { + "bits_per_byte": 1.6077376758415052, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1143988370895386, + "step": 4740, + "unclipped_grad_norm": 0.9029622673988342 + }, + { + "bits_per_byte": 1.5803602735986073, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.0954222679138184, + "step": 4750, + "unclipped_grad_norm": 0.8787683248519897 + }, + { + "bits_per_byte": 1.5545300272828713, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.077518105506897, + "step": 4760, + "unclipped_grad_norm": 0.8357442021369934 + }, + { + "bits_per_byte": 1.5563697256991507, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.0787932872772217, + "step": 4770, + "unclipped_grad_norm": 0.8196295499801636 + }, + { + "bits_per_byte": 1.593790398804484, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1047313213348389, + "step": 4780, + "unclipped_grad_norm": 0.9009830951690674 + }, + { + "bits_per_byte": 1.5906680537780697, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1025670766830444, + "step": 4790, + "unclipped_grad_norm": 0.8538159728050232 + }, + { + "bits_per_byte": 1.62577985979557, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.1269047260284424, + "step": 4800, + "unclipped_grad_norm": 0.9271508455276489 + }, + { + "bits_per_byte": 1.5952574108163438, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.105748176574707, + "step": 4810, + "unclipped_grad_norm": 0.8505629301071167 + }, + { + "bits_per_byte": 1.5898409892100704, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1019937992095947, + "step": 4820, + "unclipped_grad_norm": 0.8469997048377991 + }, + { + "bits_per_byte": 1.5923758415011373, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1037508249282837, + "step": 4830, + "unclipped_grad_norm": 0.8425012826919556 + }, + { + "bits_per_byte": 1.5390662072305417, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.0667994022369385, + "step": 4840, + "unclipped_grad_norm": 0.8439065217971802 + }, + { + "bits_per_byte": 1.6086853002477592, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1150556802749634, + "step": 4850, + "unclipped_grad_norm": 0.8472049236297607 + }, + { + "bits_per_byte": 1.5891265732783864, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.1014986038208008, + "step": 4860, + "unclipped_grad_norm": 0.8380921483039856 + }, + { + "bits_per_byte": 1.603959560967351, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.111780047416687, + "step": 4870, + "unclipped_grad_norm": 0.8809474110603333 + }, + { + "bits_per_byte": 1.5692639529649388, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.087730884552002, + "step": 4880, + "unclipped_grad_norm": 0.7983198761940002 + }, + { + "bits_per_byte": 1.525049621185223, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0570838451385498, + "step": 4890, + "unclipped_grad_norm": 0.874304473400116 + }, + { + "bits_per_byte": 1.5072996357549742, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.0447804927825928, + "step": 4900, + "unclipped_grad_norm": 0.8955963253974915 + }, + { + "bits_per_byte": 1.5786693401753242, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.094250202178955, + "step": 4910, + "unclipped_grad_norm": 0.8447177410125732 + }, + { + "bits_per_byte": 1.585648052182035, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.0990874767303467, + "step": 4920, + "unclipped_grad_norm": 0.8577683568000793 + }, + { + "bits_per_byte": 1.5737733380705616, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0908565521240234, + "step": 4930, + "unclipped_grad_norm": 0.8645251393318176 + }, + { + "bits_per_byte": 1.6512401714293001, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.14455246925354, + "step": 4940, + "unclipped_grad_norm": 0.9303934574127197 + }, + { + "bits_per_byte": 1.5141001737353912, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0494942665100098, + "step": 4950, + "unclipped_grad_norm": 0.8813614249229431 + }, + { + "bits_per_byte": 1.592528046147151, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1038563251495361, + "step": 4960, + "unclipped_grad_norm": 0.915719747543335 + }, + { + "bits_per_byte": 1.6794828183886887, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1641287803649902, + "step": 4970, + "unclipped_grad_norm": 0.9402682781219482 + }, + { + "bits_per_byte": 1.543928500735735, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.0701696872711182, + "step": 4980, + "unclipped_grad_norm": 0.9095892310142517 + }, + { + "bits_per_byte": 1.6067152389821258, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.1136901378631592, + "step": 4990, + "unclipped_grad_norm": 0.934682309627533 + }, + { + "bits_per_byte": 1.6283919322968743, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.1287152767181396, + "step": 5000, + "unclipped_grad_norm": 0.8894299268722534 + }, + { + "bits_per_byte": 1.5848010376265358, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.098500370979309, + "step": 5010, + "unclipped_grad_norm": 0.8762696385383606 + }, + { + "bits_per_byte": 1.5972207647585936, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1071090698242188, + "step": 5020, + "unclipped_grad_norm": 0.87384033203125 + }, + { + "bits_per_byte": 1.5759778116893213, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0923845767974854, + "step": 5030, + "unclipped_grad_norm": 0.875623881816864 + }, + { + "bits_per_byte": 1.5723771829108575, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0898888111114502, + "step": 5040, + "unclipped_grad_norm": 0.8759424686431885 + }, + { + "bits_per_byte": 1.6142312247901323, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.1188998222351074, + "step": 5050, + "unclipped_grad_norm": 0.886820912361145 + }, + { + "bits_per_byte": 1.5584469321562717, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.080233097076416, + "step": 5060, + "unclipped_grad_norm": 0.880901038646698 + }, + { + "bits_per_byte": 1.6299523308881525, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1297968626022339, + "step": 5070, + "unclipped_grad_norm": 0.9343949556350708 + }, + { + "bits_per_byte": 1.651922254622622, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1450252532958984, + "step": 5080, + "unclipped_grad_norm": 0.9880313277244568 + }, + { + "bits_per_byte": 1.6199948793684975, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.1228948831558228, + "step": 5090, + "unclipped_grad_norm": 1.0230180025100708 + }, + { + "bits_per_byte": 1.5480533326340287, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.073028802871704, + "step": 5100, + "unclipped_grad_norm": 0.9155668020248413 + }, + { + "bits_per_byte": 1.5785750936826513, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0941848754882812, + "step": 5110, + "unclipped_grad_norm": 0.8749560713768005 + }, + { + "bits_per_byte": 1.553970911684916, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0771305561065674, + "step": 5120, + "unclipped_grad_norm": 0.9114576578140259 + }, + { + "bits_per_byte": 1.6072428817549729, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.114055871963501, + "step": 5130, + "unclipped_grad_norm": 0.89686119556427 + }, + { + "bits_per_byte": 1.5469013928385495, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.072230339050293, + "step": 5140, + "unclipped_grad_norm": 0.9089201092720032 + }, + { + "bits_per_byte": 1.5509646549478342, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0750467777252197, + "step": 5150, + "unclipped_grad_norm": 0.9353929162025452 + }, + { + "bits_per_byte": 1.5786652125917033, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0942473411560059, + "step": 5160, + "unclipped_grad_norm": 0.9100342988967896 + }, + { + "bits_per_byte": 1.5893422395225683, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1016480922698975, + "step": 5170, + "unclipped_grad_norm": 0.8825995922088623 + }, + { + "bits_per_byte": 1.611850984902191, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.1172499656677246, + "step": 5180, + "unclipped_grad_norm": 0.8858869671821594 + }, + { + "bits_per_byte": 1.5765690880429875, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.092794418334961, + "step": 5190, + "unclipped_grad_norm": 0.893822431564331 + }, + { + "bits_per_byte": 1.5180597303062058, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0522388219833374, + "step": 5200, + "unclipped_grad_norm": 0.9365838766098022 + }, + { + "bits_per_byte": 1.6136103674205176, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.118469476699829, + "step": 5210, + "unclipped_grad_norm": 0.9187290668487549 + }, + { + "bits_per_byte": 1.578151672396227, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0938913822174072, + "step": 5220, + "unclipped_grad_norm": 0.9324937462806702 + }, + { + "bits_per_byte": 1.6149896682804372, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1194255352020264, + "step": 5230, + "unclipped_grad_norm": 0.9387596249580383 + }, + { + "bits_per_byte": 1.5729148006774545, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.090261459350586, + "step": 5240, + "unclipped_grad_norm": 0.8688122630119324 + }, + { + "bits_per_byte": 1.5356076361216906, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0644021034240723, + "step": 5250, + "unclipped_grad_norm": 0.890831708908081 + }, + { + "bits_per_byte": 1.5141386978491844, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0495209693908691, + "step": 5260, + "unclipped_grad_norm": 0.8596166968345642 + }, + { + "bits_per_byte": 1.6116535488190005, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.1171131134033203, + "step": 5270, + "unclipped_grad_norm": 0.92216956615448 + }, + { + "bits_per_byte": 1.5994467362087108, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.1086519956588745, + "step": 5280, + "unclipped_grad_norm": 0.9085080623626709 + }, + { + "bits_per_byte": 1.5769201046333985, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.093037724494934, + "step": 5290, + "unclipped_grad_norm": 0.8931360244750977 + }, + { + "bits_per_byte": 1.537008606795619, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.065373182296753, + "step": 5300, + "unclipped_grad_norm": 0.8922452330589294 + }, + { + "bits_per_byte": 1.5906943671236518, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.1025853157043457, + "step": 5310, + "unclipped_grad_norm": 0.887484073638916 + }, + { + "bits_per_byte": 1.5623235930893702, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0829201936721802, + "step": 5320, + "unclipped_grad_norm": 0.9026566743850708 + }, + { + "bits_per_byte": 1.6297041599229574, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.129624843597412, + "step": 5330, + "unclipped_grad_norm": 0.9325461387634277 + }, + { + "bits_per_byte": 1.5387120949524151, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0665539503097534, + "step": 5340, + "unclipped_grad_norm": 0.8923580050468445 + }, + { + "bits_per_byte": 1.5882112816104945, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.1008641719818115, + "step": 5350, + "unclipped_grad_norm": 0.9464812278747559 + }, + { + "bits_per_byte": 1.4963152758270173, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.037166714668274, + "step": 5360, + "unclipped_grad_norm": 0.8723886609077454 + }, + { + "bits_per_byte": 1.5320489711100373, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0619354248046875, + "step": 5370, + "unclipped_grad_norm": 0.9227972626686096 + }, + { + "bits_per_byte": 1.5933907111238788, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1044542789459229, + "step": 5380, + "unclipped_grad_norm": 0.9233055710792542 + }, + { + "bits_per_byte": 1.5140970780476757, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0494921207427979, + "step": 5390, + "unclipped_grad_norm": 0.8707529902458191 + }, + { + "bits_per_byte": 1.5792117734561455, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.0946261882781982, + "step": 5400, + "unclipped_grad_norm": 0.9207195043563843 + }, + { + "bits_per_byte": 1.5488073045754112, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0735514163970947, + "step": 5410, + "unclipped_grad_norm": 0.8852476477622986 + }, + { + "bits_per_byte": 1.5115954184082252, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0477581024169922, + "step": 5420, + "unclipped_grad_norm": 0.9259113073348999 + }, + { + "bits_per_byte": 1.5742989170515984, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0912208557128906, + "step": 5430, + "unclipped_grad_norm": 0.9123548865318298 + }, + { + "bits_per_byte": 1.5198012266288425, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0534459352493286, + "step": 5440, + "unclipped_grad_norm": 0.8873674273490906 + }, + { + "bits_per_byte": 1.5575636292614403, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0796208381652832, + "step": 5450, + "unclipped_grad_norm": 0.851129949092865 + }, + { + "bits_per_byte": 1.5371788696199733, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0654911994934082, + "step": 5460, + "unclipped_grad_norm": 0.8905500173568726 + }, + { + "bits_per_byte": 1.5449696837040585, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0708913803100586, + "step": 5470, + "unclipped_grad_norm": 0.8742530345916748 + }, + { + "bits_per_byte": 1.5184021477640735, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.052476167678833, + "step": 5480, + "unclipped_grad_norm": 0.9149489998817444 + }, + { + "bits_per_byte": 1.6054506505503316, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.1128135919570923, + "step": 5490, + "unclipped_grad_norm": 1.0173310041427612 + }, + { + "bits_per_byte": 1.5652256283400292, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.08493173122406, + "step": 5500, + "unclipped_grad_norm": 0.9233388304710388 + }, + { + "bits_per_byte": 1.5242242764437322, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0565117597579956, + "step": 5510, + "unclipped_grad_norm": 0.8623188734054565 + }, + { + "bits_per_byte": 1.4956853033769069, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0367300510406494, + "step": 5520, + "unclipped_grad_norm": 0.8464505672454834 + }, + { + "bits_per_byte": 1.501370017918526, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.040670394897461, + "step": 5530, + "unclipped_grad_norm": 0.9552764296531677 + }, + { + "bits_per_byte": 1.6303358521995766, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1300626993179321, + "step": 5540, + "unclipped_grad_norm": 0.930858314037323 + }, + { + "bits_per_byte": 1.5750513411491232, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0917423963546753, + "step": 5550, + "unclipped_grad_norm": 0.9512235522270203 + }, + { + "bits_per_byte": 1.6036597952068972, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.111572265625, + "step": 5560, + "unclipped_grad_norm": 0.9797965884208679 + }, + { + "bits_per_byte": 1.5364850916063928, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0650103092193604, + "step": 5570, + "unclipped_grad_norm": 0.8889045715332031 + }, + { + "bits_per_byte": 1.5818145588943031, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0964303016662598, + "step": 5580, + "unclipped_grad_norm": 0.91837477684021 + }, + { + "bits_per_byte": 1.571553042047923, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0893175601959229, + "step": 5590, + "unclipped_grad_norm": 0.9484079480171204 + }, + { + "bits_per_byte": 1.5589711352761015, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0805964469909668, + "step": 5600, + "unclipped_grad_norm": 0.9200659394264221 + }, + { + "bits_per_byte": 1.4898029807694462, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.032652735710144, + "step": 5610, + "unclipped_grad_norm": 0.898281455039978 + }, + { + "bits_per_byte": 1.5617056594248202, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0824918746948242, + "step": 5620, + "unclipped_grad_norm": 0.9515117406845093 + }, + { + "bits_per_byte": 1.481016903084818, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0265626907348633, + "step": 5630, + "unclipped_grad_norm": 0.8537728786468506 + }, + { + "bits_per_byte": 1.471526900410208, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0199847221374512, + "step": 5640, + "unclipped_grad_norm": 0.851723313331604 + }, + { + "bits_per_byte": 1.5525594500692852, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0761522054672241, + "step": 5650, + "unclipped_grad_norm": 0.9591006636619568 + }, + { + "bits_per_byte": 1.4925528113740916, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.0345587730407715, + "step": 5660, + "unclipped_grad_norm": 0.9846746325492859 + }, + { + "bits_per_byte": 1.5646704683430441, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0845469236373901, + "step": 5670, + "unclipped_grad_norm": 0.9337268471717834 + }, + { + "bits_per_byte": 1.5783896963850212, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0940563678741455, + "step": 5680, + "unclipped_grad_norm": 1.0177373886108398 + }, + { + "bits_per_byte": 1.5898387534356093, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1019922494888306, + "step": 5690, + "unclipped_grad_norm": 0.9374083876609802 + }, + { + "bits_per_byte": 1.5923674143512452, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.1037449836730957, + "step": 5700, + "unclipped_grad_norm": 0.9426813721656799 + }, + { + "bits_per_byte": 1.5582474322812707, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.080094814300537, + "step": 5710, + "unclipped_grad_norm": 0.968785285949707 + }, + { + "bits_per_byte": 1.5819518010496918, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0965254306793213, + "step": 5720, + "unclipped_grad_norm": 0.9166642427444458 + }, + { + "bits_per_byte": 1.5197234904706525, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.0533920526504517, + "step": 5730, + "unclipped_grad_norm": 0.9203759431838989 + }, + { + "bits_per_byte": 1.6015581672133536, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.1101155281066895, + "step": 5740, + "unclipped_grad_norm": 0.934572160243988 + }, + { + "bits_per_byte": 1.4943809869527633, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0358259677886963, + "step": 5750, + "unclipped_grad_norm": 0.8915490508079529 + }, + { + "bits_per_byte": 1.50218005620409, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0412318706512451, + "step": 5760, + "unclipped_grad_norm": 0.9438080787658691 + }, + { + "bits_per_byte": 1.5178872317073904, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.052119255065918, + "step": 5770, + "unclipped_grad_norm": 0.9708113670349121 + }, + { + "bits_per_byte": 1.6077987296825615, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.114441156387329, + "step": 5780, + "unclipped_grad_norm": 1.0216169357299805 + }, + { + "bits_per_byte": 1.454466909375221, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0081596374511719, + "step": 5790, + "unclipped_grad_norm": 0.8829967975616455 + }, + { + "bits_per_byte": 1.5674770532224742, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.0864923000335693, + "step": 5800, + "unclipped_grad_norm": 0.9498817920684814 + }, + { + "bits_per_byte": 1.4688064788388602, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.018099069595337, + "step": 5810, + "unclipped_grad_norm": 0.9071933031082153 + }, + { + "bits_per_byte": 1.4812057400354652, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.02669358253479, + "step": 5820, + "unclipped_grad_norm": 0.9959194660186768 + }, + { + "bits_per_byte": 1.5459386339590195, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0715630054473877, + "step": 5830, + "unclipped_grad_norm": 0.9805806875228882 + }, + { + "bits_per_byte": 1.5386066695874363, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0664808750152588, + "step": 5840, + "unclipped_grad_norm": 0.9813235998153687 + }, + { + "bits_per_byte": 1.4904921152514534, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.033130407333374, + "step": 5850, + "unclipped_grad_norm": 0.8884730339050293 + }, + { + "bits_per_byte": 1.505957139182339, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0438499450683594, + "step": 5860, + "unclipped_grad_norm": 0.9569032192230225 + }, + { + "bits_per_byte": 1.5707541826346652, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0887638330459595, + "step": 5870, + "unclipped_grad_norm": 0.9548988342285156 + }, + { + "bits_per_byte": 1.5056532458382643, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0436393022537231, + "step": 5880, + "unclipped_grad_norm": 0.9100088477134705 + }, + { + "bits_per_byte": 1.5621944341185723, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0828306674957275, + "step": 5890, + "unclipped_grad_norm": 1.0087047815322876 + }, + { + "bits_per_byte": 1.564086587243365, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0841422080993652, + "step": 5900, + "unclipped_grad_norm": 0.9735300540924072 + }, + { + "bits_per_byte": 1.5205851235514751, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.053989291191101, + "step": 5910, + "unclipped_grad_norm": 0.928986668586731 + }, + { + "bits_per_byte": 1.4827504882055151, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0277643203735352, + "step": 5920, + "unclipped_grad_norm": 0.9661484956741333 + }, + { + "bits_per_byte": 1.5936927126587939, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.104663610458374, + "step": 5930, + "unclipped_grad_norm": 0.9932063817977905 + }, + { + "bits_per_byte": 1.5284586612906255, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0594468116760254, + "step": 5940, + "unclipped_grad_norm": 1.0194250345230103 + }, + { + "bits_per_byte": 1.5388254315193337, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0666325092315674, + "step": 5950, + "unclipped_grad_norm": 0.9446147084236145 + }, + { + "bits_per_byte": 1.4754302186541906, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0226902961730957, + "step": 5960, + "unclipped_grad_norm": 0.9241304993629456 + }, + { + "bits_per_byte": 1.5277755462013984, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0589733123779297, + "step": 5970, + "unclipped_grad_norm": 0.9580891728401184 + }, + { + "bits_per_byte": 1.5012076662961116, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.040557861328125, + "step": 5980, + "unclipped_grad_norm": 0.954113245010376 + }, + { + "bits_per_byte": 1.4896813890352862, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0325684547424316, + "step": 5990, + "unclipped_grad_norm": 0.8945097923278809 + }, + { + "bits_per_byte": 1.5812009247960246, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0960049629211426, + "step": 6000, + "unclipped_grad_norm": 1.0032626390457153 + }, + { + "bits_per_byte": 1.556720054358958, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0790361166000366, + "step": 6010, + "unclipped_grad_norm": 0.9715147614479065 + }, + { + "bits_per_byte": 1.5090167105411882, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0459706783294678, + "step": 6020, + "unclipped_grad_norm": 0.9192096590995789 + }, + { + "bits_per_byte": 1.4969094758857484, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0375785827636719, + "step": 6030, + "unclipped_grad_norm": 0.9740291833877563 + }, + { + "bits_per_byte": 1.5263979651679873, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.058018445968628, + "step": 6040, + "unclipped_grad_norm": 0.9185873866081238 + }, + { + "bits_per_byte": 1.5301623214300724, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0606276988983154, + "step": 6050, + "unclipped_grad_norm": 0.9192250370979309 + }, + { + "bits_per_byte": 1.5237477125181913, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0561814308166504, + "step": 6060, + "unclipped_grad_norm": 0.9670439958572388 + }, + { + "bits_per_byte": 1.4068365721927425, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 0.9751448035240173, + "step": 6070, + "unclipped_grad_norm": 0.917002260684967 + }, + { + "bits_per_byte": 1.4967151354902735, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0374438762664795, + "step": 6080, + "unclipped_grad_norm": 0.9365084171295166 + }, + { + "bits_per_byte": 1.4969400887976019, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.037599802017212, + "step": 6090, + "unclipped_grad_norm": 0.9915803074836731 + }, + { + "bits_per_byte": 1.5907201645212812, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.1026031970977783, + "step": 6100, + "unclipped_grad_norm": 1.0093804597854614 + }, + { + "bits_per_byte": 1.5313995646203795, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0614852905273438, + "step": 6110, + "unclipped_grad_norm": 1.0098049640655518 + }, + { + "bits_per_byte": 1.4730523865233613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0210421085357666, + "step": 6120, + "unclipped_grad_norm": 0.9365134239196777 + }, + { + "bits_per_byte": 1.5512489422697104, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0752438306808472, + "step": 6130, + "unclipped_grad_norm": 0.9237281680107117 + }, + { + "bits_per_byte": 1.497698360305256, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0381253957748413, + "step": 6140, + "unclipped_grad_norm": 0.9138990044593811 + }, + { + "bits_per_byte": 1.4838755987074594, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0285441875457764, + "step": 6150, + "unclipped_grad_norm": 0.9429982900619507 + }, + { + "bits_per_byte": 1.551023816979731, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.0750877857208252, + "step": 6160, + "unclipped_grad_norm": 1.0046567916870117 + }, + { + "bits_per_byte": 1.4928049379402566, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.034733533859253, + "step": 6170, + "unclipped_grad_norm": 0.964278519153595 + }, + { + "bits_per_byte": 1.4789379768011883, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0251216888427734, + "step": 6180, + "unclipped_grad_norm": 0.9606515169143677 + }, + { + "bits_per_byte": 1.501445346319604, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.0407226085662842, + "step": 6190, + "unclipped_grad_norm": 1.0252128839492798 + }, + { + "bits_per_byte": 1.4915814533620186, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0338854789733887, + "step": 6200, + "unclipped_grad_norm": 0.8806238770484924 + }, + { + "bits_per_byte": 1.5186611536362729, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0526556968688965, + "step": 6210, + "unclipped_grad_norm": 0.989686906337738 + }, + { + "bits_per_byte": 1.539449556559315, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0670651197433472, + "step": 6220, + "unclipped_grad_norm": 0.9960981011390686 + }, + { + "bits_per_byte": 1.4808485320696232, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.026445984840393, + "step": 6230, + "unclipped_grad_norm": 0.9705929160118103 + }, + { + "bits_per_byte": 1.552468127281677, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0760889053344727, + "step": 6240, + "unclipped_grad_norm": 1.0799938440322876 + }, + { + "bits_per_byte": 1.4757860507588256, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0229369401931763, + "step": 6250, + "unclipped_grad_norm": 0.9792200922966003 + }, + { + "bits_per_byte": 1.445374530589406, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.0018572807312012, + "step": 6260, + "unclipped_grad_norm": 0.9422674179077148 + }, + { + "bits_per_byte": 1.5013631386124917, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.040665626525879, + "step": 6270, + "unclipped_grad_norm": 0.9675420522689819 + }, + { + "bits_per_byte": 1.5119997496204038, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.048038363456726, + "step": 6280, + "unclipped_grad_norm": 0.9240654110908508 + }, + { + "bits_per_byte": 1.5072342823476461, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0447351932525635, + "step": 6290, + "unclipped_grad_norm": 0.9724128842353821 + }, + { + "bits_per_byte": 1.4760084243263911, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0230910778045654, + "step": 6300, + "unclipped_grad_norm": 0.9724760055541992 + }, + { + "bits_per_byte": 1.539534687971492, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0671241283416748, + "step": 6310, + "unclipped_grad_norm": 1.0172570943832397 + }, + { + "bits_per_byte": 1.498671266161187, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.03879976272583, + "step": 6320, + "unclipped_grad_norm": 0.9810369610786438 + }, + { + "bits_per_byte": 1.5103010769778318, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.046860933303833, + "step": 6330, + "unclipped_grad_norm": 0.9735086560249329 + }, + { + "bits_per_byte": 1.4197871237766881, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 0.9841214418411255, + "step": 6340, + "unclipped_grad_norm": 0.9049187302589417 + }, + { + "bits_per_byte": 1.5345103868091858, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0636415481567383, + "step": 6350, + "unclipped_grad_norm": 0.9579975008964539 + }, + { + "bits_per_byte": 1.5509206273892133, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0750162601470947, + "step": 6360, + "unclipped_grad_norm": 0.989890456199646 + }, + { + "bits_per_byte": 1.4729708667468522, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0209856033325195, + "step": 6370, + "unclipped_grad_norm": 1.021422266960144 + }, + { + "bits_per_byte": 1.4625091620948676, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0137341022491455, + "step": 6380, + "unclipped_grad_norm": 1.0111291408538818 + }, + { + "bits_per_byte": 1.576993369242666, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0930885076522827, + "step": 6390, + "unclipped_grad_norm": 1.0084502696990967 + }, + { + "bits_per_byte": 1.5426105976821733, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0692561864852905, + "step": 6400, + "unclipped_grad_norm": 1.060890793800354 + }, + { + "bits_per_byte": 1.542634847235945, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0692729949951172, + "step": 6410, + "unclipped_grad_norm": 1.0786292552947998 + }, + { + "bits_per_byte": 1.4677484415707522, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0173656940460205, + "step": 6420, + "unclipped_grad_norm": 1.0185354948043823 + }, + { + "bits_per_byte": 1.5249625979638863, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.057023525238037, + "step": 6430, + "unclipped_grad_norm": 0.9899823069572449 + }, + { + "bits_per_byte": 1.447091949340922, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.0030477046966553, + "step": 6440, + "unclipped_grad_norm": 0.9853183031082153 + }, + { + "bits_per_byte": 1.4600725118974434, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.01204514503479, + "step": 6450, + "unclipped_grad_norm": 0.9329874515533447 + }, + { + "bits_per_byte": 1.4881399085356029, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.031499981880188, + "step": 6460, + "unclipped_grad_norm": 0.9553210735321045 + }, + { + "bits_per_byte": 1.5093300629310604, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0461878776550293, + "step": 6470, + "unclipped_grad_norm": 1.0880736112594604 + }, + { + "bits_per_byte": 1.4620568477230984, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.013420581817627, + "step": 6480, + "unclipped_grad_norm": 0.978254497051239 + }, + { + "bits_per_byte": 1.5301041912940807, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0605874061584473, + "step": 6490, + "unclipped_grad_norm": 1.0214275121688843 + }, + { + "bits_per_byte": 1.5703515712489953, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.088484764099121, + "step": 6500, + "unclipped_grad_norm": 1.024044156074524 + }, + { + "bits_per_byte": 1.5418132861127734, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.068703532218933, + "step": 6510, + "unclipped_grad_norm": 1.0472792387008667 + }, + { + "bits_per_byte": 1.5326505664227554, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 1.0623524188995361, + "step": 6520, + "unclipped_grad_norm": 0.9873340129852295 + }, + { + "bits_per_byte": 1.493115194642413, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0349485874176025, + "step": 6530, + "unclipped_grad_norm": 0.9528921246528625 + }, + { + "bits_per_byte": 1.4717057623671055, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.020108699798584, + "step": 6540, + "unclipped_grad_norm": 0.9596993327140808 + }, + { + "bits_per_byte": 1.532975785615537, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0625778436660767, + "step": 6550, + "unclipped_grad_norm": 0.9932572245597839 + }, + { + "bits_per_byte": 1.5382425823155597, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.0662285089492798, + "step": 6560, + "unclipped_grad_norm": 1.0132641792297363 + }, + { + "bits_per_byte": 1.5160394501065202, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0508384704589844, + "step": 6570, + "unclipped_grad_norm": 0.9644629955291748 + }, + { + "bits_per_byte": 1.503854823258192, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0423927307128906, + "step": 6580, + "unclipped_grad_norm": 0.9727261662483215 + }, + { + "bits_per_byte": 1.5231672710715294, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0557790994644165, + "step": 6590, + "unclipped_grad_norm": 1.029261589050293 + }, + { + "bits_per_byte": 1.415425385776831, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9810981154441833, + "step": 6600, + "unclipped_grad_norm": 0.9430753588676453 + }, + { + "bits_per_byte": 1.4673490978554489, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0170888900756836, + "step": 6610, + "unclipped_grad_norm": 0.9699459671974182 + }, + { + "bits_per_byte": 1.5770974187464382, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.093160629272461, + "step": 6620, + "unclipped_grad_norm": 1.053364872932434 + }, + { + "bits_per_byte": 1.520643081704816, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0540294647216797, + "step": 6630, + "unclipped_grad_norm": 1.0116336345672607 + }, + { + "bits_per_byte": 1.4843385760035823, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.028865098953247, + "step": 6640, + "unclipped_grad_norm": 0.9483272433280945 + }, + { + "bits_per_byte": 1.481005552229861, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.026554822921753, + "step": 6650, + "unclipped_grad_norm": 0.9847526550292969 + }, + { + "bits_per_byte": 1.5181830418668742, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0523242950439453, + "step": 6660, + "unclipped_grad_norm": 1.0222764015197754 + }, + { + "bits_per_byte": 1.483456133022005, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.028253436088562, + "step": 6670, + "unclipped_grad_norm": 0.9609436392784119 + }, + { + "bits_per_byte": 1.5070292790278177, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.044593095779419, + "step": 6680, + "unclipped_grad_norm": 1.0481340885162354 + }, + { + "bits_per_byte": 1.4999456576040804, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0396831035614014, + "step": 6690, + "unclipped_grad_norm": 1.0560411214828491 + }, + { + "bits_per_byte": 1.5028304945896531, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.0416827201843262, + "step": 6700, + "unclipped_grad_norm": 1.0250314474105835 + }, + { + "bits_per_byte": 1.5313854620430087, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0614755153656006, + "step": 6710, + "unclipped_grad_norm": 0.9873788356781006 + }, + { + "bits_per_byte": 1.5848861690387128, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0985593795776367, + "step": 6720, + "unclipped_grad_norm": 1.065596103668213 + }, + { + "bits_per_byte": 1.4535899698384716, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0075517892837524, + "step": 6730, + "unclipped_grad_norm": 0.9767746329307556 + }, + { + "bits_per_byte": 1.4828168735087481, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0278103351593018, + "step": 6740, + "unclipped_grad_norm": 0.9589064717292786 + }, + { + "bits_per_byte": 1.4599156637198565, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0119364261627197, + "step": 6750, + "unclipped_grad_norm": 0.9927419424057007 + }, + { + "bits_per_byte": 1.4484303183299363, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0039753913879395, + "step": 6760, + "unclipped_grad_norm": 0.9825301766395569 + }, + { + "bits_per_byte": 1.469467236183475, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 1.018557071685791, + "step": 6770, + "unclipped_grad_norm": 1.0425026416778564 + }, + { + "bits_per_byte": 1.4605650702095145, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0123865604400635, + "step": 6780, + "unclipped_grad_norm": 1.0209197998046875 + }, + { + "bits_per_byte": 1.494792713418929, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0361113548278809, + "step": 6790, + "unclipped_grad_norm": 0.9975507855415344 + }, + { + "bits_per_byte": 1.4806505800384802, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0263087749481201, + "step": 6800, + "unclipped_grad_norm": 1.0428087711334229 + }, + { + "bits_per_byte": 1.4777557680591573, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0243022441864014, + "step": 6810, + "unclipped_grad_norm": 0.9905012249946594 + }, + { + "bits_per_byte": 1.4914748241184836, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0338115692138672, + "step": 6820, + "unclipped_grad_norm": 1.0114696025848389 + }, + { + "bits_per_byte": 1.4751165222990168, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.022472858428955, + "step": 6830, + "unclipped_grad_norm": 0.9909161329269409 + }, + { + "bits_per_byte": 1.462379831141419, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.0136444568634033, + "step": 6840, + "unclipped_grad_norm": 1.0066962242126465 + }, + { + "bits_per_byte": 1.542950607382929, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0694918632507324, + "step": 6850, + "unclipped_grad_norm": 1.0904512405395508 + }, + { + "bits_per_byte": 1.5094851912821385, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.046295404434204, + "step": 6860, + "unclipped_grad_norm": 1.0577117204666138 + }, + { + "bits_per_byte": 1.4431473552607326, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0003135204315186, + "step": 6870, + "unclipped_grad_norm": 0.931652843952179 + }, + { + "bits_per_byte": 1.4721159409894133, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0203930139541626, + "step": 6880, + "unclipped_grad_norm": 1.0439471006393433 + }, + { + "bits_per_byte": 1.4601951355275085, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 1.0121301412582397, + "step": 6890, + "unclipped_grad_norm": 0.9945755004882812 + }, + { + "bits_per_byte": 1.5078214311176918, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0451421737670898, + "step": 6900, + "unclipped_grad_norm": 1.0577389001846313 + }, + { + "bits_per_byte": 1.51248938422741, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0483777523040771, + "step": 6910, + "unclipped_grad_norm": 1.042457938194275 + }, + { + "bits_per_byte": 1.4546720846777004, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.008301854133606, + "step": 6920, + "unclipped_grad_norm": 1.0035841464996338 + }, + { + "bits_per_byte": 1.4388924184870613, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9973642230033875, + "step": 6930, + "unclipped_grad_norm": 1.0526496171951294 + }, + { + "bits_per_byte": 1.5114688391771902, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0476703643798828, + "step": 6940, + "unclipped_grad_norm": 1.0117003917694092 + }, + { + "bits_per_byte": 1.4256293744264978, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9881709814071655, + "step": 6950, + "unclipped_grad_norm": 1.020249843597412 + }, + { + "bits_per_byte": 1.4392803253560824, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 0.9976330995559692, + "step": 6960, + "unclipped_grad_norm": 1.007938265800476 + }, + { + "bits_per_byte": 1.425250496646647, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 0.9879083633422852, + "step": 6970, + "unclipped_grad_norm": 1.0789695978164673 + }, + { + "bits_per_byte": 1.5310960152416067, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0612748861312866, + "step": 6980, + "unclipped_grad_norm": 1.0816211700439453 + }, + { + "bits_per_byte": 1.4834984407541174, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.0282827615737915, + "step": 6990, + "unclipped_grad_norm": 1.0542293787002563 + }, + { + "bits_per_byte": 1.494992041311279, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0362495183944702, + "step": 7000, + "unclipped_grad_norm": 1.040958046913147 + }, + { + "bits_per_byte": 1.4039343649594325, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9731331467628479, + "step": 7010, + "unclipped_grad_norm": 1.0020650625228882 + }, + { + "bits_per_byte": 1.4906596263533938, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0332465171813965, + "step": 7020, + "unclipped_grad_norm": 1.0739529132843018 + }, + { + "bits_per_byte": 1.4642052549976767, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0149097442626953, + "step": 7030, + "unclipped_grad_norm": 1.0350828170776367 + }, + { + "bits_per_byte": 1.4574465088014192, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0102249383926392, + "step": 7040, + "unclipped_grad_norm": 1.0089728832244873 + }, + { + "bits_per_byte": 1.4534154074478458, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0074307918548584, + "step": 7050, + "unclipped_grad_norm": 1.0608088970184326 + }, + { + "bits_per_byte": 1.4416395833606184, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 0.9992684125900269, + "step": 7060, + "unclipped_grad_norm": 0.9864199757575989 + }, + { + "bits_per_byte": 1.4906718371216052, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.0332549810409546, + "step": 7070, + "unclipped_grad_norm": 1.0150113105773926 + }, + { + "bits_per_byte": 1.4931912109740944, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 1.035001277923584, + "step": 7080, + "unclipped_grad_norm": 1.0090219974517822 + }, + { + "bits_per_byte": 1.4509221749582877, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0057026147842407, + "step": 7090, + "unclipped_grad_norm": 1.0420514345169067 + }, + { + "bits_per_byte": 1.4389093587781714, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9973759651184082, + "step": 7100, + "unclipped_grad_norm": 1.0253499746322632 + }, + { + "bits_per_byte": 1.41304419998431, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 0.979447603225708, + "step": 7110, + "unclipped_grad_norm": 0.9803867936134338 + }, + { + "bits_per_byte": 1.4722433801337027, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0204813480377197, + "step": 7120, + "unclipped_grad_norm": 1.0725315809249878 + }, + { + "bits_per_byte": 1.5197811046586915, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0534319877624512, + "step": 7130, + "unclipped_grad_norm": 1.0701161623001099 + }, + { + "bits_per_byte": 1.465326925846604, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0156872272491455, + "step": 7140, + "unclipped_grad_norm": 1.0326207876205444 + }, + { + "bits_per_byte": 1.5100518741167317, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.046688199043274, + "step": 7150, + "unclipped_grad_norm": 1.0673091411590576 + }, + { + "bits_per_byte": 1.5022746466620645, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.041297435760498, + "step": 7160, + "unclipped_grad_norm": 1.059691071510315 + }, + { + "bits_per_byte": 1.514605802728928, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.049844741821289, + "step": 7170, + "unclipped_grad_norm": 1.085069179534912 + }, + { + "bits_per_byte": 1.4489440305080636, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0043314695358276, + "step": 7180, + "unclipped_grad_norm": 0.9880333542823792 + }, + { + "bits_per_byte": 1.5073161460894569, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0447919368743896, + "step": 7190, + "unclipped_grad_norm": 1.0678606033325195 + }, + { + "bits_per_byte": 1.4434393818018978, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0005159378051758, + "step": 7200, + "unclipped_grad_norm": 1.0179466009140015 + }, + { + "bits_per_byte": 1.4303239848470999, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 0.9914250373840332, + "step": 7210, + "unclipped_grad_norm": 1.005952000617981 + }, + { + "bits_per_byte": 1.4493132772594661, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0045874118804932, + "step": 7220, + "unclipped_grad_norm": 1.0151865482330322 + }, + { + "bits_per_byte": 1.475039818036732, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 1.0224196910858154, + "step": 7230, + "unclipped_grad_norm": 1.0841153860092163 + }, + { + "bits_per_byte": 1.5119313005253603, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0479909181594849, + "step": 7240, + "unclipped_grad_norm": 1.09213387966156 + }, + { + "bits_per_byte": 1.494972951237033, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.03623628616333, + "step": 7250, + "unclipped_grad_norm": 1.0454415082931519 + }, + { + "bits_per_byte": 1.4613747645297765, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 1.0129477977752686, + "step": 7260, + "unclipped_grad_norm": 1.0101300477981567 + }, + { + "bits_per_byte": 1.4686286487778681, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0179758071899414, + "step": 7270, + "unclipped_grad_norm": 1.0528877973556519 + }, + { + "bits_per_byte": 1.4749450555961066, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.022354006767273, + "step": 7280, + "unclipped_grad_norm": 1.0265198945999146 + }, + { + "bits_per_byte": 1.473522931056122, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 1.0213682651519775, + "step": 7290, + "unclipped_grad_norm": 1.066664457321167 + }, + { + "bits_per_byte": 1.4559291058728565, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.0091731548309326, + "step": 7300, + "unclipped_grad_norm": 1.0080223083496094 + }, + { + "bits_per_byte": 1.445694074354709, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0020787715911865, + "step": 7310, + "unclipped_grad_norm": 1.0805282592773438 + }, + { + "bits_per_byte": 1.4250067972303744, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9877394437789917, + "step": 7320, + "unclipped_grad_norm": 1.0912657976150513 + }, + { + "bits_per_byte": 1.4783119599520478, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0246877670288086, + "step": 7330, + "unclipped_grad_norm": 1.0849530696868896 + }, + { + "bits_per_byte": 1.4380544330207323, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9967833757400513, + "step": 7340, + "unclipped_grad_norm": 1.042580485343933 + }, + { + "bits_per_byte": 1.4567071553853599, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.0097124576568604, + "step": 7350, + "unclipped_grad_norm": 1.0958631038665771 + }, + { + "bits_per_byte": 1.4481006275882324, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0037468671798706, + "step": 7360, + "unclipped_grad_norm": 1.0337251424789429 + }, + { + "bits_per_byte": 1.4027950658887918, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 0.9723434448242188, + "step": 7370, + "unclipped_grad_norm": 1.0347329378128052 + }, + { + "bits_per_byte": 1.4160821015291507, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.981553316116333, + "step": 7380, + "unclipped_grad_norm": 1.0372161865234375 + }, + { + "bits_per_byte": 1.421894943145663, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 0.9855824708938599, + "step": 7390, + "unclipped_grad_norm": 1.038377046585083 + }, + { + "bits_per_byte": 1.4461825050831596, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 1.0024173259735107, + "step": 7400, + "unclipped_grad_norm": 1.057859182357788 + }, + { + "bits_per_byte": 1.475963364871865, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 1.0230598449707031, + "step": 7410, + "unclipped_grad_norm": 1.0708297491073608 + }, + { + "bits_per_byte": 1.469500256852441, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0185799598693848, + "step": 7420, + "unclipped_grad_norm": 1.062455654144287 + }, + { + "bits_per_byte": 1.41574011402791, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9813162684440613, + "step": 7430, + "unclipped_grad_norm": 1.0484251976013184 + }, + { + "bits_per_byte": 1.3821044351027654, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9580017924308777, + "step": 7440, + "unclipped_grad_norm": 1.030645728111267 + }, + { + "bits_per_byte": 1.4987479704234719, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0388529300689697, + "step": 7450, + "unclipped_grad_norm": 1.0465614795684814 + }, + { + "bits_per_byte": 1.4339140366925356, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.993913471698761, + "step": 7460, + "unclipped_grad_norm": 1.0371458530426025 + }, + { + "bits_per_byte": 1.4012647642614147, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9712827205657959, + "step": 7470, + "unclipped_grad_norm": 1.0733681917190552 + }, + { + "bits_per_byte": 1.5010874504231586, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.040474534034729, + "step": 7480, + "unclipped_grad_norm": 1.0303142070770264 + }, + { + "bits_per_byte": 1.4650142613873354, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.0154705047607422, + "step": 7490, + "unclipped_grad_norm": 1.0710126161575317 + }, + { + "bits_per_byte": 1.472811954777455, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0208754539489746, + "step": 7500, + "unclipped_grad_norm": 1.0901119709014893 + }, + { + "bits_per_byte": 1.4645168875610401, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0151257514953613, + "step": 7510, + "unclipped_grad_norm": 1.024397611618042 + }, + { + "bits_per_byte": 1.4743771688829574, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.0219603776931763, + "step": 7520, + "unclipped_grad_norm": 1.0934079885482788 + }, + { + "bits_per_byte": 1.46691879726299, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0167906284332275, + "step": 7530, + "unclipped_grad_norm": 1.081713318824768 + }, + { + "bits_per_byte": 1.4640198577000465, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0147812366485596, + "step": 7540, + "unclipped_grad_norm": 1.0780924558639526 + }, + { + "bits_per_byte": 1.4520784143200383, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0065040588378906, + "step": 7550, + "unclipped_grad_norm": 1.04312264919281 + }, + { + "bits_per_byte": 1.3812490793886991, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 0.9574089050292969, + "step": 7560, + "unclipped_grad_norm": 1.0638409852981567 + }, + { + "bits_per_byte": 1.4099118799641457, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.9772764444351196, + "step": 7570, + "unclipped_grad_norm": 1.0123975276947021 + }, + { + "bits_per_byte": 1.3981073347742243, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9690941572189331, + "step": 7580, + "unclipped_grad_norm": 1.0492113828659058 + }, + { + "bits_per_byte": 1.472455950690169, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 1.0206286907196045, + "step": 7590, + "unclipped_grad_norm": 1.0436540842056274 + }, + { + "bits_per_byte": 1.5055485084038889, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 1.0435667037963867, + "step": 7600, + "unclipped_grad_norm": 1.140743374824524 + }, + { + "bits_per_byte": 1.4701950667619266, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.01906156539917, + "step": 7610, + "unclipped_grad_norm": 1.082565426826477 + }, + { + "bits_per_byte": 1.458967179400348, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0112789869308472, + "step": 7620, + "unclipped_grad_norm": 1.0694279670715332 + }, + { + "bits_per_byte": 1.4490995028244436, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 1.0044392347335815, + "step": 7630, + "unclipped_grad_norm": 1.0964275598526 + }, + { + "bits_per_byte": 1.4314776444690875, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9922246932983398, + "step": 7640, + "unclipped_grad_norm": 1.15219247341156 + }, + { + "bits_per_byte": 1.4066524647649938, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 0.9750171899795532, + "step": 7650, + "unclipped_grad_norm": 1.064915418624878 + }, + { + "bits_per_byte": 1.416584462852321, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9819015264511108, + "step": 7660, + "unclipped_grad_norm": 1.141569972038269 + }, + { + "bits_per_byte": 1.4608237321164121, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.0125658512115479, + "step": 7670, + "unclipped_grad_norm": 1.0876517295837402 + }, + { + "bits_per_byte": 1.4539619683122877, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0078096389770508, + "step": 7680, + "unclipped_grad_norm": 1.0938481092453003 + }, + { + "bits_per_byte": 1.433180616677931, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9934051036834717, + "step": 7690, + "unclipped_grad_norm": 1.070209264755249 + }, + { + "bits_per_byte": 1.4952543148538446, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.0364313125610352, + "step": 7700, + "unclipped_grad_norm": 1.0864635705947876 + }, + { + "bits_per_byte": 1.412102165014209, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9787946343421936, + "step": 7710, + "unclipped_grad_norm": 1.040504813194275 + }, + { + "bits_per_byte": 1.4180753804526507, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9829349517822266, + "step": 7720, + "unclipped_grad_norm": 1.0190634727478027 + }, + { + "bits_per_byte": 1.4255394275000965, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9881086349487305, + "step": 7730, + "unclipped_grad_norm": 1.0293750762939453 + }, + { + "bits_per_byte": 1.468768986620972, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 1.0180730819702148, + "step": 7740, + "unclipped_grad_norm": 1.0531129837036133 + }, + { + "bits_per_byte": 1.4037740771288283, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9730220437049866, + "step": 7750, + "unclipped_grad_norm": 1.1033527851104736 + }, + { + "bits_per_byte": 1.4111542826339787, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9781376123428345, + "step": 7760, + "unclipped_grad_norm": 1.0707266330718994 + }, + { + "bits_per_byte": 1.4583370349675868, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 1.010842204093933, + "step": 7770, + "unclipped_grad_norm": 1.0769577026367188 + }, + { + "bits_per_byte": 1.4516912813729461, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0062357187271118, + "step": 7780, + "unclipped_grad_norm": 1.1653907299041748 + }, + { + "bits_per_byte": 1.4586383485718983, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 1.011051058769226, + "step": 7790, + "unclipped_grad_norm": 1.0739812850952148 + }, + { + "bits_per_byte": 1.4426457678594913, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 0.9999658465385437, + "step": 7800, + "unclipped_grad_norm": 1.0972295999526978 + }, + { + "bits_per_byte": 1.4198621082124645, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9841734170913696, + "step": 7810, + "unclipped_grad_norm": 1.075659990310669 + }, + { + "bits_per_byte": 1.4512271001982673, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 1.0059139728546143, + "step": 7820, + "unclipped_grad_norm": 1.1029958724975586 + }, + { + "bits_per_byte": 1.4583721194283628, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0108665227890015, + "step": 7830, + "unclipped_grad_norm": 1.081362009048462 + }, + { + "bits_per_byte": 1.4242661539444337, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9872260689735413, + "step": 7840, + "unclipped_grad_norm": 1.0629743337631226 + }, + { + "bits_per_byte": 1.423362471105475, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9865996837615967, + "step": 7850, + "unclipped_grad_norm": 1.0624163150787354 + }, + { + "bits_per_byte": 1.4807702799634808, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0263917446136475, + "step": 7860, + "unclipped_grad_norm": 1.0853880643844604 + }, + { + "bits_per_byte": 1.4488509878939468, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0042669773101807, + "step": 7870, + "unclipped_grad_norm": 1.1049087047576904 + }, + { + "bits_per_byte": 1.4220910033676464, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9857183694839478, + "step": 7880, + "unclipped_grad_norm": 1.0985115766525269 + }, + { + "bits_per_byte": 1.4594506226319235, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0116140842437744, + "step": 7890, + "unclipped_grad_norm": 1.0736068487167358 + }, + { + "bits_per_byte": 1.4105420243969071, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 0.9777132272720337, + "step": 7900, + "unclipped_grad_norm": 1.1535842418670654 + }, + { + "bits_per_byte": 1.463574250651661, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0144723653793335, + "step": 7910, + "unclipped_grad_norm": 1.1884382963180542 + }, + { + "bits_per_byte": 1.4830432026772835, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0279672145843506, + "step": 7920, + "unclipped_grad_norm": 1.1013721227645874 + }, + { + "bits_per_byte": 1.435883066062264, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9952782988548279, + "step": 7930, + "unclipped_grad_norm": 1.0824542045593262 + }, + { + "bits_per_byte": 1.4594231054077853, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 1.0115950107574463, + "step": 7940, + "unclipped_grad_norm": 1.1213319301605225 + }, + { + "bits_per_byte": 1.408353201199376, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9761960506439209, + "step": 7950, + "unclipped_grad_norm": 1.0871061086654663 + }, + { + "bits_per_byte": 1.4735662706841393, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 1.0213983058929443, + "step": 7960, + "unclipped_grad_norm": 1.1091856956481934 + }, + { + "bits_per_byte": 1.4267374586460069, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9889390468597412, + "step": 7970, + "unclipped_grad_norm": 1.0478838682174683 + }, + { + "bits_per_byte": 1.472857186214632, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0209068059921265, + "step": 7980, + "unclipped_grad_norm": 1.2172881364822388 + }, + { + "bits_per_byte": 1.4136101948882995, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9798399209976196, + "step": 7990, + "unclipped_grad_norm": 1.1755717992782593 + }, + { + "bits_per_byte": 1.46345644253582, + "learning_rate": 3e-05, + "loss_nats": 1.0143907070159912, + "step": 8000, + "unclipped_grad_norm": 1.0856302976608276 + } + ] +} diff --git a/experiments/k3/attnres_spike/results/raw/formal-seed-2026073003.json b/experiments/k3/attnres_spike/results/raw/formal-seed-2026073003.json new file mode 100644 index 0000000..0df4475 --- /dev/null +++ b/experiments/k3/attnres_spike/results/raw/formal-seed-2026073003.json @@ -0,0 +1,86460 @@ +{ + "architecture": "block", + "artifacts": { + "protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", + "runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", + "scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" + }, + "batch_size": 32, + "canonical_sha256_without_self": "7e764c07e90b78c4cd0acc2e99600225f16428cbb25d5188a0a5a8fe797f8766", + "depth": 32, + "diagnostics": [ + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "initialization_negative_control": { + "comparisons": { + "detached_learned": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + }, + "uniform_value_backward": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + } + }, + "passed": true + }, + "loss_scale_gate": { + "checks": { + "attention_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "mlp_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_attention_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 5.551115123125783e-17, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_mlp_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_attention_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_mlp_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 8.06433666787865, + "forward": { + "activation_sha256": { + "attention_branch_output": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", + "mlp_branch_output": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", + "post_attention_state": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", + "post_mlp_state": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", + "pre_attention_input": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", + "pre_mlp_input": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248" + }, + "logits_sha256": "30ae47372d19e5c17da2b2a4846b2d27ef6e44fdc06045752d040a56bc6e9398", + "loss_nats": 5.5897722244262695, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.5897722244262695, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0021227982360869646, + 0.00224129855632782, + 0.0024903982412070036, + 0.0023722455371171236, + 0.0023577234242111444, + 0.0023993616923689842, + 0.002688874490559101, + 0.0029664887115359306, + 0.0029547461308538914, + 0.003052416956052184, + 0.0028214550111442804, + 0.0032544767018407583, + 0.00348988501355052, + 0.003914494998753071, + 0.0035071265883743763, + 0.004211232997477055, + 0.004072706215083599, + 0.004050548188388348, + 0.003929397091269493, + 0.004410331137478352, + 0.004824656527489424, + 0.00433384021744132, + 0.004475373309105635, + 0.004755842965096235, + 0.004843301139771938, + 0.0050001065246760845, + 0.005346097983419895, + 0.004958339501172304, + 0.0054801348596811295, + 0.005455824080854654, + 0.005286100786179304, + 0.005402065813541412 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015727665777376387, + "first_to_last_ratio": 3.444367030996094, + "imbalance_abs_log_ratio": 1.2367401520242614, + "last_quartile_mean": 4.566199140754179e-05, + "mean": 8.846047569477378e-05, + "normalized": [ + 2.4938583213844665, + 2.2321521039252294, + 1.9412946714691228, + 1.7875630514690435, + 1.605255150569201, + 1.4809330135294976, + 1.3782219007164203, + 1.3041730210393685, + 1.1983684502302745, + 1.1165155187552953, + 1.0705163494358503, + 1.028915549335682, + 0.969265519878856, + 0.9296915475356099, + 0.8760186941641476, + 0.840544602365157, + 0.8215144482528359, + 0.7958412144142111, + 0.7578169321029499, + 0.7149281644739428, + 0.6789722877716906, + 0.648159768212111, + 0.6127699836305072, + 0.5872278577962867, + 0.5712871725092085, + 0.5525140937217092, + 0.5351005407841478, + 0.5196042162209058, + 0.49974197043571306, + 0.49405312503381205, + 0.48454882285128825, + 0.4726319359854579 + ], + "peak_layer": 1, + "peak_normalized": 2.4938583213844665, + "population_cv": 0.5254866190950809, + "rest_mean": 9.469065830840177e-05, + "spike_contrast": 0.5789112734082525, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.48174895811826e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00022060789342503995, + 0.0001974572369363159, + 0.00017172785010188818, + 0.00015812867786735296, + 0.00014200163423083723, + 0.0001310040388489142, + 0.00012191816495032981, + 0.00011536776582943276, + 0.00010600824316497892, + 9.876749390969053e-05, + 9.469838551012799e-05, + 9.101835894398391e-05, + 8.574168896302581e-05, + 8.224095654441044e-05, + 7.749303040327504e-05, + 7.435497536789626e-05, + 7.267155888257548e-05, + 7.040049240458757e-05, + 6.703684630338103e-05, + 6.324288551695645e-05, + 6.0062211559852585e-05, + 5.733652142225765e-05, + 5.42059242434334e-05, + 5.1946455641882494e-05, + 5.053633503848687e-05, + 4.887565955868922e-05, + 4.733524838229641e-05, + 4.596443613991141e-05, + 4.420741242938675e-05, + 4.370417445898056e-05, + 4.286341936676763e-05, + 4.180924588581547e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00022213962802197784, + "first_to_last_ratio": 1.7678973829107683, + "imbalance_abs_log_ratio": 0.5697909211909248, + "last_quartile_mean": 0.00012565187898871955, + "mean": 0.00016188367408176418, + "normalized": [ + 1.7325269150218519, + 1.5951124185334453, + 1.452057969341735, + 1.3730142236731557, + 1.2857984704138503, + 1.2257558334519054, + 1.176224442377172, + 1.1372505911497117, + 1.088955563427776, + 1.050933480062714, + 1.0259126283978994, + 1.0049209692997896, + 0.9780904283068869, + 0.9585002626676595, + 0.9364797090707767, + 0.9194681619525875, + 0.9067005574410794, + 0.8936181540011504, + 0.876961137714969, + 0.8605080845266135, + 0.8461083257765747, + 0.8336465310750286, + 0.8211746685608088, + 0.8107906186997867, + 0.8025238658005067, + 0.7926026656489187, + 0.7851375623337778, + 0.7775608141673811, + 0.7689788128837574, + 0.7655650153667295, + 0.7607140375905382, + 0.7564070812634623 + ], + "peak_layer": 1, + "peak_normalized": 1.7325269150218519, + "population_cv": 0.2510633514944888, + "rest_mean": 0.00016719439386010722, + "spike_contrast": 0.7967120440064878, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001332057872787118, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002804678224492818, + 0.0002582226588856429, + 0.00023506447905674577, + 0.00022226858709473163, + 0.00020814978051930666, + 0.00019842985784634948, + 0.00019041153427679092, + 0.00018410230404697359, + 0.00017628412751946598, + 0.0001701289729680866, + 0.0001660785055719316, + 0.0001626802986720577, + 0.0001583368721185252, + 0.00015516554412897676, + 0.00015160077600739896, + 0.00014884688425809145, + 0.0001467800175305456, + 0.00014466218999587, + 0.00014196569100022316, + 0.0001393022103002295, + 0.00013697112444788218, + 0.00013495376333594322, + 0.0001329347724094987, + 0.0001312537642661482, + 0.00012991551193408668, + 0.00012830943160224706, + 0.0001271009532501921, + 0.00012587440141942352, + 0.0001244851155206561, + 0.00012393247743602842, + 0.0001231471833307296, + 0.00012244995741639286 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19699564762413502, + "first_to_last_ratio": 1.7678974461601602, + "imbalance_abs_log_ratio": 0.5697909569675436, + "last_quartile_mean": 0.11142934113740921, + "mean": 0.14356006286107004, + "normalized": [ + 1.73252692376823, + 1.5951124529267644, + 1.4520579427898739, + 1.3730142509369438, + 1.2857984860257277, + 1.2257558423238033, + 1.1762244498747965, + 1.1372506023188744, + 1.0889555406950477, + 1.0509335152923371, + 1.0259126331612176, + 1.004920958483027, + 0.9780904724894115, + 0.9585002702989701, + 0.9364797557971557, + 0.9194681924019411, + 0.9067005932791752, + 0.893618176643793, + 0.876961081367795, + 0.8605080517791706, + 0.8461083423470157, + 0.8336465303631113, + 0.8211746500314575, + 0.8107906024915998, + 0.8025238662049899, + 0.792602599370704, + 0.7851375422989607, + 0.7775607992253795, + 0.7689787783378558, + 0.7655650375651303, + 0.7607140142018359, + 0.7564070449079043 + ], + "peak_layer": 1, + "peak_normalized": 1.73252692376823, + "population_cv": 0.25106336091926634, + "rest_mean": 0.1482696624265777, + "spike_contrast": 0.7967120399011176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1181282252073288, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2487216740846634, + 0.2289944440126419, + 0.20845752954483032, + 0.19711001217365265, + 0.18458931148052216, + 0.17596958577632904, + 0.1688588559627533, + 0.16326376795768738, + 0.15633052587509155, + 0.15087208151817322, + 0.14728008210659027, + 0.1442665159702301, + 0.14041472971439362, + 0.13760235905647278, + 0.1344410926103592, + 0.1319989114999771, + 0.13016599416732788, + 0.1282878816127777, + 0.12589658796787262, + 0.12353459000587463, + 0.12146736681461334, + 0.11967834830284119, + 0.11788788437843323, + 0.11639714986085892, + 0.11521037667989731, + 0.11378607898950577, + 0.11271439492702484, + 0.11162667721509933, + 0.11039464175701141, + 0.10990456491708755, + 0.10920815169811249, + 0.10858984291553497 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.813340178268845e-05, + "first_to_last_ratio": 2.0396781258635253, + "imbalance_abs_log_ratio": 0.7127920139678758, + "last_quartile_mean": 3.3403996894776355e-05, + "mean": 4.817690887648496e-05, + "normalized": [ + 1.6677253734616262, + 1.5880838657379246, + 1.4689679818917765, + 1.414042908307037, + 1.3674754530876347, + 1.307279124984285, + 1.2642920390991146, + 1.2360018052494937, + 1.16456296339031, + 1.101785452839103, + 1.0797656603944665, + 1.0672809591929815, + 1.0128091896347613, + 0.9955804650846075, + 0.9623876307655963, + 0.9380984486976066, + 0.9310658554452689, + 0.9082443423860135, + 0.8811967490595315, + 0.8644115118569954, + 0.8497936463301483, + 0.8309315019236491, + 0.7889969150425813, + 0.7623309639866356, + 0.7584133539756696, + 0.7445182980676538, + 0.7180454335344151, + 0.7000361319857387, + 0.6663708133598921, + 0.6665317313809671, + 0.6558653811637546, + 0.6371080486827607 + ], + "peak_layer": 1, + "peak_normalized": 1.6677253734616262, + "population_cv": 0.28258812904813146, + "rest_mean": 4.997824995640527e-05, + "spike_contrast": 0.7693279992487513, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.844966704491526e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.034585334826261e-05, + 7.650897168787196e-05, + 7.077033660607412e-05, + 6.81242163409479e-05, + 6.588074029423296e-05, + 6.298066728049889e-05, + 6.090968236094341e-05, + 5.954674634267576e-05, + 5.610504376818426e-05, + 5.308061736286618e-05, + 5.201977182878181e-05, + 5.1418297516647726e-05, + 4.879401603830047e-05, + 4.796398934558965e-05, + 4.6364861191250384e-05, + 4.519468348007649e-05, + 4.4855874875793234e-05, + 4.3756404920713976e-05, + 4.245333548169583e-05, + 4.1644674638519064e-05, + 4.094043106306344e-05, + 4.0031711250776425e-05, + 3.8011432479834184e-05, + 3.672674938570708e-05, + 3.653801104519516e-05, + 3.586859020288102e-05, + 3.459320942056365e-05, + 3.372557694092393e-05, + 3.210368595318869e-05, + 3.21114384860266e-05, + 3.159756670356728e-05, + 3.069389640586451e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018498800636734813, + "first_to_last_ratio": 1.4771173407265819, + "imbalance_abs_log_ratio": 0.39009244570614415, + "last_quartile_mean": 0.0001252358233614359, + "mean": 0.00015057798327688943, + "normalized": [ + 1.36489476841379, + 1.32224224178262, + 1.270236278236952, + 1.2358216684832493, + 1.2002806183282966, + 1.1699184301039023, + 1.1431585124660382, + 1.1216044194279113, + 1.0931779382416982, + 1.0676137521656621, + 1.050334546837667, + 1.0352083927879365, + 1.0149316924423049, + 1.001461761419052, + 0.9840691969120363, + 0.9695910166861202, + 0.9584136856662008, + 0.9467149797350989, + 0.9322036518549593, + 0.9180752132052378, + 0.9048391519289843, + 0.8925982934025656, + 0.8798985862981495, + 0.8691051109125614, + 0.860352873872478, + 0.849675270316085, + 0.8415818308236442, + 0.8332603200076834, + 0.823928337260059, + 0.8201054366601405, + 0.8147645087933348, + 0.8099375145275801 + ], + "peak_layer": 1, + "peak_normalized": 1.36489476841379, + "population_cv": 0.15663918075507588, + "rest_mean": 0.0001538862707093358, + "spike_contrast": 0.8624111204329009, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000132713231141679, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00020552310161292553, + 0.00019910057017114013, + 0.00019126961706206203, + 0.0001860875345300883, + 0.00018073583487421274, + 0.00017616395780351013, + 0.0001721345033729449, + 0.00016888893151190132, + 0.0001646085293032229, + 0.00016075912571977824, + 0.00015815725782886147, + 0.0001558795920573175, + 0.00015282636741176248, + 0.00015079809236340225, + 0.0001481791550759226, + 0.00014599905989598483, + 0.00014431599993258715, + 0.00014255443238653243, + 0.00014036934589967132, + 0.000138241914100945, + 0.00013624885468743742, + 0.00013440565089695156, + 0.0001324933546129614, + 0.0001308680948568508, + 0.00012955020065419376, + 0.00012794238864444196, + 0.0001267236948478967, + 0.00012547065853141248, + 0.00012406546738930047, + 0.00012348982272669673, + 0.0001226855965796858, + 0.00012195875751785934 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015959660413500387, + "first_to_last_ratio": 1.2728751185715401, + "imbalance_abs_log_ratio": 0.2412782146622287, + "last_quartile_mean": 0.00012538276678242255, + "mean": 0.00014144585429676226, + "normalized": [ + 1.173955566226187, + 1.1612997099437044, + 1.147257651804958, + 1.1326524298721496, + 1.1221543736292459, + 1.1091111750587395, + 1.0952125313023318, + 1.0849401932479867, + 1.070907908686229, + 1.0558579375605581, + 1.0430997159190893, + 1.0346471150202732, + 1.0228091528121042, + 1.0150746526225596, + 1.0038187840838828, + 0.9945868690199192, + 0.986355871119351, + 0.9777591356798918, + 0.9676015085554662, + 0.9595938626090281, + 0.9511867345631487, + 0.9428977124791953, + 0.9335239262292595, + 0.9222035520824579, + 0.9142471402553919, + 0.9038764491526137, + 0.8952592406413572, + 0.8872171299760644, + 0.877779969795339, + 0.8748896658804006, + 0.8709399062198925, + 0.8672824279512239 + ], + "peak_layer": 1, + "peak_normalized": 1.173955566226187, + "population_cv": 0.09275081554992197, + "rest_mean": 0.00014320575994335942, + "spike_contrast": 0.9213481626529775, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013194236380513756, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016605114797130227, + 0.0001642610295675695, + 0.0001622748386580497, + 0.00016020899056456983, + 0.00015872408403083682, + 0.00015687917766626924, + 0.0001549132721265778, + 0.00015346029249485582, + 0.00015147548401728272, + 0.00014934672799427062, + 0.00014754213043488562, + 0.000146346545079723, + 0.00014467211440205574, + 0.00014357810141518712, + 0.00014198600547388196, + 0.00014068018936086446, + 0.00013951594883110374, + 0.00013829997624270618, + 0.00013686322199646384, + 0.0001357305736746639, + 0.00013454142026603222, + 0.0001333689724560827, + 0.00013204308925196528, + 0.00013044186925981194, + 0.00012931646779179573, + 0.00012784957652911544, + 0.00012663070810958743, + 0.000125493184896186, + 0.0001241583377122879, + 0.00012374951620586216, + 0.0001231908390764147, + 0.00012267350393813103 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031297675377572887, + "first_to_last_ratio": 2.208693462790297, + "imbalance_abs_log_ratio": 0.7924011473984336, + "last_quartile_mean": 0.00014170221402309835, + "mean": 0.00020716176595669822, + "normalized": [ + 1.8096319791310764, + 1.7504284704923412, + 1.6109023900661548, + 1.5434988529701714, + 1.4353862656069223, + 1.3677596328902204, + 1.3131374518280827, + 1.2555297512219317, + 1.1870915168673883, + 1.122656500215855, + 1.088514230207685, + 1.04408820740568, + 0.9901334868561585, + 0.9607872809839492, + 0.9219812869372472, + 0.880015778430751, + 0.8614610505244651, + 0.8397919049431777, + 0.8116424497290868, + 0.7845312039842096, + 0.7619415782348183, + 0.7436794877201931, + 0.7280047725498856, + 0.715266757195506, + 0.7065993237444729, + 0.696468071137219, + 0.6907565141722826, + 0.6863265628748565, + 0.6782234011874358, + 0.6754056248242399, + 0.6708832321424102, + 0.6674749829241257 + ], + "peak_layer": 1, + "peak_normalized": 1.8096319791310764, + "population_cv": 0.33826602846938675, + "rest_mean": 0.00021747771663487784, + "spike_contrast": 0.6964190843920172, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001514556322945282, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00037488655652850866, + 0.0003626218531280756, + 0.0003337173839099705, + 0.0003197539481334388, + 0.0002973571536131203, + 0.00028334750095382333, + 0.00027203187346458435, + 0.0002600977604743093, + 0.0002459199749864638, + 0.00023257150314748287, + 0.00022549853019881994, + 0.0002162951568607241, + 0.00020511780166998506, + 0.00019903838983736932, + 0.00019099927158094943, + 0.00018230562272947282, + 0.00017846179252956063, + 0.00017397277406416833, + 0.0001681412832112983, + 0.0001625248696655035, + 0.0001578451629029587, + 0.00015406195598188788, + 0.00015081475430633873, + 0.0001481759245507419, + 0.00014638036373071373, + 0.0001442815555492416, + 0.0001430983393220231, + 0.00014218062278814614, + 0.00014050195750314742, + 0.00013991822197567672, + 0.00013898135512135923, + 0.00013827529619447887 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027448106557130814, + 0.0027243648655712605, + 0.0027103275060653687, + 0.0027234479784965515, + 0.0027595730498433113, + 0.0027272943407297134, + 0.002728298306465149, + 0.0027007071767002344, + 0.002684502862393856, + 0.002683063969016075, + 0.0027139110025018454, + 0.002666700165718794, + 0.002679243916645646, + 0.0026584581937640905, + 0.002684473292902112, + 0.0026407993864268064, + 0.002663099905475974, + 0.0026283918414264917, + 0.002597011858597398, + 0.0026434636674821377, + 0.0025708195753395557, + 0.002691218862310052, + 0.0026598076801747084, + 0.0026596153620630503, + 0.0026765079237520695, + 0.002582729561254382, + 0.002614464145153761, + 0.0025760054122656584, + 0.002634295728057623, + 0.00259007909335196, + 0.00257018837146461, + 0.0025643527042120695 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.000156725715896755, + "first_to_last_ratio": 3.4451295459637388, + "imbalance_abs_log_ratio": 1.236961507811942, + "last_quartile_mean": 4.549196592051885e-05, + "mean": 8.812961027615529e-05, + "normalized": [ + 2.4952457548653, + 2.231881013356295, + 1.939989240144028, + 1.7866526882708733, + 1.6079164676041713, + 1.4797016884434102, + 1.3811975174028512, + 1.3042540698565042, + 1.1954344722384815, + 1.1168849900912188, + 1.0692220969851183, + 1.0304138184362992, + 0.9676518223611118, + 0.9260610329319834, + 0.8738547948342114, + 0.8395441231817105, + 0.8221010666653766, + 0.7971164256793037, + 0.7591159941756684, + 0.7162115141810864, + 0.6791638220644127, + 0.6489084022271779, + 0.6150954832405342, + 0.5868306199881643, + 0.5699702288291514, + 0.552179306852942, + 0.5335984532948914, + 0.5198944860736878, + 0.5024723171712969, + 0.4942325662695092, + 0.4850276094601851, + 0.47217611282304317 + ], + "peak_layer": 1, + "peak_normalized": 2.4952457548653, + "population_cv": 0.5254777238950282, + "rest_mean": 9.433142622583546e-05, + "spike_contrast": 0.5792322488273548, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.463980414788239e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021990503591950983, + 0.00019669480388984084, + 0.00017097049567382783, + 0.00015745700511615723, + 0.00014170505164656788, + 0.0001304055331274867, + 0.00012172439892310649, + 0.00011494340287754312, + 0.00010535317414905876, + 9.843063890002668e-05, + 9.423012670595199e-05, + 9.080996824195608e-05, + 8.527877798769623e-05, + 8.16133979242295e-05, + 7.701248250668868e-05, + 7.398869638564065e-05, + 7.24514466128312e-05, + 7.024955993983895e-05, + 6.690059672109783e-05, + 6.311944162007421e-05, + 5.985444295220077e-05, + 5.718804459320381e-05, + 5.420812522061169e-05, + 5.1717153837671503e-05, + 5.023125413572416e-05, + 4.866334711550735e-05, + 4.7025823732838035e-05, + 4.581809844239615e-05, + 4.428268948686309e-05, + 4.3556523451115936e-05, + 4.274529419490136e-05, + 4.161269680480473e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002203928706876468, + "first_to_last_ratio": 1.7624395804596433, + "imbalance_abs_log_ratio": 0.5666989745264205, + "last_quartile_mean": 0.00012504988717410015, + "mean": 0.00016078790008577926, + "normalized": [ + 1.7301295632680096, + 1.5926765618359837, + 1.4498368649995454, + 1.3715838442653223, + 1.2853282969253275, + 1.2242662006350955, + 1.1759759999659367, + 1.1358472505674493, + 1.0864560712260232, + 1.0500344477950443, + 1.0256300088344572, + 1.0049332388641516, + 0.9775387355078093, + 0.9578140668049778, + 0.9358065303644342, + 0.9184543281568608, + 0.9072826301440372, + 0.8934865935730769, + 0.8779252804708734, + 0.8613603707803614, + 0.8467225584074481, + 0.8352602527072714, + 0.8224752340358507, + 0.8113194646910352, + 0.80159772798593, + 0.7940537841119606, + 0.785769247679408, + 0.7781743501685717, + 0.7724121545166266, + 0.7676777199663887, + 0.7637141967462265, + 0.7584564239985051 + ], + "peak_layer": 1, + "peak_normalized": 1.7301295632680096, + "population_cv": 0.2498913679417364, + "rest_mean": 0.0001660440253477578, + "spike_contrast": 0.7974079368034485, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013240482367109506, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027818389935418963, + 0.00025608311989344656, + 0.00023311622499022633, + 0.00022053408611100167, + 0.00020666523778345436, + 0.0001968471915461123, + 0.00018908271158579737, + 0.0001826304942369461, + 0.0001746889902278781, + 0.00016883283387869596, + 0.00016490889538545161, + 0.00016158110520336777, + 0.00015717640053480864, + 0.00015400491247419268, + 0.0001504663669038564, + 0.00014767634274903685, + 0.00014588006888516247, + 0.00014366183313541114, + 0.00014115976227913052, + 0.00013849632523488253, + 0.00013614274212159216, + 0.0001342997420579195, + 0.00013224406575318426, + 0.00013045035302639008, + 0.00012888721539638937, + 0.00012767424050252885, + 0.0001263421872863546, + 0.0001251210196642205, + 0.00012419452832546085, + 0.00012343328853603452, + 0.00012279600196052343, + 0.00012195061572128907 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19544659927487373, + "first_to_last_ratio": 1.7624395617194093, + "imbalance_abs_log_ratio": 0.5666989638932991, + "last_quartile_mean": 0.11089549027383327, + "mean": 0.14258831785991788, + "normalized": [ + 1.7301295593799786, + 1.5926765086027146, + 1.4498368632986096, + 1.3715838909156086, + 1.2853282557768668, + 1.224266111161914, + 1.1759760185820738, + 1.1358472270183295, + 1.0864560720464596, + 1.050034483149275, + 1.025630003735948, + 1.0049332495130952, + 0.9775386792530747, + 0.9578141330250067, + 0.9358065753109587, + 0.9184542876534186, + 0.9072827263675407, + 0.8934866319653527, + 0.8779253472761699, + 0.8613603464001749, + 0.8467225704769723, + 0.8352602246328348, + 0.8224752140046494, + 0.8113194329409806, + 0.8015977187598701, + 0.7940537797484172, + 0.7857692153409586, + 0.7781743825002613, + 0.7724121455314925, + 0.7676776608495898, + 0.7637142124554999, + 0.758456472325904 + ], + "peak_layer": 1, + "peak_normalized": 1.7301295593799786, + "population_cv": 0.24989136150327743, + "rest_mean": 0.14724950271624107, + "spike_contrast": 0.7974079196861148, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11741791963577271, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24669626355171204, + 0.2270970642566681, + 0.20672979950904846, + 0.19557183980941772, + 0.18327279388904572, + 0.17456604540348053, + 0.16768044233322144, + 0.16195854544639587, + 0.1549159437417984, + 0.1497226506471634, + 0.14624285697937012, + 0.14329174160957336, + 0.13938559591770172, + 0.13657310605049133, + 0.13343508541584015, + 0.1309608519077301, + 0.1293679177761078, + 0.12740075588226318, + 0.1251818984746933, + 0.1228199228644371, + 0.12073274701833725, + 0.11909835040569305, + 0.11727535724639893, + 0.11568467319011688, + 0.11429847031831741, + 0.11322279274463654, + 0.11204151064157486, + 0.11095857620239258, + 0.11013694852590561, + 0.10946186631917953, + 0.10889672487974167, + 0.108147032558918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.812671563238837e-05, + "first_to_last_ratio": 2.0429267595554257, + "imbalance_abs_log_ratio": 0.7143834657339574, + "last_quartile_mean": 3.334760549478233e-05, + "mean": 4.8136708642232406e-05, + "normalized": [ + 1.6706878479333491, + 1.5890573277727331, + 1.4709453763829752, + 1.4135957551070697, + 1.368779351628352, + 1.3072249852707734, + 1.2659085053079855, + 1.2360067186760548, + 1.1647282180273046, + 1.1006681301372137, + 1.0750864931045292, + 1.0685694254166505, + 1.0112110373273602, + 0.9945647481091522, + 0.9602230239537092, + 0.9352227929155844, + 0.93139607887314, + 0.9092220870879134, + 0.8814974932144994, + 0.8653605860843365, + 0.8506124631158822, + 0.8320765477333729, + 0.7932580049774197, + 0.7619473309737773, + 0.7566723544753674, + 0.7419402536141336, + 0.7141691805145849, + 0.7010673290630277, + 0.6716446941281601, + 0.6659218543873533, + 0.6557479674543484, + 0.6349860372318865 + ], + "peak_layer": 1, + "peak_normalized": 1.6706878479333491, + "population_cv": 0.2830124551063321, + "rest_mean": 4.992923641111702e-05, + "spike_contrast": 0.7702312603701833, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.845705869025551e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.04214141680859e-05, + 7.649198960280046e-05, + 7.080646901158616e-05, + 6.804584700148553e-05, + 6.588853284483775e-05, + 6.292550824582577e-05, + 6.093666888773441e-05, + 5.949729529675096e-05, + 5.6066182878566906e-05, + 5.298254109220579e-05, + 5.1751125283772126e-05, + 5.143741509527899e-05, + 4.867637107963674e-05, + 4.7875073505565524e-05, + 4.622197593562305e-05, + 4.501854709815234e-05, + 4.483434167923406e-05, + 4.376695869723335e-05, + 4.24323879997246e-05, + 4.165561040281318e-05, + 4.094568430446088e-05, + 4.0053426346275955e-05, + 3.8184829463716596e-05, + 3.667763667181134e-05, + 3.642371666501276e-05, + 3.571456181816757e-05, + 3.4377753763692454e-05, + 3.3747073757695034e-05, + 3.2330764952348545e-05, + 3.205528628313914e-05, + 3.156554885208607e-05, + 3.056613786611706e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018337754227104597, + "first_to_last_ratio": 1.4712162101549686, + "imbalance_abs_log_ratio": 0.38608941256902984, + "last_quartile_mean": 0.00012464350311347516, + "mean": 0.00014952537844692415, + "normalized": [ + 1.3620834570997993, + 1.3194965005671844, + 1.2677944868152182, + 1.233561928872588, + 1.1984447257225705, + 1.1684695593357564, + 1.141528073961558, + 1.1198008373135866, + 1.0910012077210556, + 1.066096352737946, + 1.0498630645220353, + 1.0350823851472097, + 1.0145919973375974, + 1.0013407128400098, + 0.9836056688270479, + 0.9685428567287663, + 0.9591027483551056, + 0.9465833155582388, + 0.9334001555540693, + 0.9191584382428863, + 0.9057431683575706, + 0.8945627711391904, + 0.8815040843165663, + 0.8698871064160805, + 0.8596264871795505, + 0.8514885298372257, + 0.8425068993042694, + 0.8341757603627517, + 0.8278236379550317, + 0.8225948884437775, + 0.8181940461689637, + 0.812344147258793 + ], + "peak_layer": 1, + "peak_normalized": 1.3620834570997993, + "population_cv": 0.15505530299928885, + "rest_mean": 0.00015278545470209792, + "spike_contrast": 0.8634393039979241, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001319209666689858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00020366604439914227, + 0.0001972982136067003, + 0.0001895674504339695, + 0.00018444881425239146, + 0.00017919790116138756, + 0.00017471585306338966, + 0.0001706874172668904, + 0.0001674386439844966, + 0.00016313236847054213, + 0.00015940846060402691, + 0.00015698117204010487, + 0.00015477108536288142, + 0.00015170725237112492, + 0.00014972584904171526, + 0.0001470740098739043, + 0.0001448217371944338, + 0.00014341020141728222, + 0.00014153822849038988, + 0.00013956701150164008, + 0.00013743751333095133, + 0.00013543159002438188, + 0.00013375983689911664, + 0.00013180723180994391, + 0.00013007019879296422, + 0.00012853597581852227, + 0.00012731914466712624, + 0.0001259761629626155, + 0.00012473044625949115, + 0.00012378064275253564, + 0.00012299881200306118, + 0.00012234077439643443, + 0.00012146606604801491 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015816818631719798, + "first_to_last_ratio": 1.2671705792413261, + "imbalance_abs_log_ratio": 0.23678652466939742, + "last_quartile_mean": 0.00012481996418500785, + "mean": 0.0001404537983944465, + "normalized": [ + 1.172032188999453, + 1.1600674087975105, + 1.1434054243316032, + 1.1305758413418425, + 1.1202630655069494, + 1.1065457827105485, + 1.0931998253234612, + 1.0828906757132684, + 1.0683311763286112, + 1.0526488941703125, + 1.0427563459749776, + 1.0343706495522589, + 1.0219941374371992, + 1.0154952175292713, + 1.0038357654470964, + 0.9937321709742023, + 0.9862865992066487, + 0.9768881500283385, + 0.968261257859838, + 0.9607910277643221, + 0.9520546236098718, + 0.9457805296333641, + 0.9349892993508679, + 0.9232794945485685, + 0.9140411172891727, + 0.9051057888083044, + 0.8959794100889387, + 0.888100763485746, + 0.8822517666505367, + 0.8784085901067091, + 0.87543446421392, + 0.8702025472162866 + ], + "peak_layer": 1, + "peak_normalized": 1.172032188999453, + "population_cv": 0.09096684083762872, + "rest_mean": 0.00014216970130628734, + "spike_contrast": 0.9227558436510851, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013118792267050595, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016461637278553098, + 0.0001629358739592135, + 0.00016059563495218754, + 0.00015879367128945887, + 0.0001573452027514577, + 0.00015541855827905238, + 0.00015354406787082553, + 0.00015209610864985734, + 0.00015005117165856063, + 0.0001478485355619341, + 0.0001464590895920992, + 0.00014528128667734563, + 0.00014354295853991061, + 0.00014263016055338085, + 0.00014099254622124135, + 0.00013957345800008625, + 0.0001385276991641149, + 0.00013720765127800405, + 0.00013599597150459886, + 0.00013494674931280315, + 0.00013371968816500157, + 0.00013283846783451736, + 0.00013132279855199158, + 0.0001296781119890511, + 0.0001283805468119681, + 0.00012712554598692805, + 0.0001258437114302069, + 0.00012473712558858097, + 0.00012391561176627874, + 0.0001233758230227977, + 0.0001229580957442522, + 0.00012222325312905014 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0003110146244580392, + "first_to_last_ratio": 2.2098355110548322, + "imbalance_abs_log_ratio": 0.7929180833727034, + "last_quartile_mean": 0.00014074107457417995, + "mean": 0.00020577548139044666, + "normalized": [ + 1.8085529821987474, + 1.7406694730308092, + 1.619926374791616, + 1.5420513399817848, + 1.4477144166533082, + 1.3679714513872108, + 1.309931951281633, + 1.254598241859695, + 1.1926238819658395, + 1.119002359728262, + 1.082854716042971, + 1.0468411526236405, + 0.9850613364068194, + 0.9618428208148979, + 0.9182052809224994, + 0.877960480924469, + 0.8608981306240351, + 0.8408442911829569, + 0.8148282643879068, + 0.7879356949258838, + 0.7604349554811285, + 0.7438321978222967, + 0.727404890131739, + 0.7163770004088065, + 0.7034258791916428, + 0.6971552930824737, + 0.6907054382632783, + 0.684504508832364, + 0.6799757639842041, + 0.6759922505973077, + 0.6720742215597147, + 0.6678029589100581 + ], + "peak_layer": 1, + "peak_normalized": 1.8085529821987474, + "population_cv": 0.3383164219384334, + "rest_mean": 0.00021605301460820355, + "spike_contrast": 0.695555219569954, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001502768020145595, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00037215586053207517, + 0.0003581870987545699, + 0.00033334112958982587, + 0.0003173163568135351, + 0.00029790413100272417, + 0.0002814949839375913, + 0.00026955187786370516, + 0.0002581655571702868, + 0.00024541275342926383, + 0.00023026324925012887, + 0.00022282495046965778, + 0.00021541424212045968, + 0.00020270147069822997, + 0.00019792366947513074, + 0.00018894413369707763, + 0.00018066274060402066, + 0.00017715172725729644, + 0.00017302513879258186, + 0.00016767167835496366, + 0.00016213784692808986, + 0.0001564788690302521, + 0.00015306242858059704, + 0.00014968209143262357, + 0.00014741282211616635, + 0.00014474779891315848, + 0.00014345746603794396, + 0.00014213024405762553, + 0.00014085424481891096, + 0.00013992234016768634, + 0.00013910263078287244, + 0.00013829639647156, + 0.00013741747534368187 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0021227982360869646, + 0.00413061399012804, + 0.005416445434093475, + 0.006388718727976084, + 0.0023577234242111444, + 0.004388594534248114, + 0.00574506726115942, + 0.007066869642585516, + 0.0029547461308538914, + 0.004991721361875534, + 0.006190058775246143, + 0.0074937604367733, + 0.00348988501355052, + 0.006057556718587875, + 0.0074848332442343235, + 0.00897275935858488, + 0.004072706215083599, + 0.0063449000008404255, + 0.007735534105449915, + 0.00946879480034113, + 0.004824656527489424, + 0.007074959110468626, + 0.009095527231693268, + 0.010346688330173492, + 0.004843301139771938, + 0.007334613241255283, + 0.009146629832684994, + 0.010587968863546848, + 0.0054801348596811295, + 0.0077637480571866035, + 0.009536821395158768, + 0.011367839761078358 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015727686513855588, + "first_to_last_ratio": 3.444355141493678, + "imbalance_abs_log_ratio": 1.2367367001496994, + "last_quartile_mean": 4.5662209231522866e-05, + "mean": 8.846079299473786e-05, + "normalized": [ + 2.4938513501811554, + 2.232208910931446, + 1.9412909982664082, + 1.7875424925486438, + 1.6051944492471113, + 1.4809426711898561, + 1.3782256757419, + 1.304162421055249, + 1.1983748443875557, + 1.116540877408076, + 1.0705280549740306, + 1.028906183424365, + 0.9692497056236429, + 0.9297086109864628, + 0.876031508599092, + 0.840542245421429, + 0.8215073067800788, + 0.7958492991481336, + 0.7578128978794811, + 0.7149268338582802, + 0.6789684541053794, + 0.6481555926886391, + 0.6127683614407704, + 0.5872234895762031, + 0.5712935129257742, + 0.5525148673059793, + 0.5351006365732414, + 0.5196040797164857, + 0.49974182292452146, + 0.4940518464188538, + 0.4845470025727643, + 0.4726329960989905 + ], + "peak_layer": 1, + "peak_normalized": 2.4938513501811554, + "population_cv": 0.525487396006855, + "rest_mean": 9.469102304946218e-05, + "spike_contrast": 0.5789096889426617, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.481755069922656e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002206080680480227, + 0.00019746297039091587, + 0.00017172814114019275, + 0.00015812742640264332, + 0.00014199677389115095, + 0.0001310053630732, + 0.00012191893620183691, + 0.00011536724196048453, + 0.00010600918903946877, + 9.877009142655879e-05, + 9.469976066611707e-05, + 9.101785690290853e-05, + 8.574059756938368e-05, + 8.224276098189875e-05, + 7.74944419390522e-05, + 7.435503357555717e-05, + 7.267118780873716e-05, + 7.040146010695025e-05, + 6.70367298880592e-05, + 6.324299465632066e-05, + 6.0062087868573144e-05, + 5.733635771321133e-05, + 5.42059751751367e-05, + 5.1946255553048104e-05, + 5.0537077186163515e-05, + 4.88759033032693e-05, + 4.733542664325796e-05, + 4.596458893502131e-05, + 4.4207557948539034e-05, + 4.3704218114726245e-05, + 4.2863412090810016e-05, + 4.1809489630395547e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002221399099653354, + "first_to_last_ratio": 1.7678964788399845, + "imbalance_abs_log_ratio": 0.5697904098088594, + "last_quartile_mean": 0.00012565210272441618, + "mean": 0.00016188397967198398, + "normalized": [ + 1.7325277794978797, + 1.5951376332003655, + 1.4520548687114199, + 1.3729981481664384, + 1.2857750985910605, + 1.2257666436669268, + 1.1762283345939986, + 1.1372455678364286, + 1.0889526987749056, + 1.0509403055231545, + 1.0259172538151122, + 1.0049179037159137, + 0.978085345877984, + 0.9585059142501069, + 0.9364886382982085, + 0.9194669656082258, + 0.9066971379228966, + 0.893622399915035, + 0.8769613699768213, + 0.8605074489369314, + 0.8461051105300481, + 0.8336447776090111, + 0.8211750960224249, + 0.8107883690330443, + 0.8025267555268564, + 0.7926031470462279, + 0.7851370690218832, + 0.7775583575558719, + 0.7689758331264998, + 0.7655660871494063, + 0.7607136802742225, + 0.75640826022469 + ], + "peak_layer": 1, + "peak_normalized": 1.7325277794978797, + "population_cv": 0.2510639592948361, + "rest_mean": 0.00016719473286583606, + "spike_contrast": 0.796711177092359, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013320591242518275, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002804684918373823, + 0.0002582272281870246, + 0.00023506442084908485, + 0.00022226640430744737, + 0.00020814638992305845, + 0.00019843198242597282, + 0.00019041252380702645, + 0.00018410183838568628, + 0.00017628399655222893, + 0.00017013039905577898, + 0.00016607956786174327, + 0.00016268010949715972, + 0.00015833634824957699, + 0.00015516675193794072, + 0.00015160250768531114, + 0.00014884697156958282, + 0.00014677974104415625, + 0.00014466315042227507, + 0.00014196599659044296, + 0.000139302370371297, + 0.00013697086251340806, + 0.00013495373423211277, + 0.00013293509255163372, + 0.00013125364785082638, + 0.00012991622497793287, + 0.00012830975174438208, + 0.00012710111332125962, + 0.000125874241348356, + 0.00012448486813809723, + 0.00012393288488965482, + 0.00012314735795371234, + 0.00012245037942193449 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19699589163064957, + "first_to_last_ratio": 1.7678964443273588, + "imbalance_abs_log_ratio": 0.5697903902870003, + "last_quartile_mean": 0.11142954230308533, + "mean": 0.1435603315476328, + "normalized": [ + 1.7325277292733527, + 1.595137596567507, + 1.4520549137362593, + 1.3729980837787108, + 1.285775112498371, + 1.2257666266551688, + 1.1762283724960378, + 1.1372455675264566, + 1.0889526722354506, + 1.0509403711345948, + 1.0259172522938773, + 1.00491793591038, + 0.9780853203912144, + 0.9585059497750074, + 0.9364885904027979, + 0.9194669905174069, + 0.9066971317518798, + 0.8936224205946474, + 0.8769613084029552, + 0.8605074273314671, + 0.84610509802098, + 0.833644762521393, + 0.821175085273337, + 0.8107884103394608, + 0.802526775587875, + 0.7926031918850489, + 0.7851370589142022, + 0.7775583578743038, + 0.7689758340628073, + 0.7655660958716172, + 0.7607136803251955, + 0.7564082760502372 + ], + "peak_layer": 1, + "peak_normalized": 1.7325277292733527, + "population_cv": 0.2510639513231401, + "rest_mean": 0.14826996044980156, + "spike_contrast": 0.7967111822081809, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11812833547592164, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24872225522994995, + 0.22899848222732544, + 0.20845748484134674, + 0.19710806012153625, + 0.184586301445961, + 0.17597146332263947, + 0.16885973513126373, + 0.16326335072517395, + 0.156330406665802, + 0.1508733481168747, + 0.14728102087974548, + 0.14426635205745697, + 0.1404142528772354, + 0.13760343194007874, + 0.13444261252880096, + 0.13199898600578308, + 0.13016574084758759, + 0.12828873097896576, + 0.1258968561887741, + 0.12353473156690598, + 0.12146712839603424, + 0.1196783185005188, + 0.11788816750049591, + 0.11639705300331116, + 0.11521100997924805, + 0.11378637701272964, + 0.11271453648805618, + 0.111626535654068, + 0.1103944256901741, + 0.10990492254495621, + 0.10920830816030502, + 0.10859022289514542 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.813338586653117e-05, + "first_to_last_ratio": 2.039661655563216, + "imbalance_abs_log_ratio": 0.7127839389845736, + "last_quartile_mean": 3.3404258829250466e-05, + "mean": 4.817700425974181e-05, + "normalized": [ + 1.6677237328958603, + 1.5880861584886787, + 1.4689677920146058, + 1.4140322553861728, + 1.3674596064698026, + 1.3072824267629832, + 1.2642946708581568, + 1.2359968662296865, + 1.1645552208108763, + 1.101791124796405, + 1.0797798333773554, + 1.0672664620429624, + 1.012803710836977, + 0.9955853656459409, + 0.9623936542224847, + 0.9380946280712082, + 0.9310631059210642, + 0.9082494158597965, + 0.8812030087749774, + 0.8644086677601183, + 0.8497891698936241, + 0.8309347651333899, + 0.7889921814106587, + 0.7623244708429415, + 0.7584228017816982, + 0.7445258855646851, + 0.7180452956289476, + 0.7000407870401953, + 0.6663749309645193, + 0.6665299586713209, + 0.6558653663654831, + 0.637116679476424 + ], + "peak_layer": 1, + "peak_normalized": 1.6677237328958603, + "population_cv": 0.28258618188430573, + "rest_mean": 4.9978354245131074e-05, + "spike_contrast": 0.7693273401931907, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.844971433863975e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.034593338379636e-05, + 7.650923362234607e-05, + 7.077046757331118e-05, + 6.812383799115196e-05, + 6.588010728592053e-05, + 6.298095104284585e-05, + 6.090992974350229e-05, + 5.9546626289375126e-05, + 5.610478183371015e-05, + 5.3080995712662116e-05, + 5.2020557632204145e-05, + 5.141770088812336e-05, + 4.879384869127534e-05, + 4.79643204016611e-05, + 4.636524317902513e-05, + 4.519458889262751e-05, + 4.485583122004755e-05, + 4.375673597678542e-05, + 4.245372110744938e-05, + 4.164462006883696e-05, + 4.094029645784758e-05, + 4.0031947719398886e-05, + 3.8011279684724286e-05, + 3.672650927910581e-05, + 3.653853855212219e-05, + 3.5869026760337874e-05, + 3.4593271266203374e-05, + 3.3725867979228497e-05, + 3.21039478876628e-05, + 3.211141665815376e-05, + 3.1597628549207e-05, + 3.0694372981088236e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018498812278266996, + "first_to_last_ratio": 1.477115588497286, + "imbalance_abs_log_ratio": 0.3900912594562188, + "last_quartile_mean": 0.00012523605073511135, + "mean": 0.00015057817927299766, + "normalized": [ + 1.364889512785021, + 1.3222437098497068, + 1.2702392636002102, + 1.235819093505974, + 1.2002761568067795, + 1.1699209662016437, + 1.143160310272995, + 1.1216013166366416, + 1.0931712967611817, + 1.0676161315047876, + 1.0503356923443616, + 1.0352076251778792, + 1.014929888181958, + 1.0014649033450271, + 0.9840777733314531, + 0.96959004456439, + 0.958410698647572, + 0.9467186761221033, + 0.9322053376846741, + 0.9180751779000818, + 0.904837007766356, + 0.8925970349353171, + 0.879899470446981, + 0.8691033998232655, + 0.8603564893899045, + 0.8496764837250937, + 0.8415817018035472, + 0.8332582690137502, + 0.8239257185712614, + 0.820107268399535, + 0.8147646079598274, + 0.8099389729407196 + ], + "peak_layer": 1, + "peak_normalized": 1.364889512785021, + "population_cv": 0.1566386619724503, + "rest_mean": 0.000153886474436149, + "spike_contrast": 0.8624109810706347, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013271338539198042, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002055225777439773, + 0.00019910105038434267, + 0.000191270315553993, + 0.00018608738901093602, + 0.0001807353983167559, + 0.00017616456898394972, + 0.00017213499813806266, + 0.00016888868412934244, + 0.00016460774349980056, + 0.00016075969324447215, + 0.0001581576361786574, + 0.00015587967936880887, + 0.00015282629465218633, + 0.00015079876175150275, + 0.0001481806393712759, + 0.00014599910355173051, + 0.00014431573799811304, + 0.00014255517453420907, + 0.00014036978245712817, + 0.00013824208872392774, + 0.00013624870916828513, + 0.00013440563634503633, + 0.0001324936602031812, + 0.00013086800754535943, + 0.00012955091369803995, + 0.00012794273789040744, + 0.00012672384036704898, + 0.0001254705130122602, + 0.0001240652345586568, + 0.00012349025928415358, + 0.00012268577120266855, + 0.00012195913586765528 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015959453048708383, + "first_to_last_ratio": 1.2728956423420892, + "imbalance_abs_log_ratio": 0.24129433847892473, + "last_quartile_mean": 0.00012537911607068963, + "mean": 0.00014144417764327955, + "normalized": [ + 1.1740787416792278, + 1.1612764386268224, + 1.1471117856731012, + 1.1325725430810556, + 1.1221203501929577, + 1.1091452070987, + 1.0952451640185767, + 1.0850231158766837, + 1.0710066115302646, + 1.0558964823952193, + 1.0428503514266292, + 1.0346153464801762, + 1.0229188081717178, + 1.0151986196356395, + 1.003700332970425, + 0.9946300373693288, + 0.9863513080075261, + 0.9777764871930738, + 0.967654233599155, + 0.9595434059922948, + 0.9513191006674802, + 0.9429489101168639, + 0.9334504238882736, + 0.9221966853060849, + 0.914156536930625, + 0.9038869577772225, + 0.8952163547729103, + 0.8871546013922188, + 0.8776639341179028, + 0.8750860454698683, + 0.8708633986378481, + 0.8673416799041263 + ], + "peak_layer": 1, + "peak_normalized": 1.1740787416792278, + "population_cv": 0.09275162400716915, + "rest_mean": 0.00014320399646681767, + "spike_contrast": 0.9213510743517984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001319411559961736, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001660666021052748, + 0.00016425579087808728, + 0.00016225228318944573, + 0.0001601957919774577, + 0.00015871739014983177, + 0.0001568821317050606, + 0.0001549160515423864, + 0.00015347020234912634, + 0.00015148764941841364, + 0.0001493504096288234, + 0.00014750511036254466, + 0.00014634031686000526, + 0.00014468590961769223, + 0.00014359393389895558, + 0.00014196756819728762, + 0.0001406846276950091, + 0.00013951364962849766, + 0.00013830079114995897, + 0.0001368690573144704, + 0.00013572182797361165, + 0.000134558547870256, + 0.00013337463315110654, + 0.00013203112757764757, + 0.00013043935177847743, + 0.00012930211960338056, + 0.00012784954742528498, + 0.00012662314111366868, + 0.00012548285303637385, + 0.00012414045340847224, + 0.00012377582606859505, + 0.00012317855725996196, + 0.00012268043064977974 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031298474641516805, + "first_to_last_ratio": 2.209030513243648, + "imbalance_abs_log_ratio": 0.7925537374904509, + "last_quartile_mean": 0.00014168421148497146, + "mean": 0.0002071673161481158, + "normalized": [ + 1.8097158340785349, + 1.7497804411308835, + 1.611244862997228, + 1.5435020349400896, + 1.4351553463811126, + 1.3679605489917586, + 1.3130823229189843, + 1.255818245825708, + 1.1871164694634437, + 1.122716192977336, + 1.0885067728511792, + 1.0444622322416663, + 0.990152969051365, + 0.9605806666596922, + 0.9222549756846732, + 0.8800158737132093, + 0.8615285136033911, + 0.8400222785845712, + 0.8115627552032852, + 0.7845983398409414, + 0.7619432914958448, + 0.7435905859348785, + 0.7280762324918081, + 0.7153162915808863, + 0.7064441934578637, + 0.6963930075638303, + 0.6905250334602637, + 0.686293775915277, + 0.6782364888338203, + 0.6751963304984743, + 0.6708841537879804, + 0.6673229378400193 + ], + "peak_layer": 1, + "peak_normalized": 1.8097158340785349, + "population_cv": 0.3382701486201724, + "rest_mean": 0.00021748447411314205, + "spike_contrast": 0.6963929896815652, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015145466313697397, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003749139723367989, + 0.00036249731783755124, + 0.00033379727392457426, + 0.0003197631740476936, + 0.0002973172813653946, + 0.00028339671553112566, + 0.0002720277407206595, + 0.0002601644955575466, + 0.0002459317329339683, + 0.00023259010049514472, + 0.0002255030267406255, + 0.00021637843747157604, + 0.00020512733317445964, + 0.00019900091865565628, + 0.00019106108811683953, + 0.00018231052672490478, + 0.00017848054994829, + 0.00017402516095899045, + 0.00016812927788123488, + 0.0001625431323191151, + 0.00015784974675625563, + 0.00015404766600113362, + 0.00015083359903655946, + 0.00014819015632383525, + 0.00014635214756708592, + 0.00014426987036131322, + 0.00014305421791505069, + 0.00014217763964552432, + 0.00014050843310542405, + 0.0001398786116624251, + 0.00013898526958655566, + 0.0001382475020363927 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034565660171210766, + 0.004928664769977331, + 0.0060537648387253284, + 0.006960673723369837, + 0.0036145171616226435, + 0.0051584807224571705, + 0.006370538379997015, + 0.007546419743448496, + 0.00395671371370554, + 0.005665866658091545, + 0.006755253300070763, + 0.007946331985294819, + 0.004383862018585205, + 0.006652849726378918, + 0.007932198233902454, + 0.009347881190478802, + 0.004880861379206181, + 0.006906712893396616, + 0.008132148534059525, + 0.009816600009799004, + 0.005453306715935469, + 0.007622495759278536, + 0.009448722936213017, + 0.010670131072402, + 0.0055701653473079205, + 0.007738742511719465, + 0.009509840048849583, + 0.010865328833460808, + 0.005987230688333511, + 0.00815181527286768, + 0.009874933399260044, + 0.011666664853692055 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015672678182454547, + "first_to_last_ratio": 3.4451487067191406, + "imbalance_abs_log_ratio": 1.236967069490208, + "last_quartile_mean": 4.549202230919036e-05, + "mean": 8.812991916329338e-05, + "normalized": [ + 2.495342024881723, + 2.231858330120814, + 1.9399329049931513, + 1.7866550124027707, + 1.6079479837593746, + 1.479673880937976, + 1.3811985381554308, + 1.3042766606266647, + 1.1954508396552082, + 1.1168862767589318, + 1.0692251193363627, + 1.0303878333212484, + 0.9676417435164101, + 0.9260651349655372, + 0.8738635380560305, + 0.8395483633300559, + 0.8220975248023003, + 0.7970985234813915, + 0.759112342831592, + 0.7162079306537569, + 0.6791607399033079, + 0.6489075313744581, + 0.615087961024076, + 0.5868315353418163, + 0.5699679008989185, + 0.5521854623420828, + 0.5335941062986479, + 0.5198929115704046, + 0.5024718770039331, + 0.49422901772262035, + 0.4850259920431499, + 0.47217445788985424 + ], + "peak_layer": 1, + "peak_normalized": 2.495342024881723, + "population_cv": 0.5254832958744828, + "rest_mean": 9.433179891655325e-05, + "spike_contrast": 0.5792295824234721, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.4639768495690075e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021991429093759507, + 0.0001966934942174703, + 0.00017096613009925932, + 0.0001574577618157491, + 0.00014170832582749426, + 0.00013040353951510042, + 0.0001217249155160971, + 0.00011494579666759819, + 0.00010535498586250469, + 9.843109728535637e-05, + 9.423072333447635e-05, + 9.080799645744264e-05, + 8.527818863512948e-05, + 8.161404548445717e-05, + 7.701352296862751e-05, + 7.398932939395308e-05, + 7.245138840517029e-05, + 7.024822843959555e-05, + 6.690050940960646e-05, + 6.311934703262523e-05, + 5.985438110656105e-05, + 5.7188168284483254e-05, + 5.420765228336677e-05, + 5.1717415772145614e-05, + 5.02312250318937e-05, + 4.866406015935354e-05, + 4.702560545410961e-05, + 4.581812027026899e-05, + 4.428280590218492e-05, + 4.3556363380048424e-05, + 4.2745301470858976e-05, + 4.161269680480473e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00022039370378479362, + "first_to_last_ratio": 1.7624457042066763, + "imbalance_abs_log_ratio": 0.5667024491058967, + "last_quartile_mean": 0.00012504992537287762, + "mean": 0.0001607882525149762, + "normalized": [ + 1.7301666786506156, + 1.592657504255016, + 1.4498087081325215, + 1.3715848200709229, + 1.2853506396348091, + 1.224257634451794, + 1.175979576607571, + 1.1358564358860717, + 1.0864628307036615, + 1.050035766385231, + 1.0256357250890054, + 1.0049217142965674, + 0.9775345112735686, + 0.9578189361646354, + 0.9358080993289649, + 0.9184597363064851, + 0.9072850761620224, + 0.8934833681037547, + 0.8779253472396409, + 0.8613572157279928, + 0.8467159963027754, + 0.8352601414820391, + 0.8224708066612338, + 0.81131886291965, + 0.8015955184587156, + 0.7940537632065271, + 0.7857694259399116, + 0.7781749070905981, + 0.7724085609032495, + 0.7676732316943716, + 0.7637136993200662, + 0.75845476155001 + ], + "peak_layer": 1, + "peak_normalized": 1.7301666786506156, + "population_cv": 0.2498932502086342, + "rest_mean": 0.0001660444721454513, + "spike_contrast": 0.7974048446155259, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001324046665104106, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027819047681987286, + 0.0002560806169640273, + 0.0002331122086616233, + 0.00022053472639527172, + 0.00020666928321588784, + 0.00019684624567162246, + 0.0001890837011160329, + 0.00018263237143401057, + 0.00017469045997131616, + 0.0001688334159553051, + 0.0001649101759539917, + 0.00016157960635609925, + 0.00015717606584075838, + 0.00015400603297166526, + 0.00015046694898046553, + 0.00014767753600608557, + 0.00014588078192900866, + 0.00014366162940859795, + 0.00014116008242126554, + 0.00013849612150806934, + 0.0001361419854220003, + 0.00013430001854430884, + 0.00013224364374764264, + 0.00013045054220128804, + 0.00012888714263681322, + 0.00012767451698891819, + 0.0001263424928765744, + 0.00012512138346210122, + 0.00012419422273524106, + 0.00012343283742666245, + 0.0001227961911354214, + 0.00012195061572128907 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19544734060764313, + "first_to_last_ratio": 1.7624457434404193, + "imbalance_abs_log_ratio": 0.5667024713668619, + "last_quartile_mean": 0.11089552193880081, + "mean": 0.14258862822316587, + "normalized": [ + 1.7301667593069614, + 1.5926575752547714, + 1.4498087309457477, + 1.3715847721481056, + 1.2853506436856033, + 1.2242575941276408, + 1.1759796246815233, + 1.1358564592049036, + 1.0864627991256137, + 1.0500357507634608, + 1.0256357136610779, + 1.0049216567312906, + 0.9775345659226533, + 0.957818945514909, + 0.935808091557486, + 0.9184597083380731, + 0.9072850362319335, + 0.8934833286128027, + 0.8779253174360568, + 0.8613572174793438, + 0.8467160247668938, + 0.8352601831545204, + 0.8224708111632975, + 0.8113188687979516, + 0.8015955037045681, + 0.7940537757091718, + 0.7857693860910103, + 0.7781748832951502, + 0.7724085309387246, + 0.7676732205256189, + 0.7637136996793319, + 0.7584548214438025 + ], + "peak_layer": 1, + "peak_normalized": 1.7301667593069614, + "population_cv": 0.24989326289316924, + "rest_mean": 0.14724989594132812, + "spike_contrast": 0.797404859232464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11741778254508972, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24670210480690002, + 0.2270948588848114, + 0.20672623813152313, + 0.1955723911523819, + 0.18327638506889343, + 0.17456521093845367, + 0.16768132150173187, + 0.16196021437644958, + 0.15491724014282227, + 0.14972315728664398, + 0.14624398946762085, + 0.14329040050506592, + 0.13938531279563904, + 0.13657408952713013, + 0.13343559205532074, + 0.13096190989017487, + 0.12936852872371674, + 0.12740056216716766, + 0.1251821666955948, + 0.12281974405050278, + 0.12073207646608353, + 0.11909860372543335, + 0.11727498471736908, + 0.11568484455347061, + 0.11429840326309204, + 0.11322303861379623, + 0.11204177886247635, + 0.11095888912677765, + 0.11013667285442352, + 0.1094614714384079, + 0.1088968887925148, + 0.108147032558918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.812703486502869e-05, + "first_to_last_ratio": 2.0429306771418236, + "imbalance_abs_log_ratio": 0.7143853833664048, + "last_quartile_mean": 3.334769780849456e-05, + "mean": 4.813682039639389e-05, + "normalized": [ + 1.670694398734861, + 1.5890602892970849, + 1.4709330435001327, + 1.4135870318536654, + 1.3688033811669116, + 1.3072210435152478, + 1.2659107055367187, + 1.236022743120565, + 1.164719165633699, + 1.10067041168383, + 1.075093368589445, + 1.0685491843186352, + 1.0112041551575939, + 0.9945774787180313, + 0.9602353052595208, + 0.9352302198346567, + 0.9313930096398116, + 0.9092161974531191, + 0.8814894006709005, + 0.8653525310040872, + 0.8506115464001769, + 0.8320776390213925, + 0.7932455071661803, + 0.7619500965894176, + 0.7566697664554205, + 0.7419550822306056, + 0.7141621566216388, + 0.7010675908642553, + 0.6716466869044494, + 0.66592144202409, + 0.6557448579818498, + 0.6349845630520059 + ], + "peak_layer": 1, + "peak_normalized": 1.670694398734861, + "population_cv": 0.28301355753944435, + "rest_mean": 4.992937398061191e-05, + "spike_contrast": 0.7702285844110489, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.845703104161657e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.042191620916128e-05, + 7.649230974493548e-05, + 7.080603973008692e-05, + 6.804558506701142e-05, + 6.588984251720831e-05, + 6.292546459008008e-05, + 6.0936916270293295e-05, + 5.9498204791452736e-05, + 5.606587728834711e-05, + 5.2982773922849447e-05, + 5.1751576393144205e-05, + 5.143656017025933e-05, + 4.867615280090831e-05, + 4.7875797463348135e-05, + 4.622267442755401e-05, + 4.501900912146084e-05, + 4.483429802348837e-05, + 4.3766776798292994e-05, + 4.243209696142003e-05, + 4.1655319364508614e-05, + 4.094573523616418e-05, + 4.005357186542824e-05, + 3.8184316508704796e-05, + 3.667785495053977e-05, + 3.6423676647245884e-05, + 3.571535853552632e-05, + 3.437749546719715e-05, + 3.374716470716521e-05, + 3.233093593735248e-05, + 3.2055340852821246e-05, + 3.156547245453112e-05, + 3.056613786611706e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018337804431212135, + "first_to_last_ratio": 1.471219679743085, + "imbalance_abs_log_ratio": 0.38609177087911456, + "last_quartile_mean": 0.00012464355040719965, + "mean": 0.0001495256622092711, + "normalized": [ + 1.3620802882826015, + 1.3194938018514704, + 1.2677903290886852, + 1.23356455122679, + 1.198458119977671, + 1.1684656874236388, + 1.1415318441782234, + 1.1198031889553155, + 1.0910031274146845, + 1.0660983196947673, + 1.0498697336705507, + 1.0350765279986567, + 1.0145895852928877, + 1.0013439705330982, + 0.9836063325227735, + 0.968549777523238, + 0.9591067674489386, + 0.9465820057822166, + 0.9334010118478351, + 0.9191551367815048, + 0.9057362914957724, + 0.8945632145340071, + 0.8814992971854174, + 0.8698868180749819, + 0.8596244665411801, + 0.8514889576556698, + 0.8425072468470839, + 0.8341767076391075, + 0.8278204125037605, + 0.8225904077486038, + 0.8181934666462969, + 0.8123426056325708 + ], + "peak_layer": 1, + "peak_normalized": 1.3620802882826015, + "population_cv": 0.1550560200772002, + "rest_mean": 0.00015278581957789829, + "spike_contrast": 0.8634362323882038, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013192081241868435, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002036659570876509, + 0.00019729818450286984, + 0.0001895671884994954, + 0.0001844495564000681, + 0.00017920024401973933, + 0.00017471560568083078, + 0.00017068830493371934, + 0.0001674393133725971, + 0.0001631329650990665, + 0.00015940905723255128, + 0.0001569824671605602, + 0.0001547705032862723, + 0.00015170717961154878, + 0.00014972662029322237, + 0.00014707438822370023, + 0.00014482304686680436, + 0.00014341107453219593, + 0.00014153830124996603, + 0.00013956740440335125, + 0.00013743728050030768, + 0.00013543081877287477, + 0.00013376015704125166, + 0.0001318067661486566, + 0.00013007040251977742, + 0.00012853591761086136, + 0.00012731945025734603, + 0.00012597645400092006, + 0.00012473082460928708, + 0.00012378039536997676, + 0.00012299837544560432, + 0.0001223409199155867, + 0.00012146606604801491 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015817073654034175, + "first_to_last_ratio": 1.2671327323753674, + "imbalance_abs_log_ratio": 0.23675665700060805, + "last_quartile_mean": 0.00012482570491556544, + "mean": 0.00014045925900063594, + "normalized": [ + 1.1719218726461313, + 1.1599555891510773, + 1.143372783062998, + 1.1305925991754215, + 1.1202702784385838, + 1.106568447670676, + 1.0931409560371117, + 1.082952696795682, + 1.0683580206124534, + 1.05266391584401, + 1.0427272032106563, + 1.034492263470324, + 1.0219716035750168, + 1.0154196846947314, + 1.0039021031599646, + 0.9939448772529704, + 0.9863124890252286, + 0.9768583563343864, + 0.9682383265025308, + 0.9606909958387225, + 0.9520472410590707, + 0.9456933063501335, + 0.9351354973759991, + 0.9231938712502704, + 0.9140830768869257, + 0.9051693342733999, + 0.8960380266364146, + 0.8881305740575848, + 0.8822109405764296, + 0.8783208779569746, + 0.8754534745501711, + 0.8701687165279489 + ], + "peak_layer": 1, + "peak_normalized": 1.1719218726461313, + "population_cv": 0.09095723041158388, + "rest_mean": 0.00014217518737832843, + "spike_contrast": 0.9227576779061385, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013119324576109647, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016460727783851326, + 0.00016292650252580643, + 0.00016059729387052357, + 0.0001588021987117827, + 0.0001573523331899196, + 0.00015542778419330716, + 0.00015354176866821945, + 0.00015211073332466185, + 0.00015006077592261136, + 0.00014785639359615743, + 0.0001464606903027743, + 0.00014530401676893234, + 0.00014354537415783852, + 0.00014262509648688138, + 0.0001410073455190286, + 0.00013960876094643027, + 0.00013853672135155648, + 0.0001372088008793071, + 0.00013599803787656128, + 0.00013493794540409, + 0.00013372385001275688, + 0.00013283138105180115, + 0.00013134843902662396, + 0.0001296711270697415, + 0.0001283914316445589, + 0.00012713941396214068, + 0.00012585683725774288, + 0.00012474616232793778, + 0.00012391469499561936, + 0.00012336829968262464, + 0.00012296554632484913, + 0.00012222325312905014 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031102553475648165, + "first_to_last_ratio": 2.2098501973960105, + "imbalance_abs_log_ratio": 0.7929247292485178, + "last_quartile_mean": 0.00014074507635086775, + "mean": 0.00020578417843353236, + "normalized": [ + 1.8085758305333837, + 1.7408578335592815, + 1.6197863489261306, + 1.5416881776712252, + 1.4477928223214513, + 1.3679195769143782, + 1.3100342829404608, + 1.2546744848740619, + 1.192398672058626, + 1.1190508147725688, + 1.082828610043334, + 1.0467691899805913, + 0.9849335746234156, + 0.9618381641711736, + 0.9182125099812896, + 0.877817233347749, + 0.8611427657722097, + 0.8409392227987704, + 0.8149468534272446, + 0.7879793317746896, + 0.7604794717340572, + 0.7439065501103351, + 0.7274095758071225, + 0.7164574630456112, + 0.7035124756224637, + 0.6971408207181957, + 0.690610058298894, + 0.684463275349207, + 0.6799680284055707, + 0.6760380727842366, + 0.6720531720343774, + 0.6677747355978938 + ], + "peak_layer": 1, + "peak_normalized": 1.8085758305333837, + "population_cv": 0.33829873005197686, + "rest_mean": 0.0002160599310796156, + "spike_contrast": 0.6956177084463702, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015029511414468288, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003721762914210558, + 0.00035824099904857576, + 0.00033332640305161476, + 0.00031725503504276276, + 0.00029793285648338497, + 0.0002814962062984705, + 0.0002695843286346644, + 0.00025819215807132423, + 0.0002453767810948193, + 0.00023028295254334807, + 0.00022282899590209126, + 0.00021540853776969016, + 0.00020268374646548182, + 0.00019793107639998198, + 0.0001889536069938913, + 0.00018064089817926288, + 0.00017720955656841397, + 0.0001730519870761782, + 0.00016770316869951785, + 0.00016215367941185832, + 0.00015649464330635965, + 0.00015308419824577868, + 0.00014968938194215298, + 0.00014743561041541398, + 0.00014477173681370914, + 0.00014346055104397237, + 0.00014211662346497178, + 0.00014085171278566122, + 0.00013992666208650917, + 0.00013911793939769268, + 0.00013829790987074375, + 0.00013741747534368187 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.028097106143832207, + 0.014136734418570995, + 0.014224153943359852, + 0.014337832108139992, + 0.014408928342163563, + 0.009704872034490108, + 0.009773381985723972, + 0.00987160298973322, + 0.00994801800698042, + 0.007515520788729191, + 0.007602147292345762, + 0.007658426649868488, + 0.007734050042927265, + 0.0062688966281712055, + 0.006336068734526634, + 0.006399397272616625, + 0.006462901830673218, + 0.005432929843664169, + 0.005504137370735407, + 0.005560056772083044, + 0.005631812382489443, + 0.004873365629464388, + 0.004937449004501104, + 0.005005750339478254, + 0.005022916477173567, + 0.0044684382155537605, + 0.004500201437622309, + 0.004573327023535967, + 0.00463611725717783, + 0.004166214261204004, + 0.0042496914975345135, + 0.004283483140170574 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00014323742470878642, + "first_to_last_ratio": 1.6208859606473658, + "imbalance_abs_log_ratio": 0.48297288904334346, + "last_quartile_mean": 8.83698348843609e-05, + "mean": 0.00011961164659624046, + "normalized": [ + 0.879665217639624, + 1.4482702231163533, + 1.4522235545753037, + 1.2017346580032677, + 1.034069363337856, + 1.36550769437152, + 1.149442648944982, + 1.0492523703464898, + 1.0286339735981567, + 1.1048881958426662, + 1.2960670227766034, + 1.1342833655748923, + 0.9483258681651744, + 1.1110988010941807, + 1.0051223270547462, + 0.9186295907129635, + 0.9340104771989933, + 1.0935042558126027, + 0.9904421395235656, + 1.070541841916413, + 0.9831461473937306, + 1.1165574278336847, + 0.8906099662074067, + 0.8835226812669187, + 0.8406995671395937, + 0.8651479324264472, + 0.7705397043177317, + 0.6783637574582257, + 0.6658814129635636, + 0.7912689330389562, + 0.6809428819470401, + 0.6176059984003462 + ], + "peak_layer": 3, + "peak_normalized": 1.4522235545753037, + "population_cv": 0.21063690333645962, + "rest_mean": 0.00012087627045629133, + "spike_contrast": 0.9330423359872595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011278267775196582, + "top_five_layers": [ + 3, + 2, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010521820513531566, + 0.00017322998610325158, + 0.00017370285058859736, + 0.00014374146121554077, + 0.000123686739243567, + 0.0001633306237636134, + 0.00013748672790825367, + 0.00012550280371215194, + 0.00012303660332690924, + 0.0001321574964094907, + 0.00015502471069339663, + 0.00013567350106313825, + 0.00011343081860104576, + 0.0001329003571299836, + 0.00012022433656966314, + 0.00010987879795720801, + 0.0001117185311159119, + 0.00013079584459774196, + 0.00011846841516671702, + 0.00012804927246179432, + 0.00011759572953451425, + 0.00013355327246244997, + 0.00010652732453309, + 0.00010567960271146148, + 0.00010055745951831341, + 0.00010348176874686033, + 9.216552280122414e-05, + 8.114020602079108e-05, + 7.964717224240303e-05, + 9.464497998123989e-05, + 8.144869934767485e-05, + 7.387287041638047e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00015447182067873655, + "first_to_last_ratio": 1.6506746305835311, + "imbalance_abs_log_ratio": 0.5011840713694677, + "last_quartile_mean": 9.358102306578076e-05, + "mean": 0.00012785212516064348, + "normalized": [ + 0.8854544129111614, + 1.4719853585239924, + 1.4439160513495162, + 1.202741955390318, + 1.0425045283644727, + 1.3767992039076578, + 1.1738833181051227, + 1.0683704265103389, + 1.0407410271769484, + 1.136073438345882, + 1.2855554885667506, + 1.129314905977533, + 0.953735223493217, + 1.1222184469907663, + 1.0164965849441758, + 0.9298350247961144, + 0.9320412792829462, + 1.0837813144211186, + 0.9857736034320764, + 1.0482233302985868, + 0.9656597421723547, + 1.0923472818519362, + 0.884471876187184, + 0.8724974481708365, + 0.826330173068434, + 0.8536374096219558, + 0.7648976975162065, + 0.681485935203229, + 0.6604246501739865, + 0.7781176964145518, + 0.6752223987471391, + 0.6154627680834903 + ], + "peak_layer": 2, + "peak_normalized": 1.4719853585239924, + "population_cv": 0.21455917947613024, + "rest_mean": 0.00012955063343893185, + "spike_contrast": 0.9160910858365683, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011868018045788631, + "top_five_layers": [ + 2, + 3, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001132072284235619, + 0.00018819645629264414, + 0.00018460773571860045, + 0.00015377311501652002, + 0.00013328641944099218, + 0.00017602670413907617, + 0.0001500834769103676, + 0.00013659342948812991, + 0.00013306095206644386, + 0.00014524940343108028, + 0.00016436100122518837, + 0.00014438531070481986, + 0.00012193707516416907, + 0.0001434780133422464, + 0.00012996124860364944, + 0.00011888138396898285, + 0.00011916345829376951, + 0.00013856374425813556, + 0.00012603325012605637, + 0.00013401758042164147, + 0.0001234616502188146, + 0.00013965892139822245, + 0.00011308160901535302, + 0.00011155065294587985, + 0.00010564806871116161, + 0.00010913935693679377, + 9.779379615793005e-05, + 8.71294250828214e-05, + 8.443669503321871e-05, + 9.948400111170486e-05, + 8.632861863588914e-05, + 7.86882228567265e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.13698715530335903, + "first_to_last_ratio": 1.6506746463483835, + "imbalance_abs_log_ratio": 0.5011840809200188, + "last_quartile_mean": 0.08298858627676964, + "mean": 0.11338054295629263, + "normalized": [ + 0.8854544185536968, + 1.4719853260144444, + 1.4439159993116888, + 1.2027420311824657, + 1.042504519270847, + 1.3767991933658148, + 1.1738833593963587, + 1.0683704218985235, + 1.0407409783116481, + 1.1360735196220915, + 1.285555449073044, + 1.129314933056767, + 0.9537352653592515, + 1.1222184500196979, + 1.0164965450422543, + 0.9298350370138166, + 0.9320412867497836, + 1.0837813135283418, + 0.9857735942006465, + 1.0482233283425098, + 0.9656597514448471, + 1.0923472716531482, + 0.8844718771212735, + 0.8724974491223123, + 0.82633015871154, + 0.8536374112441639, + 0.7648976624892803, + 0.681485927333974, + 0.660424633911195, + 0.7781176812354046, + 0.6752224228414072, + 0.6154627835777616 + ], + "peak_layer": 2, + "peak_normalized": 1.4719853260144444, + "population_cv": 0.2145591770090877, + "rest_mean": 0.11488679713673061, + "spike_contrast": 0.9160910827435619, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10524677038192749, + "top_five_layers": [ + 2, + 3, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.10039330273866653, + 0.16689449548721313, + 0.16371197998523712, + 0.1363675445318222, + 0.11819972842931747, + 0.1561022400856018, + 0.13309553265571594, + 0.12113241851329803, + 0.11799977719783783, + 0.1288086324930191, + 0.14575697481632233, + 0.1280423402786255, + 0.10813502222299576, + 0.1272377371788025, + 0.11525093019008636, + 0.10542520135641098, + 0.1056753471493721, + 0.12287971377372742, + 0.11176754534244537, + 0.11884813010692596, + 0.10948702692985535, + 0.12385092675685883, + 0.10028190165758133, + 0.09892423450946808, + 0.09368976205587387, + 0.09678587317466736, + 0.08672451227903366, + 0.07726724445819855, + 0.07487930357456207, + 0.0882234051823616, + 0.07655708491802216, + 0.06978150457143784 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.7160250005617854e-05, + "first_to_last_ratio": 0.7388534944946782, + "imbalance_abs_log_ratio": 0.30265562600345663, + "last_quartile_mean": 6.382895981005277e-05, + "mean": 5.960399772675373e-05, + "normalized": [ + 0.46158176249046556, + 0.8375260596839308, + 0.8768757910510345, + 0.7241656982213429, + 0.6710453705877814, + 1.0089930699998053, + 0.8842011269201543, + 0.865421443073546, + 0.883514351886623, + 0.9622394496647411, + 1.1256912972870226, + 1.0210990974149952, + 0.9232791878576234, + 1.0980011773561495, + 1.0212452171652862, + 0.9544893655323485, + 1.0133393696232265, + 1.159814348379829, + 0.9921993085906312, + 1.1469178460749072, + 1.0774269793999733, + 1.3924548133021735, + 1.1792402184404436, + 1.1521666823514416, + 1.1069955376913123, + 1.1868921569761495, + 1.1034570471301723, + 0.9847635589252388, + 0.9837749617675684, + 1.1772881708824627, + 1.0498861530819708, + 0.9740133811896496 + ], + "peak_layer": 22, + "peak_normalized": 1.3924548133021735, + "population_cv": 0.17243044689052606, + "rest_mean": 5.759890990264938e-05, + "spike_contrast": 1.22279175241262, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.043147197691724e-05, + "top_five_layers": [ + 22, + 26, + 23, + 30, + 18 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.751211832219269e-05, + 4.991990135749802e-05, + 5.226530265645124e-05, + 4.316317063057795e-05, + 3.9996986743062735e-05, + 6.014002065057866e-05, + 5.2701921958941966e-05, + 5.158257772563957e-05, + 5.266098742140457e-05, + 5.735331797040999e-05, + 6.709570152452216e-05, + 6.0861588281113654e-05, + 5.503113061422482e-05, + 6.544525967910886e-05, + 6.087029760237783e-05, + 5.689138197340071e-05, + 6.0399077483452857e-05, + 6.912957178428769e-05, + 5.9139045333722606e-05, + 6.836088869022205e-05, + 6.421895523089916e-05, + 8.299587352667004e-05, + 7.028743129922077e-05, + 6.867374031571671e-05, + 6.598135951207951e-05, + 7.074351742630824e-05, + 6.577045132871717e-05, + 5.869584492756985e-05, + 5.863692058483139e-05, + 7.017108146101236e-05, + 6.2577411881648e-05, + 5.8055091358255595e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6127787956938846e-05, + "first_to_last_ratio": 0.8333328618421979, + "imbalance_abs_log_ratio": 0.18232212258347721, + "last_quartile_mean": 6.735338365615462e-05, + "mean": 6.689512088087213e-05, + "normalized": [ + 0.49022256384162527, + 0.923222562655389, + 0.9170289550360846, + 0.7742286879065733, + 0.7345865645537341, + 1.0394905128518102, + 0.9423822415780315, + 0.8911705562121757, + 0.9138181637864031, + 1.0330628394205372, + 1.1540375572503838, + 1.0455389170050469, + 0.9297420351318767, + 1.1341544713967917, + 1.0374336268991433, + 0.9867016024211971, + 1.0073449670971224, + 1.1743962796261593, + 1.0459325434797282, + 1.1475080782429012, + 1.066139536390961, + 1.3297286391604237, + 1.1245973641634917, + 1.1027270030076322, + 1.04892482254714, + 1.125623033590473, + 1.0476398534474196, + 0.9361019469456514, + 0.930671663614635, + 1.0896207315037236, + 0.9754527388499041, + 0.9007689403858299 + ], + "peak_layer": 22, + "peak_normalized": 1.3297286391604237, + "population_cv": 0.14905429467389167, + "rest_mean": 6.522988486621116e-05, + "spike_contrast": 1.1633838617941636, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.58873953600414e-05, + "top_five_layers": [ + 22, + 18, + 11, + 20, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.2793497666716576e-05, + 6.17590849287808e-05, + 6.134476279839873e-05, + 5.179212166694924e-05, + 4.9140257033286616e-05, + 6.953684351174161e-05, + 6.304077396634966e-05, + 5.961496208328754e-05, + 6.112997652962804e-05, + 6.910686352057382e-05, + 7.719948189333081e-05, + 6.994145223870873e-05, + 6.219520582817495e-05, + 7.586940046167001e-05, + 6.939924787729979e-05, + 6.600552296731621e-05, + 6.738646334270015e-05, + 7.856138108763844e-05, + 6.996778392931446e-05, + 7.676269160583615e-05, + 7.13195331627503e-05, + 8.895235805539414e-05, + 7.523007661802694e-05, + 7.37670561647974e-05, + 7.016795279923826e-05, + 7.529868889832869e-05, + 7.00819946359843e-05, + 6.26206528977491e-05, + 6.225739343790337e-05, + 7.289031054824591e-05, + 6.525302887894213e-05, + 6.025704715284519e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.866022898866504e-05, + "first_to_last_ratio": 0.40574519905788725, + "imbalance_abs_log_ratio": 0.9020299049238899, + "last_quartile_mean": 4.599001795213553e-05, + "mean": 3.414976504245715e-05, + "normalized": [ + 0.2445526901503676, + 0.5596630531980494, + 0.5376964344799271, + 0.48012169469058175, + 0.4798804572179837, + 0.6900344095639286, + 0.7152325131545565, + 0.6642055410913166, + 0.7554442803991458, + 0.9343372865664744, + 0.9240467958883766, + 0.8592444663674569, + 0.8263415713413451, + 1.0773994704320806, + 0.959597390691018, + 0.9894020700335048, + 0.9760686528076964, + 1.1936439731862338, + 1.099336047654388, + 1.1448507248259905, + 1.0495120137852647, + 1.4542665199347329, + 1.3223844598378258, + 1.2890134714336634, + 1.2402412095084396, + 1.4030995672846114, + 1.3877316376933018, + 1.246635573851661, + 1.2962845787221184, + 1.4847061188341568, + 1.3923191445524272, + 1.3227061808213745 + ], + "peak_layer": 30, + "peak_normalized": 1.4847061188341568, + "population_cv": 0.32662955778396485, + "rest_mean": 3.2435424482510255e-05, + "spike_contrast": 1.3382653305362568, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.340720406617038e-05, + "top_five_layers": [ + 30, + 22, + 26, + 31, + 27 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.351416909135878e-06, + 1.9112361769657582e-05, + 1.8362206901656464e-05, + 1.6396043065469712e-05, + 1.6387804862461053e-05, + 2.3564512957818806e-05, + 2.4425022274954244e-05, + 2.2682463168166578e-05, + 2.5798244678298943e-05, + 3.1907398806652054e-05, + 3.1555980967823416e-05, + 2.9342996640480123e-05, + 2.8219370506121777e-05, + 3.679293877212331e-05, + 3.277002542745322e-05, + 3.378784822416492e-05, + 3.333251515869051e-05, + 4.0762661228654906e-05, + 3.7542067730100825e-05, + 3.909638326149434e-05, + 3.584058868000284e-05, + 4.9662859964882955e-05, + 4.515911859925836e-05, + 4.4019507186021656e-05, + 4.2353945900686085e-05, + 4.791552055394277e-05, + 4.7390709369210526e-05, + 4.257231194060296e-05, + 4.4267813791520894e-05, + 5.070236511528492e-05, + 4.754737165058032e-05, + 4.517010529525578e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00021511870181711856, + "first_to_last_ratio": 1.067203936185183, + "imbalance_abs_log_ratio": 0.06504208450186474, + "last_quartile_mean": 0.00020157225299044512, + "mean": 0.00023711549874860793, + "normalized": [ + 0.5199215340336985, + 0.9694171998286779, + 0.9952311992308057, + 0.8139231798698612, + 0.8071491568326964, + 1.1522101416966866, + 1.0127543403881063, + 0.9872470097646113, + 0.9739878350012732, + 1.0629045356723543, + 1.2654986331851705, + 1.158397891601516, + 1.0251221075049282, + 1.1837639471542862, + 1.126357049236326, + 1.0240452993420521, + 1.0679809871494306, + 1.2152134226291609, + 1.0519957926786496, + 1.2134701299884076, + 1.1240302451392508, + 1.3270417853241223, + 1.0642149202553777, + 1.0573092573282454, + 0.9986311292560726, + 1.01615500686031, + 0.8954142860763721, + 0.793725079326075, + 0.7548043496765433, + 0.8883375219775711, + 0.764305251609244, + 0.6894397743821165 + ], + "peak_layer": 22, + "peak_normalized": 1.3270417853241223, + "population_cv": 0.17546356070810173, + "rest_mean": 0.00023209894856403548, + "spike_contrast": 1.138328593816985, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002642048697452992, + "top_five_layers": [ + 22, + 11, + 18, + 20, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00012328145385254174, + 0.00022986384283285588, + 0.0002359847421757877, + 0.00019299380073789507, + 0.00019138757488690317, + 0.00027320688241161406, + 0.00024013975053094327, + 0.00023409156710840762, + 0.00023094761127140373, + 0.0002520311390981078, + 0.00030006933957338333, + 0.0002746740938164294, + 0.00024307233979925513, + 0.00028068877873010933, + 0.0002670767134986818, + 0.0002428170118946582, + 0.00025323484442196786, + 0.0002881459367927164, + 0.00024944450706243515, + 0.0002877325750887394, + 0.0002665249921847135, + 0.00031466217478737235, + 0.0002523418515920639, + 0.00025070441188290715, + 0.0002367909182794392, + 0.00024094610125757754, + 0.00021231660502962768, + 0.00018820451805368066, + 0.00017897580983117223, + 0.00021063859458081424, + 0.00018122862093150616, + 0.00016347685595974326 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014079716056585312, + 0.014167090877890587, + 0.014255538582801819, + 0.014355218037962914, + 0.009655896574258804, + 0.009734091348946095, + 0.009828771464526653, + 0.009915035218000412, + 0.007479202933609486, + 0.007559895049780607, + 0.0076358383521437645, + 0.007706487085670233, + 0.0062490664422512054, + 0.006315518636256456, + 0.0063744354993104935, + 0.006439714692533016, + 0.0054116579703986645, + 0.005481995176523924, + 0.005541813559830189, + 0.005614723544567823, + 0.004862982779741287, + 0.004923288244754076, + 0.0049925679340958595, + 0.005004179198294878, + 0.004448203835636377, + 0.004490054678171873, + 0.00455933902412653, + 0.004625064320862293, + 0.004160413984209299, + 0.004242690280079842, + 0.004270957782864571, + 0.004303553607314825 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.6855086818832206e-05, + "first_to_last_ratio": 1.1121706932103832, + "imbalance_abs_log_ratio": 0.10631368513677814, + "last_quartile_mean": 3.313797697046539e-05, + "mean": 3.537216105087282e-05, + "normalized": [ + 1.3329674878497746, + 1.065596392820356, + 0.9206330800727252, + 0.8842664165696186, + 1.1462361065020723, + 1.065158463293282, + 0.99582829507555, + 0.9247019781352203, + 1.132583771771479, + 1.005206548698436, + 0.979948053013938, + 0.967083333200755, + 1.1057115808592233, + 1.043785177992708, + 0.995436955984306, + 0.9279460300710988, + 1.1098294348763655, + 1.0288442525987258, + 1.0234795644677463, + 0.9449806453153974, + 1.0109568163965834, + 1.0260998393764513, + 0.9416381672270038, + 0.926379336673748, + 0.9884731771330477, + 0.9341156116917169, + 0.9458688462068515, + 0.9053302302997448, + 0.9662542702797254, + 0.9495526789453999, + 0.9100381269887573, + 0.8950693296121922 + ], + "peak_layer": 1, + "peak_normalized": 1.3329674878497746, + "population_cv": 0.09165679612351446, + "rest_mean": 3.551162256001219e-05, + "spike_contrast": 0.9748658722370791, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.461906890152022e-05, + "top_five_layers": [ + 1, + 5, + 9, + 17, + 13 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.714994065579958e-05, + 3.769244722207077e-05, + 3.256478157709353e-05, + 3.127841409877874e-05, + 4.054484816151671e-05, + 3.767695670831017e-05, + 3.5224598832428455e-05, + 3.270870729465969e-05, + 4.006193557870574e-05, + 3.555632792995311e-05, + 3.4662880352698267e-05, + 3.420782741159201e-05, + 3.911140811396763e-05, + 3.692093741847202e-05, + 3.521075632306747e-05, + 3.2823456422192976e-05, + 3.9257065509445965e-05, + 3.6392444599187e-05, + 3.620268398663029e-05, + 3.342600757605396e-05, + 3.575972732505761e-05, + 3.6295368772698566e-05, + 3.330777690280229e-05, + 3.276803909102455e-05, + 3.49644324160181e-05, + 3.3041687856893986e-05, + 3.3457425161032006e-05, + 3.202348671038635e-05, + 3.417850166442804e-05, + 3.358773028594442e-05, + 3.2190015190280974e-05, + 3.166053647873923e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 7.015105893515283e-05, + "first_to_last_ratio": 0.6588274570259804, + "imbalance_abs_log_ratio": 0.4172936042156455, + "last_quartile_mean": 0.00010647865110513521, + "mean": 9.056432463694364e-05, + "normalized": [ + 0.8598786252117567, + 0.7452797309823858, + 0.6694431442719111, + 0.6417832925446045, + 0.8822867975473095, + 0.8318257491363829, + 0.804532248804658, + 0.7617644993607329, + 0.969439786050785, + 0.9144645011121496, + 0.8951531228323205, + 0.876468558935897, + 1.0407786059783084, + 1.0056397226462848, + 0.9828357943952414, + 0.9490839045924557, + 1.1213662721801265, + 1.0774120583919133, + 1.0554308943167812, + 1.0307906296953837, + 1.1314055055223455, + 1.1556700189426168, + 1.1045075200854564, + 1.0869669205578247, + 1.2145839027446308, + 1.1552275050200584, + 1.1514380979421175, + 1.1280886596086177, + 1.2254687972249847, + 1.20187962367999, + 1.1639632986601018, + 1.165142211023867 + ], + "peak_layer": 29, + "peak_normalized": 1.2254687972249847, + "population_cv": 0.1618495930711073, + "rest_mean": 8.82393912804589e-05, + "spike_contrast": 1.1686273359956587, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010311896476196125, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 7.787432696204633e-05, + 6.74957555020228e-05, + 6.062766624381766e-05, + 5.812267045257613e-05, + 7.990370795596391e-05, + 7.533373718615621e-05, + 7.286191976163536e-05, + 6.898868741700426e-05, + 8.779665949987248e-05, + 8.281785994768143e-05, + 8.106893801596016e-05, + 7.937678310554475e-05, + 9.425741154700518e-05, + 9.107508230954409e-05, + 8.900985994841903e-05, + 8.595314284320921e-05, + 0.00010155577911064029, + 9.757509542396292e-05, + 9.558438614476472e-05, + 9.335285722045228e-05, + 0.00010246497549815103, + 0.00010466247476870194, + 0.00010002897761296481, + 9.844042506301776e-05, + 0.00010999797086697072, + 0.00010462239879416302, + 0.00010427921370137483, + 0.00010216458758804947, + 0.00011098375398432836, + 0.00010884741641348228, + 0.00010541355004534125, + 0.00010552031744737178 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06221066089347005, + "first_to_last_ratio": 0.6588274447143223, + "imbalance_abs_log_ratio": 0.4172936229028725, + "last_quartile_mean": 0.09442633483558893, + "mean": 0.0803133494919166, + "normalized": [ + 0.8598786809861813, + 0.7452797356777653, + 0.6694431594543391, + 0.6417832788211475, + 0.8822868217673959, + 0.83182573854958, + 0.804532202278358, + 0.7617644434307961, + 0.9694397651274825, + 0.9144645475330049, + 0.8951531251987148, + 0.8764685422804359, + 1.0407786734529123, + 1.0056396718254574, + 0.9828357789429103, + 0.9490838578168909, + 1.121366176654653, + 1.0774119964231388, + 1.0554309636293913, + 1.030790617723408, + 1.131405440813584, + 1.1556700726993148, + 1.1045075638748008, + 1.0869669141870728, + 1.214583886452764, + 1.155227565075462, + 1.151438141297865, + 1.1280886745503538, + 1.2254688318483915, + 1.20187965027907, + 1.1639633335063522, + 1.1651421478410058 + ], + "peak_layer": 29, + "peak_normalized": 1.2254688318483915, + "population_cv": 0.16184959877781283, + "rest_mean": 0.07825157529226055, + "spike_contrast": 1.1686273385361958, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0914469301700592, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.06905973702669144, + 0.05985591188073158, + 0.05376522243022919, + 0.051543764770030975, + 0.0708594098687172, + 0.06680671125650406, + 0.0646146759390831, + 0.06117985397577286, + 0.07785895466804504, + 0.0734437108039856, + 0.07189274579286575, + 0.07039212435483932, + 0.08358842134475708, + 0.08076629042625427, + 0.07893483340740204, + 0.07622410356998444, + 0.09006067365407944, + 0.08653056621551514, + 0.08476519584655762, + 0.08278624713420868, + 0.09086696058511734, + 0.09281573444604874, + 0.08870670199394226, + 0.0872979536652565, + 0.09754730015993118, + 0.09278019517660141, + 0.09247585386037827, + 0.09060057997703552, + 0.09842150658369064, + 0.09652698040008545, + 0.09348179399967194, + 0.09357646852731705 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.077007694533677e-06, + "first_to_last_ratio": 0.4632676420751026, + "imbalance_abs_log_ratio": 0.7694503312712233, + "last_quartile_mean": 1.5276283193088602e-05, + "mean": 1.1745167029175718e-05, + "normalized": [ + 0.5860812018784138, + 0.5338863181232825, + 0.4896831424227033, + 0.48432444022666415, + 0.7227742595277804, + 0.658356157633798, + 0.6679626701195096, + 0.6773027265164295, + 0.9317627933811012, + 0.8396840560424388, + 0.8002283441953012, + 0.8237560838623014, + 0.9947731150675175, + 0.9485376018841178, + 1.012754293639493, + 0.980582798985168, + 1.1861357451339187, + 1.074396404738039, + 1.1583203932505854, + 1.1590437197037688, + 1.1716087388237089, + 1.3380122311407743, + 1.182136735603702, + 1.17274255168395, + 1.3256589211015541, + 1.1785808128912905, + 1.3283612705891032, + 1.2366027296837898, + 1.3388375403515687, + 1.3849270873253279, + 1.3156503626951777, + 1.2965347517777213 + ], + "peak_layer": 30, + "peak_normalized": 1.3849270873253279, + "population_cv": 0.2782190894097951, + "rest_mean": 1.1227440423950881e-05, + "spike_contrast": 1.2951207174852213, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4540890697389841e-05, + "top_five_layers": [ + 30, + 29, + 22, + 27, + 25 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 6.883621608722024e-06, + 6.2705839809495956e-06, + 5.751410299126292e-06, + 5.6884714467742015e-06, + 8.48910440254258e-06, + 7.732503036095295e-06, + 7.845333129807841e-06, + 7.955033652251586e-06, + 1.0943709639832377e-05, + 9.862229489954188e-06, + 9.39881556405453e-06, + 9.67515279626241e-06, + 1.168377639260143e-05, + 1.1140732567582745e-05, + 1.1894968338310719e-05, + 1.1517108760017436e-05, + 1.3931362445873674e-05, + 1.2618965229194146e-05, + 1.3604666492028628e-05, + 1.3613162082037888e-05, + 1.3760740330326371e-05, + 1.5715177141828462e-05, + 1.3884393410990015e-05, + 1.377405715174973e-05, + 1.5570085452054627e-05, + 1.38426285047899e-05, + 1.56018249981571e-05, + 1.4524105608870741e-05, + 1.572487053635996e-05, + 1.62661999638658e-05, + 1.5452533261850476e-05, + 1.5228017218760215e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.028231200616574e-05, + "first_to_last_ratio": 0.5725175422355306, + "imbalance_abs_log_ratio": 0.5577119025637401, + "last_quartile_mean": 0.00010529338851483772, + "mean": 8.693937525094952e-05, + "normalized": [ + 0.6520176082954547, + 0.61792639701702, + 0.5860559002851884, + 0.5711155588169554, + 0.8204737244004536, + 0.784378974164202, + 0.772878878140371, + 0.7422194596148288, + 0.9603819210570083, + 0.917062379210463, + 0.9022792075902993, + 0.8821977885039085, + 1.057360487646653, + 1.0255127545988867, + 1.0067830966339102, + 0.9749202156934387, + 1.1510091959216098, + 1.1091409226242501, + 1.0872456061539109, + 1.0647793564016073, + 1.1683829082644246, + 1.1939706281941995, + 1.1404893610428462, + 1.1225154239128359, + 1.2535795945499137, + 1.1915479696415308, + 1.1872717446378291, + 1.1619194447200984, + 1.2611193116864508, + 1.2370940006457103, + 1.1981927894482458, + 1.1981773904854955 + ], + "peak_layer": 29, + "peak_normalized": 1.2611193116864508, + "population_cv": 0.20676778332368514, + "rest_mean": 8.410921549916061e-05, + "spike_contrast": 1.215351222858925, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001022222379106097, + "top_five_layers": [ + 29, + 25, + 30, + 31, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 5.6686003517825156e-05, + 5.3722134907729924e-05, + 5.095133383292705e-05, + 4.965242987964302e-05, + 7.133147300919518e-05, + 6.81934179738164e-05, + 6.719360681017861e-05, + 6.452809611801058e-05, + 8.349500421900302e-05, + 7.972883031470701e-05, + 7.844359060982242e-05, + 7.669772458029911e-05, + 9.192626021103933e-05, + 8.915743819670752e-05, + 8.75290934345685e-05, + 8.475895447190851e-05, + 0.00010006802040152252, + 9.642801887821406e-05, + 9.452445374336094e-05, + 9.257125202566385e-05, + 0.00010157848009839654, + 0.00010380306048318744, + 9.915343252941966e-05, + 9.759078966453671e-05, + 0.00010898542677750811, + 0.00010359243606217206, + 0.00010322066373191774, + 0.00010101655061589554, + 0.00010964092507492751, + 0.0001075521795428358, + 0.00010417013254482299, + 0.00010416879376862198 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 5.289650744089158e-05, + "first_to_last_ratio": 0.5036607689093815, + "imbalance_abs_log_ratio": 0.6858523150899144, + "last_quartile_mean": 0.00010502407712920103, + "mean": 8.374395429200376e-05, + "normalized": [ + 0.5462883602941595, + 0.5343271256807102, + 0.5249643991287053, + 0.5130648517134513, + 0.7580131602578075, + 0.7348274620741431, + 0.7341801809099809, + 0.7074999463902408, + 0.9290893112957194, + 0.896130362600229, + 0.8896360907643062, + 0.8667732512123232, + 1.0501229826066591, + 1.0276121076179003, + 1.01304723882372, + 0.9917393510831477, + 1.161342307568102, + 1.1291245609681106, + 1.1079392656735292, + 1.0938109858524918, + 1.2035226235600145, + 1.226263478849651, + 1.1724755435916903, + 1.1553301515952579, + 1.2932755854998876, + 1.227360989677261, + 1.2282904680523223, + 1.203004103561663, + 1.301298744133899, + 1.2818290480161694, + 1.2464280682225441, + 1.251387892724203 + ], + "peak_layer": 29, + "peak_normalized": 1.301298744133899, + "population_cv": 0.24714813814810152, + "rest_mean": 8.048455465878187e-05, + "spike_contrast": 1.2591821218500583, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010134471231140196, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.574834747472778e-05, + 4.474666638998315e-05, + 4.396259464556351e-05, + 4.296607949072495e-05, + 6.347901944536716e-05, + 6.153735739644617e-05, + 6.14831515122205e-05, + 5.924884317209944e-05, + 7.780561281833798e-05, + 7.504550012527034e-05, + 7.450164412148297e-05, + 7.258701953105628e-05, + 8.794145105639473e-05, + 8.60563013702631e-05, + 8.483658166369423e-05, + 8.305217488668859e-05, + 9.725539712235332e-05, + 9.455735562369227e-05, + 9.278321522288024e-05, + 9.160005720332265e-05, + 0.00010078774357680231, + 0.00010269215272273868, + 9.818773833103478e-05, + 9.675191540736705e-05, + 0.00010830401151906699, + 0.00010278406261932105, + 0.00010286190081387758, + 0.00010074432066176087, + 0.00010897590254899114, + 0.00010734543320722878, + 0.00010438081517349929, + 0.00010479617048986256 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010133173600479495, + "first_to_last_ratio": 0.7712367006907566, + "imbalance_abs_log_ratio": 0.25975994774816064, + "last_quartile_mean": 0.00013138863323547412, + "mean": 0.00011757463107642252, + "normalized": [ + 0.9240254436843522, + 0.8516610763218788, + 0.7400083948384197, + 0.7305203183107621, + 0.9945100925181989, + 0.9162256174012862, + 0.9029060711458456, + 0.8349457218511338, + 1.057384774375838, + 0.9561549232760007, + 0.9255087968738563, + 0.9289705112696707, + 1.0651337322266232, + 1.006401362110571, + 0.966928758219108, + 0.9306903222204219, + 1.1004660046561014, + 1.0434523934672617, + 1.009934985409439, + 0.9657192405683421, + 1.061624429236619, + 1.0862247001295862, + 1.0383026763618062, + 1.022368851612807, + 1.1429833503196458, + 1.0926891991234107, + 1.0888718388877934, + 1.0734199646869733, + 1.177032649101865, + 1.140664442905499, + 1.1108444025161495, + 1.1134249543727335 + ], + "peak_layer": 29, + "peak_normalized": 1.177032649101865, + "population_cv": 0.10936728326378568, + "rest_mean": 0.00011604396611262389, + "spike_contrast": 1.0844184846181808, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012584022188093512, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00010864195064641535, + 0.00010013373685069382, + 8.700621401658282e-05, + 8.58906569192186e-05, + 0.00011692915722960606, + 0.00010772488894872367, + 0.0001061588482116349, + 9.816843521548435e-05, + 0.0001243216247530654, + 0.00011241956235608086, + 0.00010881635535042733, + 0.00010922336514340714, + 0.00012523270561359823, + 0.00011832726886495948, + 0.00011368629202479497, + 0.00010942557128146291, + 0.0001293868845095858, + 0.00012268353020772338, + 0.00011874273332068697, + 0.00011354408343322575, + 0.00012482010060921311, + 0.00012771246838383377, + 0.0001220780541189015, + 0.00012020464055240154, + 0.00013438584574032575, + 0.0001284725294681266, + 0.00012802370474673808, + 0.0001262069563381374, + 0.00013838917948305607, + 0.00013411320105660707, + 0.00013060712080914527, + 0.00013091052824165672 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 8.06433666787865, + "forward": { + "activation_sha256": { + "attention_branch_output": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", + "mlp_branch_output": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", + "post_attention_state": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", + "post_mlp_state": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", + "pre_attention_input": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", + "pre_mlp_input": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248" + }, + "logits_sha256": "30ae47372d19e5c17da2b2a4846b2d27ef6e44fdc06045752d040a56bc6e9398", + "loss_nats": 5.5897722244262695, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.5897722244262695, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0021227982360869646, + 0.00224129855632782, + 0.0024903982412070036, + 0.0023722455371171236, + 0.0023577234242111444, + 0.0023993616923689842, + 0.002688874490559101, + 0.0029664887115359306, + 0.0029547461308538914, + 0.003052416956052184, + 0.0028214550111442804, + 0.0032544767018407583, + 0.00348988501355052, + 0.003914494998753071, + 0.0035071265883743763, + 0.004211232997477055, + 0.004072706215083599, + 0.004050548188388348, + 0.003929397091269493, + 0.004410331137478352, + 0.004824656527489424, + 0.00433384021744132, + 0.004475373309105635, + 0.004755842965096235, + 0.004843301139771938, + 0.0050001065246760845, + 0.005346097983419895, + 0.004958339501172304, + 0.0054801348596811295, + 0.005455824080854654, + 0.005286100786179304, + 0.005402065813541412 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015727665777376387, + "first_to_last_ratio": 3.444367030996094, + "imbalance_abs_log_ratio": 1.2367401520242614, + "last_quartile_mean": 4.566199140754179e-05, + "mean": 8.846047569477378e-05, + "normalized": [ + 2.4938583213844665, + 2.2321521039252294, + 1.9412946714691228, + 1.7875630514690435, + 1.605255150569201, + 1.4809330135294976, + 1.3782219007164203, + 1.3041730210393685, + 1.1983684502302745, + 1.1165155187552953, + 1.0705163494358503, + 1.028915549335682, + 0.969265519878856, + 0.9296915475356099, + 0.8760186941641476, + 0.840544602365157, + 0.8215144482528359, + 0.7958412144142111, + 0.7578169321029499, + 0.7149281644739428, + 0.6789722877716906, + 0.648159768212111, + 0.6127699836305072, + 0.5872278577962867, + 0.5712871725092085, + 0.5525140937217092, + 0.5351005407841478, + 0.5196042162209058, + 0.49974197043571306, + 0.49405312503381205, + 0.48454882285128825, + 0.4726319359854579 + ], + "peak_layer": 1, + "peak_normalized": 2.4938583213844665, + "population_cv": 0.5254866190950809, + "rest_mean": 9.469065830840177e-05, + "spike_contrast": 0.5789112734082525, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.48174895811826e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00022060789342503995, + 0.0001974572369363159, + 0.00017172785010188818, + 0.00015812867786735296, + 0.00014200163423083723, + 0.0001310040388489142, + 0.00012191816495032981, + 0.00011536776582943276, + 0.00010600824316497892, + 9.876749390969053e-05, + 9.469838551012799e-05, + 9.101835894398391e-05, + 8.574168896302581e-05, + 8.224095654441044e-05, + 7.749303040327504e-05, + 7.435497536789626e-05, + 7.267155888257548e-05, + 7.040049240458757e-05, + 6.703684630338103e-05, + 6.324288551695645e-05, + 6.0062211559852585e-05, + 5.733652142225765e-05, + 5.42059242434334e-05, + 5.1946455641882494e-05, + 5.053633503848687e-05, + 4.887565955868922e-05, + 4.733524838229641e-05, + 4.596443613991141e-05, + 4.420741242938675e-05, + 4.370417445898056e-05, + 4.286341936676763e-05, + 4.180924588581547e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00022213962802197784, + "first_to_last_ratio": 1.7678973829107683, + "imbalance_abs_log_ratio": 0.5697909211909248, + "last_quartile_mean": 0.00012565187898871955, + "mean": 0.00016188367408176418, + "normalized": [ + 1.7325269150218519, + 1.5951124185334453, + 1.452057969341735, + 1.3730142236731557, + 1.2857984704138503, + 1.2257558334519054, + 1.176224442377172, + 1.1372505911497117, + 1.088955563427776, + 1.050933480062714, + 1.0259126283978994, + 1.0049209692997896, + 0.9780904283068869, + 0.9585002626676595, + 0.9364797090707767, + 0.9194681619525875, + 0.9067005574410794, + 0.8936181540011504, + 0.876961137714969, + 0.8605080845266135, + 0.8461083257765747, + 0.8336465310750286, + 0.8211746685608088, + 0.8107906186997867, + 0.8025238658005067, + 0.7926026656489187, + 0.7851375623337778, + 0.7775608141673811, + 0.7689788128837574, + 0.7655650153667295, + 0.7607140375905382, + 0.7564070812634623 + ], + "peak_layer": 1, + "peak_normalized": 1.7325269150218519, + "population_cv": 0.2510633514944888, + "rest_mean": 0.00016719439386010722, + "spike_contrast": 0.7967120440064878, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001332057872787118, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002804678224492818, + 0.0002582226588856429, + 0.00023506447905674577, + 0.00022226858709473163, + 0.00020814978051930666, + 0.00019842985784634948, + 0.00019041153427679092, + 0.00018410230404697359, + 0.00017628412751946598, + 0.0001701289729680866, + 0.0001660785055719316, + 0.0001626802986720577, + 0.0001583368721185252, + 0.00015516554412897676, + 0.00015160077600739896, + 0.00014884688425809145, + 0.0001467800175305456, + 0.00014466218999587, + 0.00014196569100022316, + 0.0001393022103002295, + 0.00013697112444788218, + 0.00013495376333594322, + 0.0001329347724094987, + 0.0001312537642661482, + 0.00012991551193408668, + 0.00012830943160224706, + 0.0001271009532501921, + 0.00012587440141942352, + 0.0001244851155206561, + 0.00012393247743602842, + 0.0001231471833307296, + 0.00012244995741639286 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19699564762413502, + "first_to_last_ratio": 1.7678974461601602, + "imbalance_abs_log_ratio": 0.5697909569675436, + "last_quartile_mean": 0.11142934113740921, + "mean": 0.14356006286107004, + "normalized": [ + 1.73252692376823, + 1.5951124529267644, + 1.4520579427898739, + 1.3730142509369438, + 1.2857984860257277, + 1.2257558423238033, + 1.1762244498747965, + 1.1372506023188744, + 1.0889555406950477, + 1.0509335152923371, + 1.0259126331612176, + 1.004920958483027, + 0.9780904724894115, + 0.9585002702989701, + 0.9364797557971557, + 0.9194681924019411, + 0.9067005932791752, + 0.893618176643793, + 0.876961081367795, + 0.8605080517791706, + 0.8461083423470157, + 0.8336465303631113, + 0.8211746500314575, + 0.8107906024915998, + 0.8025238662049899, + 0.792602599370704, + 0.7851375422989607, + 0.7775607992253795, + 0.7689787783378558, + 0.7655650375651303, + 0.7607140142018359, + 0.7564070449079043 + ], + "peak_layer": 1, + "peak_normalized": 1.73252692376823, + "population_cv": 0.25106336091926634, + "rest_mean": 0.1482696624265777, + "spike_contrast": 0.7967120399011176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1181282252073288, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2487216740846634, + 0.2289944440126419, + 0.20845752954483032, + 0.19711001217365265, + 0.18458931148052216, + 0.17596958577632904, + 0.1688588559627533, + 0.16326376795768738, + 0.15633052587509155, + 0.15087208151817322, + 0.14728008210659027, + 0.1442665159702301, + 0.14041472971439362, + 0.13760235905647278, + 0.1344410926103592, + 0.1319989114999771, + 0.13016599416732788, + 0.1282878816127777, + 0.12589658796787262, + 0.12353459000587463, + 0.12146736681461334, + 0.11967834830284119, + 0.11788788437843323, + 0.11639714986085892, + 0.11521037667989731, + 0.11378607898950577, + 0.11271439492702484, + 0.11162667721509933, + 0.11039464175701141, + 0.10990456491708755, + 0.10920815169811249, + 0.10858984291553497 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.813340178268845e-05, + "first_to_last_ratio": 2.0396781258635253, + "imbalance_abs_log_ratio": 0.7127920139678758, + "last_quartile_mean": 3.3403996894776355e-05, + "mean": 4.817690887648496e-05, + "normalized": [ + 1.6677253734616262, + 1.5880838657379246, + 1.4689679818917765, + 1.414042908307037, + 1.3674754530876347, + 1.307279124984285, + 1.2642920390991146, + 1.2360018052494937, + 1.16456296339031, + 1.101785452839103, + 1.0797656603944665, + 1.0672809591929815, + 1.0128091896347613, + 0.9955804650846075, + 0.9623876307655963, + 0.9380984486976066, + 0.9310658554452689, + 0.9082443423860135, + 0.8811967490595315, + 0.8644115118569954, + 0.8497936463301483, + 0.8309315019236491, + 0.7889969150425813, + 0.7623309639866356, + 0.7584133539756696, + 0.7445182980676538, + 0.7180454335344151, + 0.7000361319857387, + 0.6663708133598921, + 0.6665317313809671, + 0.6558653811637546, + 0.6371080486827607 + ], + "peak_layer": 1, + "peak_normalized": 1.6677253734616262, + "population_cv": 0.28258812904813146, + "rest_mean": 4.997824995640527e-05, + "spike_contrast": 0.7693279992487513, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.844966704491526e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.034585334826261e-05, + 7.650897168787196e-05, + 7.077033660607412e-05, + 6.81242163409479e-05, + 6.588074029423296e-05, + 6.298066728049889e-05, + 6.090968236094341e-05, + 5.954674634267576e-05, + 5.610504376818426e-05, + 5.308061736286618e-05, + 5.201977182878181e-05, + 5.1418297516647726e-05, + 4.879401603830047e-05, + 4.796398934558965e-05, + 4.6364861191250384e-05, + 4.519468348007649e-05, + 4.4855874875793234e-05, + 4.3756404920713976e-05, + 4.245333548169583e-05, + 4.1644674638519064e-05, + 4.094043106306344e-05, + 4.0031711250776425e-05, + 3.8011432479834184e-05, + 3.672674938570708e-05, + 3.653801104519516e-05, + 3.586859020288102e-05, + 3.459320942056365e-05, + 3.372557694092393e-05, + 3.210368595318869e-05, + 3.21114384860266e-05, + 3.159756670356728e-05, + 3.069389640586451e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018498800636734813, + "first_to_last_ratio": 1.4771173407265819, + "imbalance_abs_log_ratio": 0.39009244570614415, + "last_quartile_mean": 0.0001252358233614359, + "mean": 0.00015057798327688943, + "normalized": [ + 1.36489476841379, + 1.32224224178262, + 1.270236278236952, + 1.2358216684832493, + 1.2002806183282966, + 1.1699184301039023, + 1.1431585124660382, + 1.1216044194279113, + 1.0931779382416982, + 1.0676137521656621, + 1.050334546837667, + 1.0352083927879365, + 1.0149316924423049, + 1.001461761419052, + 0.9840691969120363, + 0.9695910166861202, + 0.9584136856662008, + 0.9467149797350989, + 0.9322036518549593, + 0.9180752132052378, + 0.9048391519289843, + 0.8925982934025656, + 0.8798985862981495, + 0.8691051109125614, + 0.860352873872478, + 0.849675270316085, + 0.8415818308236442, + 0.8332603200076834, + 0.823928337260059, + 0.8201054366601405, + 0.8147645087933348, + 0.8099375145275801 + ], + "peak_layer": 1, + "peak_normalized": 1.36489476841379, + "population_cv": 0.15663918075507588, + "rest_mean": 0.0001538862707093358, + "spike_contrast": 0.8624111204329009, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000132713231141679, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00020552310161292553, + 0.00019910057017114013, + 0.00019126961706206203, + 0.0001860875345300883, + 0.00018073583487421274, + 0.00017616395780351013, + 0.0001721345033729449, + 0.00016888893151190132, + 0.0001646085293032229, + 0.00016075912571977824, + 0.00015815725782886147, + 0.0001558795920573175, + 0.00015282636741176248, + 0.00015079809236340225, + 0.0001481791550759226, + 0.00014599905989598483, + 0.00014431599993258715, + 0.00014255443238653243, + 0.00014036934589967132, + 0.000138241914100945, + 0.00013624885468743742, + 0.00013440565089695156, + 0.0001324933546129614, + 0.0001308680948568508, + 0.00012955020065419376, + 0.00012794238864444196, + 0.0001267236948478967, + 0.00012547065853141248, + 0.00012406546738930047, + 0.00012348982272669673, + 0.0001226855965796858, + 0.00012195875751785934 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015959660413500387, + "first_to_last_ratio": 1.2728751185715401, + "imbalance_abs_log_ratio": 0.2412782146622287, + "last_quartile_mean": 0.00012538276678242255, + "mean": 0.00014144585429676226, + "normalized": [ + 1.173955566226187, + 1.1612997099437044, + 1.147257651804958, + 1.1326524298721496, + 1.1221543736292459, + 1.1091111750587395, + 1.0952125313023318, + 1.0849401932479867, + 1.070907908686229, + 1.0558579375605581, + 1.0430997159190893, + 1.0346471150202732, + 1.0228091528121042, + 1.0150746526225596, + 1.0038187840838828, + 0.9945868690199192, + 0.986355871119351, + 0.9777591356798918, + 0.9676015085554662, + 0.9595938626090281, + 0.9511867345631487, + 0.9428977124791953, + 0.9335239262292595, + 0.9222035520824579, + 0.9142471402553919, + 0.9038764491526137, + 0.8952592406413572, + 0.8872171299760644, + 0.877779969795339, + 0.8748896658804006, + 0.8709399062198925, + 0.8672824279512239 + ], + "peak_layer": 1, + "peak_normalized": 1.173955566226187, + "population_cv": 0.09275081554992197, + "rest_mean": 0.00014320575994335942, + "spike_contrast": 0.9213481626529775, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013194236380513756, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016605114797130227, + 0.0001642610295675695, + 0.0001622748386580497, + 0.00016020899056456983, + 0.00015872408403083682, + 0.00015687917766626924, + 0.0001549132721265778, + 0.00015346029249485582, + 0.00015147548401728272, + 0.00014934672799427062, + 0.00014754213043488562, + 0.000146346545079723, + 0.00014467211440205574, + 0.00014357810141518712, + 0.00014198600547388196, + 0.00014068018936086446, + 0.00013951594883110374, + 0.00013829997624270618, + 0.00013686322199646384, + 0.0001357305736746639, + 0.00013454142026603222, + 0.0001333689724560827, + 0.00013204308925196528, + 0.00013044186925981194, + 0.00012931646779179573, + 0.00012784957652911544, + 0.00012663070810958743, + 0.000125493184896186, + 0.0001241583377122879, + 0.00012374951620586216, + 0.0001231908390764147, + 0.00012267350393813103 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031297675377572887, + "first_to_last_ratio": 2.208693462790297, + "imbalance_abs_log_ratio": 0.7924011473984336, + "last_quartile_mean": 0.00014170221402309835, + "mean": 0.00020716176595669822, + "normalized": [ + 1.8096319791310764, + 1.7504284704923412, + 1.6109023900661548, + 1.5434988529701714, + 1.4353862656069223, + 1.3677596328902204, + 1.3131374518280827, + 1.2555297512219317, + 1.1870915168673883, + 1.122656500215855, + 1.088514230207685, + 1.04408820740568, + 0.9901334868561585, + 0.9607872809839492, + 0.9219812869372472, + 0.880015778430751, + 0.8614610505244651, + 0.8397919049431777, + 0.8116424497290868, + 0.7845312039842096, + 0.7619415782348183, + 0.7436794877201931, + 0.7280047725498856, + 0.715266757195506, + 0.7065993237444729, + 0.696468071137219, + 0.6907565141722826, + 0.6863265628748565, + 0.6782234011874358, + 0.6754056248242399, + 0.6708832321424102, + 0.6674749829241257 + ], + "peak_layer": 1, + "peak_normalized": 1.8096319791310764, + "population_cv": 0.33826602846938675, + "rest_mean": 0.00021747771663487784, + "spike_contrast": 0.6964190843920172, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001514556322945282, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00037488655652850866, + 0.0003626218531280756, + 0.0003337173839099705, + 0.0003197539481334388, + 0.0002973571536131203, + 0.00028334750095382333, + 0.00027203187346458435, + 0.0002600977604743093, + 0.0002459199749864638, + 0.00023257150314748287, + 0.00022549853019881994, + 0.0002162951568607241, + 0.00020511780166998506, + 0.00019903838983736932, + 0.00019099927158094943, + 0.00018230562272947282, + 0.00017846179252956063, + 0.00017397277406416833, + 0.0001681412832112983, + 0.0001625248696655035, + 0.0001578451629029587, + 0.00015406195598188788, + 0.00015081475430633873, + 0.0001481759245507419, + 0.00014638036373071373, + 0.0001442815555492416, + 0.0001430983393220231, + 0.00014218062278814614, + 0.00014050195750314742, + 0.00013991822197567672, + 0.00013898135512135923, + 0.00013827529619447887 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027448106557130814, + 0.0027243648655712605, + 0.0027103275060653687, + 0.0027234479784965515, + 0.0027595730498433113, + 0.0027272943407297134, + 0.002728298306465149, + 0.0027007071767002344, + 0.002684502862393856, + 0.002683063969016075, + 0.0027139110025018454, + 0.002666700165718794, + 0.002679243916645646, + 0.0026584581937640905, + 0.002684473292902112, + 0.0026407993864268064, + 0.002663099905475974, + 0.0026283918414264917, + 0.002597011858597398, + 0.0026434636674821377, + 0.0025708195753395557, + 0.002691218862310052, + 0.0026598076801747084, + 0.0026596153620630503, + 0.0026765079237520695, + 0.002582729561254382, + 0.002614464145153761, + 0.0025760054122656584, + 0.002634295728057623, + 0.00259007909335196, + 0.00257018837146461, + 0.0025643527042120695 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.000156725715896755, + "first_to_last_ratio": 3.4451295459637388, + "imbalance_abs_log_ratio": 1.236961507811942, + "last_quartile_mean": 4.549196592051885e-05, + "mean": 8.812961027615529e-05, + "normalized": [ + 2.4952457548653, + 2.231881013356295, + 1.939989240144028, + 1.7866526882708733, + 1.6079164676041713, + 1.4797016884434102, + 1.3811975174028512, + 1.3042540698565042, + 1.1954344722384815, + 1.1168849900912188, + 1.0692220969851183, + 1.0304138184362992, + 0.9676518223611118, + 0.9260610329319834, + 0.8738547948342114, + 0.8395441231817105, + 0.8221010666653766, + 0.7971164256793037, + 0.7591159941756684, + 0.7162115141810864, + 0.6791638220644127, + 0.6489084022271779, + 0.6150954832405342, + 0.5868306199881643, + 0.5699702288291514, + 0.552179306852942, + 0.5335984532948914, + 0.5198944860736878, + 0.5024723171712969, + 0.4942325662695092, + 0.4850276094601851, + 0.47217611282304317 + ], + "peak_layer": 1, + "peak_normalized": 2.4952457548653, + "population_cv": 0.5254777238950282, + "rest_mean": 9.433142622583546e-05, + "spike_contrast": 0.5792322488273548, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.463980414788239e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021990503591950983, + 0.00019669480388984084, + 0.00017097049567382783, + 0.00015745700511615723, + 0.00014170505164656788, + 0.0001304055331274867, + 0.00012172439892310649, + 0.00011494340287754312, + 0.00010535317414905876, + 9.843063890002668e-05, + 9.423012670595199e-05, + 9.080996824195608e-05, + 8.527877798769623e-05, + 8.16133979242295e-05, + 7.701248250668868e-05, + 7.398869638564065e-05, + 7.24514466128312e-05, + 7.024955993983895e-05, + 6.690059672109783e-05, + 6.311944162007421e-05, + 5.985444295220077e-05, + 5.718804459320381e-05, + 5.420812522061169e-05, + 5.1717153837671503e-05, + 5.023125413572416e-05, + 4.866334711550735e-05, + 4.7025823732838035e-05, + 4.581809844239615e-05, + 4.428268948686309e-05, + 4.3556523451115936e-05, + 4.274529419490136e-05, + 4.161269680480473e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002203928706876468, + "first_to_last_ratio": 1.7624395804596433, + "imbalance_abs_log_ratio": 0.5666989745264205, + "last_quartile_mean": 0.00012504988717410015, + "mean": 0.00016078790008577926, + "normalized": [ + 1.7301295632680096, + 1.5926765618359837, + 1.4498368649995454, + 1.3715838442653223, + 1.2853282969253275, + 1.2242662006350955, + 1.1759759999659367, + 1.1358472505674493, + 1.0864560712260232, + 1.0500344477950443, + 1.0256300088344572, + 1.0049332388641516, + 0.9775387355078093, + 0.9578140668049778, + 0.9358065303644342, + 0.9184543281568608, + 0.9072826301440372, + 0.8934865935730769, + 0.8779252804708734, + 0.8613603707803614, + 0.8467225584074481, + 0.8352602527072714, + 0.8224752340358507, + 0.8113194646910352, + 0.80159772798593, + 0.7940537841119606, + 0.785769247679408, + 0.7781743501685717, + 0.7724121545166266, + 0.7676777199663887, + 0.7637141967462265, + 0.7584564239985051 + ], + "peak_layer": 1, + "peak_normalized": 1.7301295632680096, + "population_cv": 0.2498913679417364, + "rest_mean": 0.0001660440253477578, + "spike_contrast": 0.7974079368034485, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013240482367109506, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027818389935418963, + 0.00025608311989344656, + 0.00023311622499022633, + 0.00022053408611100167, + 0.00020666523778345436, + 0.0001968471915461123, + 0.00018908271158579737, + 0.0001826304942369461, + 0.0001746889902278781, + 0.00016883283387869596, + 0.00016490889538545161, + 0.00016158110520336777, + 0.00015717640053480864, + 0.00015400491247419268, + 0.0001504663669038564, + 0.00014767634274903685, + 0.00014588006888516247, + 0.00014366183313541114, + 0.00014115976227913052, + 0.00013849632523488253, + 0.00013614274212159216, + 0.0001342997420579195, + 0.00013224406575318426, + 0.00013045035302639008, + 0.00012888721539638937, + 0.00012767424050252885, + 0.0001263421872863546, + 0.0001251210196642205, + 0.00012419452832546085, + 0.00012343328853603452, + 0.00012279600196052343, + 0.00012195061572128907 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19544659927487373, + "first_to_last_ratio": 1.7624395617194093, + "imbalance_abs_log_ratio": 0.5666989638932991, + "last_quartile_mean": 0.11089549027383327, + "mean": 0.14258831785991788, + "normalized": [ + 1.7301295593799786, + 1.5926765086027146, + 1.4498368632986096, + 1.3715838909156086, + 1.2853282557768668, + 1.224266111161914, + 1.1759760185820738, + 1.1358472270183295, + 1.0864560720464596, + 1.050034483149275, + 1.025630003735948, + 1.0049332495130952, + 0.9775386792530747, + 0.9578141330250067, + 0.9358065753109587, + 0.9184542876534186, + 0.9072827263675407, + 0.8934866319653527, + 0.8779253472761699, + 0.8613603464001749, + 0.8467225704769723, + 0.8352602246328348, + 0.8224752140046494, + 0.8113194329409806, + 0.8015977187598701, + 0.7940537797484172, + 0.7857692153409586, + 0.7781743825002613, + 0.7724121455314925, + 0.7676776608495898, + 0.7637142124554999, + 0.758456472325904 + ], + "peak_layer": 1, + "peak_normalized": 1.7301295593799786, + "population_cv": 0.24989136150327743, + "rest_mean": 0.14724950271624107, + "spike_contrast": 0.7974079196861148, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11741791963577271, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24669626355171204, + 0.2270970642566681, + 0.20672979950904846, + 0.19557183980941772, + 0.18327279388904572, + 0.17456604540348053, + 0.16768044233322144, + 0.16195854544639587, + 0.1549159437417984, + 0.1497226506471634, + 0.14624285697937012, + 0.14329174160957336, + 0.13938559591770172, + 0.13657310605049133, + 0.13343508541584015, + 0.1309608519077301, + 0.1293679177761078, + 0.12740075588226318, + 0.1251818984746933, + 0.1228199228644371, + 0.12073274701833725, + 0.11909835040569305, + 0.11727535724639893, + 0.11568467319011688, + 0.11429847031831741, + 0.11322279274463654, + 0.11204151064157486, + 0.11095857620239258, + 0.11013694852590561, + 0.10946186631917953, + 0.10889672487974167, + 0.108147032558918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.812671563238837e-05, + "first_to_last_ratio": 2.0429267595554257, + "imbalance_abs_log_ratio": 0.7143834657339574, + "last_quartile_mean": 3.334760549478233e-05, + "mean": 4.8136708642232406e-05, + "normalized": [ + 1.6706878479333491, + 1.5890573277727331, + 1.4709453763829752, + 1.4135957551070697, + 1.368779351628352, + 1.3072249852707734, + 1.2659085053079855, + 1.2360067186760548, + 1.1647282180273046, + 1.1006681301372137, + 1.0750864931045292, + 1.0685694254166505, + 1.0112110373273602, + 0.9945647481091522, + 0.9602230239537092, + 0.9352227929155844, + 0.93139607887314, + 0.9092220870879134, + 0.8814974932144994, + 0.8653605860843365, + 0.8506124631158822, + 0.8320765477333729, + 0.7932580049774197, + 0.7619473309737773, + 0.7566723544753674, + 0.7419402536141336, + 0.7141691805145849, + 0.7010673290630277, + 0.6716446941281601, + 0.6659218543873533, + 0.6557479674543484, + 0.6349860372318865 + ], + "peak_layer": 1, + "peak_normalized": 1.6706878479333491, + "population_cv": 0.2830124551063321, + "rest_mean": 4.992923641111702e-05, + "spike_contrast": 0.7702312603701833, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.845705869025551e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.04214141680859e-05, + 7.649198960280046e-05, + 7.080646901158616e-05, + 6.804584700148553e-05, + 6.588853284483775e-05, + 6.292550824582577e-05, + 6.093666888773441e-05, + 5.949729529675096e-05, + 5.6066182878566906e-05, + 5.298254109220579e-05, + 5.1751125283772126e-05, + 5.143741509527899e-05, + 4.867637107963674e-05, + 4.7875073505565524e-05, + 4.622197593562305e-05, + 4.501854709815234e-05, + 4.483434167923406e-05, + 4.376695869723335e-05, + 4.24323879997246e-05, + 4.165561040281318e-05, + 4.094568430446088e-05, + 4.0053426346275955e-05, + 3.8184829463716596e-05, + 3.667763667181134e-05, + 3.642371666501276e-05, + 3.571456181816757e-05, + 3.4377753763692454e-05, + 3.3747073757695034e-05, + 3.2330764952348545e-05, + 3.205528628313914e-05, + 3.156554885208607e-05, + 3.056613786611706e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018337754227104597, + "first_to_last_ratio": 1.4712162101549686, + "imbalance_abs_log_ratio": 0.38608941256902984, + "last_quartile_mean": 0.00012464350311347516, + "mean": 0.00014952537844692415, + "normalized": [ + 1.3620834570997993, + 1.3194965005671844, + 1.2677944868152182, + 1.233561928872588, + 1.1984447257225705, + 1.1684695593357564, + 1.141528073961558, + 1.1198008373135866, + 1.0910012077210556, + 1.066096352737946, + 1.0498630645220353, + 1.0350823851472097, + 1.0145919973375974, + 1.0013407128400098, + 0.9836056688270479, + 0.9685428567287663, + 0.9591027483551056, + 0.9465833155582388, + 0.9334001555540693, + 0.9191584382428863, + 0.9057431683575706, + 0.8945627711391904, + 0.8815040843165663, + 0.8698871064160805, + 0.8596264871795505, + 0.8514885298372257, + 0.8425068993042694, + 0.8341757603627517, + 0.8278236379550317, + 0.8225948884437775, + 0.8181940461689637, + 0.812344147258793 + ], + "peak_layer": 1, + "peak_normalized": 1.3620834570997993, + "population_cv": 0.15505530299928885, + "rest_mean": 0.00015278545470209792, + "spike_contrast": 0.8634393039979241, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001319209666689858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00020366604439914227, + 0.0001972982136067003, + 0.0001895674504339695, + 0.00018444881425239146, + 0.00017919790116138756, + 0.00017471585306338966, + 0.0001706874172668904, + 0.0001674386439844966, + 0.00016313236847054213, + 0.00015940846060402691, + 0.00015698117204010487, + 0.00015477108536288142, + 0.00015170725237112492, + 0.00014972584904171526, + 0.0001470740098739043, + 0.0001448217371944338, + 0.00014341020141728222, + 0.00014153822849038988, + 0.00013956701150164008, + 0.00013743751333095133, + 0.00013543159002438188, + 0.00013375983689911664, + 0.00013180723180994391, + 0.00013007019879296422, + 0.00012853597581852227, + 0.00012731914466712624, + 0.0001259761629626155, + 0.00012473044625949115, + 0.00012378064275253564, + 0.00012299881200306118, + 0.00012234077439643443, + 0.00012146606604801491 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015816818631719798, + "first_to_last_ratio": 1.2671705792413261, + "imbalance_abs_log_ratio": 0.23678652466939742, + "last_quartile_mean": 0.00012481996418500785, + "mean": 0.0001404537983944465, + "normalized": [ + 1.172032188999453, + 1.1600674087975105, + 1.1434054243316032, + 1.1305758413418425, + 1.1202630655069494, + 1.1065457827105485, + 1.0931998253234612, + 1.0828906757132684, + 1.0683311763286112, + 1.0526488941703125, + 1.0427563459749776, + 1.0343706495522589, + 1.0219941374371992, + 1.0154952175292713, + 1.0038357654470964, + 0.9937321709742023, + 0.9862865992066487, + 0.9768881500283385, + 0.968261257859838, + 0.9607910277643221, + 0.9520546236098718, + 0.9457805296333641, + 0.9349892993508679, + 0.9232794945485685, + 0.9140411172891727, + 0.9051057888083044, + 0.8959794100889387, + 0.888100763485746, + 0.8822517666505367, + 0.8784085901067091, + 0.87543446421392, + 0.8702025472162866 + ], + "peak_layer": 1, + "peak_normalized": 1.172032188999453, + "population_cv": 0.09096684083762872, + "rest_mean": 0.00014216970130628734, + "spike_contrast": 0.9227558436510851, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013118792267050595, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016461637278553098, + 0.0001629358739592135, + 0.00016059563495218754, + 0.00015879367128945887, + 0.0001573452027514577, + 0.00015541855827905238, + 0.00015354406787082553, + 0.00015209610864985734, + 0.00015005117165856063, + 0.0001478485355619341, + 0.0001464590895920992, + 0.00014528128667734563, + 0.00014354295853991061, + 0.00014263016055338085, + 0.00014099254622124135, + 0.00013957345800008625, + 0.0001385276991641149, + 0.00013720765127800405, + 0.00013599597150459886, + 0.00013494674931280315, + 0.00013371968816500157, + 0.00013283846783451736, + 0.00013132279855199158, + 0.0001296781119890511, + 0.0001283805468119681, + 0.00012712554598692805, + 0.0001258437114302069, + 0.00012473712558858097, + 0.00012391561176627874, + 0.0001233758230227977, + 0.0001229580957442522, + 0.00012222325312905014 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0003110146244580392, + "first_to_last_ratio": 2.2098355110548322, + "imbalance_abs_log_ratio": 0.7929180833727034, + "last_quartile_mean": 0.00014074107457417995, + "mean": 0.00020577548139044666, + "normalized": [ + 1.8085529821987474, + 1.7406694730308092, + 1.619926374791616, + 1.5420513399817848, + 1.4477144166533082, + 1.3679714513872108, + 1.309931951281633, + 1.254598241859695, + 1.1926238819658395, + 1.119002359728262, + 1.082854716042971, + 1.0468411526236405, + 0.9850613364068194, + 0.9618428208148979, + 0.9182052809224994, + 0.877960480924469, + 0.8608981306240351, + 0.8408442911829569, + 0.8148282643879068, + 0.7879356949258838, + 0.7604349554811285, + 0.7438321978222967, + 0.727404890131739, + 0.7163770004088065, + 0.7034258791916428, + 0.6971552930824737, + 0.6907054382632783, + 0.684504508832364, + 0.6799757639842041, + 0.6759922505973077, + 0.6720742215597147, + 0.6678029589100581 + ], + "peak_layer": 1, + "peak_normalized": 1.8085529821987474, + "population_cv": 0.3383164219384334, + "rest_mean": 0.00021605301460820355, + "spike_contrast": 0.695555219569954, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001502768020145595, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00037215586053207517, + 0.0003581870987545699, + 0.00033334112958982587, + 0.0003173163568135351, + 0.00029790413100272417, + 0.0002814949839375913, + 0.00026955187786370516, + 0.0002581655571702868, + 0.00024541275342926383, + 0.00023026324925012887, + 0.00022282495046965778, + 0.00021541424212045968, + 0.00020270147069822997, + 0.00019792366947513074, + 0.00018894413369707763, + 0.00018066274060402066, + 0.00017715172725729644, + 0.00017302513879258186, + 0.00016767167835496366, + 0.00016213784692808986, + 0.0001564788690302521, + 0.00015306242858059704, + 0.00014968209143262357, + 0.00014741282211616635, + 0.00014474779891315848, + 0.00014345746603794396, + 0.00014213024405762553, + 0.00014085424481891096, + 0.00013992234016768634, + 0.00013910263078287244, + 0.00013829639647156, + 0.00013741747534368187 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0021227982360869646, + 0.00413061399012804, + 0.005416445434093475, + 0.006388718727976084, + 0.0023577234242111444, + 0.004388594534248114, + 0.00574506726115942, + 0.007066869642585516, + 0.0029547461308538914, + 0.004991721361875534, + 0.006190058775246143, + 0.0074937604367733, + 0.00348988501355052, + 0.006057556718587875, + 0.0074848332442343235, + 0.00897275935858488, + 0.004072706215083599, + 0.0063449000008404255, + 0.007735534105449915, + 0.00946879480034113, + 0.004824656527489424, + 0.007074959110468626, + 0.009095527231693268, + 0.010346688330173492, + 0.004843301139771938, + 0.007334613241255283, + 0.009146629832684994, + 0.010587968863546848, + 0.0054801348596811295, + 0.0077637480571866035, + 0.009536821395158768, + 0.011367839761078358 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015727686513855588, + "first_to_last_ratio": 3.444355141493678, + "imbalance_abs_log_ratio": 1.2367367001496994, + "last_quartile_mean": 4.5662209231522866e-05, + "mean": 8.846079299473786e-05, + "normalized": [ + 2.4938513501811554, + 2.232208910931446, + 1.9412909982664082, + 1.7875424925486438, + 1.6051944492471113, + 1.4809426711898561, + 1.3782256757419, + 1.304162421055249, + 1.1983748443875557, + 1.116540877408076, + 1.0705280549740306, + 1.028906183424365, + 0.9692497056236429, + 0.9297086109864628, + 0.876031508599092, + 0.840542245421429, + 0.8215073067800788, + 0.7958492991481336, + 0.7578128978794811, + 0.7149268338582802, + 0.6789684541053794, + 0.6481555926886391, + 0.6127683614407704, + 0.5872234895762031, + 0.5712935129257742, + 0.5525148673059793, + 0.5351006365732414, + 0.5196040797164857, + 0.49974182292452146, + 0.4940518464188538, + 0.4845470025727643, + 0.4726329960989905 + ], + "peak_layer": 1, + "peak_normalized": 2.4938513501811554, + "population_cv": 0.525487396006855, + "rest_mean": 9.469102304946218e-05, + "spike_contrast": 0.5789096889426617, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.481755069922656e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002206080680480227, + 0.00019746297039091587, + 0.00017172814114019275, + 0.00015812742640264332, + 0.00014199677389115095, + 0.0001310053630732, + 0.00012191893620183691, + 0.00011536724196048453, + 0.00010600918903946877, + 9.877009142655879e-05, + 9.469976066611707e-05, + 9.101785690290853e-05, + 8.574059756938368e-05, + 8.224276098189875e-05, + 7.74944419390522e-05, + 7.435503357555717e-05, + 7.267118780873716e-05, + 7.040146010695025e-05, + 6.70367298880592e-05, + 6.324299465632066e-05, + 6.0062087868573144e-05, + 5.733635771321133e-05, + 5.42059751751367e-05, + 5.1946255553048104e-05, + 5.0537077186163515e-05, + 4.88759033032693e-05, + 4.733542664325796e-05, + 4.596458893502131e-05, + 4.4207557948539034e-05, + 4.3704218114726245e-05, + 4.2863412090810016e-05, + 4.1809489630395547e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002221399099653354, + "first_to_last_ratio": 1.7678964788399845, + "imbalance_abs_log_ratio": 0.5697904098088594, + "last_quartile_mean": 0.00012565210272441618, + "mean": 0.00016188397967198398, + "normalized": [ + 1.7325277794978797, + 1.5951376332003655, + 1.4520548687114199, + 1.3729981481664384, + 1.2857750985910605, + 1.2257666436669268, + 1.1762283345939986, + 1.1372455678364286, + 1.0889526987749056, + 1.0509403055231545, + 1.0259172538151122, + 1.0049179037159137, + 0.978085345877984, + 0.9585059142501069, + 0.9364886382982085, + 0.9194669656082258, + 0.9066971379228966, + 0.893622399915035, + 0.8769613699768213, + 0.8605074489369314, + 0.8461051105300481, + 0.8336447776090111, + 0.8211750960224249, + 0.8107883690330443, + 0.8025267555268564, + 0.7926031470462279, + 0.7851370690218832, + 0.7775583575558719, + 0.7689758331264998, + 0.7655660871494063, + 0.7607136802742225, + 0.75640826022469 + ], + "peak_layer": 1, + "peak_normalized": 1.7325277794978797, + "population_cv": 0.2510639592948361, + "rest_mean": 0.00016719473286583606, + "spike_contrast": 0.796711177092359, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013320591242518275, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002804684918373823, + 0.0002582272281870246, + 0.00023506442084908485, + 0.00022226640430744737, + 0.00020814638992305845, + 0.00019843198242597282, + 0.00019041252380702645, + 0.00018410183838568628, + 0.00017628399655222893, + 0.00017013039905577898, + 0.00016607956786174327, + 0.00016268010949715972, + 0.00015833634824957699, + 0.00015516675193794072, + 0.00015160250768531114, + 0.00014884697156958282, + 0.00014677974104415625, + 0.00014466315042227507, + 0.00014196599659044296, + 0.000139302370371297, + 0.00013697086251340806, + 0.00013495373423211277, + 0.00013293509255163372, + 0.00013125364785082638, + 0.00012991622497793287, + 0.00012830975174438208, + 0.00012710111332125962, + 0.000125874241348356, + 0.00012448486813809723, + 0.00012393288488965482, + 0.00012314735795371234, + 0.00012245037942193449 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19699589163064957, + "first_to_last_ratio": 1.7678964443273588, + "imbalance_abs_log_ratio": 0.5697903902870003, + "last_quartile_mean": 0.11142954230308533, + "mean": 0.1435603315476328, + "normalized": [ + 1.7325277292733527, + 1.595137596567507, + 1.4520549137362593, + 1.3729980837787108, + 1.285775112498371, + 1.2257666266551688, + 1.1762283724960378, + 1.1372455675264566, + 1.0889526722354506, + 1.0509403711345948, + 1.0259172522938773, + 1.00491793591038, + 0.9780853203912144, + 0.9585059497750074, + 0.9364885904027979, + 0.9194669905174069, + 0.9066971317518798, + 0.8936224205946474, + 0.8769613084029552, + 0.8605074273314671, + 0.84610509802098, + 0.833644762521393, + 0.821175085273337, + 0.8107884103394608, + 0.802526775587875, + 0.7926031918850489, + 0.7851370589142022, + 0.7775583578743038, + 0.7689758340628073, + 0.7655660958716172, + 0.7607136803251955, + 0.7564082760502372 + ], + "peak_layer": 1, + "peak_normalized": 1.7325277292733527, + "population_cv": 0.2510639513231401, + "rest_mean": 0.14826996044980156, + "spike_contrast": 0.7967111822081809, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11812833547592164, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24872225522994995, + 0.22899848222732544, + 0.20845748484134674, + 0.19710806012153625, + 0.184586301445961, + 0.17597146332263947, + 0.16885973513126373, + 0.16326335072517395, + 0.156330406665802, + 0.1508733481168747, + 0.14728102087974548, + 0.14426635205745697, + 0.1404142528772354, + 0.13760343194007874, + 0.13444261252880096, + 0.13199898600578308, + 0.13016574084758759, + 0.12828873097896576, + 0.1258968561887741, + 0.12353473156690598, + 0.12146712839603424, + 0.1196783185005188, + 0.11788816750049591, + 0.11639705300331116, + 0.11521100997924805, + 0.11378637701272964, + 0.11271453648805618, + 0.111626535654068, + 0.1103944256901741, + 0.10990492254495621, + 0.10920830816030502, + 0.10859022289514542 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.813338586653117e-05, + "first_to_last_ratio": 2.039661655563216, + "imbalance_abs_log_ratio": 0.7127839389845736, + "last_quartile_mean": 3.3404258829250466e-05, + "mean": 4.817700425974181e-05, + "normalized": [ + 1.6677237328958603, + 1.5880861584886787, + 1.4689677920146058, + 1.4140322553861728, + 1.3674596064698026, + 1.3072824267629832, + 1.2642946708581568, + 1.2359968662296865, + 1.1645552208108763, + 1.101791124796405, + 1.0797798333773554, + 1.0672664620429624, + 1.012803710836977, + 0.9955853656459409, + 0.9623936542224847, + 0.9380946280712082, + 0.9310631059210642, + 0.9082494158597965, + 0.8812030087749774, + 0.8644086677601183, + 0.8497891698936241, + 0.8309347651333899, + 0.7889921814106587, + 0.7623244708429415, + 0.7584228017816982, + 0.7445258855646851, + 0.7180452956289476, + 0.7000407870401953, + 0.6663749309645193, + 0.6665299586713209, + 0.6558653663654831, + 0.637116679476424 + ], + "peak_layer": 1, + "peak_normalized": 1.6677237328958603, + "population_cv": 0.28258618188430573, + "rest_mean": 4.9978354245131074e-05, + "spike_contrast": 0.7693273401931907, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.844971433863975e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.034593338379636e-05, + 7.650923362234607e-05, + 7.077046757331118e-05, + 6.812383799115196e-05, + 6.588010728592053e-05, + 6.298095104284585e-05, + 6.090992974350229e-05, + 5.9546626289375126e-05, + 5.610478183371015e-05, + 5.3080995712662116e-05, + 5.2020557632204145e-05, + 5.141770088812336e-05, + 4.879384869127534e-05, + 4.79643204016611e-05, + 4.636524317902513e-05, + 4.519458889262751e-05, + 4.485583122004755e-05, + 4.375673597678542e-05, + 4.245372110744938e-05, + 4.164462006883696e-05, + 4.094029645784758e-05, + 4.0031947719398886e-05, + 3.8011279684724286e-05, + 3.672650927910581e-05, + 3.653853855212219e-05, + 3.5869026760337874e-05, + 3.4593271266203374e-05, + 3.3725867979228497e-05, + 3.21039478876628e-05, + 3.211141665815376e-05, + 3.1597628549207e-05, + 3.0694372981088236e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018498812278266996, + "first_to_last_ratio": 1.477115588497286, + "imbalance_abs_log_ratio": 0.3900912594562188, + "last_quartile_mean": 0.00012523605073511135, + "mean": 0.00015057817927299766, + "normalized": [ + 1.364889512785021, + 1.3222437098497068, + 1.2702392636002102, + 1.235819093505974, + 1.2002761568067795, + 1.1699209662016437, + 1.143160310272995, + 1.1216013166366416, + 1.0931712967611817, + 1.0676161315047876, + 1.0503356923443616, + 1.0352076251778792, + 1.014929888181958, + 1.0014649033450271, + 0.9840777733314531, + 0.96959004456439, + 0.958410698647572, + 0.9467186761221033, + 0.9322053376846741, + 0.9180751779000818, + 0.904837007766356, + 0.8925970349353171, + 0.879899470446981, + 0.8691033998232655, + 0.8603564893899045, + 0.8496764837250937, + 0.8415817018035472, + 0.8332582690137502, + 0.8239257185712614, + 0.820107268399535, + 0.8147646079598274, + 0.8099389729407196 + ], + "peak_layer": 1, + "peak_normalized": 1.364889512785021, + "population_cv": 0.1566386619724503, + "rest_mean": 0.000153886474436149, + "spike_contrast": 0.8624109810706347, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013271338539198042, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002055225777439773, + 0.00019910105038434267, + 0.000191270315553993, + 0.00018608738901093602, + 0.0001807353983167559, + 0.00017616456898394972, + 0.00017213499813806266, + 0.00016888868412934244, + 0.00016460774349980056, + 0.00016075969324447215, + 0.0001581576361786574, + 0.00015587967936880887, + 0.00015282629465218633, + 0.00015079876175150275, + 0.0001481806393712759, + 0.00014599910355173051, + 0.00014431573799811304, + 0.00014255517453420907, + 0.00014036978245712817, + 0.00013824208872392774, + 0.00013624870916828513, + 0.00013440563634503633, + 0.0001324936602031812, + 0.00013086800754535943, + 0.00012955091369803995, + 0.00012794273789040744, + 0.00012672384036704898, + 0.0001254705130122602, + 0.0001240652345586568, + 0.00012349025928415358, + 0.00012268577120266855, + 0.00012195913586765528 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015959453048708383, + "first_to_last_ratio": 1.2728956423420892, + "imbalance_abs_log_ratio": 0.24129433847892473, + "last_quartile_mean": 0.00012537911607068963, + "mean": 0.00014144417764327955, + "normalized": [ + 1.1740787416792278, + 1.1612764386268224, + 1.1471117856731012, + 1.1325725430810556, + 1.1221203501929577, + 1.1091452070987, + 1.0952451640185767, + 1.0850231158766837, + 1.0710066115302646, + 1.0558964823952193, + 1.0428503514266292, + 1.0346153464801762, + 1.0229188081717178, + 1.0151986196356395, + 1.003700332970425, + 0.9946300373693288, + 0.9863513080075261, + 0.9777764871930738, + 0.967654233599155, + 0.9595434059922948, + 0.9513191006674802, + 0.9429489101168639, + 0.9334504238882736, + 0.9221966853060849, + 0.914156536930625, + 0.9038869577772225, + 0.8952163547729103, + 0.8871546013922188, + 0.8776639341179028, + 0.8750860454698683, + 0.8708633986378481, + 0.8673416799041263 + ], + "peak_layer": 1, + "peak_normalized": 1.1740787416792278, + "population_cv": 0.09275162400716915, + "rest_mean": 0.00014320399646681767, + "spike_contrast": 0.9213510743517984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001319411559961736, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001660666021052748, + 0.00016425579087808728, + 0.00016225228318944573, + 0.0001601957919774577, + 0.00015871739014983177, + 0.0001568821317050606, + 0.0001549160515423864, + 0.00015347020234912634, + 0.00015148764941841364, + 0.0001493504096288234, + 0.00014750511036254466, + 0.00014634031686000526, + 0.00014468590961769223, + 0.00014359393389895558, + 0.00014196756819728762, + 0.0001406846276950091, + 0.00013951364962849766, + 0.00013830079114995897, + 0.0001368690573144704, + 0.00013572182797361165, + 0.000134558547870256, + 0.00013337463315110654, + 0.00013203112757764757, + 0.00013043935177847743, + 0.00012930211960338056, + 0.00012784954742528498, + 0.00012662314111366868, + 0.00012548285303637385, + 0.00012414045340847224, + 0.00012377582606859505, + 0.00012317855725996196, + 0.00012268043064977974 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031298474641516805, + "first_to_last_ratio": 2.209030513243648, + "imbalance_abs_log_ratio": 0.7925537374904509, + "last_quartile_mean": 0.00014168421148497146, + "mean": 0.0002071673161481158, + "normalized": [ + 1.8097158340785349, + 1.7497804411308835, + 1.611244862997228, + 1.5435020349400896, + 1.4351553463811126, + 1.3679605489917586, + 1.3130823229189843, + 1.255818245825708, + 1.1871164694634437, + 1.122716192977336, + 1.0885067728511792, + 1.0444622322416663, + 0.990152969051365, + 0.9605806666596922, + 0.9222549756846732, + 0.8800158737132093, + 0.8615285136033911, + 0.8400222785845712, + 0.8115627552032852, + 0.7845983398409414, + 0.7619432914958448, + 0.7435905859348785, + 0.7280762324918081, + 0.7153162915808863, + 0.7064441934578637, + 0.6963930075638303, + 0.6905250334602637, + 0.686293775915277, + 0.6782364888338203, + 0.6751963304984743, + 0.6708841537879804, + 0.6673229378400193 + ], + "peak_layer": 1, + "peak_normalized": 1.8097158340785349, + "population_cv": 0.3382701486201724, + "rest_mean": 0.00021748447411314205, + "spike_contrast": 0.6963929896815652, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015145466313697397, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003749139723367989, + 0.00036249731783755124, + 0.00033379727392457426, + 0.0003197631740476936, + 0.0002973172813653946, + 0.00028339671553112566, + 0.0002720277407206595, + 0.0002601644955575466, + 0.0002459317329339683, + 0.00023259010049514472, + 0.0002255030267406255, + 0.00021637843747157604, + 0.00020512733317445964, + 0.00019900091865565628, + 0.00019106108811683953, + 0.00018231052672490478, + 0.00017848054994829, + 0.00017402516095899045, + 0.00016812927788123488, + 0.0001625431323191151, + 0.00015784974675625563, + 0.00015404766600113362, + 0.00015083359903655946, + 0.00014819015632383525, + 0.00014635214756708592, + 0.00014426987036131322, + 0.00014305421791505069, + 0.00014217763964552432, + 0.00014050843310542405, + 0.0001398786116624251, + 0.00013898526958655566, + 0.0001382475020363927 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034565660171210766, + 0.004928664769977331, + 0.0060537648387253284, + 0.006960673723369837, + 0.0036145171616226435, + 0.0051584807224571705, + 0.006370538379997015, + 0.007546419743448496, + 0.00395671371370554, + 0.005665866658091545, + 0.006755253300070763, + 0.007946331985294819, + 0.004383862018585205, + 0.006652849726378918, + 0.007932198233902454, + 0.009347881190478802, + 0.004880861379206181, + 0.006906712893396616, + 0.008132148534059525, + 0.009816600009799004, + 0.005453306715935469, + 0.007622495759278536, + 0.009448722936213017, + 0.010670131072402, + 0.0055701653473079205, + 0.007738742511719465, + 0.009509840048849583, + 0.010865328833460808, + 0.005987230688333511, + 0.00815181527286768, + 0.009874933399260044, + 0.011666664853692055 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015672678182454547, + "first_to_last_ratio": 3.4451487067191406, + "imbalance_abs_log_ratio": 1.236967069490208, + "last_quartile_mean": 4.549202230919036e-05, + "mean": 8.812991916329338e-05, + "normalized": [ + 2.495342024881723, + 2.231858330120814, + 1.9399329049931513, + 1.7866550124027707, + 1.6079479837593746, + 1.479673880937976, + 1.3811985381554308, + 1.3042766606266647, + 1.1954508396552082, + 1.1168862767589318, + 1.0692251193363627, + 1.0303878333212484, + 0.9676417435164101, + 0.9260651349655372, + 0.8738635380560305, + 0.8395483633300559, + 0.8220975248023003, + 0.7970985234813915, + 0.759112342831592, + 0.7162079306537569, + 0.6791607399033079, + 0.6489075313744581, + 0.615087961024076, + 0.5868315353418163, + 0.5699679008989185, + 0.5521854623420828, + 0.5335941062986479, + 0.5198929115704046, + 0.5024718770039331, + 0.49422901772262035, + 0.4850259920431499, + 0.47217445788985424 + ], + "peak_layer": 1, + "peak_normalized": 2.495342024881723, + "population_cv": 0.5254832958744828, + "rest_mean": 9.433179891655325e-05, + "spike_contrast": 0.5792295824234721, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.4639768495690075e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021991429093759507, + 0.0001966934942174703, + 0.00017096613009925932, + 0.0001574577618157491, + 0.00014170832582749426, + 0.00013040353951510042, + 0.0001217249155160971, + 0.00011494579666759819, + 0.00010535498586250469, + 9.843109728535637e-05, + 9.423072333447635e-05, + 9.080799645744264e-05, + 8.527818863512948e-05, + 8.161404548445717e-05, + 7.701352296862751e-05, + 7.398932939395308e-05, + 7.245138840517029e-05, + 7.024822843959555e-05, + 6.690050940960646e-05, + 6.311934703262523e-05, + 5.985438110656105e-05, + 5.7188168284483254e-05, + 5.420765228336677e-05, + 5.1717415772145614e-05, + 5.02312250318937e-05, + 4.866406015935354e-05, + 4.702560545410961e-05, + 4.581812027026899e-05, + 4.428280590218492e-05, + 4.3556363380048424e-05, + 4.2745301470858976e-05, + 4.161269680480473e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00022039370378479362, + "first_to_last_ratio": 1.7624457042066763, + "imbalance_abs_log_ratio": 0.5667024491058967, + "last_quartile_mean": 0.00012504992537287762, + "mean": 0.0001607882525149762, + "normalized": [ + 1.7301666786506156, + 1.592657504255016, + 1.4498087081325215, + 1.3715848200709229, + 1.2853506396348091, + 1.224257634451794, + 1.175979576607571, + 1.1358564358860717, + 1.0864628307036615, + 1.050035766385231, + 1.0256357250890054, + 1.0049217142965674, + 0.9775345112735686, + 0.9578189361646354, + 0.9358080993289649, + 0.9184597363064851, + 0.9072850761620224, + 0.8934833681037547, + 0.8779253472396409, + 0.8613572157279928, + 0.8467159963027754, + 0.8352601414820391, + 0.8224708066612338, + 0.81131886291965, + 0.8015955184587156, + 0.7940537632065271, + 0.7857694259399116, + 0.7781749070905981, + 0.7724085609032495, + 0.7676732316943716, + 0.7637136993200662, + 0.75845476155001 + ], + "peak_layer": 1, + "peak_normalized": 1.7301666786506156, + "population_cv": 0.2498932502086342, + "rest_mean": 0.0001660444721454513, + "spike_contrast": 0.7974048446155259, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001324046665104106, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027819047681987286, + 0.0002560806169640273, + 0.0002331122086616233, + 0.00022053472639527172, + 0.00020666928321588784, + 0.00019684624567162246, + 0.0001890837011160329, + 0.00018263237143401057, + 0.00017469045997131616, + 0.0001688334159553051, + 0.0001649101759539917, + 0.00016157960635609925, + 0.00015717606584075838, + 0.00015400603297166526, + 0.00015046694898046553, + 0.00014767753600608557, + 0.00014588078192900866, + 0.00014366162940859795, + 0.00014116008242126554, + 0.00013849612150806934, + 0.0001361419854220003, + 0.00013430001854430884, + 0.00013224364374764264, + 0.00013045054220128804, + 0.00012888714263681322, + 0.00012767451698891819, + 0.0001263424928765744, + 0.00012512138346210122, + 0.00012419422273524106, + 0.00012343283742666245, + 0.0001227961911354214, + 0.00012195061572128907 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19544734060764313, + "first_to_last_ratio": 1.7624457434404193, + "imbalance_abs_log_ratio": 0.5667024713668619, + "last_quartile_mean": 0.11089552193880081, + "mean": 0.14258862822316587, + "normalized": [ + 1.7301667593069614, + 1.5926575752547714, + 1.4498087309457477, + 1.3715847721481056, + 1.2853506436856033, + 1.2242575941276408, + 1.1759796246815233, + 1.1358564592049036, + 1.0864627991256137, + 1.0500357507634608, + 1.0256357136610779, + 1.0049216567312906, + 0.9775345659226533, + 0.957818945514909, + 0.935808091557486, + 0.9184597083380731, + 0.9072850362319335, + 0.8934833286128027, + 0.8779253174360568, + 0.8613572174793438, + 0.8467160247668938, + 0.8352601831545204, + 0.8224708111632975, + 0.8113188687979516, + 0.8015955037045681, + 0.7940537757091718, + 0.7857693860910103, + 0.7781748832951502, + 0.7724085309387246, + 0.7676732205256189, + 0.7637136996793319, + 0.7584548214438025 + ], + "peak_layer": 1, + "peak_normalized": 1.7301667593069614, + "population_cv": 0.24989326289316924, + "rest_mean": 0.14724989594132812, + "spike_contrast": 0.797404859232464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11741778254508972, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24670210480690002, + 0.2270948588848114, + 0.20672623813152313, + 0.1955723911523819, + 0.18327638506889343, + 0.17456521093845367, + 0.16768132150173187, + 0.16196021437644958, + 0.15491724014282227, + 0.14972315728664398, + 0.14624398946762085, + 0.14329040050506592, + 0.13938531279563904, + 0.13657408952713013, + 0.13343559205532074, + 0.13096190989017487, + 0.12936852872371674, + 0.12740056216716766, + 0.1251821666955948, + 0.12281974405050278, + 0.12073207646608353, + 0.11909860372543335, + 0.11727498471736908, + 0.11568484455347061, + 0.11429840326309204, + 0.11322303861379623, + 0.11204177886247635, + 0.11095888912677765, + 0.11013667285442352, + 0.1094614714384079, + 0.1088968887925148, + 0.108147032558918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.812703486502869e-05, + "first_to_last_ratio": 2.0429306771418236, + "imbalance_abs_log_ratio": 0.7143853833664048, + "last_quartile_mean": 3.334769780849456e-05, + "mean": 4.813682039639389e-05, + "normalized": [ + 1.670694398734861, + 1.5890602892970849, + 1.4709330435001327, + 1.4135870318536654, + 1.3688033811669116, + 1.3072210435152478, + 1.2659107055367187, + 1.236022743120565, + 1.164719165633699, + 1.10067041168383, + 1.075093368589445, + 1.0685491843186352, + 1.0112041551575939, + 0.9945774787180313, + 0.9602353052595208, + 0.9352302198346567, + 0.9313930096398116, + 0.9092161974531191, + 0.8814894006709005, + 0.8653525310040872, + 0.8506115464001769, + 0.8320776390213925, + 0.7932455071661803, + 0.7619500965894176, + 0.7566697664554205, + 0.7419550822306056, + 0.7141621566216388, + 0.7010675908642553, + 0.6716466869044494, + 0.66592144202409, + 0.6557448579818498, + 0.6349845630520059 + ], + "peak_layer": 1, + "peak_normalized": 1.670694398734861, + "population_cv": 0.28301355753944435, + "rest_mean": 4.992937398061191e-05, + "spike_contrast": 0.7702285844110489, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.845703104161657e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.042191620916128e-05, + 7.649230974493548e-05, + 7.080603973008692e-05, + 6.804558506701142e-05, + 6.588984251720831e-05, + 6.292546459008008e-05, + 6.0936916270293295e-05, + 5.9498204791452736e-05, + 5.606587728834711e-05, + 5.2982773922849447e-05, + 5.1751576393144205e-05, + 5.143656017025933e-05, + 4.867615280090831e-05, + 4.7875797463348135e-05, + 4.622267442755401e-05, + 4.501900912146084e-05, + 4.483429802348837e-05, + 4.3766776798292994e-05, + 4.243209696142003e-05, + 4.1655319364508614e-05, + 4.094573523616418e-05, + 4.005357186542824e-05, + 3.8184316508704796e-05, + 3.667785495053977e-05, + 3.6423676647245884e-05, + 3.571535853552632e-05, + 3.437749546719715e-05, + 3.374716470716521e-05, + 3.233093593735248e-05, + 3.2055340852821246e-05, + 3.156547245453112e-05, + 3.056613786611706e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018337804431212135, + "first_to_last_ratio": 1.471219679743085, + "imbalance_abs_log_ratio": 0.38609177087911456, + "last_quartile_mean": 0.00012464355040719965, + "mean": 0.0001495256622092711, + "normalized": [ + 1.3620802882826015, + 1.3194938018514704, + 1.2677903290886852, + 1.23356455122679, + 1.198458119977671, + 1.1684656874236388, + 1.1415318441782234, + 1.1198031889553155, + 1.0910031274146845, + 1.0660983196947673, + 1.0498697336705507, + 1.0350765279986567, + 1.0145895852928877, + 1.0013439705330982, + 0.9836063325227735, + 0.968549777523238, + 0.9591067674489386, + 0.9465820057822166, + 0.9334010118478351, + 0.9191551367815048, + 0.9057362914957724, + 0.8945632145340071, + 0.8814992971854174, + 0.8698868180749819, + 0.8596244665411801, + 0.8514889576556698, + 0.8425072468470839, + 0.8341767076391075, + 0.8278204125037605, + 0.8225904077486038, + 0.8181934666462969, + 0.8123426056325708 + ], + "peak_layer": 1, + "peak_normalized": 1.3620802882826015, + "population_cv": 0.1550560200772002, + "rest_mean": 0.00015278581957789829, + "spike_contrast": 0.8634362323882038, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013192081241868435, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002036659570876509, + 0.00019729818450286984, + 0.0001895671884994954, + 0.0001844495564000681, + 0.00017920024401973933, + 0.00017471560568083078, + 0.00017068830493371934, + 0.0001674393133725971, + 0.0001631329650990665, + 0.00015940905723255128, + 0.0001569824671605602, + 0.0001547705032862723, + 0.00015170717961154878, + 0.00014972662029322237, + 0.00014707438822370023, + 0.00014482304686680436, + 0.00014341107453219593, + 0.00014153830124996603, + 0.00013956740440335125, + 0.00013743728050030768, + 0.00013543081877287477, + 0.00013376015704125166, + 0.0001318067661486566, + 0.00013007040251977742, + 0.00012853591761086136, + 0.00012731945025734603, + 0.00012597645400092006, + 0.00012473082460928708, + 0.00012378039536997676, + 0.00012299837544560432, + 0.0001223409199155867, + 0.00012146606604801491 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015817073654034175, + "first_to_last_ratio": 1.2671327323753674, + "imbalance_abs_log_ratio": 0.23675665700060805, + "last_quartile_mean": 0.00012482570491556544, + "mean": 0.00014045925900063594, + "normalized": [ + 1.1719218726461313, + 1.1599555891510773, + 1.143372783062998, + 1.1305925991754215, + 1.1202702784385838, + 1.106568447670676, + 1.0931409560371117, + 1.082952696795682, + 1.0683580206124534, + 1.05266391584401, + 1.0427272032106563, + 1.034492263470324, + 1.0219716035750168, + 1.0154196846947314, + 1.0039021031599646, + 0.9939448772529704, + 0.9863124890252286, + 0.9768583563343864, + 0.9682383265025308, + 0.9606909958387225, + 0.9520472410590707, + 0.9456933063501335, + 0.9351354973759991, + 0.9231938712502704, + 0.9140830768869257, + 0.9051693342733999, + 0.8960380266364146, + 0.8881305740575848, + 0.8822109405764296, + 0.8783208779569746, + 0.8754534745501711, + 0.8701687165279489 + ], + "peak_layer": 1, + "peak_normalized": 1.1719218726461313, + "population_cv": 0.09095723041158388, + "rest_mean": 0.00014217518737832843, + "spike_contrast": 0.9227576779061385, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013119324576109647, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016460727783851326, + 0.00016292650252580643, + 0.00016059729387052357, + 0.0001588021987117827, + 0.0001573523331899196, + 0.00015542778419330716, + 0.00015354176866821945, + 0.00015211073332466185, + 0.00015006077592261136, + 0.00014785639359615743, + 0.0001464606903027743, + 0.00014530401676893234, + 0.00014354537415783852, + 0.00014262509648688138, + 0.0001410073455190286, + 0.00013960876094643027, + 0.00013853672135155648, + 0.0001372088008793071, + 0.00013599803787656128, + 0.00013493794540409, + 0.00013372385001275688, + 0.00013283138105180115, + 0.00013134843902662396, + 0.0001296711270697415, + 0.0001283914316445589, + 0.00012713941396214068, + 0.00012585683725774288, + 0.00012474616232793778, + 0.00012391469499561936, + 0.00012336829968262464, + 0.00012296554632484913, + 0.00012222325312905014 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031102553475648165, + "first_to_last_ratio": 2.2098501973960105, + "imbalance_abs_log_ratio": 0.7929247292485178, + "last_quartile_mean": 0.00014074507635086775, + "mean": 0.00020578417843353236, + "normalized": [ + 1.8085758305333837, + 1.7408578335592815, + 1.6197863489261306, + 1.5416881776712252, + 1.4477928223214513, + 1.3679195769143782, + 1.3100342829404608, + 1.2546744848740619, + 1.192398672058626, + 1.1190508147725688, + 1.082828610043334, + 1.0467691899805913, + 0.9849335746234156, + 0.9618381641711736, + 0.9182125099812896, + 0.877817233347749, + 0.8611427657722097, + 0.8409392227987704, + 0.8149468534272446, + 0.7879793317746896, + 0.7604794717340572, + 0.7439065501103351, + 0.7274095758071225, + 0.7164574630456112, + 0.7035124756224637, + 0.6971408207181957, + 0.690610058298894, + 0.684463275349207, + 0.6799680284055707, + 0.6760380727842366, + 0.6720531720343774, + 0.6677747355978938 + ], + "peak_layer": 1, + "peak_normalized": 1.8085758305333837, + "population_cv": 0.33829873005197686, + "rest_mean": 0.0002160599310796156, + "spike_contrast": 0.6956177084463702, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015029511414468288, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003721762914210558, + 0.00035824099904857576, + 0.00033332640305161476, + 0.00031725503504276276, + 0.00029793285648338497, + 0.0002814962062984705, + 0.0002695843286346644, + 0.00025819215807132423, + 0.0002453767810948193, + 0.00023028295254334807, + 0.00022282899590209126, + 0.00021540853776969016, + 0.00020268374646548182, + 0.00019793107639998198, + 0.0001889536069938913, + 0.00018064089817926288, + 0.00017720955656841397, + 0.0001730519870761782, + 0.00016770316869951785, + 0.00016215367941185832, + 0.00015649464330635965, + 0.00015308419824577868, + 0.00014968938194215298, + 0.00014743561041541398, + 0.00014477173681370914, + 0.00014346055104397237, + 0.00014211662346497178, + 0.00014085171278566122, + 0.00013992666208650917, + 0.00013911793939769268, + 0.00013829790987074375, + 0.00013741747534368187 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.028097106143832207, + 0.014136734418570995, + 0.014224153943359852, + 0.014337832108139992, + 0.014408928342163563, + 0.009704872034490108, + 0.009773381985723972, + 0.00987160298973322, + 0.00994801800698042, + 0.007515520788729191, + 0.007602147292345762, + 0.007658426649868488, + 0.007734050042927265, + 0.0062688966281712055, + 0.006336068734526634, + 0.006399397272616625, + 0.006462901830673218, + 0.005432929843664169, + 0.005504137370735407, + 0.005560056772083044, + 0.005631812382489443, + 0.004873365629464388, + 0.004937449004501104, + 0.005005750339478254, + 0.005022916477173567, + 0.0044684382155537605, + 0.004500201437622309, + 0.004573327023535967, + 0.00463611725717783, + 0.004166214261204004, + 0.0042496914975345135, + 0.004283483140170574 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00014323742470878642, + "first_to_last_ratio": 1.6208859606473658, + "imbalance_abs_log_ratio": 0.48297288904334346, + "last_quartile_mean": 8.83698348843609e-05, + "mean": 0.00011961164659624046, + "normalized": [ + 0.879665217639624, + 1.4482702231163533, + 1.4522235545753037, + 1.2017346580032677, + 1.034069363337856, + 1.36550769437152, + 1.149442648944982, + 1.0492523703464898, + 1.0286339735981567, + 1.1048881958426662, + 1.2960670227766034, + 1.1342833655748923, + 0.9483258681651744, + 1.1110988010941807, + 1.0051223270547462, + 0.9186295907129635, + 0.9340104771989933, + 1.0935042558126027, + 0.9904421395235656, + 1.070541841916413, + 0.9831461473937306, + 1.1165574278336847, + 0.8906099662074067, + 0.8835226812669187, + 0.8406995671395937, + 0.8651479324264472, + 0.7705397043177317, + 0.6783637574582257, + 0.6658814129635636, + 0.7912689330389562, + 0.6809428819470401, + 0.6176059984003462 + ], + "peak_layer": 3, + "peak_normalized": 1.4522235545753037, + "population_cv": 0.21063690333645962, + "rest_mean": 0.00012087627045629133, + "spike_contrast": 0.9330423359872595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011278267775196582, + "top_five_layers": [ + 3, + 2, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010521820513531566, + 0.00017322998610325158, + 0.00017370285058859736, + 0.00014374146121554077, + 0.000123686739243567, + 0.0001633306237636134, + 0.00013748672790825367, + 0.00012550280371215194, + 0.00012303660332690924, + 0.0001321574964094907, + 0.00015502471069339663, + 0.00013567350106313825, + 0.00011343081860104576, + 0.0001329003571299836, + 0.00012022433656966314, + 0.00010987879795720801, + 0.0001117185311159119, + 0.00013079584459774196, + 0.00011846841516671702, + 0.00012804927246179432, + 0.00011759572953451425, + 0.00013355327246244997, + 0.00010652732453309, + 0.00010567960271146148, + 0.00010055745951831341, + 0.00010348176874686033, + 9.216552280122414e-05, + 8.114020602079108e-05, + 7.964717224240303e-05, + 9.464497998123989e-05, + 8.144869934767485e-05, + 7.387287041638047e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00015447182067873655, + "first_to_last_ratio": 1.6506746305835311, + "imbalance_abs_log_ratio": 0.5011840713694677, + "last_quartile_mean": 9.358102306578076e-05, + "mean": 0.00012785212516064348, + "normalized": [ + 0.8854544129111614, + 1.4719853585239924, + 1.4439160513495162, + 1.202741955390318, + 1.0425045283644727, + 1.3767992039076578, + 1.1738833181051227, + 1.0683704265103389, + 1.0407410271769484, + 1.136073438345882, + 1.2855554885667506, + 1.129314905977533, + 0.953735223493217, + 1.1222184469907663, + 1.0164965849441758, + 0.9298350247961144, + 0.9320412792829462, + 1.0837813144211186, + 0.9857736034320764, + 1.0482233302985868, + 0.9656597421723547, + 1.0923472818519362, + 0.884471876187184, + 0.8724974481708365, + 0.826330173068434, + 0.8536374096219558, + 0.7648976975162065, + 0.681485935203229, + 0.6604246501739865, + 0.7781176964145518, + 0.6752223987471391, + 0.6154627680834903 + ], + "peak_layer": 2, + "peak_normalized": 1.4719853585239924, + "population_cv": 0.21455917947613024, + "rest_mean": 0.00012955063343893185, + "spike_contrast": 0.9160910858365683, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011868018045788631, + "top_five_layers": [ + 2, + 3, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001132072284235619, + 0.00018819645629264414, + 0.00018460773571860045, + 0.00015377311501652002, + 0.00013328641944099218, + 0.00017602670413907617, + 0.0001500834769103676, + 0.00013659342948812991, + 0.00013306095206644386, + 0.00014524940343108028, + 0.00016436100122518837, + 0.00014438531070481986, + 0.00012193707516416907, + 0.0001434780133422464, + 0.00012996124860364944, + 0.00011888138396898285, + 0.00011916345829376951, + 0.00013856374425813556, + 0.00012603325012605637, + 0.00013401758042164147, + 0.0001234616502188146, + 0.00013965892139822245, + 0.00011308160901535302, + 0.00011155065294587985, + 0.00010564806871116161, + 0.00010913935693679377, + 9.779379615793005e-05, + 8.71294250828214e-05, + 8.443669503321871e-05, + 9.948400111170486e-05, + 8.632861863588914e-05, + 7.86882228567265e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.13698715530335903, + "first_to_last_ratio": 1.6506746463483835, + "imbalance_abs_log_ratio": 0.5011840809200188, + "last_quartile_mean": 0.08298858627676964, + "mean": 0.11338054295629263, + "normalized": [ + 0.8854544185536968, + 1.4719853260144444, + 1.4439159993116888, + 1.2027420311824657, + 1.042504519270847, + 1.3767991933658148, + 1.1738833593963587, + 1.0683704218985235, + 1.0407409783116481, + 1.1360735196220915, + 1.285555449073044, + 1.129314933056767, + 0.9537352653592515, + 1.1222184500196979, + 1.0164965450422543, + 0.9298350370138166, + 0.9320412867497836, + 1.0837813135283418, + 0.9857735942006465, + 1.0482233283425098, + 0.9656597514448471, + 1.0923472716531482, + 0.8844718771212735, + 0.8724974491223123, + 0.82633015871154, + 0.8536374112441639, + 0.7648976624892803, + 0.681485927333974, + 0.660424633911195, + 0.7781176812354046, + 0.6752224228414072, + 0.6154627835777616 + ], + "peak_layer": 2, + "peak_normalized": 1.4719853260144444, + "population_cv": 0.2145591770090877, + "rest_mean": 0.11488679713673061, + "spike_contrast": 0.9160910827435619, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10524677038192749, + "top_five_layers": [ + 2, + 3, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.10039330273866653, + 0.16689449548721313, + 0.16371197998523712, + 0.1363675445318222, + 0.11819972842931747, + 0.1561022400856018, + 0.13309553265571594, + 0.12113241851329803, + 0.11799977719783783, + 0.1288086324930191, + 0.14575697481632233, + 0.1280423402786255, + 0.10813502222299576, + 0.1272377371788025, + 0.11525093019008636, + 0.10542520135641098, + 0.1056753471493721, + 0.12287971377372742, + 0.11176754534244537, + 0.11884813010692596, + 0.10948702692985535, + 0.12385092675685883, + 0.10028190165758133, + 0.09892423450946808, + 0.09368976205587387, + 0.09678587317466736, + 0.08672451227903366, + 0.07726724445819855, + 0.07487930357456207, + 0.0882234051823616, + 0.07655708491802216, + 0.06978150457143784 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.7160250005617854e-05, + "first_to_last_ratio": 0.7388534944946782, + "imbalance_abs_log_ratio": 0.30265562600345663, + "last_quartile_mean": 6.382895981005277e-05, + "mean": 5.960399772675373e-05, + "normalized": [ + 0.46158176249046556, + 0.8375260596839308, + 0.8768757910510345, + 0.7241656982213429, + 0.6710453705877814, + 1.0089930699998053, + 0.8842011269201543, + 0.865421443073546, + 0.883514351886623, + 0.9622394496647411, + 1.1256912972870226, + 1.0210990974149952, + 0.9232791878576234, + 1.0980011773561495, + 1.0212452171652862, + 0.9544893655323485, + 1.0133393696232265, + 1.159814348379829, + 0.9921993085906312, + 1.1469178460749072, + 1.0774269793999733, + 1.3924548133021735, + 1.1792402184404436, + 1.1521666823514416, + 1.1069955376913123, + 1.1868921569761495, + 1.1034570471301723, + 0.9847635589252388, + 0.9837749617675684, + 1.1772881708824627, + 1.0498861530819708, + 0.9740133811896496 + ], + "peak_layer": 22, + "peak_normalized": 1.3924548133021735, + "population_cv": 0.17243044689052606, + "rest_mean": 5.759890990264938e-05, + "spike_contrast": 1.22279175241262, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.043147197691724e-05, + "top_five_layers": [ + 22, + 26, + 23, + 30, + 18 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.751211832219269e-05, + 4.991990135749802e-05, + 5.226530265645124e-05, + 4.316317063057795e-05, + 3.9996986743062735e-05, + 6.014002065057866e-05, + 5.2701921958941966e-05, + 5.158257772563957e-05, + 5.266098742140457e-05, + 5.735331797040999e-05, + 6.709570152452216e-05, + 6.0861588281113654e-05, + 5.503113061422482e-05, + 6.544525967910886e-05, + 6.087029760237783e-05, + 5.689138197340071e-05, + 6.0399077483452857e-05, + 6.912957178428769e-05, + 5.9139045333722606e-05, + 6.836088869022205e-05, + 6.421895523089916e-05, + 8.299587352667004e-05, + 7.028743129922077e-05, + 6.867374031571671e-05, + 6.598135951207951e-05, + 7.074351742630824e-05, + 6.577045132871717e-05, + 5.869584492756985e-05, + 5.863692058483139e-05, + 7.017108146101236e-05, + 6.2577411881648e-05, + 5.8055091358255595e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6127787956938846e-05, + "first_to_last_ratio": 0.8333328618421979, + "imbalance_abs_log_ratio": 0.18232212258347721, + "last_quartile_mean": 6.735338365615462e-05, + "mean": 6.689512088087213e-05, + "normalized": [ + 0.49022256384162527, + 0.923222562655389, + 0.9170289550360846, + 0.7742286879065733, + 0.7345865645537341, + 1.0394905128518102, + 0.9423822415780315, + 0.8911705562121757, + 0.9138181637864031, + 1.0330628394205372, + 1.1540375572503838, + 1.0455389170050469, + 0.9297420351318767, + 1.1341544713967917, + 1.0374336268991433, + 0.9867016024211971, + 1.0073449670971224, + 1.1743962796261593, + 1.0459325434797282, + 1.1475080782429012, + 1.066139536390961, + 1.3297286391604237, + 1.1245973641634917, + 1.1027270030076322, + 1.04892482254714, + 1.125623033590473, + 1.0476398534474196, + 0.9361019469456514, + 0.930671663614635, + 1.0896207315037236, + 0.9754527388499041, + 0.9007689403858299 + ], + "peak_layer": 22, + "peak_normalized": 1.3297286391604237, + "population_cv": 0.14905429467389167, + "rest_mean": 6.522988486621116e-05, + "spike_contrast": 1.1633838617941636, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.58873953600414e-05, + "top_five_layers": [ + 22, + 18, + 11, + 20, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.2793497666716576e-05, + 6.17590849287808e-05, + 6.134476279839873e-05, + 5.179212166694924e-05, + 4.9140257033286616e-05, + 6.953684351174161e-05, + 6.304077396634966e-05, + 5.961496208328754e-05, + 6.112997652962804e-05, + 6.910686352057382e-05, + 7.719948189333081e-05, + 6.994145223870873e-05, + 6.219520582817495e-05, + 7.586940046167001e-05, + 6.939924787729979e-05, + 6.600552296731621e-05, + 6.738646334270015e-05, + 7.856138108763844e-05, + 6.996778392931446e-05, + 7.676269160583615e-05, + 7.13195331627503e-05, + 8.895235805539414e-05, + 7.523007661802694e-05, + 7.37670561647974e-05, + 7.016795279923826e-05, + 7.529868889832869e-05, + 7.00819946359843e-05, + 6.26206528977491e-05, + 6.225739343790337e-05, + 7.289031054824591e-05, + 6.525302887894213e-05, + 6.025704715284519e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.866022898866504e-05, + "first_to_last_ratio": 0.40574519905788725, + "imbalance_abs_log_ratio": 0.9020299049238899, + "last_quartile_mean": 4.599001795213553e-05, + "mean": 3.414976504245715e-05, + "normalized": [ + 0.2445526901503676, + 0.5596630531980494, + 0.5376964344799271, + 0.48012169469058175, + 0.4798804572179837, + 0.6900344095639286, + 0.7152325131545565, + 0.6642055410913166, + 0.7554442803991458, + 0.9343372865664744, + 0.9240467958883766, + 0.8592444663674569, + 0.8263415713413451, + 1.0773994704320806, + 0.959597390691018, + 0.9894020700335048, + 0.9760686528076964, + 1.1936439731862338, + 1.099336047654388, + 1.1448507248259905, + 1.0495120137852647, + 1.4542665199347329, + 1.3223844598378258, + 1.2890134714336634, + 1.2402412095084396, + 1.4030995672846114, + 1.3877316376933018, + 1.246635573851661, + 1.2962845787221184, + 1.4847061188341568, + 1.3923191445524272, + 1.3227061808213745 + ], + "peak_layer": 30, + "peak_normalized": 1.4847061188341568, + "population_cv": 0.32662955778396485, + "rest_mean": 3.2435424482510255e-05, + "spike_contrast": 1.3382653305362568, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.340720406617038e-05, + "top_five_layers": [ + 30, + 22, + 26, + 31, + 27 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.351416909135878e-06, + 1.9112361769657582e-05, + 1.8362206901656464e-05, + 1.6396043065469712e-05, + 1.6387804862461053e-05, + 2.3564512957818806e-05, + 2.4425022274954244e-05, + 2.2682463168166578e-05, + 2.5798244678298943e-05, + 3.1907398806652054e-05, + 3.1555980967823416e-05, + 2.9342996640480123e-05, + 2.8219370506121777e-05, + 3.679293877212331e-05, + 3.277002542745322e-05, + 3.378784822416492e-05, + 3.333251515869051e-05, + 4.0762661228654906e-05, + 3.7542067730100825e-05, + 3.909638326149434e-05, + 3.584058868000284e-05, + 4.9662859964882955e-05, + 4.515911859925836e-05, + 4.4019507186021656e-05, + 4.2353945900686085e-05, + 4.791552055394277e-05, + 4.7390709369210526e-05, + 4.257231194060296e-05, + 4.4267813791520894e-05, + 5.070236511528492e-05, + 4.754737165058032e-05, + 4.517010529525578e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00021511870181711856, + "first_to_last_ratio": 1.067203936185183, + "imbalance_abs_log_ratio": 0.06504208450186474, + "last_quartile_mean": 0.00020157225299044512, + "mean": 0.00023711549874860793, + "normalized": [ + 0.5199215340336985, + 0.9694171998286779, + 0.9952311992308057, + 0.8139231798698612, + 0.8071491568326964, + 1.1522101416966866, + 1.0127543403881063, + 0.9872470097646113, + 0.9739878350012732, + 1.0629045356723543, + 1.2654986331851705, + 1.158397891601516, + 1.0251221075049282, + 1.1837639471542862, + 1.126357049236326, + 1.0240452993420521, + 1.0679809871494306, + 1.2152134226291609, + 1.0519957926786496, + 1.2134701299884076, + 1.1240302451392508, + 1.3270417853241223, + 1.0642149202553777, + 1.0573092573282454, + 0.9986311292560726, + 1.01615500686031, + 0.8954142860763721, + 0.793725079326075, + 0.7548043496765433, + 0.8883375219775711, + 0.764305251609244, + 0.6894397743821165 + ], + "peak_layer": 22, + "peak_normalized": 1.3270417853241223, + "population_cv": 0.17546356070810173, + "rest_mean": 0.00023209894856403548, + "spike_contrast": 1.138328593816985, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002642048697452992, + "top_five_layers": [ + 22, + 11, + 18, + 20, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00012328145385254174, + 0.00022986384283285588, + 0.0002359847421757877, + 0.00019299380073789507, + 0.00019138757488690317, + 0.00027320688241161406, + 0.00024013975053094327, + 0.00023409156710840762, + 0.00023094761127140373, + 0.0002520311390981078, + 0.00030006933957338333, + 0.0002746740938164294, + 0.00024307233979925513, + 0.00028068877873010933, + 0.0002670767134986818, + 0.0002428170118946582, + 0.00025323484442196786, + 0.0002881459367927164, + 0.00024944450706243515, + 0.0002877325750887394, + 0.0002665249921847135, + 0.00031466217478737235, + 0.0002523418515920639, + 0.00025070441188290715, + 0.0002367909182794392, + 0.00024094610125757754, + 0.00021231660502962768, + 0.00018820451805368066, + 0.00017897580983117223, + 0.00021063859458081424, + 0.00018122862093150616, + 0.00016347685595974326 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014079716056585312, + 0.014167090877890587, + 0.014255538582801819, + 0.014355218037962914, + 0.009655896574258804, + 0.009734091348946095, + 0.009828771464526653, + 0.009915035218000412, + 0.007479202933609486, + 0.007559895049780607, + 0.0076358383521437645, + 0.007706487085670233, + 0.0062490664422512054, + 0.006315518636256456, + 0.0063744354993104935, + 0.006439714692533016, + 0.0054116579703986645, + 0.005481995176523924, + 0.005541813559830189, + 0.005614723544567823, + 0.004862982779741287, + 0.004923288244754076, + 0.0049925679340958595, + 0.005004179198294878, + 0.004448203835636377, + 0.004490054678171873, + 0.00455933902412653, + 0.004625064320862293, + 0.004160413984209299, + 0.004242690280079842, + 0.004270957782864571, + 0.004303553607314825 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.6855086818832206e-05, + "first_to_last_ratio": 1.1121706932103832, + "imbalance_abs_log_ratio": 0.10631368513677814, + "last_quartile_mean": 3.313797697046539e-05, + "mean": 3.537216105087282e-05, + "normalized": [ + 1.3329674878497746, + 1.065596392820356, + 0.9206330800727252, + 0.8842664165696186, + 1.1462361065020723, + 1.065158463293282, + 0.99582829507555, + 0.9247019781352203, + 1.132583771771479, + 1.005206548698436, + 0.979948053013938, + 0.967083333200755, + 1.1057115808592233, + 1.043785177992708, + 0.995436955984306, + 0.9279460300710988, + 1.1098294348763655, + 1.0288442525987258, + 1.0234795644677463, + 0.9449806453153974, + 1.0109568163965834, + 1.0260998393764513, + 0.9416381672270038, + 0.926379336673748, + 0.9884731771330477, + 0.9341156116917169, + 0.9458688462068515, + 0.9053302302997448, + 0.9662542702797254, + 0.9495526789453999, + 0.9100381269887573, + 0.8950693296121922 + ], + "peak_layer": 1, + "peak_normalized": 1.3329674878497746, + "population_cv": 0.09165679612351446, + "rest_mean": 3.551162256001219e-05, + "spike_contrast": 0.9748658722370791, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.461906890152022e-05, + "top_five_layers": [ + 1, + 5, + 9, + 17, + 13 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.714994065579958e-05, + 3.769244722207077e-05, + 3.256478157709353e-05, + 3.127841409877874e-05, + 4.054484816151671e-05, + 3.767695670831017e-05, + 3.5224598832428455e-05, + 3.270870729465969e-05, + 4.006193557870574e-05, + 3.555632792995311e-05, + 3.4662880352698267e-05, + 3.420782741159201e-05, + 3.911140811396763e-05, + 3.692093741847202e-05, + 3.521075632306747e-05, + 3.2823456422192976e-05, + 3.9257065509445965e-05, + 3.6392444599187e-05, + 3.620268398663029e-05, + 3.342600757605396e-05, + 3.575972732505761e-05, + 3.6295368772698566e-05, + 3.330777690280229e-05, + 3.276803909102455e-05, + 3.49644324160181e-05, + 3.3041687856893986e-05, + 3.3457425161032006e-05, + 3.202348671038635e-05, + 3.417850166442804e-05, + 3.358773028594442e-05, + 3.2190015190280974e-05, + 3.166053647873923e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 7.015105893515283e-05, + "first_to_last_ratio": 0.6588274570259804, + "imbalance_abs_log_ratio": 0.4172936042156455, + "last_quartile_mean": 0.00010647865110513521, + "mean": 9.056432463694364e-05, + "normalized": [ + 0.8598786252117567, + 0.7452797309823858, + 0.6694431442719111, + 0.6417832925446045, + 0.8822867975473095, + 0.8318257491363829, + 0.804532248804658, + 0.7617644993607329, + 0.969439786050785, + 0.9144645011121496, + 0.8951531228323205, + 0.876468558935897, + 1.0407786059783084, + 1.0056397226462848, + 0.9828357943952414, + 0.9490839045924557, + 1.1213662721801265, + 1.0774120583919133, + 1.0554308943167812, + 1.0307906296953837, + 1.1314055055223455, + 1.1556700189426168, + 1.1045075200854564, + 1.0869669205578247, + 1.2145839027446308, + 1.1552275050200584, + 1.1514380979421175, + 1.1280886596086177, + 1.2254687972249847, + 1.20187962367999, + 1.1639632986601018, + 1.165142211023867 + ], + "peak_layer": 29, + "peak_normalized": 1.2254687972249847, + "population_cv": 0.1618495930711073, + "rest_mean": 8.82393912804589e-05, + "spike_contrast": 1.1686273359956587, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010311896476196125, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 7.787432696204633e-05, + 6.74957555020228e-05, + 6.062766624381766e-05, + 5.812267045257613e-05, + 7.990370795596391e-05, + 7.533373718615621e-05, + 7.286191976163536e-05, + 6.898868741700426e-05, + 8.779665949987248e-05, + 8.281785994768143e-05, + 8.106893801596016e-05, + 7.937678310554475e-05, + 9.425741154700518e-05, + 9.107508230954409e-05, + 8.900985994841903e-05, + 8.595314284320921e-05, + 0.00010155577911064029, + 9.757509542396292e-05, + 9.558438614476472e-05, + 9.335285722045228e-05, + 0.00010246497549815103, + 0.00010466247476870194, + 0.00010002897761296481, + 9.844042506301776e-05, + 0.00010999797086697072, + 0.00010462239879416302, + 0.00010427921370137483, + 0.00010216458758804947, + 0.00011098375398432836, + 0.00010884741641348228, + 0.00010541355004534125, + 0.00010552031744737178 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06221066089347005, + "first_to_last_ratio": 0.6588274447143223, + "imbalance_abs_log_ratio": 0.4172936229028725, + "last_quartile_mean": 0.09442633483558893, + "mean": 0.0803133494919166, + "normalized": [ + 0.8598786809861813, + 0.7452797356777653, + 0.6694431594543391, + 0.6417832788211475, + 0.8822868217673959, + 0.83182573854958, + 0.804532202278358, + 0.7617644434307961, + 0.9694397651274825, + 0.9144645475330049, + 0.8951531251987148, + 0.8764685422804359, + 1.0407786734529123, + 1.0056396718254574, + 0.9828357789429103, + 0.9490838578168909, + 1.121366176654653, + 1.0774119964231388, + 1.0554309636293913, + 1.030790617723408, + 1.131405440813584, + 1.1556700726993148, + 1.1045075638748008, + 1.0869669141870728, + 1.214583886452764, + 1.155227565075462, + 1.151438141297865, + 1.1280886745503538, + 1.2254688318483915, + 1.20187965027907, + 1.1639633335063522, + 1.1651421478410058 + ], + "peak_layer": 29, + "peak_normalized": 1.2254688318483915, + "population_cv": 0.16184959877781283, + "rest_mean": 0.07825157529226055, + "spike_contrast": 1.1686273385361958, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0914469301700592, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.06905973702669144, + 0.05985591188073158, + 0.05376522243022919, + 0.051543764770030975, + 0.0708594098687172, + 0.06680671125650406, + 0.0646146759390831, + 0.06117985397577286, + 0.07785895466804504, + 0.0734437108039856, + 0.07189274579286575, + 0.07039212435483932, + 0.08358842134475708, + 0.08076629042625427, + 0.07893483340740204, + 0.07622410356998444, + 0.09006067365407944, + 0.08653056621551514, + 0.08476519584655762, + 0.08278624713420868, + 0.09086696058511734, + 0.09281573444604874, + 0.08870670199394226, + 0.0872979536652565, + 0.09754730015993118, + 0.09278019517660141, + 0.09247585386037827, + 0.09060057997703552, + 0.09842150658369064, + 0.09652698040008545, + 0.09348179399967194, + 0.09357646852731705 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.077007694533677e-06, + "first_to_last_ratio": 0.4632676420751026, + "imbalance_abs_log_ratio": 0.7694503312712233, + "last_quartile_mean": 1.5276283193088602e-05, + "mean": 1.1745167029175718e-05, + "normalized": [ + 0.5860812018784138, + 0.5338863181232825, + 0.4896831424227033, + 0.48432444022666415, + 0.7227742595277804, + 0.658356157633798, + 0.6679626701195096, + 0.6773027265164295, + 0.9317627933811012, + 0.8396840560424388, + 0.8002283441953012, + 0.8237560838623014, + 0.9947731150675175, + 0.9485376018841178, + 1.012754293639493, + 0.980582798985168, + 1.1861357451339187, + 1.074396404738039, + 1.1583203932505854, + 1.1590437197037688, + 1.1716087388237089, + 1.3380122311407743, + 1.182136735603702, + 1.17274255168395, + 1.3256589211015541, + 1.1785808128912905, + 1.3283612705891032, + 1.2366027296837898, + 1.3388375403515687, + 1.3849270873253279, + 1.3156503626951777, + 1.2965347517777213 + ], + "peak_layer": 30, + "peak_normalized": 1.3849270873253279, + "population_cv": 0.2782190894097951, + "rest_mean": 1.1227440423950881e-05, + "spike_contrast": 1.2951207174852213, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4540890697389841e-05, + "top_five_layers": [ + 30, + 29, + 22, + 27, + 25 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 6.883621608722024e-06, + 6.2705839809495956e-06, + 5.751410299126292e-06, + 5.6884714467742015e-06, + 8.48910440254258e-06, + 7.732503036095295e-06, + 7.845333129807841e-06, + 7.955033652251586e-06, + 1.0943709639832377e-05, + 9.862229489954188e-06, + 9.39881556405453e-06, + 9.67515279626241e-06, + 1.168377639260143e-05, + 1.1140732567582745e-05, + 1.1894968338310719e-05, + 1.1517108760017436e-05, + 1.3931362445873674e-05, + 1.2618965229194146e-05, + 1.3604666492028628e-05, + 1.3613162082037888e-05, + 1.3760740330326371e-05, + 1.5715177141828462e-05, + 1.3884393410990015e-05, + 1.377405715174973e-05, + 1.5570085452054627e-05, + 1.38426285047899e-05, + 1.56018249981571e-05, + 1.4524105608870741e-05, + 1.572487053635996e-05, + 1.62661999638658e-05, + 1.5452533261850476e-05, + 1.5228017218760215e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.028231200616574e-05, + "first_to_last_ratio": 0.5725175422355306, + "imbalance_abs_log_ratio": 0.5577119025637401, + "last_quartile_mean": 0.00010529338851483772, + "mean": 8.693937525094952e-05, + "normalized": [ + 0.6520176082954547, + 0.61792639701702, + 0.5860559002851884, + 0.5711155588169554, + 0.8204737244004536, + 0.784378974164202, + 0.772878878140371, + 0.7422194596148288, + 0.9603819210570083, + 0.917062379210463, + 0.9022792075902993, + 0.8821977885039085, + 1.057360487646653, + 1.0255127545988867, + 1.0067830966339102, + 0.9749202156934387, + 1.1510091959216098, + 1.1091409226242501, + 1.0872456061539109, + 1.0647793564016073, + 1.1683829082644246, + 1.1939706281941995, + 1.1404893610428462, + 1.1225154239128359, + 1.2535795945499137, + 1.1915479696415308, + 1.1872717446378291, + 1.1619194447200984, + 1.2611193116864508, + 1.2370940006457103, + 1.1981927894482458, + 1.1981773904854955 + ], + "peak_layer": 29, + "peak_normalized": 1.2611193116864508, + "population_cv": 0.20676778332368514, + "rest_mean": 8.410921549916061e-05, + "spike_contrast": 1.215351222858925, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001022222379106097, + "top_five_layers": [ + 29, + 25, + 30, + 31, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 5.6686003517825156e-05, + 5.3722134907729924e-05, + 5.095133383292705e-05, + 4.965242987964302e-05, + 7.133147300919518e-05, + 6.81934179738164e-05, + 6.719360681017861e-05, + 6.452809611801058e-05, + 8.349500421900302e-05, + 7.972883031470701e-05, + 7.844359060982242e-05, + 7.669772458029911e-05, + 9.192626021103933e-05, + 8.915743819670752e-05, + 8.75290934345685e-05, + 8.475895447190851e-05, + 0.00010006802040152252, + 9.642801887821406e-05, + 9.452445374336094e-05, + 9.257125202566385e-05, + 0.00010157848009839654, + 0.00010380306048318744, + 9.915343252941966e-05, + 9.759078966453671e-05, + 0.00010898542677750811, + 0.00010359243606217206, + 0.00010322066373191774, + 0.00010101655061589554, + 0.00010964092507492751, + 0.0001075521795428358, + 0.00010417013254482299, + 0.00010416879376862198 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 5.289650744089158e-05, + "first_to_last_ratio": 0.5036607689093815, + "imbalance_abs_log_ratio": 0.6858523150899144, + "last_quartile_mean": 0.00010502407712920103, + "mean": 8.374395429200376e-05, + "normalized": [ + 0.5462883602941595, + 0.5343271256807102, + 0.5249643991287053, + 0.5130648517134513, + 0.7580131602578075, + 0.7348274620741431, + 0.7341801809099809, + 0.7074999463902408, + 0.9290893112957194, + 0.896130362600229, + 0.8896360907643062, + 0.8667732512123232, + 1.0501229826066591, + 1.0276121076179003, + 1.01304723882372, + 0.9917393510831477, + 1.161342307568102, + 1.1291245609681106, + 1.1079392656735292, + 1.0938109858524918, + 1.2035226235600145, + 1.226263478849651, + 1.1724755435916903, + 1.1553301515952579, + 1.2932755854998876, + 1.227360989677261, + 1.2282904680523223, + 1.203004103561663, + 1.301298744133899, + 1.2818290480161694, + 1.2464280682225441, + 1.251387892724203 + ], + "peak_layer": 29, + "peak_normalized": 1.301298744133899, + "population_cv": 0.24714813814810152, + "rest_mean": 8.048455465878187e-05, + "spike_contrast": 1.2591821218500583, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010134471231140196, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.574834747472778e-05, + 4.474666638998315e-05, + 4.396259464556351e-05, + 4.296607949072495e-05, + 6.347901944536716e-05, + 6.153735739644617e-05, + 6.14831515122205e-05, + 5.924884317209944e-05, + 7.780561281833798e-05, + 7.504550012527034e-05, + 7.450164412148297e-05, + 7.258701953105628e-05, + 8.794145105639473e-05, + 8.60563013702631e-05, + 8.483658166369423e-05, + 8.305217488668859e-05, + 9.725539712235332e-05, + 9.455735562369227e-05, + 9.278321522288024e-05, + 9.160005720332265e-05, + 0.00010078774357680231, + 0.00010269215272273868, + 9.818773833103478e-05, + 9.675191540736705e-05, + 0.00010830401151906699, + 0.00010278406261932105, + 0.00010286190081387758, + 0.00010074432066176087, + 0.00010897590254899114, + 0.00010734543320722878, + 0.00010438081517349929, + 0.00010479617048986256 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010133173600479495, + "first_to_last_ratio": 0.7712367006907566, + "imbalance_abs_log_ratio": 0.25975994774816064, + "last_quartile_mean": 0.00013138863323547412, + "mean": 0.00011757463107642252, + "normalized": [ + 0.9240254436843522, + 0.8516610763218788, + 0.7400083948384197, + 0.7305203183107621, + 0.9945100925181989, + 0.9162256174012862, + 0.9029060711458456, + 0.8349457218511338, + 1.057384774375838, + 0.9561549232760007, + 0.9255087968738563, + 0.9289705112696707, + 1.0651337322266232, + 1.006401362110571, + 0.966928758219108, + 0.9306903222204219, + 1.1004660046561014, + 1.0434523934672617, + 1.009934985409439, + 0.9657192405683421, + 1.061624429236619, + 1.0862247001295862, + 1.0383026763618062, + 1.022368851612807, + 1.1429833503196458, + 1.0926891991234107, + 1.0888718388877934, + 1.0734199646869733, + 1.177032649101865, + 1.140664442905499, + 1.1108444025161495, + 1.1134249543727335 + ], + "peak_layer": 29, + "peak_normalized": 1.177032649101865, + "population_cv": 0.10936728326378568, + "rest_mean": 0.00011604396611262389, + "spike_contrast": 1.0844184846181808, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012584022188093512, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00010864195064641535, + 0.00010013373685069382, + 8.700621401658282e-05, + 8.58906569192186e-05, + 0.00011692915722960606, + 0.00010772488894872367, + 0.0001061588482116349, + 9.816843521548435e-05, + 0.0001243216247530654, + 0.00011241956235608086, + 0.00010881635535042733, + 0.00010922336514340714, + 0.00012523270561359823, + 0.00011832726886495948, + 0.00011368629202479497, + 0.00010942557128146291, + 0.0001293868845095858, + 0.00012268353020772338, + 0.00011874273332068697, + 0.00011354408343322575, + 0.00012482010060921311, + 0.00012771246838383377, + 0.0001220780541189015, + 0.00012020464055240154, + 0.00013438584574032575, + 0.0001284725294681266, + 0.00012802370474673808, + 0.0001262069563381374, + 0.00013838917948305607, + 0.00013411320105660707, + 0.00013060712080914527, + 0.00013091052824165672 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 8.06433666787865, + "forward": { + "activation_sha256": { + "attention_branch_output": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", + "mlp_branch_output": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", + "post_attention_state": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", + "post_mlp_state": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", + "pre_attention_input": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", + "pre_mlp_input": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248" + }, + "logits_sha256": "30ae47372d19e5c17da2b2a4846b2d27ef6e44fdc06045752d040a56bc6e9398", + "loss_nats": 5.5897722244262695, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.5897722244262695, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "276350f9764eff97176a7a0c9acae7ac426b2757485f30de8d7552c1f85aba55", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0021227982360869646, + 0.00224129855632782, + 0.0024903982412070036, + 0.0023722455371171236, + 0.0023577234242111444, + 0.0023993616923689842, + 0.002688874490559101, + 0.0029664887115359306, + 0.0029547461308538914, + 0.003052416956052184, + 0.0028214550111442804, + 0.0032544767018407583, + 0.00348988501355052, + 0.003914494998753071, + 0.0035071265883743763, + 0.004211232997477055, + 0.004072706215083599, + 0.004050548188388348, + 0.003929397091269493, + 0.004410331137478352, + 0.004824656527489424, + 0.00433384021744132, + 0.004475373309105635, + 0.004755842965096235, + 0.004843301139771938, + 0.0050001065246760845, + 0.005346097983419895, + 0.004958339501172304, + 0.0054801348596811295, + 0.005455824080854654, + 0.005286100786179304, + 0.005402065813541412 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015727665777376387, + "first_to_last_ratio": 3.444367030996094, + "imbalance_abs_log_ratio": 1.2367401520242614, + "last_quartile_mean": 4.566199140754179e-05, + "mean": 8.846047569477378e-05, + "normalized": [ + 2.4938583213844665, + 2.2321521039252294, + 1.9412946714691228, + 1.7875630514690435, + 1.605255150569201, + 1.4809330135294976, + 1.3782219007164203, + 1.3041730210393685, + 1.1983684502302745, + 1.1165155187552953, + 1.0705163494358503, + 1.028915549335682, + 0.969265519878856, + 0.9296915475356099, + 0.8760186941641476, + 0.840544602365157, + 0.8215144482528359, + 0.7958412144142111, + 0.7578169321029499, + 0.7149281644739428, + 0.6789722877716906, + 0.648159768212111, + 0.6127699836305072, + 0.5872278577962867, + 0.5712871725092085, + 0.5525140937217092, + 0.5351005407841478, + 0.5196042162209058, + 0.49974197043571306, + 0.49405312503381205, + 0.48454882285128825, + 0.4726319359854579 + ], + "peak_layer": 1, + "peak_normalized": 2.4938583213844665, + "population_cv": 0.5254866190950809, + "rest_mean": 9.469065830840177e-05, + "spike_contrast": 0.5789112734082525, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.48174895811826e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00022060789342503995, + 0.0001974572369363159, + 0.00017172785010188818, + 0.00015812867786735296, + 0.00014200163423083723, + 0.0001310040388489142, + 0.00012191816495032981, + 0.00011536776582943276, + 0.00010600824316497892, + 9.876749390969053e-05, + 9.469838551012799e-05, + 9.101835894398391e-05, + 8.574168896302581e-05, + 8.224095654441044e-05, + 7.749303040327504e-05, + 7.435497536789626e-05, + 7.267155888257548e-05, + 7.040049240458757e-05, + 6.703684630338103e-05, + 6.324288551695645e-05, + 6.0062211559852585e-05, + 5.733652142225765e-05, + 5.42059242434334e-05, + 5.1946455641882494e-05, + 5.053633503848687e-05, + 4.887565955868922e-05, + 4.733524838229641e-05, + 4.596443613991141e-05, + 4.420741242938675e-05, + 4.370417445898056e-05, + 4.286341936676763e-05, + 4.180924588581547e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00022213962802197784, + "first_to_last_ratio": 1.7678973829107683, + "imbalance_abs_log_ratio": 0.5697909211909248, + "last_quartile_mean": 0.00012565187898871955, + "mean": 0.00016188367408176418, + "normalized": [ + 1.7325269150218519, + 1.5951124185334453, + 1.452057969341735, + 1.3730142236731557, + 1.2857984704138503, + 1.2257558334519054, + 1.176224442377172, + 1.1372505911497117, + 1.088955563427776, + 1.050933480062714, + 1.0259126283978994, + 1.0049209692997896, + 0.9780904283068869, + 0.9585002626676595, + 0.9364797090707767, + 0.9194681619525875, + 0.9067005574410794, + 0.8936181540011504, + 0.876961137714969, + 0.8605080845266135, + 0.8461083257765747, + 0.8336465310750286, + 0.8211746685608088, + 0.8107906186997867, + 0.8025238658005067, + 0.7926026656489187, + 0.7851375623337778, + 0.7775608141673811, + 0.7689788128837574, + 0.7655650153667295, + 0.7607140375905382, + 0.7564070812634623 + ], + "peak_layer": 1, + "peak_normalized": 1.7325269150218519, + "population_cv": 0.2510633514944888, + "rest_mean": 0.00016719439386010722, + "spike_contrast": 0.7967120440064878, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001332057872787118, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002804678224492818, + 0.0002582226588856429, + 0.00023506447905674577, + 0.00022226858709473163, + 0.00020814978051930666, + 0.00019842985784634948, + 0.00019041153427679092, + 0.00018410230404697359, + 0.00017628412751946598, + 0.0001701289729680866, + 0.0001660785055719316, + 0.0001626802986720577, + 0.0001583368721185252, + 0.00015516554412897676, + 0.00015160077600739896, + 0.00014884688425809145, + 0.0001467800175305456, + 0.00014466218999587, + 0.00014196569100022316, + 0.0001393022103002295, + 0.00013697112444788218, + 0.00013495376333594322, + 0.0001329347724094987, + 0.0001312537642661482, + 0.00012991551193408668, + 0.00012830943160224706, + 0.0001271009532501921, + 0.00012587440141942352, + 0.0001244851155206561, + 0.00012393247743602842, + 0.0001231471833307296, + 0.00012244995741639286 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19699564762413502, + "first_to_last_ratio": 1.7678974461601602, + "imbalance_abs_log_ratio": 0.5697909569675436, + "last_quartile_mean": 0.11142934113740921, + "mean": 0.14356006286107004, + "normalized": [ + 1.73252692376823, + 1.5951124529267644, + 1.4520579427898739, + 1.3730142509369438, + 1.2857984860257277, + 1.2257558423238033, + 1.1762244498747965, + 1.1372506023188744, + 1.0889555406950477, + 1.0509335152923371, + 1.0259126331612176, + 1.004920958483027, + 0.9780904724894115, + 0.9585002702989701, + 0.9364797557971557, + 0.9194681924019411, + 0.9067005932791752, + 0.893618176643793, + 0.876961081367795, + 0.8605080517791706, + 0.8461083423470157, + 0.8336465303631113, + 0.8211746500314575, + 0.8107906024915998, + 0.8025238662049899, + 0.792602599370704, + 0.7851375422989607, + 0.7775607992253795, + 0.7689787783378558, + 0.7655650375651303, + 0.7607140142018359, + 0.7564070449079043 + ], + "peak_layer": 1, + "peak_normalized": 1.73252692376823, + "population_cv": 0.25106336091926634, + "rest_mean": 0.1482696624265777, + "spike_contrast": 0.7967120399011176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1181282252073288, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2487216740846634, + 0.2289944440126419, + 0.20845752954483032, + 0.19711001217365265, + 0.18458931148052216, + 0.17596958577632904, + 0.1688588559627533, + 0.16326376795768738, + 0.15633052587509155, + 0.15087208151817322, + 0.14728008210659027, + 0.1442665159702301, + 0.14041472971439362, + 0.13760235905647278, + 0.1344410926103592, + 0.1319989114999771, + 0.13016599416732788, + 0.1282878816127777, + 0.12589658796787262, + 0.12353459000587463, + 0.12146736681461334, + 0.11967834830284119, + 0.11788788437843323, + 0.11639714986085892, + 0.11521037667989731, + 0.11378607898950577, + 0.11271439492702484, + 0.11162667721509933, + 0.11039464175701141, + 0.10990456491708755, + 0.10920815169811249, + 0.10858984291553497 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.813340178268845e-05, + "first_to_last_ratio": 2.0396781258635253, + "imbalance_abs_log_ratio": 0.7127920139678758, + "last_quartile_mean": 3.3403996894776355e-05, + "mean": 4.817690887648496e-05, + "normalized": [ + 1.6677253734616262, + 1.5880838657379246, + 1.4689679818917765, + 1.414042908307037, + 1.3674754530876347, + 1.307279124984285, + 1.2642920390991146, + 1.2360018052494937, + 1.16456296339031, + 1.101785452839103, + 1.0797656603944665, + 1.0672809591929815, + 1.0128091896347613, + 0.9955804650846075, + 0.9623876307655963, + 0.9380984486976066, + 0.9310658554452689, + 0.9082443423860135, + 0.8811967490595315, + 0.8644115118569954, + 0.8497936463301483, + 0.8309315019236491, + 0.7889969150425813, + 0.7623309639866356, + 0.7584133539756696, + 0.7445182980676538, + 0.7180454335344151, + 0.7000361319857387, + 0.6663708133598921, + 0.6665317313809671, + 0.6558653811637546, + 0.6371080486827607 + ], + "peak_layer": 1, + "peak_normalized": 1.6677253734616262, + "population_cv": 0.28258812904813146, + "rest_mean": 4.997824995640527e-05, + "spike_contrast": 0.7693279992487513, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.844966704491526e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.034585334826261e-05, + 7.650897168787196e-05, + 7.077033660607412e-05, + 6.81242163409479e-05, + 6.588074029423296e-05, + 6.298066728049889e-05, + 6.090968236094341e-05, + 5.954674634267576e-05, + 5.610504376818426e-05, + 5.308061736286618e-05, + 5.201977182878181e-05, + 5.1418297516647726e-05, + 4.879401603830047e-05, + 4.796398934558965e-05, + 4.6364861191250384e-05, + 4.519468348007649e-05, + 4.4855874875793234e-05, + 4.3756404920713976e-05, + 4.245333548169583e-05, + 4.1644674638519064e-05, + 4.094043106306344e-05, + 4.0031711250776425e-05, + 3.8011432479834184e-05, + 3.672674938570708e-05, + 3.653801104519516e-05, + 3.586859020288102e-05, + 3.459320942056365e-05, + 3.372557694092393e-05, + 3.210368595318869e-05, + 3.21114384860266e-05, + 3.159756670356728e-05, + 3.069389640586451e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018498800636734813, + "first_to_last_ratio": 1.4771173407265819, + "imbalance_abs_log_ratio": 0.39009244570614415, + "last_quartile_mean": 0.0001252358233614359, + "mean": 0.00015057798327688943, + "normalized": [ + 1.36489476841379, + 1.32224224178262, + 1.270236278236952, + 1.2358216684832493, + 1.2002806183282966, + 1.1699184301039023, + 1.1431585124660382, + 1.1216044194279113, + 1.0931779382416982, + 1.0676137521656621, + 1.050334546837667, + 1.0352083927879365, + 1.0149316924423049, + 1.001461761419052, + 0.9840691969120363, + 0.9695910166861202, + 0.9584136856662008, + 0.9467149797350989, + 0.9322036518549593, + 0.9180752132052378, + 0.9048391519289843, + 0.8925982934025656, + 0.8798985862981495, + 0.8691051109125614, + 0.860352873872478, + 0.849675270316085, + 0.8415818308236442, + 0.8332603200076834, + 0.823928337260059, + 0.8201054366601405, + 0.8147645087933348, + 0.8099375145275801 + ], + "peak_layer": 1, + "peak_normalized": 1.36489476841379, + "population_cv": 0.15663918075507588, + "rest_mean": 0.0001538862707093358, + "spike_contrast": 0.8624111204329009, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000132713231141679, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00020552310161292553, + 0.00019910057017114013, + 0.00019126961706206203, + 0.0001860875345300883, + 0.00018073583487421274, + 0.00017616395780351013, + 0.0001721345033729449, + 0.00016888893151190132, + 0.0001646085293032229, + 0.00016075912571977824, + 0.00015815725782886147, + 0.0001558795920573175, + 0.00015282636741176248, + 0.00015079809236340225, + 0.0001481791550759226, + 0.00014599905989598483, + 0.00014431599993258715, + 0.00014255443238653243, + 0.00014036934589967132, + 0.000138241914100945, + 0.00013624885468743742, + 0.00013440565089695156, + 0.0001324933546129614, + 0.0001308680948568508, + 0.00012955020065419376, + 0.00012794238864444196, + 0.0001267236948478967, + 0.00012547065853141248, + 0.00012406546738930047, + 0.00012348982272669673, + 0.0001226855965796858, + 0.00012195875751785934 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015959660413500387, + "first_to_last_ratio": 1.2728751185715401, + "imbalance_abs_log_ratio": 0.2412782146622287, + "last_quartile_mean": 0.00012538276678242255, + "mean": 0.00014144585429676226, + "normalized": [ + 1.173955566226187, + 1.1612997099437044, + 1.147257651804958, + 1.1326524298721496, + 1.1221543736292459, + 1.1091111750587395, + 1.0952125313023318, + 1.0849401932479867, + 1.070907908686229, + 1.0558579375605581, + 1.0430997159190893, + 1.0346471150202732, + 1.0228091528121042, + 1.0150746526225596, + 1.0038187840838828, + 0.9945868690199192, + 0.986355871119351, + 0.9777591356798918, + 0.9676015085554662, + 0.9595938626090281, + 0.9511867345631487, + 0.9428977124791953, + 0.9335239262292595, + 0.9222035520824579, + 0.9142471402553919, + 0.9038764491526137, + 0.8952592406413572, + 0.8872171299760644, + 0.877779969795339, + 0.8748896658804006, + 0.8709399062198925, + 0.8672824279512239 + ], + "peak_layer": 1, + "peak_normalized": 1.173955566226187, + "population_cv": 0.09275081554992197, + "rest_mean": 0.00014320575994335942, + "spike_contrast": 0.9213481626529775, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013194236380513756, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016605114797130227, + 0.0001642610295675695, + 0.0001622748386580497, + 0.00016020899056456983, + 0.00015872408403083682, + 0.00015687917766626924, + 0.0001549132721265778, + 0.00015346029249485582, + 0.00015147548401728272, + 0.00014934672799427062, + 0.00014754213043488562, + 0.000146346545079723, + 0.00014467211440205574, + 0.00014357810141518712, + 0.00014198600547388196, + 0.00014068018936086446, + 0.00013951594883110374, + 0.00013829997624270618, + 0.00013686322199646384, + 0.0001357305736746639, + 0.00013454142026603222, + 0.0001333689724560827, + 0.00013204308925196528, + 0.00013044186925981194, + 0.00012931646779179573, + 0.00012784957652911544, + 0.00012663070810958743, + 0.000125493184896186, + 0.0001241583377122879, + 0.00012374951620586216, + 0.0001231908390764147, + 0.00012267350393813103 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031297675377572887, + "first_to_last_ratio": 2.208693462790297, + "imbalance_abs_log_ratio": 0.7924011473984336, + "last_quartile_mean": 0.00014170221402309835, + "mean": 0.00020716176595669822, + "normalized": [ + 1.8096319791310764, + 1.7504284704923412, + 1.6109023900661548, + 1.5434988529701714, + 1.4353862656069223, + 1.3677596328902204, + 1.3131374518280827, + 1.2555297512219317, + 1.1870915168673883, + 1.122656500215855, + 1.088514230207685, + 1.04408820740568, + 0.9901334868561585, + 0.9607872809839492, + 0.9219812869372472, + 0.880015778430751, + 0.8614610505244651, + 0.8397919049431777, + 0.8116424497290868, + 0.7845312039842096, + 0.7619415782348183, + 0.7436794877201931, + 0.7280047725498856, + 0.715266757195506, + 0.7065993237444729, + 0.696468071137219, + 0.6907565141722826, + 0.6863265628748565, + 0.6782234011874358, + 0.6754056248242399, + 0.6708832321424102, + 0.6674749829241257 + ], + "peak_layer": 1, + "peak_normalized": 1.8096319791310764, + "population_cv": 0.33826602846938675, + "rest_mean": 0.00021747771663487784, + "spike_contrast": 0.6964190843920172, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001514556322945282, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00037488655652850866, + 0.0003626218531280756, + 0.0003337173839099705, + 0.0003197539481334388, + 0.0002973571536131203, + 0.00028334750095382333, + 0.00027203187346458435, + 0.0002600977604743093, + 0.0002459199749864638, + 0.00023257150314748287, + 0.00022549853019881994, + 0.0002162951568607241, + 0.00020511780166998506, + 0.00019903838983736932, + 0.00019099927158094943, + 0.00018230562272947282, + 0.00017846179252956063, + 0.00017397277406416833, + 0.0001681412832112983, + 0.0001625248696655035, + 0.0001578451629029587, + 0.00015406195598188788, + 0.00015081475430633873, + 0.0001481759245507419, + 0.00014638036373071373, + 0.0001442815555492416, + 0.0001430983393220231, + 0.00014218062278814614, + 0.00014050195750314742, + 0.00013991822197567672, + 0.00013898135512135923, + 0.00013827529619447887 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "08aec19bd41bfe00a4f8b30e53420fcee9de8867ddeef28bf4b8be73f77fe41d", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027448106557130814, + 0.0027243648655712605, + 0.0027103275060653687, + 0.0027234479784965515, + 0.0027595730498433113, + 0.0027272943407297134, + 0.002728298306465149, + 0.0027007071767002344, + 0.002684502862393856, + 0.002683063969016075, + 0.0027139110025018454, + 0.002666700165718794, + 0.002679243916645646, + 0.0026584581937640905, + 0.002684473292902112, + 0.0026407993864268064, + 0.002663099905475974, + 0.0026283918414264917, + 0.002597011858597398, + 0.0026434636674821377, + 0.0025708195753395557, + 0.002691218862310052, + 0.0026598076801747084, + 0.0026596153620630503, + 0.0026765079237520695, + 0.002582729561254382, + 0.002614464145153761, + 0.0025760054122656584, + 0.002634295728057623, + 0.00259007909335196, + 0.00257018837146461, + 0.0025643527042120695 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.000156725715896755, + "first_to_last_ratio": 3.4451295459637388, + "imbalance_abs_log_ratio": 1.236961507811942, + "last_quartile_mean": 4.549196592051885e-05, + "mean": 8.812961027615529e-05, + "normalized": [ + 2.4952457548653, + 2.231881013356295, + 1.939989240144028, + 1.7866526882708733, + 1.6079164676041713, + 1.4797016884434102, + 1.3811975174028512, + 1.3042540698565042, + 1.1954344722384815, + 1.1168849900912188, + 1.0692220969851183, + 1.0304138184362992, + 0.9676518223611118, + 0.9260610329319834, + 0.8738547948342114, + 0.8395441231817105, + 0.8221010666653766, + 0.7971164256793037, + 0.7591159941756684, + 0.7162115141810864, + 0.6791638220644127, + 0.6489084022271779, + 0.6150954832405342, + 0.5868306199881643, + 0.5699702288291514, + 0.552179306852942, + 0.5335984532948914, + 0.5198944860736878, + 0.5024723171712969, + 0.4942325662695092, + 0.4850276094601851, + 0.47217611282304317 + ], + "peak_layer": 1, + "peak_normalized": 2.4952457548653, + "population_cv": 0.5254777238950282, + "rest_mean": 9.433142622583546e-05, + "spike_contrast": 0.5792322488273548, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.463980414788239e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021990503591950983, + 0.00019669480388984084, + 0.00017097049567382783, + 0.00015745700511615723, + 0.00014170505164656788, + 0.0001304055331274867, + 0.00012172439892310649, + 0.00011494340287754312, + 0.00010535317414905876, + 9.843063890002668e-05, + 9.423012670595199e-05, + 9.080996824195608e-05, + 8.527877798769623e-05, + 8.16133979242295e-05, + 7.701248250668868e-05, + 7.398869638564065e-05, + 7.24514466128312e-05, + 7.024955993983895e-05, + 6.690059672109783e-05, + 6.311944162007421e-05, + 5.985444295220077e-05, + 5.718804459320381e-05, + 5.420812522061169e-05, + 5.1717153837671503e-05, + 5.023125413572416e-05, + 4.866334711550735e-05, + 4.7025823732838035e-05, + 4.581809844239615e-05, + 4.428268948686309e-05, + 4.3556523451115936e-05, + 4.274529419490136e-05, + 4.161269680480473e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002203928706876468, + "first_to_last_ratio": 1.7624395804596433, + "imbalance_abs_log_ratio": 0.5666989745264205, + "last_quartile_mean": 0.00012504988717410015, + "mean": 0.00016078790008577926, + "normalized": [ + 1.7301295632680096, + 1.5926765618359837, + 1.4498368649995454, + 1.3715838442653223, + 1.2853282969253275, + 1.2242662006350955, + 1.1759759999659367, + 1.1358472505674493, + 1.0864560712260232, + 1.0500344477950443, + 1.0256300088344572, + 1.0049332388641516, + 0.9775387355078093, + 0.9578140668049778, + 0.9358065303644342, + 0.9184543281568608, + 0.9072826301440372, + 0.8934865935730769, + 0.8779252804708734, + 0.8613603707803614, + 0.8467225584074481, + 0.8352602527072714, + 0.8224752340358507, + 0.8113194646910352, + 0.80159772798593, + 0.7940537841119606, + 0.785769247679408, + 0.7781743501685717, + 0.7724121545166266, + 0.7676777199663887, + 0.7637141967462265, + 0.7584564239985051 + ], + "peak_layer": 1, + "peak_normalized": 1.7301295632680096, + "population_cv": 0.2498913679417364, + "rest_mean": 0.0001660440253477578, + "spike_contrast": 0.7974079368034485, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013240482367109506, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027818389935418963, + 0.00025608311989344656, + 0.00023311622499022633, + 0.00022053408611100167, + 0.00020666523778345436, + 0.0001968471915461123, + 0.00018908271158579737, + 0.0001826304942369461, + 0.0001746889902278781, + 0.00016883283387869596, + 0.00016490889538545161, + 0.00016158110520336777, + 0.00015717640053480864, + 0.00015400491247419268, + 0.0001504663669038564, + 0.00014767634274903685, + 0.00014588006888516247, + 0.00014366183313541114, + 0.00014115976227913052, + 0.00013849632523488253, + 0.00013614274212159216, + 0.0001342997420579195, + 0.00013224406575318426, + 0.00013045035302639008, + 0.00012888721539638937, + 0.00012767424050252885, + 0.0001263421872863546, + 0.0001251210196642205, + 0.00012419452832546085, + 0.00012343328853603452, + 0.00012279600196052343, + 0.00012195061572128907 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19544659927487373, + "first_to_last_ratio": 1.7624395617194093, + "imbalance_abs_log_ratio": 0.5666989638932991, + "last_quartile_mean": 0.11089549027383327, + "mean": 0.14258831785991788, + "normalized": [ + 1.7301295593799786, + 1.5926765086027146, + 1.4498368632986096, + 1.3715838909156086, + 1.2853282557768668, + 1.224266111161914, + 1.1759760185820738, + 1.1358472270183295, + 1.0864560720464596, + 1.050034483149275, + 1.025630003735948, + 1.0049332495130952, + 0.9775386792530747, + 0.9578141330250067, + 0.9358065753109587, + 0.9184542876534186, + 0.9072827263675407, + 0.8934866319653527, + 0.8779253472761699, + 0.8613603464001749, + 0.8467225704769723, + 0.8352602246328348, + 0.8224752140046494, + 0.8113194329409806, + 0.8015977187598701, + 0.7940537797484172, + 0.7857692153409586, + 0.7781743825002613, + 0.7724121455314925, + 0.7676776608495898, + 0.7637142124554999, + 0.758456472325904 + ], + "peak_layer": 1, + "peak_normalized": 1.7301295593799786, + "population_cv": 0.24989136150327743, + "rest_mean": 0.14724950271624107, + "spike_contrast": 0.7974079196861148, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11741791963577271, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24669626355171204, + 0.2270970642566681, + 0.20672979950904846, + 0.19557183980941772, + 0.18327279388904572, + 0.17456604540348053, + 0.16768044233322144, + 0.16195854544639587, + 0.1549159437417984, + 0.1497226506471634, + 0.14624285697937012, + 0.14329174160957336, + 0.13938559591770172, + 0.13657310605049133, + 0.13343508541584015, + 0.1309608519077301, + 0.1293679177761078, + 0.12740075588226318, + 0.1251818984746933, + 0.1228199228644371, + 0.12073274701833725, + 0.11909835040569305, + 0.11727535724639893, + 0.11568467319011688, + 0.11429847031831741, + 0.11322279274463654, + 0.11204151064157486, + 0.11095857620239258, + 0.11013694852590561, + 0.10946186631917953, + 0.10889672487974167, + 0.108147032558918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.812671563238837e-05, + "first_to_last_ratio": 2.0429267595554257, + "imbalance_abs_log_ratio": 0.7143834657339574, + "last_quartile_mean": 3.334760549478233e-05, + "mean": 4.8136708642232406e-05, + "normalized": [ + 1.6706878479333491, + 1.5890573277727331, + 1.4709453763829752, + 1.4135957551070697, + 1.368779351628352, + 1.3072249852707734, + 1.2659085053079855, + 1.2360067186760548, + 1.1647282180273046, + 1.1006681301372137, + 1.0750864931045292, + 1.0685694254166505, + 1.0112110373273602, + 0.9945647481091522, + 0.9602230239537092, + 0.9352227929155844, + 0.93139607887314, + 0.9092220870879134, + 0.8814974932144994, + 0.8653605860843365, + 0.8506124631158822, + 0.8320765477333729, + 0.7932580049774197, + 0.7619473309737773, + 0.7566723544753674, + 0.7419402536141336, + 0.7141691805145849, + 0.7010673290630277, + 0.6716446941281601, + 0.6659218543873533, + 0.6557479674543484, + 0.6349860372318865 + ], + "peak_layer": 1, + "peak_normalized": 1.6706878479333491, + "population_cv": 0.2830124551063321, + "rest_mean": 4.992923641111702e-05, + "spike_contrast": 0.7702312603701833, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.845705869025551e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.04214141680859e-05, + 7.649198960280046e-05, + 7.080646901158616e-05, + 6.804584700148553e-05, + 6.588853284483775e-05, + 6.292550824582577e-05, + 6.093666888773441e-05, + 5.949729529675096e-05, + 5.6066182878566906e-05, + 5.298254109220579e-05, + 5.1751125283772126e-05, + 5.143741509527899e-05, + 4.867637107963674e-05, + 4.7875073505565524e-05, + 4.622197593562305e-05, + 4.501854709815234e-05, + 4.483434167923406e-05, + 4.376695869723335e-05, + 4.24323879997246e-05, + 4.165561040281318e-05, + 4.094568430446088e-05, + 4.0053426346275955e-05, + 3.8184829463716596e-05, + 3.667763667181134e-05, + 3.642371666501276e-05, + 3.571456181816757e-05, + 3.4377753763692454e-05, + 3.3747073757695034e-05, + 3.2330764952348545e-05, + 3.205528628313914e-05, + 3.156554885208607e-05, + 3.056613786611706e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018337754227104597, + "first_to_last_ratio": 1.4712162101549686, + "imbalance_abs_log_ratio": 0.38608941256902984, + "last_quartile_mean": 0.00012464350311347516, + "mean": 0.00014952537844692415, + "normalized": [ + 1.3620834570997993, + 1.3194965005671844, + 1.2677944868152182, + 1.233561928872588, + 1.1984447257225705, + 1.1684695593357564, + 1.141528073961558, + 1.1198008373135866, + 1.0910012077210556, + 1.066096352737946, + 1.0498630645220353, + 1.0350823851472097, + 1.0145919973375974, + 1.0013407128400098, + 0.9836056688270479, + 0.9685428567287663, + 0.9591027483551056, + 0.9465833155582388, + 0.9334001555540693, + 0.9191584382428863, + 0.9057431683575706, + 0.8945627711391904, + 0.8815040843165663, + 0.8698871064160805, + 0.8596264871795505, + 0.8514885298372257, + 0.8425068993042694, + 0.8341757603627517, + 0.8278236379550317, + 0.8225948884437775, + 0.8181940461689637, + 0.812344147258793 + ], + "peak_layer": 1, + "peak_normalized": 1.3620834570997993, + "population_cv": 0.15505530299928885, + "rest_mean": 0.00015278545470209792, + "spike_contrast": 0.8634393039979241, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001319209666689858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00020366604439914227, + 0.0001972982136067003, + 0.0001895674504339695, + 0.00018444881425239146, + 0.00017919790116138756, + 0.00017471585306338966, + 0.0001706874172668904, + 0.0001674386439844966, + 0.00016313236847054213, + 0.00015940846060402691, + 0.00015698117204010487, + 0.00015477108536288142, + 0.00015170725237112492, + 0.00014972584904171526, + 0.0001470740098739043, + 0.0001448217371944338, + 0.00014341020141728222, + 0.00014153822849038988, + 0.00013956701150164008, + 0.00013743751333095133, + 0.00013543159002438188, + 0.00013375983689911664, + 0.00013180723180994391, + 0.00013007019879296422, + 0.00012853597581852227, + 0.00012731914466712624, + 0.0001259761629626155, + 0.00012473044625949115, + 0.00012378064275253564, + 0.00012299881200306118, + 0.00012234077439643443, + 0.00012146606604801491 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015816818631719798, + "first_to_last_ratio": 1.2671705792413261, + "imbalance_abs_log_ratio": 0.23678652466939742, + "last_quartile_mean": 0.00012481996418500785, + "mean": 0.0001404537983944465, + "normalized": [ + 1.172032188999453, + 1.1600674087975105, + 1.1434054243316032, + 1.1305758413418425, + 1.1202630655069494, + 1.1065457827105485, + 1.0931998253234612, + 1.0828906757132684, + 1.0683311763286112, + 1.0526488941703125, + 1.0427563459749776, + 1.0343706495522589, + 1.0219941374371992, + 1.0154952175292713, + 1.0038357654470964, + 0.9937321709742023, + 0.9862865992066487, + 0.9768881500283385, + 0.968261257859838, + 0.9607910277643221, + 0.9520546236098718, + 0.9457805296333641, + 0.9349892993508679, + 0.9232794945485685, + 0.9140411172891727, + 0.9051057888083044, + 0.8959794100889387, + 0.888100763485746, + 0.8822517666505367, + 0.8784085901067091, + 0.87543446421392, + 0.8702025472162866 + ], + "peak_layer": 1, + "peak_normalized": 1.172032188999453, + "population_cv": 0.09096684083762872, + "rest_mean": 0.00014216970130628734, + "spike_contrast": 0.9227558436510851, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013118792267050595, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016461637278553098, + 0.0001629358739592135, + 0.00016059563495218754, + 0.00015879367128945887, + 0.0001573452027514577, + 0.00015541855827905238, + 0.00015354406787082553, + 0.00015209610864985734, + 0.00015005117165856063, + 0.0001478485355619341, + 0.0001464590895920992, + 0.00014528128667734563, + 0.00014354295853991061, + 0.00014263016055338085, + 0.00014099254622124135, + 0.00013957345800008625, + 0.0001385276991641149, + 0.00013720765127800405, + 0.00013599597150459886, + 0.00013494674931280315, + 0.00013371968816500157, + 0.00013283846783451736, + 0.00013132279855199158, + 0.0001296781119890511, + 0.0001283805468119681, + 0.00012712554598692805, + 0.0001258437114302069, + 0.00012473712558858097, + 0.00012391561176627874, + 0.0001233758230227977, + 0.0001229580957442522, + 0.00012222325312905014 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0003110146244580392, + "first_to_last_ratio": 2.2098355110548322, + "imbalance_abs_log_ratio": 0.7929180833727034, + "last_quartile_mean": 0.00014074107457417995, + "mean": 0.00020577548139044666, + "normalized": [ + 1.8085529821987474, + 1.7406694730308092, + 1.619926374791616, + 1.5420513399817848, + 1.4477144166533082, + 1.3679714513872108, + 1.309931951281633, + 1.254598241859695, + 1.1926238819658395, + 1.119002359728262, + 1.082854716042971, + 1.0468411526236405, + 0.9850613364068194, + 0.9618428208148979, + 0.9182052809224994, + 0.877960480924469, + 0.8608981306240351, + 0.8408442911829569, + 0.8148282643879068, + 0.7879356949258838, + 0.7604349554811285, + 0.7438321978222967, + 0.727404890131739, + 0.7163770004088065, + 0.7034258791916428, + 0.6971552930824737, + 0.6907054382632783, + 0.684504508832364, + 0.6799757639842041, + 0.6759922505973077, + 0.6720742215597147, + 0.6678029589100581 + ], + "peak_layer": 1, + "peak_normalized": 1.8085529821987474, + "population_cv": 0.3383164219384334, + "rest_mean": 0.00021605301460820355, + "spike_contrast": 0.695555219569954, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001502768020145595, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00037215586053207517, + 0.0003581870987545699, + 0.00033334112958982587, + 0.0003173163568135351, + 0.00029790413100272417, + 0.0002814949839375913, + 0.00026955187786370516, + 0.0002581655571702868, + 0.00024541275342926383, + 0.00023026324925012887, + 0.00022282495046965778, + 0.00021541424212045968, + 0.00020270147069822997, + 0.00019792366947513074, + 0.00018894413369707763, + 0.00018066274060402066, + 0.00017715172725729644, + 0.00017302513879258186, + 0.00016767167835496366, + 0.00016213784692808986, + 0.0001564788690302521, + 0.00015306242858059704, + 0.00014968209143262357, + 0.00014741282211616635, + 0.00014474779891315848, + 0.00014345746603794396, + 0.00014213024405762553, + 0.00014085424481891096, + 0.00013992234016768634, + 0.00013910263078287244, + 0.00013829639647156, + 0.00013741747534368187 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b86d0e064c62f21a6ce3be8d010c5b21fe8aec7ea511fe91741a6c488e69da6d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0021227982360869646, + 0.00413061399012804, + 0.005416445434093475, + 0.006388718727976084, + 0.0023577234242111444, + 0.004388594534248114, + 0.00574506726115942, + 0.007066869642585516, + 0.0029547461308538914, + 0.004991721361875534, + 0.006190058775246143, + 0.0074937604367733, + 0.00348988501355052, + 0.006057556718587875, + 0.0074848332442343235, + 0.00897275935858488, + 0.004072706215083599, + 0.0063449000008404255, + 0.007735534105449915, + 0.00946879480034113, + 0.004824656527489424, + 0.007074959110468626, + 0.009095527231693268, + 0.010346688330173492, + 0.004843301139771938, + 0.007334613241255283, + 0.009146629832684994, + 0.010587968863546848, + 0.0054801348596811295, + 0.0077637480571866035, + 0.009536821395158768, + 0.011367839761078358 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015727686513855588, + "first_to_last_ratio": 3.444355141493678, + "imbalance_abs_log_ratio": 1.2367367001496994, + "last_quartile_mean": 4.5662209231522866e-05, + "mean": 8.846079299473786e-05, + "normalized": [ + 2.4938513501811554, + 2.232208910931446, + 1.9412909982664082, + 1.7875424925486438, + 1.6051944492471113, + 1.4809426711898561, + 1.3782256757419, + 1.304162421055249, + 1.1983748443875557, + 1.116540877408076, + 1.0705280549740306, + 1.028906183424365, + 0.9692497056236429, + 0.9297086109864628, + 0.876031508599092, + 0.840542245421429, + 0.8215073067800788, + 0.7958492991481336, + 0.7578128978794811, + 0.7149268338582802, + 0.6789684541053794, + 0.6481555926886391, + 0.6127683614407704, + 0.5872234895762031, + 0.5712935129257742, + 0.5525148673059793, + 0.5351006365732414, + 0.5196040797164857, + 0.49974182292452146, + 0.4940518464188538, + 0.4845470025727643, + 0.4726329960989905 + ], + "peak_layer": 1, + "peak_normalized": 2.4938513501811554, + "population_cv": 0.525487396006855, + "rest_mean": 9.469102304946218e-05, + "spike_contrast": 0.5789096889426617, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.481755069922656e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002206080680480227, + 0.00019746297039091587, + 0.00017172814114019275, + 0.00015812742640264332, + 0.00014199677389115095, + 0.0001310053630732, + 0.00012191893620183691, + 0.00011536724196048453, + 0.00010600918903946877, + 9.877009142655879e-05, + 9.469976066611707e-05, + 9.101785690290853e-05, + 8.574059756938368e-05, + 8.224276098189875e-05, + 7.74944419390522e-05, + 7.435503357555717e-05, + 7.267118780873716e-05, + 7.040146010695025e-05, + 6.70367298880592e-05, + 6.324299465632066e-05, + 6.0062087868573144e-05, + 5.733635771321133e-05, + 5.42059751751367e-05, + 5.1946255553048104e-05, + 5.0537077186163515e-05, + 4.88759033032693e-05, + 4.733542664325796e-05, + 4.596458893502131e-05, + 4.4207557948539034e-05, + 4.3704218114726245e-05, + 4.2863412090810016e-05, + 4.1809489630395547e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0002221399099653354, + "first_to_last_ratio": 1.7678964788399845, + "imbalance_abs_log_ratio": 0.5697904098088594, + "last_quartile_mean": 0.00012565210272441618, + "mean": 0.00016188397967198398, + "normalized": [ + 1.7325277794978797, + 1.5951376332003655, + 1.4520548687114199, + 1.3729981481664384, + 1.2857750985910605, + 1.2257666436669268, + 1.1762283345939986, + 1.1372455678364286, + 1.0889526987749056, + 1.0509403055231545, + 1.0259172538151122, + 1.0049179037159137, + 0.978085345877984, + 0.9585059142501069, + 0.9364886382982085, + 0.9194669656082258, + 0.9066971379228966, + 0.893622399915035, + 0.8769613699768213, + 0.8605074489369314, + 0.8461051105300481, + 0.8336447776090111, + 0.8211750960224249, + 0.8107883690330443, + 0.8025267555268564, + 0.7926031470462279, + 0.7851370690218832, + 0.7775583575558719, + 0.7689758331264998, + 0.7655660871494063, + 0.7607136802742225, + 0.75640826022469 + ], + "peak_layer": 1, + "peak_normalized": 1.7325277794978797, + "population_cv": 0.2510639592948361, + "rest_mean": 0.00016719473286583606, + "spike_contrast": 0.796711177092359, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013320591242518275, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002804684918373823, + 0.0002582272281870246, + 0.00023506442084908485, + 0.00022226640430744737, + 0.00020814638992305845, + 0.00019843198242597282, + 0.00019041252380702645, + 0.00018410183838568628, + 0.00017628399655222893, + 0.00017013039905577898, + 0.00016607956786174327, + 0.00016268010949715972, + 0.00015833634824957699, + 0.00015516675193794072, + 0.00015160250768531114, + 0.00014884697156958282, + 0.00014677974104415625, + 0.00014466315042227507, + 0.00014196599659044296, + 0.000139302370371297, + 0.00013697086251340806, + 0.00013495373423211277, + 0.00013293509255163372, + 0.00013125364785082638, + 0.00012991622497793287, + 0.00012830975174438208, + 0.00012710111332125962, + 0.000125874241348356, + 0.00012448486813809723, + 0.00012393288488965482, + 0.00012314735795371234, + 0.00012245037942193449 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19699589163064957, + "first_to_last_ratio": 1.7678964443273588, + "imbalance_abs_log_ratio": 0.5697903902870003, + "last_quartile_mean": 0.11142954230308533, + "mean": 0.1435603315476328, + "normalized": [ + 1.7325277292733527, + 1.595137596567507, + 1.4520549137362593, + 1.3729980837787108, + 1.285775112498371, + 1.2257666266551688, + 1.1762283724960378, + 1.1372455675264566, + 1.0889526722354506, + 1.0509403711345948, + 1.0259172522938773, + 1.00491793591038, + 0.9780853203912144, + 0.9585059497750074, + 0.9364885904027979, + 0.9194669905174069, + 0.9066971317518798, + 0.8936224205946474, + 0.8769613084029552, + 0.8605074273314671, + 0.84610509802098, + 0.833644762521393, + 0.821175085273337, + 0.8107884103394608, + 0.802526775587875, + 0.7926031918850489, + 0.7851370589142022, + 0.7775583578743038, + 0.7689758340628073, + 0.7655660958716172, + 0.7607136803251955, + 0.7564082760502372 + ], + "peak_layer": 1, + "peak_normalized": 1.7325277292733527, + "population_cv": 0.2510639513231401, + "rest_mean": 0.14826996044980156, + "spike_contrast": 0.7967111822081809, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11812833547592164, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24872225522994995, + 0.22899848222732544, + 0.20845748484134674, + 0.19710806012153625, + 0.184586301445961, + 0.17597146332263947, + 0.16885973513126373, + 0.16326335072517395, + 0.156330406665802, + 0.1508733481168747, + 0.14728102087974548, + 0.14426635205745697, + 0.1404142528772354, + 0.13760343194007874, + 0.13444261252880096, + 0.13199898600578308, + 0.13016574084758759, + 0.12828873097896576, + 0.1258968561887741, + 0.12353473156690598, + 0.12146712839603424, + 0.1196783185005188, + 0.11788816750049591, + 0.11639705300331116, + 0.11521100997924805, + 0.11378637701272964, + 0.11271453648805618, + 0.111626535654068, + 0.1103944256901741, + 0.10990492254495621, + 0.10920830816030502, + 0.10859022289514542 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.813338586653117e-05, + "first_to_last_ratio": 2.039661655563216, + "imbalance_abs_log_ratio": 0.7127839389845736, + "last_quartile_mean": 3.3404258829250466e-05, + "mean": 4.817700425974181e-05, + "normalized": [ + 1.6677237328958603, + 1.5880861584886787, + 1.4689677920146058, + 1.4140322553861728, + 1.3674596064698026, + 1.3072824267629832, + 1.2642946708581568, + 1.2359968662296865, + 1.1645552208108763, + 1.101791124796405, + 1.0797798333773554, + 1.0672664620429624, + 1.012803710836977, + 0.9955853656459409, + 0.9623936542224847, + 0.9380946280712082, + 0.9310631059210642, + 0.9082494158597965, + 0.8812030087749774, + 0.8644086677601183, + 0.8497891698936241, + 0.8309347651333899, + 0.7889921814106587, + 0.7623244708429415, + 0.7584228017816982, + 0.7445258855646851, + 0.7180452956289476, + 0.7000407870401953, + 0.6663749309645193, + 0.6665299586713209, + 0.6558653663654831, + 0.637116679476424 + ], + "peak_layer": 1, + "peak_normalized": 1.6677237328958603, + "population_cv": 0.28258618188430573, + "rest_mean": 4.9978354245131074e-05, + "spike_contrast": 0.7693273401931907, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.844971433863975e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.034593338379636e-05, + 7.650923362234607e-05, + 7.077046757331118e-05, + 6.812383799115196e-05, + 6.588010728592053e-05, + 6.298095104284585e-05, + 6.090992974350229e-05, + 5.9546626289375126e-05, + 5.610478183371015e-05, + 5.3080995712662116e-05, + 5.2020557632204145e-05, + 5.141770088812336e-05, + 4.879384869127534e-05, + 4.79643204016611e-05, + 4.636524317902513e-05, + 4.519458889262751e-05, + 4.485583122004755e-05, + 4.375673597678542e-05, + 4.245372110744938e-05, + 4.164462006883696e-05, + 4.094029645784758e-05, + 4.0031947719398886e-05, + 3.8011279684724286e-05, + 3.672650927910581e-05, + 3.653853855212219e-05, + 3.5869026760337874e-05, + 3.4593271266203374e-05, + 3.3725867979228497e-05, + 3.21039478876628e-05, + 3.211141665815376e-05, + 3.1597628549207e-05, + 3.0694372981088236e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018498812278266996, + "first_to_last_ratio": 1.477115588497286, + "imbalance_abs_log_ratio": 0.3900912594562188, + "last_quartile_mean": 0.00012523605073511135, + "mean": 0.00015057817927299766, + "normalized": [ + 1.364889512785021, + 1.3222437098497068, + 1.2702392636002102, + 1.235819093505974, + 1.2002761568067795, + 1.1699209662016437, + 1.143160310272995, + 1.1216013166366416, + 1.0931712967611817, + 1.0676161315047876, + 1.0503356923443616, + 1.0352076251778792, + 1.014929888181958, + 1.0014649033450271, + 0.9840777733314531, + 0.96959004456439, + 0.958410698647572, + 0.9467186761221033, + 0.9322053376846741, + 0.9180751779000818, + 0.904837007766356, + 0.8925970349353171, + 0.879899470446981, + 0.8691033998232655, + 0.8603564893899045, + 0.8496764837250937, + 0.8415817018035472, + 0.8332582690137502, + 0.8239257185712614, + 0.820107268399535, + 0.8147646079598274, + 0.8099389729407196 + ], + "peak_layer": 1, + "peak_normalized": 1.364889512785021, + "population_cv": 0.1566386619724503, + "rest_mean": 0.000153886474436149, + "spike_contrast": 0.8624109810706347, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013271338539198042, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002055225777439773, + 0.00019910105038434267, + 0.000191270315553993, + 0.00018608738901093602, + 0.0001807353983167559, + 0.00017616456898394972, + 0.00017213499813806266, + 0.00016888868412934244, + 0.00016460774349980056, + 0.00016075969324447215, + 0.0001581576361786574, + 0.00015587967936880887, + 0.00015282629465218633, + 0.00015079876175150275, + 0.0001481806393712759, + 0.00014599910355173051, + 0.00014431573799811304, + 0.00014255517453420907, + 0.00014036978245712817, + 0.00013824208872392774, + 0.00013624870916828513, + 0.00013440563634503633, + 0.0001324936602031812, + 0.00013086800754535943, + 0.00012955091369803995, + 0.00012794273789040744, + 0.00012672384036704898, + 0.0001254705130122602, + 0.0001240652345586568, + 0.00012349025928415358, + 0.00012268577120266855, + 0.00012195913586765528 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015959453048708383, + "first_to_last_ratio": 1.2728956423420892, + "imbalance_abs_log_ratio": 0.24129433847892473, + "last_quartile_mean": 0.00012537911607068963, + "mean": 0.00014144417764327955, + "normalized": [ + 1.1740787416792278, + 1.1612764386268224, + 1.1471117856731012, + 1.1325725430810556, + 1.1221203501929577, + 1.1091452070987, + 1.0952451640185767, + 1.0850231158766837, + 1.0710066115302646, + 1.0558964823952193, + 1.0428503514266292, + 1.0346153464801762, + 1.0229188081717178, + 1.0151986196356395, + 1.003700332970425, + 0.9946300373693288, + 0.9863513080075261, + 0.9777764871930738, + 0.967654233599155, + 0.9595434059922948, + 0.9513191006674802, + 0.9429489101168639, + 0.9334504238882736, + 0.9221966853060849, + 0.914156536930625, + 0.9038869577772225, + 0.8952163547729103, + 0.8871546013922188, + 0.8776639341179028, + 0.8750860454698683, + 0.8708633986378481, + 0.8673416799041263 + ], + "peak_layer": 1, + "peak_normalized": 1.1740787416792278, + "population_cv": 0.09275162400716915, + "rest_mean": 0.00014320399646681767, + "spike_contrast": 0.9213510743517984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001319411559961736, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001660666021052748, + 0.00016425579087808728, + 0.00016225228318944573, + 0.0001601957919774577, + 0.00015871739014983177, + 0.0001568821317050606, + 0.0001549160515423864, + 0.00015347020234912634, + 0.00015148764941841364, + 0.0001493504096288234, + 0.00014750511036254466, + 0.00014634031686000526, + 0.00014468590961769223, + 0.00014359393389895558, + 0.00014196756819728762, + 0.0001406846276950091, + 0.00013951364962849766, + 0.00013830079114995897, + 0.0001368690573144704, + 0.00013572182797361165, + 0.000134558547870256, + 0.00013337463315110654, + 0.00013203112757764757, + 0.00013043935177847743, + 0.00012930211960338056, + 0.00012784954742528498, + 0.00012662314111366868, + 0.00012548285303637385, + 0.00012414045340847224, + 0.00012377582606859505, + 0.00012317855725996196, + 0.00012268043064977974 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031298474641516805, + "first_to_last_ratio": 2.209030513243648, + "imbalance_abs_log_ratio": 0.7925537374904509, + "last_quartile_mean": 0.00014168421148497146, + "mean": 0.0002071673161481158, + "normalized": [ + 1.8097158340785349, + 1.7497804411308835, + 1.611244862997228, + 1.5435020349400896, + 1.4351553463811126, + 1.3679605489917586, + 1.3130823229189843, + 1.255818245825708, + 1.1871164694634437, + 1.122716192977336, + 1.0885067728511792, + 1.0444622322416663, + 0.990152969051365, + 0.9605806666596922, + 0.9222549756846732, + 0.8800158737132093, + 0.8615285136033911, + 0.8400222785845712, + 0.8115627552032852, + 0.7845983398409414, + 0.7619432914958448, + 0.7435905859348785, + 0.7280762324918081, + 0.7153162915808863, + 0.7064441934578637, + 0.6963930075638303, + 0.6905250334602637, + 0.686293775915277, + 0.6782364888338203, + 0.6751963304984743, + 0.6708841537879804, + 0.6673229378400193 + ], + "peak_layer": 1, + "peak_normalized": 1.8097158340785349, + "population_cv": 0.3382701486201724, + "rest_mean": 0.00021748447411314205, + "spike_contrast": 0.6963929896815652, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015145466313697397, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003749139723367989, + 0.00036249731783755124, + 0.00033379727392457426, + 0.0003197631740476936, + 0.0002973172813653946, + 0.00028339671553112566, + 0.0002720277407206595, + 0.0002601644955575466, + 0.0002459317329339683, + 0.00023259010049514472, + 0.0002255030267406255, + 0.00021637843747157604, + 0.00020512733317445964, + 0.00019900091865565628, + 0.00019106108811683953, + 0.00018231052672490478, + 0.00017848054994829, + 0.00017402516095899045, + 0.00016812927788123488, + 0.0001625431323191151, + 0.00015784974675625563, + 0.00015404766600113362, + 0.00015083359903655946, + 0.00014819015632383525, + 0.00014635214756708592, + 0.00014426987036131322, + 0.00014305421791505069, + 0.00014217763964552432, + 0.00014050843310542405, + 0.0001398786116624251, + 0.00013898526958655566, + 0.0001382475020363927 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4c7ce131fe83b6fe58fa4ddc70f94a155d25a4ef1c2e6acbb1fd86c6ed0e665d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034565660171210766, + 0.004928664769977331, + 0.0060537648387253284, + 0.006960673723369837, + 0.0036145171616226435, + 0.0051584807224571705, + 0.006370538379997015, + 0.007546419743448496, + 0.00395671371370554, + 0.005665866658091545, + 0.006755253300070763, + 0.007946331985294819, + 0.004383862018585205, + 0.006652849726378918, + 0.007932198233902454, + 0.009347881190478802, + 0.004880861379206181, + 0.006906712893396616, + 0.008132148534059525, + 0.009816600009799004, + 0.005453306715935469, + 0.007622495759278536, + 0.009448722936213017, + 0.010670131072402, + 0.0055701653473079205, + 0.007738742511719465, + 0.009509840048849583, + 0.010865328833460808, + 0.005987230688333511, + 0.00815181527286768, + 0.009874933399260044, + 0.011666664853692055 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015672678182454547, + "first_to_last_ratio": 3.4451487067191406, + "imbalance_abs_log_ratio": 1.236967069490208, + "last_quartile_mean": 4.549202230919036e-05, + "mean": 8.812991916329338e-05, + "normalized": [ + 2.495342024881723, + 2.231858330120814, + 1.9399329049931513, + 1.7866550124027707, + 1.6079479837593746, + 1.479673880937976, + 1.3811985381554308, + 1.3042766606266647, + 1.1954508396552082, + 1.1168862767589318, + 1.0692251193363627, + 1.0303878333212484, + 0.9676417435164101, + 0.9260651349655372, + 0.8738635380560305, + 0.8395483633300559, + 0.8220975248023003, + 0.7970985234813915, + 0.759112342831592, + 0.7162079306537569, + 0.6791607399033079, + 0.6489075313744581, + 0.615087961024076, + 0.5868315353418163, + 0.5699679008989185, + 0.5521854623420828, + 0.5335941062986479, + 0.5198929115704046, + 0.5024718770039331, + 0.49422901772262035, + 0.4850259920431499, + 0.47217445788985424 + ], + "peak_layer": 1, + "peak_normalized": 2.495342024881723, + "population_cv": 0.5254832958744828, + "rest_mean": 9.433179891655325e-05, + "spike_contrast": 0.5792295824234721, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.4639768495690075e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021991429093759507, + 0.0001966934942174703, + 0.00017096613009925932, + 0.0001574577618157491, + 0.00014170832582749426, + 0.00013040353951510042, + 0.0001217249155160971, + 0.00011494579666759819, + 0.00010535498586250469, + 9.843109728535637e-05, + 9.423072333447635e-05, + 9.080799645744264e-05, + 8.527818863512948e-05, + 8.161404548445717e-05, + 7.701352296862751e-05, + 7.398932939395308e-05, + 7.245138840517029e-05, + 7.024822843959555e-05, + 6.690050940960646e-05, + 6.311934703262523e-05, + 5.985438110656105e-05, + 5.7188168284483254e-05, + 5.420765228336677e-05, + 5.1717415772145614e-05, + 5.02312250318937e-05, + 4.866406015935354e-05, + 4.702560545410961e-05, + 4.581812027026899e-05, + 4.428280590218492e-05, + 4.3556363380048424e-05, + 4.2745301470858976e-05, + 4.161269680480473e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00022039370378479362, + "first_to_last_ratio": 1.7624457042066763, + "imbalance_abs_log_ratio": 0.5667024491058967, + "last_quartile_mean": 0.00012504992537287762, + "mean": 0.0001607882525149762, + "normalized": [ + 1.7301666786506156, + 1.592657504255016, + 1.4498087081325215, + 1.3715848200709229, + 1.2853506396348091, + 1.224257634451794, + 1.175979576607571, + 1.1358564358860717, + 1.0864628307036615, + 1.050035766385231, + 1.0256357250890054, + 1.0049217142965674, + 0.9775345112735686, + 0.9578189361646354, + 0.9358080993289649, + 0.9184597363064851, + 0.9072850761620224, + 0.8934833681037547, + 0.8779253472396409, + 0.8613572157279928, + 0.8467159963027754, + 0.8352601414820391, + 0.8224708066612338, + 0.81131886291965, + 0.8015955184587156, + 0.7940537632065271, + 0.7857694259399116, + 0.7781749070905981, + 0.7724085609032495, + 0.7676732316943716, + 0.7637136993200662, + 0.75845476155001 + ], + "peak_layer": 1, + "peak_normalized": 1.7301666786506156, + "population_cv": 0.2498932502086342, + "rest_mean": 0.0001660444721454513, + "spike_contrast": 0.7974048446155259, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001324046665104106, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027819047681987286, + 0.0002560806169640273, + 0.0002331122086616233, + 0.00022053472639527172, + 0.00020666928321588784, + 0.00019684624567162246, + 0.0001890837011160329, + 0.00018263237143401057, + 0.00017469045997131616, + 0.0001688334159553051, + 0.0001649101759539917, + 0.00016157960635609925, + 0.00015717606584075838, + 0.00015400603297166526, + 0.00015046694898046553, + 0.00014767753600608557, + 0.00014588078192900866, + 0.00014366162940859795, + 0.00014116008242126554, + 0.00013849612150806934, + 0.0001361419854220003, + 0.00013430001854430884, + 0.00013224364374764264, + 0.00013045054220128804, + 0.00012888714263681322, + 0.00012767451698891819, + 0.0001263424928765744, + 0.00012512138346210122, + 0.00012419422273524106, + 0.00012343283742666245, + 0.0001227961911354214, + 0.00012195061572128907 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19544734060764313, + "first_to_last_ratio": 1.7624457434404193, + "imbalance_abs_log_ratio": 0.5667024713668619, + "last_quartile_mean": 0.11089552193880081, + "mean": 0.14258862822316587, + "normalized": [ + 1.7301667593069614, + 1.5926575752547714, + 1.4498087309457477, + 1.3715847721481056, + 1.2853506436856033, + 1.2242575941276408, + 1.1759796246815233, + 1.1358564592049036, + 1.0864627991256137, + 1.0500357507634608, + 1.0256357136610779, + 1.0049216567312906, + 0.9775345659226533, + 0.957818945514909, + 0.935808091557486, + 0.9184597083380731, + 0.9072850362319335, + 0.8934833286128027, + 0.8779253174360568, + 0.8613572174793438, + 0.8467160247668938, + 0.8352601831545204, + 0.8224708111632975, + 0.8113188687979516, + 0.8015955037045681, + 0.7940537757091718, + 0.7857693860910103, + 0.7781748832951502, + 0.7724085309387246, + 0.7676732205256189, + 0.7637136996793319, + 0.7584548214438025 + ], + "peak_layer": 1, + "peak_normalized": 1.7301667593069614, + "population_cv": 0.24989326289316924, + "rest_mean": 0.14724989594132812, + "spike_contrast": 0.797404859232464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.11741778254508972, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.24670210480690002, + 0.2270948588848114, + 0.20672623813152313, + 0.1955723911523819, + 0.18327638506889343, + 0.17456521093845367, + 0.16768132150173187, + 0.16196021437644958, + 0.15491724014282227, + 0.14972315728664398, + 0.14624398946762085, + 0.14329040050506592, + 0.13938531279563904, + 0.13657408952713013, + 0.13343559205532074, + 0.13096190989017487, + 0.12936852872371674, + 0.12740056216716766, + 0.1251821666955948, + 0.12281974405050278, + 0.12073207646608353, + 0.11909860372543335, + 0.11727498471736908, + 0.11568484455347061, + 0.11429840326309204, + 0.11322303861379623, + 0.11204177886247635, + 0.11095888912677765, + 0.11013667285442352, + 0.1094614714384079, + 0.1088968887925148, + 0.108147032558918 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.812703486502869e-05, + "first_to_last_ratio": 2.0429306771418236, + "imbalance_abs_log_ratio": 0.7143853833664048, + "last_quartile_mean": 3.334769780849456e-05, + "mean": 4.813682039639389e-05, + "normalized": [ + 1.670694398734861, + 1.5890602892970849, + 1.4709330435001327, + 1.4135870318536654, + 1.3688033811669116, + 1.3072210435152478, + 1.2659107055367187, + 1.236022743120565, + 1.164719165633699, + 1.10067041168383, + 1.075093368589445, + 1.0685491843186352, + 1.0112041551575939, + 0.9945774787180313, + 0.9602353052595208, + 0.9352302198346567, + 0.9313930096398116, + 0.9092161974531191, + 0.8814894006709005, + 0.8653525310040872, + 0.8506115464001769, + 0.8320776390213925, + 0.7932455071661803, + 0.7619500965894176, + 0.7566697664554205, + 0.7419550822306056, + 0.7141621566216388, + 0.7010675908642553, + 0.6716466869044494, + 0.66592144202409, + 0.6557448579818498, + 0.6349845630520059 + ], + "peak_layer": 1, + "peak_normalized": 1.670694398734861, + "population_cv": 0.28301355753944435, + "rest_mean": 4.992937398061191e-05, + "spike_contrast": 0.7702285844110489, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.845703104161657e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.042191620916128e-05, + 7.649230974493548e-05, + 7.080603973008692e-05, + 6.804558506701142e-05, + 6.588984251720831e-05, + 6.292546459008008e-05, + 6.0936916270293295e-05, + 5.9498204791452736e-05, + 5.606587728834711e-05, + 5.2982773922849447e-05, + 5.1751576393144205e-05, + 5.143656017025933e-05, + 4.867615280090831e-05, + 4.7875797463348135e-05, + 4.622267442755401e-05, + 4.501900912146084e-05, + 4.483429802348837e-05, + 4.3766776798292994e-05, + 4.243209696142003e-05, + 4.1655319364508614e-05, + 4.094573523616418e-05, + 4.005357186542824e-05, + 3.8184316508704796e-05, + 3.667785495053977e-05, + 3.6423676647245884e-05, + 3.571535853552632e-05, + 3.437749546719715e-05, + 3.374716470716521e-05, + 3.233093593735248e-05, + 3.2055340852821246e-05, + 3.156547245453112e-05, + 3.056613786611706e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00018337804431212135, + "first_to_last_ratio": 1.471219679743085, + "imbalance_abs_log_ratio": 0.38609177087911456, + "last_quartile_mean": 0.00012464355040719965, + "mean": 0.0001495256622092711, + "normalized": [ + 1.3620802882826015, + 1.3194938018514704, + 1.2677903290886852, + 1.23356455122679, + 1.198458119977671, + 1.1684656874236388, + 1.1415318441782234, + 1.1198031889553155, + 1.0910031274146845, + 1.0660983196947673, + 1.0498697336705507, + 1.0350765279986567, + 1.0145895852928877, + 1.0013439705330982, + 0.9836063325227735, + 0.968549777523238, + 0.9591067674489386, + 0.9465820057822166, + 0.9334010118478351, + 0.9191551367815048, + 0.9057362914957724, + 0.8945632145340071, + 0.8814992971854174, + 0.8698868180749819, + 0.8596244665411801, + 0.8514889576556698, + 0.8425072468470839, + 0.8341767076391075, + 0.8278204125037605, + 0.8225904077486038, + 0.8181934666462969, + 0.8123426056325708 + ], + "peak_layer": 1, + "peak_normalized": 1.3620802882826015, + "population_cv": 0.1550560200772002, + "rest_mean": 0.00015278581957789829, + "spike_contrast": 0.8634362323882038, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013192081241868435, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002036659570876509, + 0.00019729818450286984, + 0.0001895671884994954, + 0.0001844495564000681, + 0.00017920024401973933, + 0.00017471560568083078, + 0.00017068830493371934, + 0.0001674393133725971, + 0.0001631329650990665, + 0.00015940905723255128, + 0.0001569824671605602, + 0.0001547705032862723, + 0.00015170717961154878, + 0.00014972662029322237, + 0.00014707438822370023, + 0.00014482304686680436, + 0.00014341107453219593, + 0.00014153830124996603, + 0.00013956740440335125, + 0.00013743728050030768, + 0.00013543081877287477, + 0.00013376015704125166, + 0.0001318067661486566, + 0.00013007040251977742, + 0.00012853591761086136, + 0.00012731945025734603, + 0.00012597645400092006, + 0.00012473082460928708, + 0.00012378039536997676, + 0.00012299837544560432, + 0.0001223409199155867, + 0.00012146606604801491 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00015817073654034175, + "first_to_last_ratio": 1.2671327323753674, + "imbalance_abs_log_ratio": 0.23675665700060805, + "last_quartile_mean": 0.00012482570491556544, + "mean": 0.00014045925900063594, + "normalized": [ + 1.1719218726461313, + 1.1599555891510773, + 1.143372783062998, + 1.1305925991754215, + 1.1202702784385838, + 1.106568447670676, + 1.0931409560371117, + 1.082952696795682, + 1.0683580206124534, + 1.05266391584401, + 1.0427272032106563, + 1.034492263470324, + 1.0219716035750168, + 1.0154196846947314, + 1.0039021031599646, + 0.9939448772529704, + 0.9863124890252286, + 0.9768583563343864, + 0.9682383265025308, + 0.9606909958387225, + 0.9520472410590707, + 0.9456933063501335, + 0.9351354973759991, + 0.9231938712502704, + 0.9140830768869257, + 0.9051693342733999, + 0.8960380266364146, + 0.8881305740575848, + 0.8822109405764296, + 0.8783208779569746, + 0.8754534745501711, + 0.8701687165279489 + ], + "peak_layer": 1, + "peak_normalized": 1.1719218726461313, + "population_cv": 0.09095723041158388, + "rest_mean": 0.00014217518737832843, + "spike_contrast": 0.9227576779061385, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013119324576109647, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016460727783851326, + 0.00016292650252580643, + 0.00016059729387052357, + 0.0001588021987117827, + 0.0001573523331899196, + 0.00015542778419330716, + 0.00015354176866821945, + 0.00015211073332466185, + 0.00015006077592261136, + 0.00014785639359615743, + 0.0001464606903027743, + 0.00014530401676893234, + 0.00014354537415783852, + 0.00014262509648688138, + 0.0001410073455190286, + 0.00013960876094643027, + 0.00013853672135155648, + 0.0001372088008793071, + 0.00013599803787656128, + 0.00013493794540409, + 0.00013372385001275688, + 0.00013283138105180115, + 0.00013134843902662396, + 0.0001296711270697415, + 0.0001283914316445589, + 0.00012713941396214068, + 0.00012585683725774288, + 0.00012474616232793778, + 0.00012391469499561936, + 0.00012336829968262464, + 0.00012296554632484913, + 0.00012222325312905014 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00031102553475648165, + "first_to_last_ratio": 2.2098501973960105, + "imbalance_abs_log_ratio": 0.7929247292485178, + "last_quartile_mean": 0.00014074507635086775, + "mean": 0.00020578417843353236, + "normalized": [ + 1.8085758305333837, + 1.7408578335592815, + 1.6197863489261306, + 1.5416881776712252, + 1.4477928223214513, + 1.3679195769143782, + 1.3100342829404608, + 1.2546744848740619, + 1.192398672058626, + 1.1190508147725688, + 1.082828610043334, + 1.0467691899805913, + 0.9849335746234156, + 0.9618381641711736, + 0.9182125099812896, + 0.877817233347749, + 0.8611427657722097, + 0.8409392227987704, + 0.8149468534272446, + 0.7879793317746896, + 0.7604794717340572, + 0.7439065501103351, + 0.7274095758071225, + 0.7164574630456112, + 0.7035124756224637, + 0.6971408207181957, + 0.690610058298894, + 0.684463275349207, + 0.6799680284055707, + 0.6760380727842366, + 0.6720531720343774, + 0.6677747355978938 + ], + "peak_layer": 1, + "peak_normalized": 1.8085758305333837, + "population_cv": 0.33829873005197686, + "rest_mean": 0.0002160599310796156, + "spike_contrast": 0.6956177084463702, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015029511414468288, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003721762914210558, + 0.00035824099904857576, + 0.00033332640305161476, + 0.00031725503504276276, + 0.00029793285648338497, + 0.0002814962062984705, + 0.0002695843286346644, + 0.00025819215807132423, + 0.0002453767810948193, + 0.00023028295254334807, + 0.00022282899590209126, + 0.00021540853776969016, + 0.00020268374646548182, + 0.00019793107639998198, + 0.0001889536069938913, + 0.00018064089817926288, + 0.00017720955656841397, + 0.0001730519870761782, + 0.00016770316869951785, + 0.00016215367941185832, + 0.00015649464330635965, + 0.00015308419824577868, + 0.00014968938194215298, + 0.00014743561041541398, + 0.00014477173681370914, + 0.00014346055104397237, + 0.00014211662346497178, + 0.00014085171278566122, + 0.00013992666208650917, + 0.00013911793939769268, + 0.00013829790987074375, + 0.00013741747534368187 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "43ba487a4502ed530074a46d41ef4bc5ad3ecf78c91632d9d64d6f90fd9859c7", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.028097106143832207, + 0.014136734418570995, + 0.014224153943359852, + 0.014337832108139992, + 0.014408928342163563, + 0.009704872034490108, + 0.009773381985723972, + 0.00987160298973322, + 0.00994801800698042, + 0.007515520788729191, + 0.007602147292345762, + 0.007658426649868488, + 0.007734050042927265, + 0.0062688966281712055, + 0.006336068734526634, + 0.006399397272616625, + 0.006462901830673218, + 0.005432929843664169, + 0.005504137370735407, + 0.005560056772083044, + 0.005631812382489443, + 0.004873365629464388, + 0.004937449004501104, + 0.005005750339478254, + 0.005022916477173567, + 0.0044684382155537605, + 0.004500201437622309, + 0.004573327023535967, + 0.00463611725717783, + 0.004166214261204004, + 0.0042496914975345135, + 0.004283483140170574 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00014323742470878642, + "first_to_last_ratio": 1.6208859606473658, + "imbalance_abs_log_ratio": 0.48297288904334346, + "last_quartile_mean": 8.83698348843609e-05, + "mean": 0.00011961164659624046, + "normalized": [ + 0.879665217639624, + 1.4482702231163533, + 1.4522235545753037, + 1.2017346580032677, + 1.034069363337856, + 1.36550769437152, + 1.149442648944982, + 1.0492523703464898, + 1.0286339735981567, + 1.1048881958426662, + 1.2960670227766034, + 1.1342833655748923, + 0.9483258681651744, + 1.1110988010941807, + 1.0051223270547462, + 0.9186295907129635, + 0.9340104771989933, + 1.0935042558126027, + 0.9904421395235656, + 1.070541841916413, + 0.9831461473937306, + 1.1165574278336847, + 0.8906099662074067, + 0.8835226812669187, + 0.8406995671395937, + 0.8651479324264472, + 0.7705397043177317, + 0.6783637574582257, + 0.6658814129635636, + 0.7912689330389562, + 0.6809428819470401, + 0.6176059984003462 + ], + "peak_layer": 3, + "peak_normalized": 1.4522235545753037, + "population_cv": 0.21063690333645962, + "rest_mean": 0.00012087627045629133, + "spike_contrast": 0.9330423359872595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011278267775196582, + "top_five_layers": [ + 3, + 2, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010521820513531566, + 0.00017322998610325158, + 0.00017370285058859736, + 0.00014374146121554077, + 0.000123686739243567, + 0.0001633306237636134, + 0.00013748672790825367, + 0.00012550280371215194, + 0.00012303660332690924, + 0.0001321574964094907, + 0.00015502471069339663, + 0.00013567350106313825, + 0.00011343081860104576, + 0.0001329003571299836, + 0.00012022433656966314, + 0.00010987879795720801, + 0.0001117185311159119, + 0.00013079584459774196, + 0.00011846841516671702, + 0.00012804927246179432, + 0.00011759572953451425, + 0.00013355327246244997, + 0.00010652732453309, + 0.00010567960271146148, + 0.00010055745951831341, + 0.00010348176874686033, + 9.216552280122414e-05, + 8.114020602079108e-05, + 7.964717224240303e-05, + 9.464497998123989e-05, + 8.144869934767485e-05, + 7.387287041638047e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00015447182067873655, + "first_to_last_ratio": 1.6506746305835311, + "imbalance_abs_log_ratio": 0.5011840713694677, + "last_quartile_mean": 9.358102306578076e-05, + "mean": 0.00012785212516064348, + "normalized": [ + 0.8854544129111614, + 1.4719853585239924, + 1.4439160513495162, + 1.202741955390318, + 1.0425045283644727, + 1.3767992039076578, + 1.1738833181051227, + 1.0683704265103389, + 1.0407410271769484, + 1.136073438345882, + 1.2855554885667506, + 1.129314905977533, + 0.953735223493217, + 1.1222184469907663, + 1.0164965849441758, + 0.9298350247961144, + 0.9320412792829462, + 1.0837813144211186, + 0.9857736034320764, + 1.0482233302985868, + 0.9656597421723547, + 1.0923472818519362, + 0.884471876187184, + 0.8724974481708365, + 0.826330173068434, + 0.8536374096219558, + 0.7648976975162065, + 0.681485935203229, + 0.6604246501739865, + 0.7781176964145518, + 0.6752223987471391, + 0.6154627680834903 + ], + "peak_layer": 2, + "peak_normalized": 1.4719853585239924, + "population_cv": 0.21455917947613024, + "rest_mean": 0.00012955063343893185, + "spike_contrast": 0.9160910858365683, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011868018045788631, + "top_five_layers": [ + 2, + 3, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001132072284235619, + 0.00018819645629264414, + 0.00018460773571860045, + 0.00015377311501652002, + 0.00013328641944099218, + 0.00017602670413907617, + 0.0001500834769103676, + 0.00013659342948812991, + 0.00013306095206644386, + 0.00014524940343108028, + 0.00016436100122518837, + 0.00014438531070481986, + 0.00012193707516416907, + 0.0001434780133422464, + 0.00012996124860364944, + 0.00011888138396898285, + 0.00011916345829376951, + 0.00013856374425813556, + 0.00012603325012605637, + 0.00013401758042164147, + 0.0001234616502188146, + 0.00013965892139822245, + 0.00011308160901535302, + 0.00011155065294587985, + 0.00010564806871116161, + 0.00010913935693679377, + 9.779379615793005e-05, + 8.71294250828214e-05, + 8.443669503321871e-05, + 9.948400111170486e-05, + 8.632861863588914e-05, + 7.86882228567265e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.13698715530335903, + "first_to_last_ratio": 1.6506746463483835, + "imbalance_abs_log_ratio": 0.5011840809200188, + "last_quartile_mean": 0.08298858627676964, + "mean": 0.11338054295629263, + "normalized": [ + 0.8854544185536968, + 1.4719853260144444, + 1.4439159993116888, + 1.2027420311824657, + 1.042504519270847, + 1.3767991933658148, + 1.1738833593963587, + 1.0683704218985235, + 1.0407409783116481, + 1.1360735196220915, + 1.285555449073044, + 1.129314933056767, + 0.9537352653592515, + 1.1222184500196979, + 1.0164965450422543, + 0.9298350370138166, + 0.9320412867497836, + 1.0837813135283418, + 0.9857735942006465, + 1.0482233283425098, + 0.9656597514448471, + 1.0923472716531482, + 0.8844718771212735, + 0.8724974491223123, + 0.82633015871154, + 0.8536374112441639, + 0.7648976624892803, + 0.681485927333974, + 0.660424633911195, + 0.7781176812354046, + 0.6752224228414072, + 0.6154627835777616 + ], + "peak_layer": 2, + "peak_normalized": 1.4719853260144444, + "population_cv": 0.2145591770090877, + "rest_mean": 0.11488679713673061, + "spike_contrast": 0.9160910827435619, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10524677038192749, + "top_five_layers": [ + 2, + 3, + 6, + 11, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.10039330273866653, + 0.16689449548721313, + 0.16371197998523712, + 0.1363675445318222, + 0.11819972842931747, + 0.1561022400856018, + 0.13309553265571594, + 0.12113241851329803, + 0.11799977719783783, + 0.1288086324930191, + 0.14575697481632233, + 0.1280423402786255, + 0.10813502222299576, + 0.1272377371788025, + 0.11525093019008636, + 0.10542520135641098, + 0.1056753471493721, + 0.12287971377372742, + 0.11176754534244537, + 0.11884813010692596, + 0.10948702692985535, + 0.12385092675685883, + 0.10028190165758133, + 0.09892423450946808, + 0.09368976205587387, + 0.09678587317466736, + 0.08672451227903366, + 0.07726724445819855, + 0.07487930357456207, + 0.0882234051823616, + 0.07655708491802216, + 0.06978150457143784 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.7160250005617854e-05, + "first_to_last_ratio": 0.7388534944946782, + "imbalance_abs_log_ratio": 0.30265562600345663, + "last_quartile_mean": 6.382895981005277e-05, + "mean": 5.960399772675373e-05, + "normalized": [ + 0.46158176249046556, + 0.8375260596839308, + 0.8768757910510345, + 0.7241656982213429, + 0.6710453705877814, + 1.0089930699998053, + 0.8842011269201543, + 0.865421443073546, + 0.883514351886623, + 0.9622394496647411, + 1.1256912972870226, + 1.0210990974149952, + 0.9232791878576234, + 1.0980011773561495, + 1.0212452171652862, + 0.9544893655323485, + 1.0133393696232265, + 1.159814348379829, + 0.9921993085906312, + 1.1469178460749072, + 1.0774269793999733, + 1.3924548133021735, + 1.1792402184404436, + 1.1521666823514416, + 1.1069955376913123, + 1.1868921569761495, + 1.1034570471301723, + 0.9847635589252388, + 0.9837749617675684, + 1.1772881708824627, + 1.0498861530819708, + 0.9740133811896496 + ], + "peak_layer": 22, + "peak_normalized": 1.3924548133021735, + "population_cv": 0.17243044689052606, + "rest_mean": 5.759890990264938e-05, + "spike_contrast": 1.22279175241262, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.043147197691724e-05, + "top_five_layers": [ + 22, + 26, + 23, + 30, + 18 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.751211832219269e-05, + 4.991990135749802e-05, + 5.226530265645124e-05, + 4.316317063057795e-05, + 3.9996986743062735e-05, + 6.014002065057866e-05, + 5.2701921958941966e-05, + 5.158257772563957e-05, + 5.266098742140457e-05, + 5.735331797040999e-05, + 6.709570152452216e-05, + 6.0861588281113654e-05, + 5.503113061422482e-05, + 6.544525967910886e-05, + 6.087029760237783e-05, + 5.689138197340071e-05, + 6.0399077483452857e-05, + 6.912957178428769e-05, + 5.9139045333722606e-05, + 6.836088869022205e-05, + 6.421895523089916e-05, + 8.299587352667004e-05, + 7.028743129922077e-05, + 6.867374031571671e-05, + 6.598135951207951e-05, + 7.074351742630824e-05, + 6.577045132871717e-05, + 5.869584492756985e-05, + 5.863692058483139e-05, + 7.017108146101236e-05, + 6.2577411881648e-05, + 5.8055091358255595e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6127787956938846e-05, + "first_to_last_ratio": 0.8333328618421979, + "imbalance_abs_log_ratio": 0.18232212258347721, + "last_quartile_mean": 6.735338365615462e-05, + "mean": 6.689512088087213e-05, + "normalized": [ + 0.49022256384162527, + 0.923222562655389, + 0.9170289550360846, + 0.7742286879065733, + 0.7345865645537341, + 1.0394905128518102, + 0.9423822415780315, + 0.8911705562121757, + 0.9138181637864031, + 1.0330628394205372, + 1.1540375572503838, + 1.0455389170050469, + 0.9297420351318767, + 1.1341544713967917, + 1.0374336268991433, + 0.9867016024211971, + 1.0073449670971224, + 1.1743962796261593, + 1.0459325434797282, + 1.1475080782429012, + 1.066139536390961, + 1.3297286391604237, + 1.1245973641634917, + 1.1027270030076322, + 1.04892482254714, + 1.125623033590473, + 1.0476398534474196, + 0.9361019469456514, + 0.930671663614635, + 1.0896207315037236, + 0.9754527388499041, + 0.9007689403858299 + ], + "peak_layer": 22, + "peak_normalized": 1.3297286391604237, + "population_cv": 0.14905429467389167, + "rest_mean": 6.522988486621116e-05, + "spike_contrast": 1.1633838617941636, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.58873953600414e-05, + "top_five_layers": [ + 22, + 18, + 11, + 20, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.2793497666716576e-05, + 6.17590849287808e-05, + 6.134476279839873e-05, + 5.179212166694924e-05, + 4.9140257033286616e-05, + 6.953684351174161e-05, + 6.304077396634966e-05, + 5.961496208328754e-05, + 6.112997652962804e-05, + 6.910686352057382e-05, + 7.719948189333081e-05, + 6.994145223870873e-05, + 6.219520582817495e-05, + 7.586940046167001e-05, + 6.939924787729979e-05, + 6.600552296731621e-05, + 6.738646334270015e-05, + 7.856138108763844e-05, + 6.996778392931446e-05, + 7.676269160583615e-05, + 7.13195331627503e-05, + 8.895235805539414e-05, + 7.523007661802694e-05, + 7.37670561647974e-05, + 7.016795279923826e-05, + 7.529868889832869e-05, + 7.00819946359843e-05, + 6.26206528977491e-05, + 6.225739343790337e-05, + 7.289031054824591e-05, + 6.525302887894213e-05, + 6.025704715284519e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.866022898866504e-05, + "first_to_last_ratio": 0.40574519905788725, + "imbalance_abs_log_ratio": 0.9020299049238899, + "last_quartile_mean": 4.599001795213553e-05, + "mean": 3.414976504245715e-05, + "normalized": [ + 0.2445526901503676, + 0.5596630531980494, + 0.5376964344799271, + 0.48012169469058175, + 0.4798804572179837, + 0.6900344095639286, + 0.7152325131545565, + 0.6642055410913166, + 0.7554442803991458, + 0.9343372865664744, + 0.9240467958883766, + 0.8592444663674569, + 0.8263415713413451, + 1.0773994704320806, + 0.959597390691018, + 0.9894020700335048, + 0.9760686528076964, + 1.1936439731862338, + 1.099336047654388, + 1.1448507248259905, + 1.0495120137852647, + 1.4542665199347329, + 1.3223844598378258, + 1.2890134714336634, + 1.2402412095084396, + 1.4030995672846114, + 1.3877316376933018, + 1.246635573851661, + 1.2962845787221184, + 1.4847061188341568, + 1.3923191445524272, + 1.3227061808213745 + ], + "peak_layer": 30, + "peak_normalized": 1.4847061188341568, + "population_cv": 0.32662955778396485, + "rest_mean": 3.2435424482510255e-05, + "spike_contrast": 1.3382653305362568, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.340720406617038e-05, + "top_five_layers": [ + 30, + 22, + 26, + 31, + 27 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.351416909135878e-06, + 1.9112361769657582e-05, + 1.8362206901656464e-05, + 1.6396043065469712e-05, + 1.6387804862461053e-05, + 2.3564512957818806e-05, + 2.4425022274954244e-05, + 2.2682463168166578e-05, + 2.5798244678298943e-05, + 3.1907398806652054e-05, + 3.1555980967823416e-05, + 2.9342996640480123e-05, + 2.8219370506121777e-05, + 3.679293877212331e-05, + 3.277002542745322e-05, + 3.378784822416492e-05, + 3.333251515869051e-05, + 4.0762661228654906e-05, + 3.7542067730100825e-05, + 3.909638326149434e-05, + 3.584058868000284e-05, + 4.9662859964882955e-05, + 4.515911859925836e-05, + 4.4019507186021656e-05, + 4.2353945900686085e-05, + 4.791552055394277e-05, + 4.7390709369210526e-05, + 4.257231194060296e-05, + 4.4267813791520894e-05, + 5.070236511528492e-05, + 4.754737165058032e-05, + 4.517010529525578e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00021511870181711856, + "first_to_last_ratio": 1.067203936185183, + "imbalance_abs_log_ratio": 0.06504208450186474, + "last_quartile_mean": 0.00020157225299044512, + "mean": 0.00023711549874860793, + "normalized": [ + 0.5199215340336985, + 0.9694171998286779, + 0.9952311992308057, + 0.8139231798698612, + 0.8071491568326964, + 1.1522101416966866, + 1.0127543403881063, + 0.9872470097646113, + 0.9739878350012732, + 1.0629045356723543, + 1.2654986331851705, + 1.158397891601516, + 1.0251221075049282, + 1.1837639471542862, + 1.126357049236326, + 1.0240452993420521, + 1.0679809871494306, + 1.2152134226291609, + 1.0519957926786496, + 1.2134701299884076, + 1.1240302451392508, + 1.3270417853241223, + 1.0642149202553777, + 1.0573092573282454, + 0.9986311292560726, + 1.01615500686031, + 0.8954142860763721, + 0.793725079326075, + 0.7548043496765433, + 0.8883375219775711, + 0.764305251609244, + 0.6894397743821165 + ], + "peak_layer": 22, + "peak_normalized": 1.3270417853241223, + "population_cv": 0.17546356070810173, + "rest_mean": 0.00023209894856403548, + "spike_contrast": 1.138328593816985, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002642048697452992, + "top_five_layers": [ + 22, + 11, + 18, + 20, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00012328145385254174, + 0.00022986384283285588, + 0.0002359847421757877, + 0.00019299380073789507, + 0.00019138757488690317, + 0.00027320688241161406, + 0.00024013975053094327, + 0.00023409156710840762, + 0.00023094761127140373, + 0.0002520311390981078, + 0.00030006933957338333, + 0.0002746740938164294, + 0.00024307233979925513, + 0.00028068877873010933, + 0.0002670767134986818, + 0.0002428170118946582, + 0.00025323484442196786, + 0.0002881459367927164, + 0.00024944450706243515, + 0.0002877325750887394, + 0.0002665249921847135, + 0.00031466217478737235, + 0.0002523418515920639, + 0.00025070441188290715, + 0.0002367909182794392, + 0.00024094610125757754, + 0.00021231660502962768, + 0.00018820451805368066, + 0.00017897580983117223, + 0.00021063859458081424, + 0.00018122862093150616, + 0.00016347685595974326 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "a0cfcabdbb790e473dbbef021706294309c661b6e6673d5270b8d06f78dc5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014079716056585312, + 0.014167090877890587, + 0.014255538582801819, + 0.014355218037962914, + 0.009655896574258804, + 0.009734091348946095, + 0.009828771464526653, + 0.009915035218000412, + 0.007479202933609486, + 0.007559895049780607, + 0.0076358383521437645, + 0.007706487085670233, + 0.0062490664422512054, + 0.006315518636256456, + 0.0063744354993104935, + 0.006439714692533016, + 0.0054116579703986645, + 0.005481995176523924, + 0.005541813559830189, + 0.005614723544567823, + 0.004862982779741287, + 0.004923288244754076, + 0.0049925679340958595, + 0.005004179198294878, + 0.004448203835636377, + 0.004490054678171873, + 0.00455933902412653, + 0.004625064320862293, + 0.004160413984209299, + 0.004242690280079842, + 0.004270957782864571, + 0.004303553607314825 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.6855086818832206e-05, + "first_to_last_ratio": 1.1121706932103832, + "imbalance_abs_log_ratio": 0.10631368513677814, + "last_quartile_mean": 3.313797697046539e-05, + "mean": 3.537216105087282e-05, + "normalized": [ + 1.3329674878497746, + 1.065596392820356, + 0.9206330800727252, + 0.8842664165696186, + 1.1462361065020723, + 1.065158463293282, + 0.99582829507555, + 0.9247019781352203, + 1.132583771771479, + 1.005206548698436, + 0.979948053013938, + 0.967083333200755, + 1.1057115808592233, + 1.043785177992708, + 0.995436955984306, + 0.9279460300710988, + 1.1098294348763655, + 1.0288442525987258, + 1.0234795644677463, + 0.9449806453153974, + 1.0109568163965834, + 1.0260998393764513, + 0.9416381672270038, + 0.926379336673748, + 0.9884731771330477, + 0.9341156116917169, + 0.9458688462068515, + 0.9053302302997448, + 0.9662542702797254, + 0.9495526789453999, + 0.9100381269887573, + 0.8950693296121922 + ], + "peak_layer": 1, + "peak_normalized": 1.3329674878497746, + "population_cv": 0.09165679612351446, + "rest_mean": 3.551162256001219e-05, + "spike_contrast": 0.9748658722370791, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.461906890152022e-05, + "top_five_layers": [ + 1, + 5, + 9, + 17, + 13 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.714994065579958e-05, + 3.769244722207077e-05, + 3.256478157709353e-05, + 3.127841409877874e-05, + 4.054484816151671e-05, + 3.767695670831017e-05, + 3.5224598832428455e-05, + 3.270870729465969e-05, + 4.006193557870574e-05, + 3.555632792995311e-05, + 3.4662880352698267e-05, + 3.420782741159201e-05, + 3.911140811396763e-05, + 3.692093741847202e-05, + 3.521075632306747e-05, + 3.2823456422192976e-05, + 3.9257065509445965e-05, + 3.6392444599187e-05, + 3.620268398663029e-05, + 3.342600757605396e-05, + 3.575972732505761e-05, + 3.6295368772698566e-05, + 3.330777690280229e-05, + 3.276803909102455e-05, + 3.49644324160181e-05, + 3.3041687856893986e-05, + 3.3457425161032006e-05, + 3.202348671038635e-05, + 3.417850166442804e-05, + 3.358773028594442e-05, + 3.2190015190280974e-05, + 3.166053647873923e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 7.015105893515283e-05, + "first_to_last_ratio": 0.6588274570259804, + "imbalance_abs_log_ratio": 0.4172936042156455, + "last_quartile_mean": 0.00010647865110513521, + "mean": 9.056432463694364e-05, + "normalized": [ + 0.8598786252117567, + 0.7452797309823858, + 0.6694431442719111, + 0.6417832925446045, + 0.8822867975473095, + 0.8318257491363829, + 0.804532248804658, + 0.7617644993607329, + 0.969439786050785, + 0.9144645011121496, + 0.8951531228323205, + 0.876468558935897, + 1.0407786059783084, + 1.0056397226462848, + 0.9828357943952414, + 0.9490839045924557, + 1.1213662721801265, + 1.0774120583919133, + 1.0554308943167812, + 1.0307906296953837, + 1.1314055055223455, + 1.1556700189426168, + 1.1045075200854564, + 1.0869669205578247, + 1.2145839027446308, + 1.1552275050200584, + 1.1514380979421175, + 1.1280886596086177, + 1.2254687972249847, + 1.20187962367999, + 1.1639632986601018, + 1.165142211023867 + ], + "peak_layer": 29, + "peak_normalized": 1.2254687972249847, + "population_cv": 0.1618495930711073, + "rest_mean": 8.82393912804589e-05, + "spike_contrast": 1.1686273359956587, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010311896476196125, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 7.787432696204633e-05, + 6.74957555020228e-05, + 6.062766624381766e-05, + 5.812267045257613e-05, + 7.990370795596391e-05, + 7.533373718615621e-05, + 7.286191976163536e-05, + 6.898868741700426e-05, + 8.779665949987248e-05, + 8.281785994768143e-05, + 8.106893801596016e-05, + 7.937678310554475e-05, + 9.425741154700518e-05, + 9.107508230954409e-05, + 8.900985994841903e-05, + 8.595314284320921e-05, + 0.00010155577911064029, + 9.757509542396292e-05, + 9.558438614476472e-05, + 9.335285722045228e-05, + 0.00010246497549815103, + 0.00010466247476870194, + 0.00010002897761296481, + 9.844042506301776e-05, + 0.00010999797086697072, + 0.00010462239879416302, + 0.00010427921370137483, + 0.00010216458758804947, + 0.00011098375398432836, + 0.00010884741641348228, + 0.00010541355004534125, + 0.00010552031744737178 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06221066089347005, + "first_to_last_ratio": 0.6588274447143223, + "imbalance_abs_log_ratio": 0.4172936229028725, + "last_quartile_mean": 0.09442633483558893, + "mean": 0.0803133494919166, + "normalized": [ + 0.8598786809861813, + 0.7452797356777653, + 0.6694431594543391, + 0.6417832788211475, + 0.8822868217673959, + 0.83182573854958, + 0.804532202278358, + 0.7617644434307961, + 0.9694397651274825, + 0.9144645475330049, + 0.8951531251987148, + 0.8764685422804359, + 1.0407786734529123, + 1.0056396718254574, + 0.9828357789429103, + 0.9490838578168909, + 1.121366176654653, + 1.0774119964231388, + 1.0554309636293913, + 1.030790617723408, + 1.131405440813584, + 1.1556700726993148, + 1.1045075638748008, + 1.0869669141870728, + 1.214583886452764, + 1.155227565075462, + 1.151438141297865, + 1.1280886745503538, + 1.2254688318483915, + 1.20187965027907, + 1.1639633335063522, + 1.1651421478410058 + ], + "peak_layer": 29, + "peak_normalized": 1.2254688318483915, + "population_cv": 0.16184959877781283, + "rest_mean": 0.07825157529226055, + "spike_contrast": 1.1686273385361958, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0914469301700592, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.06905973702669144, + 0.05985591188073158, + 0.05376522243022919, + 0.051543764770030975, + 0.0708594098687172, + 0.06680671125650406, + 0.0646146759390831, + 0.06117985397577286, + 0.07785895466804504, + 0.0734437108039856, + 0.07189274579286575, + 0.07039212435483932, + 0.08358842134475708, + 0.08076629042625427, + 0.07893483340740204, + 0.07622410356998444, + 0.09006067365407944, + 0.08653056621551514, + 0.08476519584655762, + 0.08278624713420868, + 0.09086696058511734, + 0.09281573444604874, + 0.08870670199394226, + 0.0872979536652565, + 0.09754730015993118, + 0.09278019517660141, + 0.09247585386037827, + 0.09060057997703552, + 0.09842150658369064, + 0.09652698040008545, + 0.09348179399967194, + 0.09357646852731705 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.077007694533677e-06, + "first_to_last_ratio": 0.4632676420751026, + "imbalance_abs_log_ratio": 0.7694503312712233, + "last_quartile_mean": 1.5276283193088602e-05, + "mean": 1.1745167029175718e-05, + "normalized": [ + 0.5860812018784138, + 0.5338863181232825, + 0.4896831424227033, + 0.48432444022666415, + 0.7227742595277804, + 0.658356157633798, + 0.6679626701195096, + 0.6773027265164295, + 0.9317627933811012, + 0.8396840560424388, + 0.8002283441953012, + 0.8237560838623014, + 0.9947731150675175, + 0.9485376018841178, + 1.012754293639493, + 0.980582798985168, + 1.1861357451339187, + 1.074396404738039, + 1.1583203932505854, + 1.1590437197037688, + 1.1716087388237089, + 1.3380122311407743, + 1.182136735603702, + 1.17274255168395, + 1.3256589211015541, + 1.1785808128912905, + 1.3283612705891032, + 1.2366027296837898, + 1.3388375403515687, + 1.3849270873253279, + 1.3156503626951777, + 1.2965347517777213 + ], + "peak_layer": 30, + "peak_normalized": 1.3849270873253279, + "population_cv": 0.2782190894097951, + "rest_mean": 1.1227440423950881e-05, + "spike_contrast": 1.2951207174852213, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4540890697389841e-05, + "top_five_layers": [ + 30, + 29, + 22, + 27, + 25 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 6.883621608722024e-06, + 6.2705839809495956e-06, + 5.751410299126292e-06, + 5.6884714467742015e-06, + 8.48910440254258e-06, + 7.732503036095295e-06, + 7.845333129807841e-06, + 7.955033652251586e-06, + 1.0943709639832377e-05, + 9.862229489954188e-06, + 9.39881556405453e-06, + 9.67515279626241e-06, + 1.168377639260143e-05, + 1.1140732567582745e-05, + 1.1894968338310719e-05, + 1.1517108760017436e-05, + 1.3931362445873674e-05, + 1.2618965229194146e-05, + 1.3604666492028628e-05, + 1.3613162082037888e-05, + 1.3760740330326371e-05, + 1.5715177141828462e-05, + 1.3884393410990015e-05, + 1.377405715174973e-05, + 1.5570085452054627e-05, + 1.38426285047899e-05, + 1.56018249981571e-05, + 1.4524105608870741e-05, + 1.572487053635996e-05, + 1.62661999638658e-05, + 1.5452533261850476e-05, + 1.5228017218760215e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.028231200616574e-05, + "first_to_last_ratio": 0.5725175422355306, + "imbalance_abs_log_ratio": 0.5577119025637401, + "last_quartile_mean": 0.00010529338851483772, + "mean": 8.693937525094952e-05, + "normalized": [ + 0.6520176082954547, + 0.61792639701702, + 0.5860559002851884, + 0.5711155588169554, + 0.8204737244004536, + 0.784378974164202, + 0.772878878140371, + 0.7422194596148288, + 0.9603819210570083, + 0.917062379210463, + 0.9022792075902993, + 0.8821977885039085, + 1.057360487646653, + 1.0255127545988867, + 1.0067830966339102, + 0.9749202156934387, + 1.1510091959216098, + 1.1091409226242501, + 1.0872456061539109, + 1.0647793564016073, + 1.1683829082644246, + 1.1939706281941995, + 1.1404893610428462, + 1.1225154239128359, + 1.2535795945499137, + 1.1915479696415308, + 1.1872717446378291, + 1.1619194447200984, + 1.2611193116864508, + 1.2370940006457103, + 1.1981927894482458, + 1.1981773904854955 + ], + "peak_layer": 29, + "peak_normalized": 1.2611193116864508, + "population_cv": 0.20676778332368514, + "rest_mean": 8.410921549916061e-05, + "spike_contrast": 1.215351222858925, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001022222379106097, + "top_five_layers": [ + 29, + 25, + 30, + 31, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 5.6686003517825156e-05, + 5.3722134907729924e-05, + 5.095133383292705e-05, + 4.965242987964302e-05, + 7.133147300919518e-05, + 6.81934179738164e-05, + 6.719360681017861e-05, + 6.452809611801058e-05, + 8.349500421900302e-05, + 7.972883031470701e-05, + 7.844359060982242e-05, + 7.669772458029911e-05, + 9.192626021103933e-05, + 8.915743819670752e-05, + 8.75290934345685e-05, + 8.475895447190851e-05, + 0.00010006802040152252, + 9.642801887821406e-05, + 9.452445374336094e-05, + 9.257125202566385e-05, + 0.00010157848009839654, + 0.00010380306048318744, + 9.915343252941966e-05, + 9.759078966453671e-05, + 0.00010898542677750811, + 0.00010359243606217206, + 0.00010322066373191774, + 0.00010101655061589554, + 0.00010964092507492751, + 0.0001075521795428358, + 0.00010417013254482299, + 0.00010416879376862198 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 5.289650744089158e-05, + "first_to_last_ratio": 0.5036607689093815, + "imbalance_abs_log_ratio": 0.6858523150899144, + "last_quartile_mean": 0.00010502407712920103, + "mean": 8.374395429200376e-05, + "normalized": [ + 0.5462883602941595, + 0.5343271256807102, + 0.5249643991287053, + 0.5130648517134513, + 0.7580131602578075, + 0.7348274620741431, + 0.7341801809099809, + 0.7074999463902408, + 0.9290893112957194, + 0.896130362600229, + 0.8896360907643062, + 0.8667732512123232, + 1.0501229826066591, + 1.0276121076179003, + 1.01304723882372, + 0.9917393510831477, + 1.161342307568102, + 1.1291245609681106, + 1.1079392656735292, + 1.0938109858524918, + 1.2035226235600145, + 1.226263478849651, + 1.1724755435916903, + 1.1553301515952579, + 1.2932755854998876, + 1.227360989677261, + 1.2282904680523223, + 1.203004103561663, + 1.301298744133899, + 1.2818290480161694, + 1.2464280682225441, + 1.251387892724203 + ], + "peak_layer": 29, + "peak_normalized": 1.301298744133899, + "population_cv": 0.24714813814810152, + "rest_mean": 8.048455465878187e-05, + "spike_contrast": 1.2591821218500583, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010134471231140196, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 4.574834747472778e-05, + 4.474666638998315e-05, + 4.396259464556351e-05, + 4.296607949072495e-05, + 6.347901944536716e-05, + 6.153735739644617e-05, + 6.14831515122205e-05, + 5.924884317209944e-05, + 7.780561281833798e-05, + 7.504550012527034e-05, + 7.450164412148297e-05, + 7.258701953105628e-05, + 8.794145105639473e-05, + 8.60563013702631e-05, + 8.483658166369423e-05, + 8.305217488668859e-05, + 9.725539712235332e-05, + 9.455735562369227e-05, + 9.278321522288024e-05, + 9.160005720332265e-05, + 0.00010078774357680231, + 0.00010269215272273868, + 9.818773833103478e-05, + 9.675191540736705e-05, + 0.00010830401151906699, + 0.00010278406261932105, + 0.00010286190081387758, + 0.00010074432066176087, + 0.00010897590254899114, + 0.00010734543320722878, + 0.00010438081517349929, + 0.00010479617048986256 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010133173600479495, + "first_to_last_ratio": 0.7712367006907566, + "imbalance_abs_log_ratio": 0.25975994774816064, + "last_quartile_mean": 0.00013138863323547412, + "mean": 0.00011757463107642252, + "normalized": [ + 0.9240254436843522, + 0.8516610763218788, + 0.7400083948384197, + 0.7305203183107621, + 0.9945100925181989, + 0.9162256174012862, + 0.9029060711458456, + 0.8349457218511338, + 1.057384774375838, + 0.9561549232760007, + 0.9255087968738563, + 0.9289705112696707, + 1.0651337322266232, + 1.006401362110571, + 0.966928758219108, + 0.9306903222204219, + 1.1004660046561014, + 1.0434523934672617, + 1.009934985409439, + 0.9657192405683421, + 1.061624429236619, + 1.0862247001295862, + 1.0383026763618062, + 1.022368851612807, + 1.1429833503196458, + 1.0926891991234107, + 1.0888718388877934, + 1.0734199646869733, + 1.177032649101865, + 1.140664442905499, + 1.1108444025161495, + 1.1134249543727335 + ], + "peak_layer": 29, + "peak_normalized": 1.177032649101865, + "population_cv": 0.10936728326378568, + "rest_mean": 0.00011604396611262389, + "spike_contrast": 1.0844184846181808, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012584022188093512, + "top_five_layers": [ + 29, + 25, + 30, + 32, + 31 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00010864195064641535, + 0.00010013373685069382, + 8.700621401658282e-05, + 8.58906569192186e-05, + 0.00011692915722960606, + 0.00010772488894872367, + 0.0001061588482116349, + 9.816843521548435e-05, + 0.0001243216247530654, + 0.00011241956235608086, + 0.00010881635535042733, + 0.00010922336514340714, + 0.00012523270561359823, + 0.00011832726886495948, + 0.00011368629202479497, + 0.00010942557128146291, + 0.0001293868845095858, + 0.00012268353020772338, + 0.00011874273332068697, + 0.00011354408343322575, + 0.00012482010060921311, + 0.00012771246838383377, + 0.0001220780541189015, + 0.00012020464055240154, + 0.00013438584574032575, + 0.0001284725294681266, + 0.00012802370474673808, + 0.0001262069563381374, + 0.00013838917948305607, + 0.00013411320105660707, + 0.00013060712080914527, + 0.00013091052824165672 + ] + } + } + } + } + } + }, + "step": 0 + }, + { + "modes": { + "learned": { + "bits_per_byte": 5.18286091123436, + "forward": { + "activation_sha256": { + "attention_branch_output": "5f9ceb4daf4a04de3e80e18a04e2c787acd0577d8e42d246715f712ea2c503dc", + "mlp_branch_output": "78bdb1dc451fca8b12adc84e1182b5c9328709671c2cc7151ff16c36d6a3b757", + "post_attention_state": "ae4066fc37f2b8857b68206f2c50d90301cff538a7afaed7e0b4959067354e04", + "post_mlp_state": "0a445d8d97762f2c9a1505daa03132d58d2d7cce48cebe69f58dcee8b68ddc3c", + "pre_attention_input": "bbef604712053bff665de420d5353433a19666047adbe681202c1aa4c9b40d0a", + "pre_mlp_input": "4e4a33d69f29ecf2b2276b1faef8a4abb641becd86149078c17228416d2e6c90" + }, + "logits_sha256": "2d69f7bc604d2338c84b2c31752b4e64390f685f11c1122bb80ad52194829404", + "loss_nats": 3.5924854278564453, + "mixer_summary_sha256": "57b7f21bb725bbab9d3242fe4186ef8ea689dbc30e549430632de5bdaef4d2ca" + }, + "loss_nats": 3.5924854278564453, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6920576691627502, + "group": 1, + "latest_source_mass": 0.47704219818115234, + "layer": 1, + "max_source_mass": 0.5229578018188477, + "mixer_index": 1, + "normalized_entropy": 0.9984281673102747, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5229578018188477, + "median": 0.5233346819877625, + "p05": 0.5158256888389587, + "p95": 0.5291492938995361 + }, + { + "label": "current_group_partial", + "mean": 0.47704219818115234, + "median": 0.47666531801223755, + "p05": 0.4708506762981415, + "p95": 0.48417431116104126 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6922386288642883, + "group": 1, + "latest_source_mass": 0.479327529668808, + "layer": 2, + "max_source_mass": 0.5206724405288696, + "mixer_index": 2, + "normalized_entropy": 0.9986892369742845, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5206724405288696, + "median": 0.5204353928565979, + "p05": 0.5125095844268799, + "p95": 0.529628336429596 + }, + { + "label": "current_group_partial", + "mean": 0.479327529668808, + "median": 0.4795646071434021, + "p05": 0.47037166357040405, + "p95": 0.48749038577079773 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6930604577064514, + "group": 1, + "latest_source_mass": 0.4976575970649719, + "layer": 2, + "max_source_mass": 0.5023424625396729, + "mixer_index": 3, + "normalized_entropy": 0.9998748853693327, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5023424625396729, + "median": 0.5015497207641602, + "p05": 0.4933284819126129, + "p95": 0.5133330225944519 + }, + { + "label": "current_group_partial", + "mean": 0.4976575970649719, + "median": 0.49845027923583984, + "p05": 0.4866669774055481, + "p95": 0.5066715478897095 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6924410462379456, + "group": 1, + "latest_source_mass": 0.48233455419540405, + "layer": 3, + "max_source_mass": 0.517665445804596, + "mixer_index": 4, + "normalized_entropy": 0.9989812635154495, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.517665445804596, + "median": 0.5173795223236084, + "p05": 0.5080559253692627, + "p95": 0.528046727180481 + }, + { + "label": "current_group_partial", + "mean": 0.48233455419540405, + "median": 0.4826204478740692, + "p05": 0.4719533324241638, + "p95": 0.4919440448284149 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6929901838302612, + "group": 1, + "latest_source_mass": 0.4986313283443451, + "layer": 3, + "max_source_mass": 0.5013687014579773, + "mixer_index": 5, + "normalized_entropy": 0.999773501596649, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5013687014579773, + "median": 0.5013427734375, + "p05": 0.4864077866077423, + "p95": 0.5143497586250305 + }, + { + "label": "current_group_partial", + "mean": 0.4986313283443451, + "median": 0.4986572265625, + "p05": 0.4856501817703247, + "p95": 0.5135922431945801 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6929076313972473, + "group": 1, + "latest_source_mass": 0.49019312858581543, + "layer": 4, + "max_source_mass": 0.5098068714141846, + "mixer_index": 6, + "normalized_entropy": 0.9996544036109265, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5098068714141846, + "median": 0.5095012187957764, + "p05": 0.5024833679199219, + "p95": 0.5181499719619751 + }, + { + "label": "current_group_partial", + "mean": 0.49019312858581543, + "median": 0.49049875140190125, + "p05": 0.4818500280380249, + "p95": 0.4975166320800781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6927430629730225, + "group": 1, + "latest_source_mass": 0.4872150421142578, + "layer": 4, + "max_source_mass": 0.5127849578857422, + "mixer_index": 7, + "normalized_entropy": 0.9994169815614105, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5127849578857422, + "median": 0.5123876333236694, + "p05": 0.5027084350585938, + "p95": 0.5231767296791077 + }, + { + "label": "current_group_partial", + "mean": 0.4872150421142578, + "median": 0.48761239647865295, + "p05": 0.47682327032089233, + "p95": 0.49729159474372864 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6928876638412476, + "group": 2, + "latest_source_mass": 0.48993486166000366, + "layer": 5, + "max_source_mass": 0.5100650787353516, + "mixer_index": 8, + "normalized_entropy": 0.999625596516907, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5100650787353516, + "median": 0.5104688405990601, + "p05": 0.5006189346313477, + "p95": 0.5182909369468689 + }, + { + "label": "completed_group_1", + "mean": 0.48993486166000366, + "median": 0.48953112959861755, + "p05": 0.4817090630531311, + "p95": 0.49938106536865234 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.098104476928711, + "group": 2, + "latest_source_mass": 0.34344637393951416, + "layer": 5, + "max_source_mass": 0.34344637393951416, + "mixer_index": 9, + "normalized_entropy": 0.9995377698350576, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33572152256965637, + "median": 0.33574378490448, + "p05": 0.3252582252025604, + "p95": 0.3447045683860779 + }, + { + "label": "completed_group_1", + "mean": 0.32083210349082947, + "median": 0.3210340738296509, + "p05": 0.3132280111312866, + "p95": 0.3294433057308197 + }, + { + "label": "current_group_partial", + "mean": 0.34344637393951416, + "median": 0.3437310457229614, + "p05": 0.335769921541214, + "p95": 0.3489701747894287 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.098517656326294, + "group": 2, + "latest_source_mass": 0.33111652731895447, + "layer": 6, + "max_source_mass": 0.3379058539867401, + "mixer_index": 10, + "normalized_entropy": 0.9999138619303717, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3379058539867401, + "median": 0.33803731203079224, + "p05": 0.3307536244392395, + "p95": 0.34437015652656555 + }, + { + "label": "completed_group_1", + "mean": 0.3309776186943054, + "median": 0.33117008209228516, + "p05": 0.3264409899711609, + "p95": 0.3352589011192322 + }, + { + "label": "current_group_partial", + "mean": 0.33111652731895447, + "median": 0.3308069109916687, + "p05": 0.32785022258758545, + "p95": 0.33529508113861084 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0984238386154175, + "group": 2, + "latest_source_mass": 0.3280537724494934, + "layer": 6, + "max_source_mass": 0.34072375297546387, + "mixer_index": 11, + "normalized_entropy": 0.9998284653697795, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34072375297546387, + "median": 0.3406665623188019, + "p05": 0.33292704820632935, + "p95": 0.3486446142196655 + }, + { + "label": "completed_group_1", + "mean": 0.3312225341796875, + "median": 0.3312021493911743, + "p05": 0.32623448967933655, + "p95": 0.33618849515914917 + }, + { + "label": "current_group_partial", + "mean": 0.3280537724494934, + "median": 0.32822102308273315, + "p05": 0.323047935962677, + "p95": 0.3323698341846466 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0985376834869385, + "group": 2, + "latest_source_mass": 0.3316706418991089, + "layer": 7, + "max_source_mass": 0.3372243046760559, + "mixer_index": 12, + "normalized_entropy": 0.9999320914375883, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3372243046760559, + "median": 0.3373582065105438, + "p05": 0.33067408204078674, + "p95": 0.3436351418495178 + }, + { + "label": "completed_group_1", + "mean": 0.33110499382019043, + "median": 0.331229567527771, + "p05": 0.32648319005966187, + "p95": 0.3351142108440399 + }, + { + "label": "current_group_partial", + "mean": 0.3316706418991089, + "median": 0.3314104676246643, + "p05": 0.32845842838287354, + "p95": 0.33538299798965454 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0983879566192627, + "group": 2, + "latest_source_mass": 0.3259334862232208, + "layer": 7, + "max_source_mass": 0.34103792905807495, + "mixer_index": 13, + "normalized_entropy": 0.9997958041693498, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34103792905807495, + "median": 0.34118473529815674, + "p05": 0.33348870277404785, + "p95": 0.3480936288833618 + }, + { + "label": "completed_group_1", + "mean": 0.33302855491638184, + "median": 0.3329259753227234, + "p05": 0.32893261313438416, + "p95": 0.33756527304649353 + }, + { + "label": "current_group_partial", + "mean": 0.3259334862232208, + "median": 0.32614004611968994, + "p05": 0.32082775235176086, + "p95": 0.33071696758270264 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0984759330749512, + "group": 2, + "latest_source_mass": 0.3290713131427765, + "layer": 8, + "max_source_mass": 0.3378061056137085, + "mixer_index": 14, + "normalized_entropy": 0.9998758837903371, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3378061056137085, + "median": 0.3386826515197754, + "p05": 0.32766789197921753, + "p95": 0.34578830003738403 + }, + { + "label": "completed_group_1", + "mean": 0.3331226110458374, + "median": 0.3330760598182678, + "p05": 0.32885679602622986, + "p95": 0.33746954798698425 + }, + { + "label": "current_group_partial", + "mean": 0.3290713131427765, + "median": 0.3284408450126648, + "p05": 0.3237078785896301, + "p95": 0.33546704053878784 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0982115268707275, + "group": 2, + "latest_source_mass": 0.32475554943084717, + "layer": 8, + "max_source_mass": 0.3427826762199402, + "mixer_index": 15, + "normalized_entropy": 0.9996352108914892, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3427826762199402, + "median": 0.34300696849823, + "p05": 0.33116698265075684, + "p95": 0.3554348945617676 + }, + { + "label": "completed_group_1", + "mean": 0.33246177434921265, + "median": 0.3327745795249939, + "p05": 0.3261578679084778, + "p95": 0.33856457471847534 + }, + { + "label": "current_group_partial", + "mean": 0.32475554943084717, + "median": 0.32512229681015015, + "p05": 0.3160964250564575, + "p95": 0.33136001229286194 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.098510980606079, + "group": 3, + "latest_source_mass": 0.32809075713157654, + "layer": 9, + "max_source_mass": 0.3366020917892456, + "mixer_index": 16, + "normalized_entropy": 0.9999077854279661, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3366020917892456, + "median": 0.33599138259887695, + "p05": 0.33130791783332825, + "p95": 0.343323677778244 + }, + { + "label": "completed_group_1", + "mean": 0.33530712127685547, + "median": 0.335271418094635, + "p05": 0.33171460032463074, + "p95": 0.33948856592178345 + }, + { + "label": "completed_group_2", + "mean": 0.32809075713157654, + "median": 0.328441321849823, + "p05": 0.3236806392669678, + "p95": 0.3313118815422058 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3860652446746826, + "group": 3, + "latest_source_mass": 0.24453917145729065, + "layer": 9, + "max_source_mass": 0.25563085079193115, + "mixer_index": 17, + "normalized_entropy": 0.9998347274203562, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25563085079193115, + "median": 0.2557356357574463, + "p05": 0.24995456635951996, + "p95": 0.2610935866832733 + }, + { + "label": "completed_group_1", + "mean": 0.25276094675064087, + "median": 0.2525555491447449, + "p05": 0.24839484691619873, + "p95": 0.25784528255462646 + }, + { + "label": "completed_group_2", + "mean": 0.24706900119781494, + "median": 0.24680209159851074, + "p05": 0.24331113696098328, + "p95": 0.2516176700592041 + }, + { + "label": "current_group_partial", + "mean": 0.24453917145729065, + "median": 0.24438345432281494, + "p05": 0.24019667506217957, + "p95": 0.24867366254329681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3861669301986694, + "group": 3, + "latest_source_mass": 0.24664369225502014, + "layer": 10, + "max_source_mass": 0.25406360626220703, + "mixer_index": 18, + "normalized_entropy": 0.9999080780209492, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25406360626220703, + "median": 0.25364670157432556, + "p05": 0.24951469898223877, + "p95": 0.25949251651763916 + }, + { + "label": "completed_group_1", + "mean": 0.25270599126815796, + "median": 0.2528834342956543, + "p05": 0.24992677569389343, + "p95": 0.25502878427505493 + }, + { + "label": "completed_group_2", + "mean": 0.24658671021461487, + "median": 0.2470710575580597, + "p05": 0.2430468648672104, + "p95": 0.24908794462680817 + }, + { + "label": "current_group_partial", + "mean": 0.24664369225502014, + "median": 0.2466917634010315, + "p05": 0.24438677728176117, + "p95": 0.24867767095565796 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.386000394821167, + "group": 3, + "latest_source_mass": 0.24240648746490479, + "layer": 10, + "max_source_mass": 0.2551618814468384, + "mixer_index": 19, + "normalized_entropy": 0.9997879481393215, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2551618814468384, + "median": 0.2564138174057007, + "p05": 0.24672001600265503, + "p95": 0.2620469629764557 + }, + { + "label": "completed_group_1", + "mean": 0.2541123628616333, + "median": 0.25428205728530884, + "p05": 0.25075286626815796, + "p95": 0.2573728561401367 + }, + { + "label": "completed_group_2", + "mean": 0.2483193278312683, + "median": 0.24777266383171082, + "p05": 0.24365577101707458, + "p95": 0.25304728746414185 + }, + { + "label": "current_group_partial", + "mean": 0.24240648746490479, + "median": 0.24237973988056183, + "p05": 0.2394917905330658, + "p95": 0.2452111691236496 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.386232852935791, + "group": 3, + "latest_source_mass": 0.24722051620483398, + "layer": 11, + "max_source_mass": 0.25246676802635193, + "mixer_index": 20, + "normalized_entropy": 0.9999556312239127, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25183945894241333, + "median": 0.2516742944717407, + "p05": 0.2478763908147812, + "p95": 0.2557043135166168 + }, + { + "label": "completed_group_1", + "mean": 0.25246676802635193, + "median": 0.2523234784603119, + "p05": 0.25062504410743713, + "p95": 0.254791259765625 + }, + { + "label": "completed_group_2", + "mean": 0.24847325682640076, + "median": 0.24844470620155334, + "p05": 0.24627941846847534, + "p95": 0.250301718711853 + }, + { + "label": "current_group_partial", + "mean": 0.24722051620483398, + "median": 0.24719130992889404, + "p05": 0.2453235685825348, + "p95": 0.24936428666114807 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.385960340499878, + "group": 3, + "latest_source_mass": 0.24082177877426147, + "layer": 11, + "max_source_mass": 0.25490617752075195, + "mixer_index": 21, + "normalized_entropy": 0.9997590550539766, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25454509258270264, + "median": 0.2548168897628784, + "p05": 0.24703091382980347, + "p95": 0.2618085741996765 + }, + { + "label": "completed_group_1", + "mean": 0.25490617752075195, + "median": 0.2547938823699951, + "p05": 0.2522759437561035, + "p95": 0.2576809227466583 + }, + { + "label": "completed_group_2", + "mean": 0.24972698092460632, + "median": 0.2496279776096344, + "p05": 0.24634553492069244, + "p95": 0.25290369987487793 + }, + { + "label": "current_group_partial", + "mean": 0.24082177877426147, + "median": 0.24057629704475403, + "p05": 0.23598811030387878, + "p95": 0.245779350399971 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.386134147644043, + "group": 3, + "latest_source_mass": 0.24431857466697693, + "layer": 12, + "max_source_mass": 0.25437670946121216, + "mixer_index": 22, + "normalized_entropy": 0.9998844304064556, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25200381875038147, + "median": 0.25220245122909546, + "p05": 0.2462744265794754, + "p95": 0.25811436772346497 + }, + { + "label": "completed_group_1", + "mean": 0.25437670946121216, + "median": 0.2543410062789917, + "p05": 0.25215157866477966, + "p95": 0.25676217675209045 + }, + { + "label": "completed_group_2", + "mean": 0.24930089712142944, + "median": 0.24924488365650177, + "p05": 0.24625414609909058, + "p95": 0.2522015869617462 + }, + { + "label": "current_group_partial", + "mean": 0.24431857466697693, + "median": 0.24446552991867065, + "p05": 0.24055814743041992, + "p95": 0.24748677015304565 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3859115839004517, + "group": 3, + "latest_source_mass": 0.2395908236503601, + "layer": 12, + "max_source_mass": 0.2563633620738983, + "mixer_index": 23, + "normalized_entropy": 0.9997238846018751, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2541099786758423, + "median": 0.25436341762542725, + "p05": 0.2479730099439621, + "p95": 0.2596602141857147 + }, + { + "label": "completed_group_1", + "mean": 0.2563633620738983, + "median": 0.25615423917770386, + "p05": 0.2534457743167877, + "p95": 0.26022869348526 + }, + { + "label": "completed_group_2", + "mean": 0.2499358206987381, + "median": 0.24995511770248413, + "p05": 0.247016042470932, + "p95": 0.2527387738227844 + }, + { + "label": "current_group_partial", + "mean": 0.2395908236503601, + "median": 0.24023187160491943, + "p05": 0.23504012823104858, + "p95": 0.24278664588928223 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3861732482910156, + "group": 4, + "latest_source_mass": 0.24519044160842896, + "layer": 13, + "max_source_mass": 0.25510814785957336, + "mixer_index": 24, + "normalized_entropy": 0.999912635561197, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2492598295211792, + "median": 0.24941539764404297, + "p05": 0.24585920572280884, + "p95": 0.2526070773601532 + }, + { + "label": "completed_group_1", + "mean": 0.25510814785957336, + "median": 0.25510847568511963, + "p05": 0.253096342086792, + "p95": 0.2572459578514099 + }, + { + "label": "completed_group_2", + "mean": 0.2504415512084961, + "median": 0.2503618597984314, + "p05": 0.24890245497226715, + "p95": 0.2520940899848938 + }, + { + "label": "completed_group_3", + "mean": 0.24519044160842896, + "median": 0.24486954510211945, + "p05": 0.2423042356967926, + "p95": 0.24860209226608276 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6091704368591309, + "group": 4, + "latest_source_mass": 0.19571462273597717, + "layer": 13, + "max_source_mass": 0.20561829209327698, + "mixer_index": 25, + "normalized_entropy": 0.9998338080811301, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20294126868247986, + "median": 0.20319771766662598, + "p05": 0.19784998893737793, + "p95": 0.20785459876060486 + }, + { + "label": "completed_group_1", + "mean": 0.20561829209327698, + "median": 0.2055630385875702, + "p05": 0.203423410654068, + "p95": 0.20820140838623047 + }, + { + "label": "completed_group_2", + "mean": 0.20076946914196014, + "median": 0.20075580477714539, + "p05": 0.19847290217876434, + "p95": 0.2027682065963745 + }, + { + "label": "completed_group_3", + "mean": 0.19495636224746704, + "median": 0.19467401504516602, + "p05": 0.19074591994285583, + "p95": 0.19898995757102966 + }, + { + "label": "current_group_partial", + "mean": 0.19571462273597717, + "median": 0.19596168398857117, + "p05": 0.19247503578662872, + "p95": 0.19751161336898804 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093817949295044, + "group": 4, + "latest_source_mass": 0.2000274658203125, + "layer": 14, + "max_source_mass": 0.20290037989616394, + "mixer_index": 26, + "normalized_entropy": 0.9999651322339542, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1999536156654358, + "median": 0.1995687186717987, + "p05": 0.19656820595264435, + "p95": 0.20360995829105377 + }, + { + "label": "completed_group_1", + "mean": 0.20290037989616394, + "median": 0.20293280482292175, + "p05": 0.20150557160377502, + "p95": 0.204205721616745 + }, + { + "label": "completed_group_2", + "mean": 0.19905918836593628, + "median": 0.19910168647766113, + "p05": 0.19694079458713531, + "p95": 0.2005346566438675 + }, + { + "label": "completed_group_3", + "mean": 0.1980593204498291, + "median": 0.1983218491077423, + "p05": 0.19644758105278015, + "p95": 0.19932645559310913 + }, + { + "label": "current_group_partial", + "mean": 0.2000274658203125, + "median": 0.19992563128471375, + "p05": 0.19864776730537415, + "p95": 0.20164194703102112 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6089024543762207, + "group": 4, + "latest_source_mass": 0.1908060908317566, + "layer": 14, + "max_source_mass": 0.20689502358436584, + "mixer_index": 27, + "normalized_entropy": 0.999667301202648, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20434537529945374, + "median": 0.2042810022830963, + "p05": 0.19885064661502838, + "p95": 0.21045605838298798 + }, + { + "label": "completed_group_1", + "mean": 0.20689502358436584, + "median": 0.20666636526584625, + "p05": 0.20447728037834167, + "p95": 0.21027149260044098 + }, + { + "label": "completed_group_2", + "mean": 0.20309436321258545, + "median": 0.203084334731102, + "p05": 0.20090451836585999, + "p95": 0.20521050691604614 + }, + { + "label": "completed_group_3", + "mean": 0.19485914707183838, + "median": 0.19490095973014832, + "p05": 0.19095906615257263, + "p95": 0.19829493761062622 + }, + { + "label": "current_group_partial", + "mean": 0.1908060908317566, + "median": 0.19071418046951294, + "p05": 0.18832355737686157, + "p95": 0.1931709200143814 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.609398365020752, + "group": 4, + "latest_source_mass": 0.19768188893795013, + "layer": 15, + "max_source_mass": 0.2025333046913147, + "mixer_index": 28, + "normalized_entropy": 0.9999754278105152, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20015040040016174, + "median": 0.20033058524131775, + "p05": 0.19777344167232513, + "p95": 0.20235712826251984 + }, + { + "label": "completed_group_1", + "mean": 0.2025333046913147, + "median": 0.2024737000465393, + "p05": 0.20170874893665314, + "p95": 0.20358291268348694 + }, + { + "label": "completed_group_2", + "mean": 0.2003234475851059, + "median": 0.20028962194919586, + "p05": 0.1992899477481842, + "p95": 0.20118050277233124 + }, + { + "label": "completed_group_3", + "mean": 0.19931095838546753, + "median": 0.1992638111114502, + "p05": 0.19840562343597412, + "p95": 0.2002609521150589 + }, + { + "label": "current_group_partial", + "mean": 0.19768188893795013, + "median": 0.19755831360816956, + "p05": 0.1966717392206192, + "p95": 0.19881953299045563 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6085304021835327, + "group": 4, + "latest_source_mass": 0.18813428282737732, + "layer": 15, + "max_source_mass": 0.21047377586364746, + "mixer_index": 29, + "normalized_entropy": 0.9994361321778514, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20286625623703003, + "median": 0.20224162936210632, + "p05": 0.19363349676132202, + "p95": 0.21225287020206451 + }, + { + "label": "completed_group_1", + "mean": 0.21047377586364746, + "median": 0.2103899121284485, + "p05": 0.20706231892108917, + "p95": 0.2142515927553177 + }, + { + "label": "completed_group_2", + "mean": 0.20388861000537872, + "median": 0.2042773962020874, + "p05": 0.2002551406621933, + "p95": 0.20698434114456177 + }, + { + "label": "completed_group_3", + "mean": 0.19463706016540527, + "median": 0.194687619805336, + "p05": 0.1895509660243988, + "p95": 0.19856791198253632 + }, + { + "label": "current_group_partial", + "mean": 0.18813428282737732, + "median": 0.18814176321029663, + "p05": 0.18432703614234924, + "p95": 0.19110313057899475 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.60928213596344, + "group": 4, + "latest_source_mass": 0.19474157691001892, + "layer": 16, + "max_source_mass": 0.20464840531349182, + "mixer_index": 30, + "normalized_entropy": 0.9999032106367963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20026026666164398, + "median": 0.20021198689937592, + "p05": 0.19762656092643738, + "p95": 0.20328035950660706 + }, + { + "label": "completed_group_1", + "mean": 0.20464840531349182, + "median": 0.20486173033714294, + "p05": 0.20261231064796448, + "p95": 0.20639576017856598 + }, + { + "label": "completed_group_2", + "mean": 0.20178081095218658, + "median": 0.20188090205192566, + "p05": 0.2006548047065735, + "p95": 0.2025863528251648 + }, + { + "label": "completed_group_3", + "mean": 0.1985689103603363, + "median": 0.19860434532165527, + "p05": 0.19688552618026733, + "p95": 0.20015479624271393 + }, + { + "label": "current_group_partial", + "mean": 0.19474157691001892, + "median": 0.19454869627952576, + "p05": 0.19281607866287231, + "p95": 0.1971154808998108 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6085617542266846, + "group": 4, + "latest_source_mass": 0.18782781064510345, + "layer": 16, + "max_source_mass": 0.2110326886177063, + "mixer_index": 31, + "normalized_entropy": 0.9994556122975314, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20295968651771545, + "median": 0.20242132246494293, + "p05": 0.1979435831308365, + "p95": 0.209271639585495 + }, + { + "label": "completed_group_1", + "mean": 0.2110326886177063, + "median": 0.21132414042949677, + "p05": 0.20678338408470154, + "p95": 0.2153906226158142 + }, + { + "label": "completed_group_2", + "mean": 0.20332126319408417, + "median": 0.20331284403800964, + "p05": 0.2010943740606308, + "p95": 0.2055126279592514 + }, + { + "label": "completed_group_3", + "mean": 0.194858580827713, + "median": 0.19520977139472961, + "p05": 0.190799281001091, + "p95": 0.1979817897081375 + }, + { + "label": "current_group_partial", + "mean": 0.18782781064510345, + "median": 0.18755686283111572, + "p05": 0.18426461517810822, + "p95": 0.19186276197433472 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093013286590576, + "group": 5, + "latest_source_mass": 0.19532112777233124, + "layer": 17, + "max_source_mass": 0.20404377579689026, + "mixer_index": 32, + "normalized_entropy": 0.999915135729072, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20157882571220398, + "median": 0.20241472125053406, + "p05": 0.196755051612854, + "p95": 0.20509284734725952 + }, + { + "label": "completed_group_1", + "mean": 0.20404377579689026, + "median": 0.20405352115631104, + "p05": 0.2024468630552292, + "p95": 0.20564454793930054 + }, + { + "label": "completed_group_2", + "mean": 0.20028471946716309, + "median": 0.20018786191940308, + "p05": 0.19867847859859467, + "p95": 0.20205119252204895 + }, + { + "label": "completed_group_3", + "mean": 0.19877153635025024, + "median": 0.19872286915779114, + "p05": 0.19718100130558014, + "p95": 0.20058035850524902 + }, + { + "label": "completed_group_4", + "mean": 0.19532112777233124, + "median": 0.1952555775642395, + "p05": 0.19347216188907623, + "p95": 0.19731438159942627 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.79081130027771, + "group": 5, + "latest_source_mass": 0.16283263266086578, + "layer": 17, + "max_source_mass": 0.17698298394680023, + "mixer_index": 33, + "normalized_entropy": 0.9994708168330465, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1695082038640976, + "median": 0.17041431367397308, + "p05": 0.1590532660484314, + "p95": 0.17703557014465332 + }, + { + "label": "completed_group_1", + "mean": 0.17698298394680023, + "median": 0.17722181975841522, + "p05": 0.17372196912765503, + "p95": 0.1802559494972229 + }, + { + "label": "completed_group_2", + "mean": 0.17063109576702118, + "median": 0.17032179236412048, + "p05": 0.1679578572511673, + "p95": 0.173906147480011 + }, + { + "label": "completed_group_3", + "mean": 0.16265574097633362, + "median": 0.162393718957901, + "p05": 0.1576516330242157, + "p95": 0.16904693841934204 + }, + { + "label": "completed_group_4", + "mean": 0.15738935768604279, + "median": 0.15682470798492432, + "p05": 0.15282297134399414, + "p95": 0.16380727291107178 + }, + { + "label": "current_group_partial", + "mean": 0.16283263266086578, + "median": 0.16261553764343262, + "p05": 0.1596994698047638, + "p95": 0.16662785410881042 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917133569717407, + "group": 5, + "latest_source_mass": 0.16537019610404968, + "layer": 18, + "max_source_mass": 0.1689736545085907, + "mixer_index": 34, + "normalized_entropy": 0.9999742642597368, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16738522052764893, + "median": 0.1673279106616974, + "p05": 0.16558583080768585, + "p95": 0.1691284030675888 + }, + { + "label": "completed_group_1", + "mean": 0.1689736545085907, + "median": 0.1689835786819458, + "p05": 0.1681547313928604, + "p95": 0.1699315309524536 + }, + { + "label": "completed_group_2", + "mean": 0.16775795817375183, + "median": 0.167745441198349, + "p05": 0.16722241044044495, + "p95": 0.16836081445217133 + }, + { + "label": "completed_group_3", + "mean": 0.16576024889945984, + "median": 0.16582953929901123, + "p05": 0.1648658663034439, + "p95": 0.16642220318317413 + }, + { + "label": "completed_group_4", + "mean": 0.16475272178649902, + "median": 0.16466641426086426, + "p05": 0.16412994265556335, + "p95": 0.16559217870235443 + }, + { + "label": "current_group_partial", + "mean": 0.16537019610404968, + "median": 0.16532129049301147, + "p05": 0.16465029120445251, + "p95": 0.16635482013225555 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7902145385742188, + "group": 5, + "latest_source_mass": 0.15478186309337616, + "layer": 18, + "max_source_mass": 0.18038460612297058, + "mixer_index": 35, + "normalized_entropy": 0.9991377577848093, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17057639360427856, + "median": 0.17081734538078308, + "p05": 0.16390566527843475, + "p95": 0.17745791375637054 + }, + { + "label": "completed_group_1", + "mean": 0.18038460612297058, + "median": 0.18108245730400085, + "p05": 0.17430555820465088, + "p95": 0.18539397418498993 + }, + { + "label": "completed_group_2", + "mean": 0.17244961857795715, + "median": 0.17250855267047882, + "p05": 0.17035990953445435, + "p95": 0.17447513341903687 + }, + { + "label": "completed_group_3", + "mean": 0.164292573928833, + "median": 0.16445159912109375, + "p05": 0.16052639484405518, + "p95": 0.167647585272789 + }, + { + "label": "completed_group_4", + "mean": 0.15751495957374573, + "median": 0.1570037603378296, + "p05": 0.1535545289516449, + "p95": 0.16239827871322632 + }, + { + "label": "current_group_partial", + "mean": 0.15478186309337616, + "median": 0.1544024646282196, + "p05": 0.15216419100761414, + "p95": 0.15823917090892792 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917354106903076, + "group": 5, + "latest_source_mass": 0.165629044175148, + "layer": 19, + "max_source_mass": 0.16850554943084717, + "mixer_index": 36, + "normalized_entropy": 0.9999865726744239, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16660895943641663, + "median": 0.1668481081724167, + "p05": 0.1639198660850525, + "p95": 0.16857405006885529 + }, + { + "label": "completed_group_1", + "mean": 0.16850554943084717, + "median": 0.16847547888755798, + "p05": 0.16779136657714844, + "p95": 0.16932576894760132 + }, + { + "label": "completed_group_2", + "mean": 0.1666441708803177, + "median": 0.1666129231452942, + "p05": 0.1657959520816803, + "p95": 0.16746148467063904 + }, + { + "label": "completed_group_3", + "mean": 0.16635674238204956, + "median": 0.1662396490573883, + "p05": 0.16566412150859833, + "p95": 0.16728293895721436 + }, + { + "label": "completed_group_4", + "mean": 0.16625551879405975, + "median": 0.16612455248832703, + "p05": 0.1655116081237793, + "p95": 0.16721341013908386 + }, + { + "label": "current_group_partial", + "mean": 0.165629044175148, + "median": 0.16562071442604065, + "p05": 0.16496792435646057, + "p95": 0.16613325476646423 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7898752689361572, + "group": 5, + "latest_source_mass": 0.15259043872356415, + "layer": 19, + "max_source_mass": 0.1802286058664322, + "mixer_index": 37, + "normalized_entropy": 0.9989484077945409, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17184747755527496, + "median": 0.171621635556221, + "p05": 0.16667789220809937, + "p95": 0.17803548276424408 + }, + { + "label": "completed_group_1", + "mean": 0.1802286058664322, + "median": 0.1802484691143036, + "p05": 0.17482617497444153, + "p95": 0.18618299067020416 + }, + { + "label": "completed_group_2", + "mean": 0.17470473051071167, + "median": 0.17439083755016327, + "p05": 0.17253704369068146, + "p95": 0.17793725430965424 + }, + { + "label": "completed_group_3", + "mean": 0.16429948806762695, + "median": 0.16437724232673645, + "p05": 0.16081778705120087, + "p95": 0.1669623851776123 + }, + { + "label": "completed_group_4", + "mean": 0.15632927417755127, + "median": 0.15611344575881958, + "p05": 0.15268553793430328, + "p95": 0.16074182093143463 + }, + { + "label": "current_group_partial", + "mean": 0.15259043872356415, + "median": 0.15249013900756836, + "p05": 0.14960677921772003, + "p95": 0.1559593677520752 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917178869247437, + "group": 5, + "latest_source_mass": 0.16608723998069763, + "layer": 20, + "max_source_mass": 0.16851314902305603, + "mixer_index": 38, + "normalized_entropy": 0.9999767924746454, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1659121811389923, + "median": 0.1657598316669464, + "p05": 0.16222882270812988, + "p95": 0.16958965361118317 + }, + { + "label": "completed_group_1", + "mean": 0.16851314902305603, + "median": 0.16853493452072144, + "p05": 0.1675589680671692, + "p95": 0.1694892942905426 + }, + { + "label": "completed_group_2", + "mean": 0.16693057119846344, + "median": 0.16706672310829163, + "p05": 0.1656341552734375, + "p95": 0.16794949769973755 + }, + { + "label": "completed_group_3", + "mean": 0.16679996252059937, + "median": 0.16667509078979492, + "p05": 0.16564559936523438, + "p95": 0.16818198561668396 + }, + { + "label": "completed_group_4", + "mean": 0.16575691103935242, + "median": 0.16555160284042358, + "p05": 0.16459843516349792, + "p95": 0.16740773618221283 + }, + { + "label": "current_group_partial", + "mean": 0.16608723998069763, + "median": 0.1660577356815338, + "p05": 0.16539554297924042, + "p95": 0.1667378544807434 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7883799076080322, + "group": 5, + "latest_source_mass": 0.1484336256980896, + "layer": 20, + "max_source_mass": 0.18512636423110962, + "mixer_index": 39, + "normalized_entropy": 0.9981138307467806, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1728566437959671, + "median": 0.17455406486988068, + "p05": 0.1564135104417801, + "p95": 0.1859586089849472 + }, + { + "label": "completed_group_1", + "mean": 0.18512636423110962, + "median": 0.18562474846839905, + "p05": 0.1771477460861206, + "p95": 0.1933506578207016 + }, + { + "label": "completed_group_2", + "mean": 0.17585304379463196, + "median": 0.17591087520122528, + "p05": 0.1723366677761078, + "p95": 0.17899538576602936 + }, + { + "label": "completed_group_3", + "mean": 0.16398006677627563, + "median": 0.16363000869750977, + "p05": 0.15788918733596802, + "p95": 0.17107567191123962 + }, + { + "label": "completed_group_4", + "mean": 0.1537502557039261, + "median": 0.1523861587047577, + "p05": 0.1478816270828247, + "p95": 0.16301551461219788 + }, + { + "label": "current_group_partial", + "mean": 0.1484336256980896, + "median": 0.14806321263313293, + "p05": 0.1446857899427414, + "p95": 0.1533387005329132 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917424440383911, + "group": 6, + "latest_source_mass": 0.16565825045108795, + "layer": 21, + "max_source_mass": 0.1680906116962433, + "mixer_index": 40, + "normalized_entropy": 0.9999904980607296, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1667865514755249, + "median": 0.16677325963974, + "p05": 0.16519494354724884, + "p95": 0.1684957593679428 + }, + { + "label": "completed_group_1", + "mean": 0.1680906116962433, + "median": 0.16817617416381836, + "p05": 0.16728028655052185, + "p95": 0.16889086365699768 + }, + { + "label": "completed_group_2", + "mean": 0.16619735956192017, + "median": 0.1661883294582367, + "p05": 0.1652708500623703, + "p95": 0.1671273112297058 + }, + { + "label": "completed_group_3", + "mean": 0.16709285974502563, + "median": 0.16709713637828827, + "p05": 0.1664952039718628, + "p95": 0.16763292253017426 + }, + { + "label": "completed_group_4", + "mean": 0.16617438197135925, + "median": 0.16625460982322693, + "p05": 0.16550303995609283, + "p95": 0.1667831838130951 + }, + { + "label": "completed_group_5", + "mean": 0.16565825045108795, + "median": 0.16577762365341187, + "p05": 0.16497491300106049, + "p95": 0.16629165410995483 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9426109790802002, + "group": 6, + "latest_source_mass": 0.12981724739074707, + "layer": 21, + "max_source_mass": 0.16000477969646454, + "mixer_index": 41, + "normalized_entropy": 0.9983045620185934, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14892546832561493, + "median": 0.14995476603507996, + "p05": 0.1375032216310501, + "p95": 0.158592090010643 + }, + { + "label": "completed_group_1", + "mean": 0.16000477969646454, + "median": 0.16008985042572021, + "p05": 0.153324156999588, + "p95": 0.16691897809505463 + }, + { + "label": "completed_group_2", + "mean": 0.15354624390602112, + "median": 0.1537785828113556, + "p05": 0.14996647834777832, + "p95": 0.1563865840435028 + }, + { + "label": "completed_group_3", + "mean": 0.14328594505786896, + "median": 0.14265790581703186, + "p05": 0.13877804577350616, + "p95": 0.1482013314962387 + }, + { + "label": "completed_group_4", + "mean": 0.13467344641685486, + "median": 0.13396033644676208, + "p05": 0.13011276721954346, + "p95": 0.14165091514587402 + }, + { + "label": "completed_group_5", + "mean": 0.12974688410758972, + "median": 0.12941759824752808, + "p05": 0.1261180341243744, + "p95": 0.13459308445453644 + }, + { + "label": "current_group_partial", + "mean": 0.12981724739074707, + "median": 0.12967640161514282, + "p05": 0.1270051896572113, + "p95": 0.13326594233512878 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458978176116943, + "group": 6, + "latest_source_mass": 0.143035888671875, + "layer": 22, + "max_source_mass": 0.1434222161769867, + "mixer_index": 42, + "normalized_entropy": 0.9999936628915652, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1419999748468399, + "median": 0.1419375240802765, + "p05": 0.14045022428035736, + "p95": 0.14377202093601227 + }, + { + "label": "completed_group_1", + "mean": 0.1434222161769867, + "median": 0.14351919293403625, + "p05": 0.14237116277217865, + "p95": 0.1442951261997223 + }, + { + "label": "completed_group_2", + "mean": 0.14279979467391968, + "median": 0.1427600383758545, + "p05": 0.14219357073307037, + "p95": 0.14337173104286194 + }, + { + "label": "completed_group_3", + "mean": 0.14312142133712769, + "median": 0.14313778281211853, + "p05": 0.14236494898796082, + "p95": 0.14370372891426086 + }, + { + "label": "completed_group_4", + "mean": 0.14279043674468994, + "median": 0.14272844791412354, + "p05": 0.14216938614845276, + "p95": 0.14356710016727448 + }, + { + "label": "completed_group_5", + "mean": 0.1428302675485611, + "median": 0.14281237125396729, + "p05": 0.1421329379081726, + "p95": 0.1434640884399414 + }, + { + "label": "current_group_partial", + "mean": 0.143035888671875, + "median": 0.14297914505004883, + "p05": 0.14242705702781677, + "p95": 0.14364232122898102 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9415357112884521, + "group": 6, + "latest_source_mass": 0.1259511113166809, + "layer": 22, + "max_source_mass": 0.16313126683235168, + "mixer_index": 43, + "normalized_entropy": 0.9977519836828105, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1496729701757431, + "median": 0.15015238523483276, + "p05": 0.13863591849803925, + "p95": 0.15954872965812683 + }, + { + "label": "completed_group_1", + "mean": 0.16313126683235168, + "median": 0.163755863904953, + "p05": 0.15348099172115326, + "p95": 0.17192509770393372 + }, + { + "label": "completed_group_2", + "mean": 0.15414540469646454, + "median": 0.1541890650987625, + "p05": 0.1515030711889267, + "p95": 0.15697401762008667 + }, + { + "label": "completed_group_3", + "mean": 0.1440294086933136, + "median": 0.14383843541145325, + "p05": 0.13977289199829102, + "p95": 0.14864303171634674 + }, + { + "label": "completed_group_4", + "mean": 0.1343868225812912, + "median": 0.13351139426231384, + "p05": 0.12925155460834503, + "p95": 0.1419946253299713 + }, + { + "label": "completed_group_5", + "mean": 0.12868301570415497, + "median": 0.12839242815971375, + "p05": 0.12411807477474213, + "p95": 0.13453525304794312 + }, + { + "label": "current_group_partial", + "mean": 0.1259511113166809, + "median": 0.1258702129125595, + "p05": 0.12331026792526245, + "p95": 0.12794387340545654 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458801746368408, + "group": 6, + "latest_source_mass": 0.14443498849868774, + "layer": 23, + "max_source_mass": 0.14443498849868774, + "mixer_index": 44, + "normalized_entropy": 0.9999845961960335, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1429852843284607, + "median": 0.1430346667766571, + "p05": 0.14120227098464966, + "p95": 0.1445307582616806 + }, + { + "label": "completed_group_1", + "mean": 0.1429750919342041, + "median": 0.14302998781204224, + "p05": 0.14226141571998596, + "p95": 0.14353078603744507 + }, + { + "label": "completed_group_2", + "mean": 0.14214137196540833, + "median": 0.14214855432510376, + "p05": 0.14139141142368317, + "p95": 0.14300128817558289 + }, + { + "label": "completed_group_3", + "mean": 0.1414642482995987, + "median": 0.14139702916145325, + "p05": 0.14092163741588593, + "p95": 0.14213398098945618 + }, + { + "label": "completed_group_4", + "mean": 0.1421438455581665, + "median": 0.14223212003707886, + "p05": 0.141431987285614, + "p95": 0.14265216886997223 + }, + { + "label": "completed_group_5", + "mean": 0.14385518431663513, + "median": 0.1438562422990799, + "p05": 0.14321480691432953, + "p95": 0.14455664157867432 + }, + { + "label": "current_group_partial", + "mean": 0.14443498849868774, + "median": 0.14441588521003723, + "p05": 0.14399737119674683, + "p95": 0.1448911726474762 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9424511194229126, + "group": 6, + "latest_source_mass": 0.12740188837051392, + "layer": 23, + "max_source_mass": 0.16045089066028595, + "mixer_index": 45, + "normalized_entropy": 0.9982224104057015, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14928652346134186, + "median": 0.1492544412612915, + "p05": 0.14078925549983978, + "p95": 0.15894511342048645 + }, + { + "label": "completed_group_1", + "mean": 0.16045089066028595, + "median": 0.16077518463134766, + "p05": 0.15384821593761444, + "p95": 0.16740038990974426 + }, + { + "label": "completed_group_2", + "mean": 0.15314346551895142, + "median": 0.1533186435699463, + "p05": 0.15041443705558777, + "p95": 0.15549692511558533 + }, + { + "label": "completed_group_3", + "mean": 0.1440504640340805, + "median": 0.14415611326694489, + "p05": 0.13973577320575714, + "p95": 0.1474868506193161 + }, + { + "label": "completed_group_4", + "mean": 0.1356543004512787, + "median": 0.13497784733772278, + "p05": 0.13136392831802368, + "p95": 0.14105865359306335 + }, + { + "label": "completed_group_5", + "mean": 0.13001246750354767, + "median": 0.13024744391441345, + "p05": 0.126283660531044, + "p95": 0.1334703266620636 + }, + { + "label": "current_group_partial", + "mean": 0.12740188837051392, + "median": 0.1274060606956482, + "p05": 0.12529851496219635, + "p95": 0.12901271879673004 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458882808685303, + "group": 6, + "latest_source_mass": 0.1432376205921173, + "layer": 24, + "max_source_mass": 0.14391477406024933, + "mixer_index": 46, + "normalized_entropy": 0.9999887619750616, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14319148659706116, + "median": 0.14321468770503998, + "p05": 0.14192207157611847, + "p95": 0.1444668471813202 + }, + { + "label": "completed_group_1", + "mean": 0.14391477406024933, + "median": 0.1440032571554184, + "p05": 0.14299027621746063, + "p95": 0.14467763900756836 + }, + { + "label": "completed_group_2", + "mean": 0.14284169673919678, + "median": 0.14285600185394287, + "p05": 0.14201918244361877, + "p95": 0.1436670869588852 + }, + { + "label": "completed_group_3", + "mean": 0.1420421302318573, + "median": 0.1421079933643341, + "p05": 0.1414509415626526, + "p95": 0.14248287677764893 + }, + { + "label": "completed_group_4", + "mean": 0.14134714007377625, + "median": 0.14131714403629303, + "p05": 0.14094975590705872, + "p95": 0.14189478754997253 + }, + { + "label": "completed_group_5", + "mean": 0.14342516660690308, + "median": 0.14335885643959045, + "p05": 0.14284199476242065, + "p95": 0.14415055513381958 + }, + { + "label": "current_group_partial", + "mean": 0.1432376205921173, + "median": 0.14320111274719238, + "p05": 0.14274808764457703, + "p95": 0.1438399851322174 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9395697116851807, + "group": 6, + "latest_source_mass": 0.12089881300926208, + "layer": 24, + "max_source_mass": 0.16719450056552887, + "mixer_index": 47, + "normalized_entropy": 0.9967416597455897, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15016257762908936, + "median": 0.15012264251708984, + "p05": 0.13872724771499634, + "p95": 0.1616549789905548 + }, + { + "label": "completed_group_1", + "mean": 0.16719450056552887, + "median": 0.1678844839334488, + "p05": 0.15553203225135803, + "p95": 0.1781744658946991 + }, + { + "label": "completed_group_2", + "mean": 0.15599417686462402, + "median": 0.15601703524589539, + "p05": 0.15278863906860352, + "p95": 0.15932069718837738 + }, + { + "label": "completed_group_3", + "mean": 0.14574658870697021, + "median": 0.14581912755966187, + "p05": 0.14079399406909943, + "p95": 0.15032999217510223 + }, + { + "label": "completed_group_4", + "mean": 0.13377559185028076, + "median": 0.1329595148563385, + "p05": 0.12759888172149658, + "p95": 0.14201229810714722 + }, + { + "label": "completed_group_5", + "mean": 0.12622776627540588, + "median": 0.12609627842903137, + "p05": 0.1209360659122467, + "p95": 0.13270525634288788 + }, + { + "label": "current_group_partial", + "mean": 0.12089881300926208, + "median": 0.1206001490354538, + "p05": 0.1178487241268158, + "p95": 0.12392976135015488 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458978176116943, + "group": 7, + "latest_source_mass": 0.14348836243152618, + "layer": 25, + "max_source_mass": 0.14348836243152618, + "mixer_index": 48, + "normalized_entropy": 0.9999936628915652, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428530067205429, + "median": 0.14293229579925537, + "p05": 0.14151178300380707, + "p95": 0.14395615458488464 + }, + { + "label": "completed_group_1", + "mean": 0.14342239499092102, + "median": 0.1434272974729538, + "p05": 0.14267732203006744, + "p95": 0.14410075545310974 + }, + { + "label": "completed_group_2", + "mean": 0.14335417747497559, + "median": 0.14335879683494568, + "p05": 0.1428845077753067, + "p95": 0.14383281767368317 + }, + { + "label": "completed_group_3", + "mean": 0.14274315536022186, + "median": 0.14269700646400452, + "p05": 0.1423313021659851, + "p95": 0.14328503608703613 + }, + { + "label": "completed_group_4", + "mean": 0.1419316530227661, + "median": 0.14191937446594238, + "p05": 0.14147570729255676, + "p95": 0.14250396192073822 + }, + { + "label": "completed_group_5", + "mean": 0.14220724999904633, + "median": 0.14221754670143127, + "p05": 0.14166903495788574, + "p95": 0.1429031491279602 + }, + { + "label": "completed_group_6", + "mean": 0.14348836243152618, + "median": 0.14348852634429932, + "p05": 0.14303231239318848, + "p95": 0.14401927590370178 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.074862480163574, + "group": 7, + "latest_source_mass": 0.11496111750602722, + "layer": 25, + "max_source_mass": 0.14486080408096313, + "mixer_index": 49, + "normalized_entropy": 0.997797936886188, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13159659504890442, + "median": 0.13219013810157776, + "p05": 0.12017668038606644, + "p95": 0.14146660268306732 + }, + { + "label": "completed_group_1", + "mean": 0.14486080408096313, + "median": 0.14564016461372375, + "p05": 0.13528235256671906, + "p95": 0.15322431921958923 + }, + { + "label": "completed_group_2", + "mean": 0.13573890924453735, + "median": 0.1359761655330658, + "p05": 0.13284167647361755, + "p95": 0.13837191462516785 + }, + { + "label": "completed_group_3", + "mean": 0.12842535972595215, + "median": 0.1282735913991928, + "p05": 0.12434759736061096, + "p95": 0.13281521201133728 + }, + { + "label": "completed_group_4", + "mean": 0.12030278891324997, + "median": 0.12004269659519196, + "p05": 0.11514662206172943, + "p95": 0.12723737955093384 + }, + { + "label": "completed_group_5", + "mean": 0.11495959758758545, + "median": 0.11503666639328003, + "p05": 0.11052325367927551, + "p95": 0.12032671272754669 + }, + { + "label": "completed_group_6", + "mean": 0.1091548278927803, + "median": 0.1089196503162384, + "p05": 0.10650476068258286, + "p95": 0.11220317333936691 + }, + { + "label": "current_group_partial", + "mean": 0.11496111750602722, + "median": 0.11521677672863007, + "p05": 0.11175674200057983, + "p95": 0.11837965995073318 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079418659210205, + "group": 7, + "latest_source_mass": 0.12366288155317307, + "layer": 26, + "max_source_mass": 0.1261361837387085, + "mixer_index": 50, + "normalized_entropy": 0.9999889958581801, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12459750473499298, + "median": 0.12467166036367416, + "p05": 0.12324866652488708, + "p95": 0.12578506767749786 + }, + { + "label": "completed_group_1", + "mean": 0.12504151463508606, + "median": 0.12500853836536407, + "p05": 0.12452991306781769, + "p95": 0.12561221420764923 + }, + { + "label": "completed_group_2", + "mean": 0.12489250302314758, + "median": 0.12485124170780182, + "p05": 0.12434293329715729, + "p95": 0.1255818009376526 + }, + { + "label": "completed_group_3", + "mean": 0.1250421404838562, + "median": 0.12497402727603912, + "p05": 0.12459105253219604, + "p95": 0.12559659779071808 + }, + { + "label": "completed_group_4", + "mean": 0.12461285293102264, + "median": 0.12454816699028015, + "p05": 0.12421505898237228, + "p95": 0.1250859797000885 + }, + { + "label": "completed_group_5", + "mean": 0.12601442635059357, + "median": 0.12595048546791077, + "p05": 0.1256113201379776, + "p95": 0.12648876011371613 + }, + { + "label": "completed_group_6", + "mean": 0.1261361837387085, + "median": 0.1261894702911377, + "p05": 0.1256546974182129, + "p95": 0.12649866938591003 + }, + { + "label": "current_group_partial", + "mean": 0.12366288155317307, + "median": 0.12370622158050537, + "p05": 0.12311097979545593, + "p95": 0.12407678365707397 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0741164684295654, + "group": 7, + "latest_source_mass": 0.11192375421524048, + "layer": 26, + "max_source_mass": 0.14626389741897583, + "mixer_index": 51, + "normalized_entropy": 0.9974391810764882, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1323997974395752, + "median": 0.1329696774482727, + "p05": 0.12004873901605606, + "p95": 0.14371390640735626 + }, + { + "label": "completed_group_1", + "mean": 0.14626389741897583, + "median": 0.1470441222190857, + "p05": 0.1354091465473175, + "p95": 0.1561085730791092 + }, + { + "label": "completed_group_2", + "mean": 0.13691633939743042, + "median": 0.13697296380996704, + "p05": 0.13360059261322021, + "p95": 0.14020705223083496 + }, + { + "label": "completed_group_3", + "mean": 0.12836162745952606, + "median": 0.12827181816101074, + "p05": 0.12401524186134338, + "p95": 0.13266678154468536 + }, + { + "label": "completed_group_4", + "mean": 0.12002508342266083, + "median": 0.11955710500478745, + "p05": 0.11444184184074402, + "p95": 0.12734319269657135 + }, + { + "label": "completed_group_5", + "mean": 0.11502262204885483, + "median": 0.11501411348581314, + "p05": 0.1104419082403183, + "p95": 0.12084008753299713 + }, + { + "label": "completed_group_6", + "mean": 0.10908685624599457, + "median": 0.10856537520885468, + "p05": 0.10614658892154694, + "p95": 0.11223960667848587 + }, + { + "label": "current_group_partial", + "mean": 0.11192375421524048, + "median": 0.11169644445180893, + "p05": 0.10886920988559723, + "p95": 0.11444390565156937 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0793213844299316, + "group": 7, + "latest_source_mass": 0.12797673046588898, + "layer": 27, + "max_source_mass": 0.12797673046588898, + "mixer_index": 52, + "normalized_entropy": 0.9999422165771455, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12429114431142807, + "median": 0.12408939003944397, + "p05": 0.12283186614513397, + "p95": 0.12610697746276855 + }, + { + "label": "completed_group_1", + "mean": 0.12243181467056274, + "median": 0.12224709987640381, + "p05": 0.12128561735153198, + "p95": 0.12399756163358688 + }, + { + "label": "completed_group_2", + "mean": 0.12357296049594879, + "median": 0.12363500148057938, + "p05": 0.12287977337837219, + "p95": 0.1240825355052948 + }, + { + "label": "completed_group_3", + "mean": 0.12375843524932861, + "median": 0.12379264831542969, + "p05": 0.1232200488448143, + "p95": 0.12428808957338333 + }, + { + "label": "completed_group_4", + "mean": 0.12469147890806198, + "median": 0.12474392354488373, + "p05": 0.1235789805650711, + "p95": 0.12562167644500732 + }, + { + "label": "completed_group_5", + "mean": 0.12560917437076569, + "median": 0.12565192580223083, + "p05": 0.12473294138908386, + "p95": 0.12630751729011536 + }, + { + "label": "completed_group_6", + "mean": 0.12766827642917633, + "median": 0.12772443890571594, + "p05": 0.12707561254501343, + "p95": 0.12822651863098145 + }, + { + "label": "current_group_partial", + "mean": 0.12797673046588898, + "median": 0.12799106538295746, + "p05": 0.1274825632572174, + "p95": 0.12843307852745056 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0734808444976807, + "group": 7, + "latest_source_mass": 0.10855817049741745, + "layer": 27, + "max_source_mass": 0.14781729876995087, + "mixer_index": 53, + "normalized_entropy": 0.9971335105783546, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1330580711364746, + "median": 0.13408532738685608, + "p05": 0.12270413339138031, + "p95": 0.14177323877811432 + }, + { + "label": "completed_group_1", + "mean": 0.14781729876995087, + "median": 0.14838829636573792, + "p05": 0.13672368228435516, + "p95": 0.15774855017662048 + }, + { + "label": "completed_group_2", + "mean": 0.1374434530735016, + "median": 0.1373273879289627, + "p05": 0.13421031832695007, + "p95": 0.140645369887352 + }, + { + "label": "completed_group_3", + "mean": 0.12885303795337677, + "median": 0.12835927307605743, + "p05": 0.12514662742614746, + "p95": 0.1329982876777649 + }, + { + "label": "completed_group_4", + "mean": 0.11949820816516876, + "median": 0.11871285736560822, + "p05": 0.11462017893791199, + "p95": 0.12620851397514343 + }, + { + "label": "completed_group_5", + "mean": 0.11495119333267212, + "median": 0.11446745693683624, + "p05": 0.11059992760419846, + "p95": 0.12061949074268341 + }, + { + "label": "completed_group_6", + "mean": 0.10982058197259903, + "median": 0.1093798577785492, + "p05": 0.1069897711277008, + "p95": 0.112794429063797 + }, + { + "label": "current_group_partial", + "mean": 0.10855817049741745, + "median": 0.10835569351911545, + "p05": 0.10530182719230652, + "p95": 0.11158856004476547 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.07940673828125, + "group": 7, + "latest_source_mass": 0.12688271701335907, + "layer": 28, + "max_source_mass": 0.12688271701335907, + "mixer_index": 54, + "normalized_entropy": 0.9999832631031514, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12457560002803802, + "median": 0.12460930645465851, + "p05": 0.12360189855098724, + "p95": 0.1253867745399475 + }, + { + "label": "completed_group_1", + "mean": 0.12431882321834564, + "median": 0.12448525428771973, + "p05": 0.12329473346471786, + "p95": 0.12526866793632507 + }, + { + "label": "completed_group_2", + "mean": 0.12479399144649506, + "median": 0.12486443668603897, + "p05": 0.12387760728597641, + "p95": 0.1253962516784668 + }, + { + "label": "completed_group_3", + "mean": 0.12437158077955246, + "median": 0.12438835948705673, + "p05": 0.1238550916314125, + "p95": 0.12478173524141312 + }, + { + "label": "completed_group_4", + "mean": 0.12397614121437073, + "median": 0.12399939447641373, + "p05": 0.12361378967761993, + "p95": 0.12437361478805542 + }, + { + "label": "completed_group_5", + "mean": 0.12488298863172531, + "median": 0.12472546845674515, + "p05": 0.12454084306955338, + "p95": 0.12553870677947998 + }, + { + "label": "completed_group_6", + "mean": 0.12619814276695251, + "median": 0.12612387537956238, + "p05": 0.12555824220180511, + "p95": 0.12703651189804077 + }, + { + "label": "current_group_partial", + "mean": 0.12688271701335907, + "median": 0.12681353092193604, + "p05": 0.1262284815311432, + "p95": 0.12773586809635162 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.07299542427063, + "group": 7, + "latest_source_mass": 0.10567984730005264, + "layer": 28, + "max_source_mass": 0.14826728403568268, + "mixer_index": 55, + "normalized_entropy": 0.9969000727935836, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13274571299552917, + "median": 0.13371163606643677, + "p05": 0.12133019417524338, + "p95": 0.14282676577568054 + }, + { + "label": "completed_group_1", + "mean": 0.14826728403568268, + "median": 0.1491231620311737, + "p05": 0.1354067623615265, + "p95": 0.15911932289600372 + }, + { + "label": "completed_group_2", + "mean": 0.13701491057872772, + "median": 0.1368696093559265, + "p05": 0.13440042734146118, + "p95": 0.14018136262893677 + }, + { + "label": "completed_group_3", + "mean": 0.12918588519096375, + "median": 0.12887251377105713, + "p05": 0.12538790702819824, + "p95": 0.13307984173297882 + }, + { + "label": "completed_group_4", + "mean": 0.12125477194786072, + "median": 0.12069438397884369, + "p05": 0.11560913175344467, + "p95": 0.12839479744434357 + }, + { + "label": "completed_group_5", + "mean": 0.1169566810131073, + "median": 0.11671148240566254, + "p05": 0.11176177859306335, + "p95": 0.1237177848815918 + }, + { + "label": "completed_group_6", + "mean": 0.10889489948749542, + "median": 0.10865428298711777, + "p05": 0.10550769418478012, + "p95": 0.11299075931310654 + }, + { + "label": "current_group_partial", + "mean": 0.10567984730005264, + "median": 0.1053324043750763, + "p05": 0.10240665078163147, + "p95": 0.10906655341386795 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079298734664917, + "group": 8, + "latest_source_mass": 0.1289007067680359, + "layer": 29, + "max_source_mass": 0.1289007067680359, + "mixer_index": 56, + "normalized_entropy": 0.9999313243425908, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12391228973865509, + "median": 0.12379568815231323, + "p05": 0.12216563522815704, + "p95": 0.1260690689086914 + }, + { + "label": "completed_group_1", + "mean": 0.12222036719322205, + "median": 0.12197070568799973, + "p05": 0.12120269238948822, + "p95": 0.12389923632144928 + }, + { + "label": "completed_group_2", + "mean": 0.12298265099525452, + "median": 0.12302859127521515, + "p05": 0.12237658351659775, + "p95": 0.12344501912593842 + }, + { + "label": "completed_group_3", + "mean": 0.12416766583919525, + "median": 0.12414702773094177, + "p05": 0.12359493225812912, + "p95": 0.1248319149017334 + }, + { + "label": "completed_group_4", + "mean": 0.125657856464386, + "median": 0.1257546842098236, + "p05": 0.1244942918419838, + "p95": 0.12650099396705627 + }, + { + "label": "completed_group_5", + "mean": 0.12553520500659943, + "median": 0.1257142722606659, + "p05": 0.12441182136535645, + "p95": 0.12623384594917297 + }, + { + "label": "completed_group_6", + "mean": 0.126623272895813, + "median": 0.1266922652721405, + "p05": 0.1259365975856781, + "p95": 0.12718230485916138 + }, + { + "label": "completed_group_7", + "mean": 0.1289007067680359, + "median": 0.12883605062961578, + "p05": 0.1284138262271881, + "p95": 0.1295773684978485 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.189734935760498, + "group": 8, + "latest_source_mass": 0.09586615115404129, + "layer": 29, + "max_source_mass": 0.13466420769691467, + "mixer_index": 57, + "normalized_entropy": 0.9965913172222015, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1203557699918747, + "median": 0.12168776988983154, + "p05": 0.10881253331899643, + "p95": 0.1301186978816986 + }, + { + "label": "completed_group_1", + "mean": 0.13466420769691467, + "median": 0.13564801216125488, + "p05": 0.1213466078042984, + "p95": 0.14536651968955994 + }, + { + "label": "completed_group_2", + "mean": 0.12461012601852417, + "median": 0.12456075102090836, + "p05": 0.1213802695274353, + "p95": 0.12872551381587982 + }, + { + "label": "completed_group_3", + "mean": 0.11655421555042267, + "median": 0.11625538021326065, + "p05": 0.11300642788410187, + "p95": 0.12002266943454742 + }, + { + "label": "completed_group_4", + "mean": 0.10905998945236206, + "median": 0.10815867781639099, + "p05": 0.10389839112758636, + "p95": 0.11589331924915314 + }, + { + "label": "completed_group_5", + "mean": 0.10487955808639526, + "median": 0.10438832640647888, + "p05": 0.09981175512075424, + "p95": 0.11163480579853058 + }, + { + "label": "completed_group_6", + "mean": 0.09900732338428497, + "median": 0.09865365922451019, + "p05": 0.09571661055088043, + "p95": 0.10329514741897583 + }, + { + "label": "completed_group_7", + "mean": 0.0950026735663414, + "median": 0.09439435601234436, + "p05": 0.09206795692443848, + "p95": 0.09867438673973083 + }, + { + "label": "current_group_partial", + "mean": 0.09586615115404129, + "median": 0.0960603654384613, + "p05": 0.09171032160520554, + "p95": 0.09956925362348557 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.197068691253662, + "group": 8, + "latest_source_mass": 0.11299411952495575, + "layer": 30, + "max_source_mass": 0.11490922421216965, + "mixer_index": 58, + "normalized_entropy": 0.9999290531863855, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1099812239408493, + "median": 0.11007699370384216, + "p05": 0.10831384360790253, + "p95": 0.11145084351301193 + }, + { + "label": "completed_group_1", + "mean": 0.10853178799152374, + "median": 0.10820644348859787, + "p05": 0.10737297683954239, + "p95": 0.11033640801906586 + }, + { + "label": "completed_group_2", + "mean": 0.10949578881263733, + "median": 0.10946135222911835, + "p05": 0.10865518450737, + "p95": 0.11031430959701538 + }, + { + "label": "completed_group_3", + "mean": 0.11001666635274887, + "median": 0.11020137369632721, + "p05": 0.10912945121526718, + "p95": 0.11061263829469681 + }, + { + "label": "completed_group_4", + "mean": 0.1107073426246643, + "median": 0.11061955988407135, + "p05": 0.11008889228105545, + "p95": 0.11154733598232269 + }, + { + "label": "completed_group_5", + "mean": 0.11118210107088089, + "median": 0.11119748651981354, + "p05": 0.11032600700855255, + "p95": 0.11185683310031891 + }, + { + "label": "completed_group_6", + "mean": 0.11218173801898956, + "median": 0.11228874325752258, + "p05": 0.11149641126394272, + "p95": 0.11289148032665253 + }, + { + "label": "completed_group_7", + "mean": 0.11490922421216965, + "median": 0.11485384404659271, + "p05": 0.11418338119983673, + "p95": 0.1157391220331192 + }, + { + "label": "current_group_partial", + "mean": 0.11299411952495575, + "median": 0.11296258866786957, + "p05": 0.11219073832035065, + "p95": 0.1138962060213089 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1907877922058105, + "group": 8, + "latest_source_mass": 0.09503427892923355, + "layer": 30, + "max_source_mass": 0.13228625059127808, + "mixer_index": 59, + "normalized_entropy": 0.9970704928404667, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11948459595441818, + "median": 0.1209583580493927, + "p05": 0.10761230438947678, + "p95": 0.12862204015254974 + }, + { + "label": "completed_group_1", + "mean": 0.13228625059127808, + "median": 0.1333099603652954, + "p05": 0.12083890289068222, + "p95": 0.1419546753168106 + }, + { + "label": "completed_group_2", + "mean": 0.12348943948745728, + "median": 0.12335763871669769, + "p05": 0.11937301605939865, + "p95": 0.12711547315120697 + }, + { + "label": "completed_group_3", + "mean": 0.11669500172138214, + "median": 0.11618717014789581, + "p05": 0.11331704258918762, + "p95": 0.12055575847625732 + }, + { + "label": "completed_group_4", + "mean": 0.11027230322360992, + "median": 0.10946162790060043, + "p05": 0.10527626425027847, + "p95": 0.11702649295330048 + }, + { + "label": "completed_group_5", + "mean": 0.1054498553276062, + "median": 0.10493277758359909, + "p05": 0.10078853368759155, + "p95": 0.11155365407466888 + }, + { + "label": "completed_group_6", + "mean": 0.10031206905841827, + "median": 0.09992779791355133, + "p05": 0.09708504378795624, + "p95": 0.1038607582449913 + }, + { + "label": "completed_group_7", + "mean": 0.09697622060775757, + "median": 0.09662114828824997, + "p05": 0.09399978816509247, + "p95": 0.09983330965042114 + }, + { + "label": "current_group_partial", + "mean": 0.09503427892923355, + "median": 0.0953635647892952, + "p05": 0.09135627001523972, + "p95": 0.09895467758178711 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1971726417541504, + "group": 8, + "latest_source_mass": 0.11179381608963013, + "layer": 31, + "max_source_mass": 0.11264219880104065, + "mixer_index": 60, + "normalized_entropy": 0.9999763630979714, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11100216209888458, + "median": 0.11096040159463882, + "p05": 0.10992560535669327, + "p95": 0.1121426597237587 + }, + { + "label": "completed_group_1", + "mean": 0.11012112349271774, + "median": 0.11014193296432495, + "p05": 0.10946580022573471, + "p95": 0.110725536942482 + }, + { + "label": "completed_group_2", + "mean": 0.1099993959069252, + "median": 0.10994885116815567, + "p05": 0.10935711860656738, + "p95": 0.11085627228021622 + }, + { + "label": "completed_group_3", + "mean": 0.1100020483136177, + "median": 0.10996703803539276, + "p05": 0.1096094623208046, + "p95": 0.11043745279312134 + }, + { + "label": "completed_group_4", + "mean": 0.10997181385755539, + "median": 0.11000630259513855, + "p05": 0.10946355760097504, + "p95": 0.11040553450584412 + }, + { + "label": "completed_group_5", + "mean": 0.11260797083377838, + "median": 0.11259360611438751, + "p05": 0.11212003231048584, + "p95": 0.11315739899873734 + }, + { + "label": "completed_group_6", + "mean": 0.11264219880104065, + "median": 0.11264394223690033, + "p05": 0.11231723427772522, + "p95": 0.11297592520713806 + }, + { + "label": "completed_group_7", + "mean": 0.11185946315526962, + "median": 0.11186999082565308, + "p05": 0.11150093376636505, + "p95": 0.11219023168087006 + }, + { + "label": "current_group_partial", + "mean": 0.11179381608963013, + "median": 0.11179733276367188, + "p05": 0.11130586266517639, + "p95": 0.11215443909168243 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.190375804901123, + "group": 8, + "latest_source_mass": 0.09452830255031586, + "layer": 31, + "max_source_mass": 0.13351142406463623, + "mixer_index": 61, + "normalized_entropy": 0.9968829893376673, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11983220279216766, + "median": 0.12115231156349182, + "p05": 0.10900401324033737, + "p95": 0.12850891053676605 + }, + { + "label": "completed_group_1", + "mean": 0.13351142406463623, + "median": 0.1344168484210968, + "p05": 0.1215919628739357, + "p95": 0.14321453869342804 + }, + { + "label": "completed_group_2", + "mean": 0.12386792153120041, + "median": 0.123808853328228, + "p05": 0.1205662339925766, + "p95": 0.12756995856761932 + }, + { + "label": "completed_group_3", + "mean": 0.11615028977394104, + "median": 0.11574137210845947, + "p05": 0.11283998191356659, + "p95": 0.11950776726007462 + }, + { + "label": "completed_group_4", + "mean": 0.10967587679624557, + "median": 0.10882684588432312, + "p05": 0.10498950630426407, + "p95": 0.11603635549545288 + }, + { + "label": "completed_group_5", + "mean": 0.10584976524114609, + "median": 0.10532175004482269, + "p05": 0.10128854960203171, + "p95": 0.11199010908603668 + }, + { + "label": "completed_group_6", + "mean": 0.10062196850776672, + "median": 0.10019652545452118, + "p05": 0.09754259884357452, + "p95": 0.10435834527015686 + }, + { + "label": "completed_group_7", + "mean": 0.09596226364374161, + "median": 0.0954611748456955, + "p05": 0.09298955649137497, + "p95": 0.09913706034421921 + }, + { + "label": "current_group_partial", + "mean": 0.09452830255031586, + "median": 0.09470570832490921, + "p05": 0.09095818549394608, + "p95": 0.09791092574596405 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1969356536865234, + "group": 8, + "latest_source_mass": 0.11499489098787308, + "layer": 32, + "max_source_mass": 0.11499489098787308, + "mixer_index": 62, + "normalized_entropy": 0.9998685051802731, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10960094630718231, + "median": 0.10919953882694244, + "p05": 0.10751543194055557, + "p95": 0.11286121606826782 + }, + { + "label": "completed_group_1", + "mean": 0.10724945366382599, + "median": 0.1071096658706665, + "p05": 0.10547052323818207, + "p95": 0.10940680652856827 + }, + { + "label": "completed_group_2", + "mean": 0.10892991721630096, + "median": 0.10898078978061676, + "p05": 0.10812374949455261, + "p95": 0.10961546003818512 + }, + { + "label": "completed_group_3", + "mean": 0.1096237450838089, + "median": 0.1097160205245018, + "p05": 0.10855352133512497, + "p95": 0.11059606075286865 + }, + { + "label": "completed_group_4", + "mean": 0.11032412201166153, + "median": 0.11054199188947678, + "p05": 0.1088760644197464, + "p95": 0.11135599762201309 + }, + { + "label": "completed_group_5", + "mean": 0.11112386733293533, + "median": 0.11126254498958588, + "p05": 0.10977865755558014, + "p95": 0.11215295642614365 + }, + { + "label": "completed_group_6", + "mean": 0.11386257410049438, + "median": 0.11392036080360413, + "p05": 0.11291038990020752, + "p95": 0.11467544734477997 + }, + { + "label": "completed_group_7", + "mean": 0.11429048329591751, + "median": 0.11428040266036987, + "p05": 0.11370178312063217, + "p95": 0.11495206505060196 + }, + { + "label": "current_group_partial", + "mean": 0.11499489098787308, + "median": 0.11495819687843323, + "p05": 0.11425958573818207, + "p95": 0.1156669408082962 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.188795566558838, + "group": 8, + "latest_source_mass": 0.09158481657505035, + "layer": 32, + "max_source_mass": 0.13515985012054443, + "mixer_index": 63, + "normalized_entropy": 0.9961637918743834, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12045153975486755, + "median": 0.12207483500242233, + "p05": 0.10754543542861938, + "p95": 0.13057786226272583 + }, + { + "label": "completed_group_1", + "mean": 0.13515985012054443, + "median": 0.13634194433689117, + "p05": 0.12046528607606888, + "p95": 0.14655664563179016 + }, + { + "label": "completed_group_2", + "mean": 0.12505561113357544, + "median": 0.12510301172733307, + "p05": 0.12129884958267212, + "p95": 0.12965641915798187 + }, + { + "label": "completed_group_3", + "mean": 0.11722971498966217, + "median": 0.11687646061182022, + "p05": 0.11354094743728638, + "p95": 0.12092718482017517 + }, + { + "label": "completed_group_4", + "mean": 0.11039334535598755, + "median": 0.10915638506412506, + "p05": 0.1051320806145668, + "p95": 0.11790860444307327 + }, + { + "label": "completed_group_5", + "mean": 0.10568177700042725, + "median": 0.10491170734167099, + "p05": 0.10036594420671463, + "p95": 0.11337733268737793 + }, + { + "label": "completed_group_6", + "mean": 0.09951679408550262, + "median": 0.09897811710834503, + "p05": 0.09581314772367477, + "p95": 0.10455189645290375 + }, + { + "label": "completed_group_7", + "mean": 0.09492656588554382, + "median": 0.09424832463264465, + "p05": 0.09176740050315857, + "p95": 0.09912464767694473 + }, + { + "label": "current_group_partial", + "mean": 0.09158481657505035, + "median": 0.09171414375305176, + "p05": 0.087994284927845, + "p95": 0.09507401287555695 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1922647953033447, + "group": null, + "latest_source_mass": 0.12173722684383392, + "layer": null, + "max_source_mass": 0.12246126681566238, + "mixer_index": null, + "normalized_entropy": 0.9977427059190792, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09991010278463364, + "median": 0.09913325309753418, + "p05": 0.09100741893053055, + "p95": 0.1105685830116272 + }, + { + "label": "completed_group_1", + "mean": 0.0929517075419426, + "median": 0.09252890944480896, + "p05": 0.08571945130825043, + "p95": 0.09959838539361954 + }, + { + "label": "completed_group_2", + "mean": 0.10029369592666626, + "median": 0.10029015690088272, + "p05": 0.0971769392490387, + "p95": 0.10383395105600357 + }, + { + "label": "completed_group_3", + "mean": 0.10801893472671509, + "median": 0.10761593282222748, + "p05": 0.10540572553873062, + "p95": 0.1119115948677063 + }, + { + "label": "completed_group_4", + "mean": 0.11530699580907822, + "median": 0.11600182950496674, + "p05": 0.10986640304327011, + "p95": 0.11992081254720688 + }, + { + "label": "completed_group_5", + "mean": 0.11817377060651779, + "median": 0.11768905073404312, + "p05": 0.11345858126878738, + "p95": 0.1227451041340828 + }, + { + "label": "completed_group_6", + "mean": 0.12114628404378891, + "median": 0.12136594951152802, + "p05": 0.11863197386264801, + "p95": 0.12347076088190079 + }, + { + "label": "completed_group_7", + "mean": 0.12246126681566238, + "median": 0.12239877134561539, + "p05": 0.12033852934837341, + "p95": 0.12512150406837463 + }, + { + "label": "completed_group_8", + "mean": 0.12173722684383392, + "median": 0.12140394747257233, + "p05": 0.11953023076057434, + "p95": 0.12511639297008514 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "0e9274e8b27f225d1a13bd767f85857d08623357c1eea15d895fa78720bf942c", + "optimizer_state_before": "0e9274e8b27f225d1a13bd767f85857d08623357c1eea15d895fa78720bf942c", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5f9ceb4daf4a04de3e80e18a04e2c787acd0577d8e42d246715f712ea2c503dc", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.002723898272961378, + 0.0025367254856973886, + 0.0027546093333512545, + 0.002648916095495224, + 0.0027809101156890392, + 0.0025255572982132435, + 0.003397067543119192, + 0.0033452848438173532, + 0.003559867851436138, + 0.0038245979230850935, + 0.003386941971257329, + 0.00461281044408679, + 0.004887574352324009, + 0.005150536075234413, + 0.0054018739610910416, + 0.006125702988356352, + 0.005625482182949781, + 0.006412550341337919, + 0.0072620403952896595, + 0.008774910122156143, + 0.008495661430060863, + 0.007638709619641304, + 0.009623398073017597, + 0.008996927179396152, + 0.00961991585791111, + 0.011907265521585941, + 0.014190610498189926, + 0.013635334558784962, + 0.014384749345481396, + 0.01446358859539032, + 0.013604369945824146, + 0.015084062702953815 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.492743683684239e-05, + "first_to_last_ratio": 3.25801775252393, + "imbalance_abs_log_ratio": 1.1811189590673807, + "last_quartile_mean": 4.581754296850704e-06, + "mean": 8.43460854582645e-06, + "normalized": [ + 2.149273354999072, + 2.053516748208887, + 1.936844350800347, + 1.827189890602358, + 1.6952544178724434, + 1.5885589988048698, + 1.4979335388418822, + 1.4097022043150367, + 1.2877506109767536, + 1.206962383078768, + 1.1247429419702293, + 1.0539031177503007, + 0.9733757280016656, + 0.9100554611124945, + 0.8555710730539545, + 0.808590391475426, + 0.7636731240457068, + 0.7224612895073799, + 0.6861986404994214, + 0.6566175563965578, + 0.639518909427233, + 0.6187999065817441, + 0.6011551600640996, + 0.5866791280052756, + 0.5802231951082482, + 0.5655891936865475, + 0.5545366219021041, + 0.5449216791487539, + 0.534973976358655, + 0.5278906409101619, + 0.5216937669853605, + 0.515841999508263 + ], + "peak_layer": 1, + "peak_normalized": 2.149273354999072, + "population_cv": 0.5023641496635137, + "rest_mean": 9.051154595201499e-06, + "spike_contrast": 0.5640451530800011, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.105259879201185e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8128279407392256e-05, + 1.732060991344042e-05, + 1.6336523913196288e-05, + 1.5411631466122344e-05, + 1.4298807400336955e-05, + 1.3398873306869064e-05, + 1.2634483027795795e-05, + 1.1890286259585992e-05, + 1.0861672308237758e-05, + 1.0180255230807234e-05, + 9.486766430200078e-06, + 8.889260243449826e-06, + 8.21004323370289e-06, + 7.675961569475476e-06, + 7.216407084342791e-06, + 6.820143426011782e-06, + 6.4412838582939e-06, + 6.093678166507743e-06, + 5.787816917290911e-06, + 5.5383120525220875e-06, + 5.394091658672551e-06, + 5.219334980210988e-06, + 5.070508450444322e-06, + 4.9484087867313065e-06, + 4.893955519946758e-06, + 4.770523446495645e-06, + 4.677299330069218e-06, + 4.5962010517541785e-06, + 4.512296072789468e-06, + 4.452550911082653e-06, + 4.400282705319114e-06, + 4.350925337348599e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.9674976455426076e-05, + "first_to_last_ratio": 3.267906982188047, + "imbalance_abs_log_ratio": 1.1841497133699193, + "last_quartile_mean": 1.8260916476720013e-05, + "mean": 3.383400519396673e-05, + "normalized": [ + 2.1128677469499655, + 2.0289069422167634, + 1.9237165333750805, + 1.8241483568045902, + 1.696610198326043, + 1.5957581812315693, + 1.5065087922293456, + 1.421542312419023, + 1.2976047137215567, + 1.216584484419669, + 1.1353408195559018, + 1.064739590499697, + 0.982436165819818, + 0.9182427418043383, + 0.8622268120476726, + 0.8145227282026447, + 0.7677725464688829, + 0.7258005310001772, + 0.6882849152330417, + 0.657779832149612, + 0.6393622043713836, + 0.6177095995408469, + 0.5992999823253519, + 0.5844668394405714, + 0.5774961982840346, + 0.5625252092056405, + 0.5512933787722648, + 0.5414231827170685, + 0.5312049841613803, + 0.5240801531127671, + 0.517855677058088, + 0.5118876465352097 + ], + "peak_layer": 1, + "peak_normalized": 2.1128677469499655, + "population_cv": 0.49917871029499944, + "rest_mean": 3.631725222577289e-05, + "spike_contrast": 0.5623903233438747, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.042447122221347e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.148677832446992e-05, + 6.864604802103713e-05, + 6.508703518193215e-05, + 6.171824497869238e-05, + 5.740311826230027e-05, + 5.3990890592103824e-05, + 5.0971226301044226e-05, + 4.8096469981828704e-05, + 4.390316462377086e-05, + 4.1161925764754415e-05, + 3.841312718577683e-05, + 3.602440483518876e-05, + 3.323975033708848e-05, + 3.1067829695530236e-05, + 2.9172586437198333e-05, + 2.7558566216612235e-05, + 2.597682032501325e-05, + 2.4556738935643807e-05, + 2.3287435396923684e-05, + 2.2255326257436536e-05, + 2.163218414352741e-05, + 2.0899589799228124e-05, + 2.0276718714740127e-05, + 1.9774854081333615e-05, + 1.953900937223807e-05, + 1.9032480850000866e-05, + 1.8652463040780276e-05, + 1.8318514776183292e-05, + 1.7972792193177156e-05, + 1.773173062247224e-05, + 1.752113166730851e-05, + 1.731920929159969e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05292036524042487, + "first_to_last_ratio": 3.26790701098967, + "imbalance_abs_log_ratio": 1.1841497221833963, + "last_quartile_mean": 0.01619396300520748, + "mean": 0.030004334141267464, + "normalized": [ + 2.1128676597267897, + 2.0289069034480245, + 1.9237164254948962, + 1.8241483367966431, + 1.6966102148820104, + 1.595758204107866, + 1.5065087963694095, + 1.421542357002479, + 1.297604704701885, + 1.2165845161095536, + 1.135340842386519, + 1.0647396547459889, + 0.9824361648697959, + 0.9182427925601578, + 0.8622268653413654, + 0.8145227226086523, + 0.7677725509880283, + 0.7258005535410471, + 0.688284911679496, + 0.6577798121305034, + 0.6393622162716224, + 0.6177096110666057, + 0.5993000234249083, + 0.5844668186664959, + 0.577496193281449, + 0.5625251726924361, + 0.5512933687361069, + 0.5414231479385886, + 0.5312049732084057, + 0.5240801430831734, + 0.5178556786782627, + 0.5118876634608343 + ], + "peak_layer": 1, + "peak_normalized": 2.1128676597267897, + "population_cv": 0.49917869887275884, + "rest_mean": 0.03220650239813107, + "spike_contrast": 0.5623903313156743, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01811262555420399, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0633951872587204, + 0.06087600067257881, + 0.05771983042359352, + 0.05473235622048378, + 0.050905659794807434, + 0.0478796623647213, + 0.04520179331302643, + 0.0426524318754673, + 0.03893376514315605, + 0.03650280833244324, + 0.03406514599919319, + 0.031946804374456406, + 0.02947734296321869, + 0.027551263570785522, + 0.025870542973279953, + 0.024439211934804916, + 0.023036504164338112, + 0.021777162328362465, + 0.020651530474424362, + 0.019736245274543762, + 0.01918363757431507, + 0.018533965572714806, + 0.017981598153710365, + 0.01753653772175312, + 0.017327388748526573, + 0.016878193244338036, + 0.016541190445423126, + 0.0162450410425663, + 0.015938451513648033, + 0.015724675729870796, + 0.01553791482001543, + 0.015358848497271538 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.549642255824438e-06, + "first_to_last_ratio": 3.68048280550862, + "imbalance_abs_log_ratio": 1.3030439407243277, + "last_quartile_mean": 1.2361536505522963e-06, + "mean": 2.4386981998247848e-06, + "normalized": [ + 2.466847232801335, + 2.237590961217267, + 2.067896996004643, + 1.9400809535365267, + 1.7430260468181578, + 1.597805887165436, + 1.487469389543883, + 1.3841051490161118, + 1.2636941636501926, + 1.1797245287558995, + 1.1016637980695667, + 1.0304914123685331, + 0.9505007964227872, + 0.8872713444393213, + 0.8269839442149156, + 0.7815777037800856, + 0.7317613226304623, + 0.6893227271477795, + 0.652141132025827, + 0.6215326520683091, + 0.6024681473460322, + 0.5840809531927201, + 0.5656423861818819, + 0.5511940730185599, + 0.5440686297089605, + 0.529931304143241, + 0.5180806767693359, + 0.508433674599848, + 0.4992370473637191, + 0.4907378696025909, + 0.48494560322542063, + 0.4796914931706492 + ], + "peak_layer": 1, + "peak_normalized": 2.466847232801335, + "population_cv": 0.5676326111755923, + "rest_mean": 2.633120777351939e-06, + "spike_contrast": 0.527441161500707, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3888162811781512e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.015895905875368e-06, + 5.456809049064759e-06, + 5.042976681579603e-06, + 4.73127192890388e-06, + 4.2507144826231524e-06, + 3.896566340699792e-06, + 3.6274889225751394e-06, + 3.3754147352738073e-06, + 3.081768682022812e-06, + 2.8769920845661545e-06, + 2.6866255211643875e-06, + 2.513057552278042e-06, + 2.3179845811682753e-06, + 2.1637870304402895e-06, + 2.016764256040915e-06, + 1.9060321392316837e-06, + 1.7845450202003121e-06, + 1.6810500937936013e-06, + 1.5903754047030816e-06, + 1.5157305597313098e-06, + 1.4692379863845417e-06, + 1.424397169103031e-06, + 1.3794310689263511e-06, + 1.344195993624453e-06, + 1.3268191878523794e-06, + 1.2923425174449221e-06, + 1.2634424138013856e-06, + 1.2399162869769498e-06, + 1.2174884886917425e-06, + 1.1967615591856884e-06, + 1.1826359695987776e-06, + 1.1698227808665251e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.834020455746213e-05, + "first_to_last_ratio": 2.7330556661570378, + "imbalance_abs_log_ratio": 1.0054202748896328, + "last_quartile_mean": 1.7687237459540484e-05, + "mean": 2.944969719465007e-05, + "normalized": [ + 1.944393539840578, + 1.8698217429189048, + 1.7778954351065075, + 1.6896780508522091, + 1.5856589524406433, + 1.4966311985306986, + 1.4199003100173502, + 1.3476205250759918, + 1.2499825935748032, + 1.1821154937659675, + 1.1141820572947714, + 1.0564873202395735, + 0.996375680392802, + 0.9433030151418219, + 0.8970944032316949, + 0.8560097734780557, + 0.8186006518874163, + 0.7814267969140296, + 0.7477627937456819, + 0.7193270337078107, + 0.7043434099536711, + 0.683052553167036, + 0.6644263425171573, + 0.6491784221352348, + 0.6422634118610044, + 0.6262862823209598, + 0.6139376570332481, + 0.6029150238009798, + 0.5913772622967535, + 0.5831537397393776, + 0.5759145816028887, + 0.568883945414377 + ], + "peak_layer": 1, + "peak_normalized": 1.944393539840578, + "population_cv": 0.42167498055732977, + "rest_mean": 3.125674791388317e-05, + "spike_contrast": 0.6299959088848456, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.969162331079133e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.726180097553879e-05, + 5.506568413693458e-05, + 5.235848220763728e-05, + 4.9760506954044104e-05, + 4.6697176003362983e-05, + 4.4075335608795285e-05, + 4.1815634176600724e-05, + 3.968701639678329e-05, + 3.68116088793613e-05, + 3.481294334051199e-05, + 3.2812324207043275e-05, + 3.111323167104274e-05, + 2.9342962079681456e-05, + 2.7779988158727065e-05, + 2.6419158530188724e-05, + 2.520922862458974e-05, + 2.410754132142756e-05, + 2.3012782548903488e-05, + 2.2021387849235907e-05, + 2.118396332662087e-05, + 2.074270014418289e-05, + 2.0115690858801827e-05, + 1.9567154595279135e-05, + 1.9118107957183383e-05, + 1.8914462998509407e-05, + 1.8443941371515393e-05, + 1.8080278096022084e-05, + 1.7755664885044098e-05, + 1.741588130244054e-05, + 1.7173701053252444e-05, + 1.696051003818866e-05, + 1.6753459931351244e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.911581188731361e-05, + "first_to_last_ratio": 2.1442152520739666, + "imbalance_abs_log_ratio": 0.7627736355916307, + "last_quartile_mean": 1.824248374759918e-05, + "mean": 2.586868231446715e-05, + "normalized": [ + 1.7871656412737131, + 1.7134487296081482, + 1.6333736796405236, + 1.556430797867884, + 1.4684197123663512, + 1.3787761642028489, + 1.3133890592163948, + 1.2457273633361867, + 1.1743095829446748, + 1.1167737823796342, + 1.0549744950561857, + 1.006411191932248, + 0.9696254431287762, + 0.9309287088943796, + 0.9010845828506268, + 0.8670296369391367, + 0.844384842806383, + 0.8202754309623025, + 0.7995759366031269, + 0.778130175607516, + 0.7700498510754012, + 0.754237267577021, + 0.7412791029837059, + 0.7326331543280178, + 0.7310833132902649, + 0.7218522629025708, + 0.7147141764679059, + 0.7081022667260183, + 0.6994302307319604, + 0.6934167828145053, + 0.6888024881554871, + 0.684164145330101 + ], + "peak_layer": 1, + "peak_normalized": 1.7871656412737131, + "population_cv": 0.3324134582891191, + "rest_mean": 2.7086155737547585e-05, + "spike_contrast": 0.7123316433969433, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9294325829832816e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.623162021744065e-05, + 4.432466084836051e-05, + 4.2253224819432944e-05, + 4.0262813854496926e-05, + 3.798608304350637e-05, + 3.566712257452309e-05, + 3.39756443281658e-05, + 3.2225325412582606e-05, + 3.0377841540030204e-05, + 2.888946619350463e-05, + 2.7290800062473863e-05, + 2.603453140181955e-05, + 2.5082932552322745e-05, + 2.4081899027805775e-05, + 2.3309870812227018e-05, + 2.242891423520632e-05, + 2.1843123249709606e-05, + 2.1219444533926435e-05, + 2.0683975890278816e-05, + 2.0129202312091365e-05, + 1.9920174963772297e-05, + 1.9511124264681712e-05, + 1.9175913621438667e-05, + 1.8952254322357476e-05, + 1.891216197691392e-05, + 1.8673366867005825e-05, + 1.848871397669427e-05, + 1.831767258408945e-05, + 1.8093338439939544e-05, + 1.7937778466148302e-05, + 1.7818412743508816e-05, + 1.7698424926493317e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011420926875871373, + "first_to_last_ratio": 4.906198539252964, + "imbalance_abs_log_ratio": 1.590499413678891, + "last_quartile_mean": 2.3278566459339345e-05, + "mean": 5.893888680930104e-05, + "normalized": [ + 2.34070421869906, + 2.2660705441771873, + 2.12832569527821, + 2.0019442205229447, + 1.8447664945236562, + 1.7382507097156406, + 1.640255140843167, + 1.5417428136771152, + 1.39214202838135, + 1.3090459121430928, + 1.2137431403699552, + 1.1419557102859084, + 1.0196179370749445, + 0.9468482835406217, + 0.8718957397707048, + 0.8113961329283941, + 0.7366247508392765, + 0.6792650123242505, + 0.625986195283176, + 0.5827208972589306, + 0.5455979369449757, + 0.513063459434176, + 0.4857088480531883, + 0.4626395282739635, + 0.44411249043994555, + 0.42204051663631037, + 0.40670988685687265, + 0.39333811790230955, + 0.3823094762270729, + 0.3761622622739262, + 0.3695001417649948, + 0.36551575755467886 + ], + "peak_layer": 1, + "peak_normalized": 2.34070421869906, + "population_cv": 0.6231669472526075, + "rest_mean": 6.450288741756662e-05, + "spike_contrast": 0.4479378316450088, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.8893283524666912e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00013795850099995732, + 0.00013355967530515045, + 0.00012544114724732935, + 0.00011799236381193623, + 0.00010872848361032084, + 0.00010245056182611734, + 9.667481208452955e-05, + 9.086860518436879e-05, + 8.205130143323913e-05, + 7.715370884397998e-05, + 7.153666956583038e-05, + 6.730559834977612e-05, + 6.0095146181993186e-05, + 5.5806183809181675e-05, + 5.1388564315857366e-05, + 4.7822784836171195e-05, + 4.34158428106457e-05, + 4.003512367489748e-05, + 3.689492950798012e-05, + 3.434492100495845e-05, + 3.215693504898809e-05, + 3.023938916157931e-05, + 2.8627138817682862e-05, + 2.7267458790447563e-05, + 2.617549580463674e-05, + 2.4874598238966428e-05, + 2.397102798568085e-05, + 2.3182910808827728e-05, + 2.2532894945470616e-05, + 2.2170584998093545e-05, + 2.1777927031507716e-05, + 2.154309186153114e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "78bdb1dc451fca8b12adc84e1182b5c9328709671c2cc7151ff16c36d6a3b757", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.004827107302844524, + 0.004943604115396738, + 0.00549353240057826, + 0.005891259294003248, + 0.0059933834709227085, + 0.006044833920896053, + 0.0061737713403999805, + 0.0059272525832057, + 0.0063678608275949955, + 0.0061998614110052586, + 0.006479958072304726, + 0.006899545900523663, + 0.007097913417965174, + 0.006852242164313793, + 0.006816149223595858, + 0.007103090640157461, + 0.007109561935067177, + 0.007626448292285204, + 0.007445307914167643, + 0.007442969363182783, + 0.007471293676644564, + 0.007774693425744772, + 0.007449399214237928, + 0.007351893465965986, + 0.008440978825092316, + 0.008385274559259415, + 0.008216616697609425, + 0.007803319953382015, + 0.007909612730145454, + 0.007761957123875618, + 0.008686874993145466, + 0.008338849991559982 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4170842405292206e-05, + "first_to_last_ratio": 3.1401414420956977, + "imbalance_abs_log_ratio": 1.144267844159062, + "last_quartile_mean": 4.5128038550501515e-06, + "mean": 8.044083415370551e-06, + "normalized": [ + 2.1698039188085403, + 2.0423624648735483, + 1.924499098358531, + 1.829561680165496, + 1.6732631766752304, + 1.5731330769781084, + 1.4808173823619837, + 1.39974215233296, + 1.2679140487801928, + 1.180926373046176, + 1.1065316311690123, + 1.0359566369769586, + 0.9549370343414074, + 0.8983380270359873, + 0.8480121389826683, + 0.7997697342488901, + 0.7579715143085217, + 0.7199721252488075, + 0.6886613084032159, + 0.6633099718066915, + 0.648790261068007, + 0.6305361658191545, + 0.6153213953215836, + 0.6017960242412196, + 0.5932141420323949, + 0.5815513841228961, + 0.5714478326558409, + 0.56366655527055, + 0.5536526068712324, + 0.5470993720351313, + 0.540939542718517, + 0.5365012229405448 + ], + "peak_layer": 1, + "peak_normalized": 2.1698039188085403, + "population_cv": 0.494390404654856, + "rest_mean": 8.613229729908872e-06, + "spike_contrast": 0.5770998188522956, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.970693316863617e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.745408371789381e-05, + 1.6428934031864628e-05, + 1.548083128000144e-05, + 1.4717146768816747e-05, + 1.3459868569043465e-05, + 1.2654413694690447e-05, + 1.1911818546650466e-05, + 1.1259642633376643e-05, + 1.0199206371908076e-05, + 9.499470252194442e-06, + 8.901032742869575e-06, + 8.333321602549404e-06, + 7.681593160668854e-06, + 7.226306024676887e-06, + 6.821480383223388e-06, + 6.43341445538681e-06, + 6.0971860875724815e-06, + 5.791515832243022e-06, + 5.539649009733694e-06, + 5.335720743460115e-06, + 5.218922979111085e-06, + 5.072085514257196e-06, + 4.949696631229017e-06, + 4.840897418034729e-06, + 4.771864041686058e-06, + 4.678047844208777e-06, + 4.5967740334162954e-06, + 4.534180789050879e-06, + 4.4536277528095525e-06, + 4.400912985147443e-06, + 4.351362804300152e-06, + 4.3156605897820555e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.672785118804313e-05, + "first_to_last_ratio": 3.1553631949775576, + "imbalance_abs_log_ratio": 1.149103606634747, + "last_quartile_mean": 1.797823188098846e-05, + "mean": 3.227320695486924e-05, + "normalized": [ + 2.1337751341856186, + 2.022538217328439, + 1.9170288286790025, + 1.8286007981160775, + 1.6766598792294076, + 1.5813157540855372, + 1.4920540136500582, + 1.4099335912034971, + 1.276811677467349, + 1.1911592605052752, + 1.116819754454456, + 1.0462850360359126, + 0.9632481639863413, + 0.9048770313780828, + 0.8541222938622952, + 0.8042705353803223, + 0.7611962767945593, + 0.7217531538590994, + 0.6897404870714406, + 0.6635480643338956, + 0.6476672777722129, + 0.6283660375169337, + 0.6128277641309886, + 0.5988914175972425, + 0.5898193555984405, + 0.5780461908459694, + 0.5676768961249729, + 0.5596908776317308, + 0.5494806934414415, + 0.5429427329556125, + 0.5366812854926843, + 0.532171519285104 + ], + "peak_layer": 1, + "peak_normalized": 2.1337751341856186, + "population_cv": 0.49221664771975665, + "rest_mean": 3.457109293473574e-05, + "spike_contrast": 0.5746021018511365, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9864622663590127e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.886376650072634e-05, + 6.527379446197301e-05, + 6.186866812640801e-05, + 5.901481199543923e-05, + 5.411119127529673e-05, + 5.103413059259765e-05, + 4.815336797037162e-05, + 4.5503078581532463e-05, + 4.1206807509297505e-05, + 3.8442529330495745e-05, + 3.60433550667949e-05, + 3.3766973501769826e-05, + 3.1087107345229015e-05, + 2.9203283702372573e-05, + 2.7565265554585494e-05, + 2.5956389436032623e-05, + 2.4566244974266738e-05, + 2.3293288904824294e-05, + 2.2260137484408915e-05, + 2.14148240047507e-05, + 2.0902300093439408e-05, + 2.027938717219513e-05, + 1.9777917259489186e-05, + 1.9328146663610823e-05, + 1.9035362129216082e-05, + 1.865540434664581e-05, + 1.8320753952139057e-05, + 1.806301952456124e-05, + 1.77335041371407e-05, + 1.7522503185318783e-05, + 1.7320426195510663e-05, + 1.7174881577375345e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.050306825432926416, + "first_to_last_ratio": 3.1553631873240913, + "imbalance_abs_log_ratio": 1.1491036042092049, + "last_quartile_mean": 0.01594327576458454, + "mean": 0.02862020255997777, + "normalized": [ + 2.1337750803890514, + 2.022538074577747, + 1.9170287596851208, + 1.8286008602762933, + 1.6766599442378622, + 1.5813157057540668, + 1.4920541125825328, + 1.40993364991894, + 1.2768116947254893, + 1.1911592601257468, + 1.1168198090844537, + 1.0462850213817882, + 0.9632482114755018, + 0.9048770222788713, + 0.8541222536408491, + 0.8042704905420582, + 0.7611962477207449, + 0.7217531610115661, + 0.6897404919902839, + 0.663548124834154, + 0.647667267417541, + 0.6283660528035011, + 0.6128277842992133, + 0.5988913662696712, + 0.5898193333225623, + 0.578046223698357, + 0.5676769216043295, + 0.5596908382375889, + 0.5494807254443873, + 0.5429427375551722, + 0.5366812808757728, + 0.5321714922387812 + ], + "peak_layer": 1, + "peak_normalized": 2.1337750803890514, + "population_cv": 0.49221664059540937, + "rest_mean": 0.03065799087010048, + "spike_contrast": 0.5746020918316423, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01761614568531513, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.061069075018167496, + 0.05788544937968254, + 0.054865751415491104, + 0.05233492702245712, + 0.04798634722828865, + 0.0452575758099556, + 0.04270289093255997, + 0.04035258665680885, + 0.036542609333992004, + 0.034091219305992126, + 0.03196360915899277, + 0.02994488924741745, + 0.027568358927965164, + 0.025897763669490814, + 0.024445151910185814, + 0.023018384352326393, + 0.021785590797662735, + 0.02065672166645527, + 0.01974051259458065, + 0.018990881741046906, + 0.018536368384957314, + 0.01798396371304989, + 0.01753925532102585, + 0.01714039221405983, + 0.01688074879348278, + 0.0165438000112772, + 0.01624702848494053, + 0.016018465161323547, + 0.015726249665021896, + 0.015539131127297878, + 0.015359926968812943, + 0.015230855904519558 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.399239884378403e-06, + "first_to_last_ratio": 3.612110266791686, + "imbalance_abs_log_ratio": 1.284292162993447, + "last_quartile_mean": 1.2179140611578987e-06, + "mean": 2.349132376622265e-06, + "normalized": [ + 2.4955076017050923, + 2.2436808131458656, + 2.0874700594685764, + 1.9621517331257625, + 1.7419934380554116, + 1.5859298059239806, + 1.4811826529402687, + 1.383750925243364, + 1.24754299116381, + 1.1572103620070064, + 1.0826808130566368, + 1.0127662347026543, + 0.9261323573573094, + 0.8680208953912559, + 0.8137959587886114, + 0.7661051062366301, + 0.7198218777908413, + 0.6807070982717376, + 0.6465769322329894, + 0.6214722858706658, + 0.6055471528554645, + 0.5897479442686585, + 0.5729654410868715, + 0.5596177904436604, + 0.5505791556557464, + 0.5392724776945974, + 0.5286699021580167, + 0.5214282796203655, + 0.5112551634910032, + 0.5041753262585503, + 0.49807684781126477, + 0.49416457617733067 + ], + "peak_layer": 1, + "peak_normalized": 2.4955076017050923, + "population_cv": 0.5694378379798026, + "rest_mean": 2.5337169770662313e-06, + "spike_contrast": 0.5337516172744564, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3523755342248478e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.862277703272412e-06, + 5.270703240967123e-06, + 4.903743501927238e-06, + 4.6093541641312186e-06, + 4.092173185199499e-06, + 3.725559054146288e-06, + 3.479494125713245e-06, + 3.2506140996702015e-06, + 2.9306436317710904e-06, + 2.718440327953431e-06, + 2.543360551499063e-06, + 2.3791219518898288e-06, + 2.175607505705557e-06, + 2.0390959889482474e-06, + 1.9117144347546855e-06, + 1.7996823089561076e-06, + 1.6909568785195006e-06, + 1.5990710835467326e-06, + 1.5188948054856155e-06, + 1.459920667912229e-06, + 1.4225104223442031e-06, + 1.385395989927929e-06, + 1.3459716683428269e-06, + 1.3146162700650166e-06, + 1.2933833204442635e-06, + 1.266822437173687e-06, + 1.2419155837051221e-06, + 1.224904053742648e-06, + 1.201006057272025e-06, + 1.1843745824080543e-06, + 1.1700484492394025e-06, + 1.160858005277987e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.614919907908188e-05, + "first_to_last_ratio": 2.6506059611493034, + "imbalance_abs_log_ratio": 0.9747882784430122, + "last_quartile_mean": 1.7410810869478155e-05, + "mean": 2.826825851798276e-05, + "normalized": [ + 1.9567333803186102, + 1.8597972147876323, + 1.766445715132986, + 1.6894057707724273, + 1.5636321818586363, + 1.4819572930902338, + 1.4065416062160392, + 1.3358454162549211, + 1.2332908204028759, + 1.1620147315245914, + 1.1014502996752689, + 1.0446856471155146, + 0.9835188591006802, + 0.9354808101194146, + 0.8921368408217227, + 0.8494656881105699, + 0.8144325093861817, + 0.7792466985899428, + 0.7495668388307936, + 0.7246367192704585, + 0.7117068137537608, + 0.6923040729201976, + 0.6764241600957448, + 0.6619694138511188, + 0.6525606633107036, + 0.6397070094825148, + 0.6281961575410799, + 0.6192140303718157, + 0.607592629568897, + 0.6000373420122876, + 0.5927022513148152, + 0.5873004143975631 + ], + "peak_layer": 1, + "peak_normalized": 1.9567333803186102, + "population_cv": 0.41380193863129383, + "rest_mean": 2.9948685955282093e-05, + "spike_contrast": 0.6408945749815472, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.919395035656635e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.531344504561275e-05, + 5.25732284586411e-05, + 4.9934344133362174e-05, + 4.77565590699669e-05, + 4.420115874381736e-05, + 4.1892351873684675e-05, + 3.97604817408137e-05, + 3.77620235667564e-05, + 3.486298373900354e-05, + 3.284813283244148e-05, + 3.1136081815930083e-05, + 2.9531443942687474e-05, + 2.7802365366369486e-05, + 2.6444413379067555e-05, + 2.521915484976489e-05, + 2.4012915673665702e-05, + 2.3022588720778003e-05, + 2.2027947125025094e-05, + 2.118894917657599e-05, + 2.0484218111960217e-05, + 2.0118712200201117e-05, + 1.957023050636053e-05, + 1.912133302539587e-05, + 1.8712722521740943e-05, + 1.8446753529133275e-05, + 1.8083403119817376e-05, + 1.7758011381374672e-05, + 1.7504102288512513e-05, + 1.7175585526274517e-05, + 1.6962010704446584e-05, + 1.6754660464357585e-05, + 1.6601959941908717e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.735190557563328e-05, + "first_to_last_ratio": 2.064507245129547, + "imbalance_abs_log_ratio": 0.7248915757248915, + "last_quartile_mean": 1.8092407117364928e-05, + "mean": 2.5017036136887327e-05, + "normalized": [ + 1.7806348537056282, + 1.6958575783596996, + 1.6145104713051759, + 1.5397532195284758, + 1.4341567363816627, + 1.3605904748672402, + 1.2904091560066684, + 1.2285577888193309, + 1.1575565189704986, + 1.0915619889550103, + 1.0406567529663657, + 0.9956018385584979, + 0.9640474313507927, + 0.9320995174703498, + 0.8974595875050125, + 0.8691926915133404, + 0.8490635018802836, + 0.8275552938519668, + 0.8049190624458126, + 0.7866702995626351, + 0.780171913500315, + 0.7668318773324383, + 0.7578168517732956, + 0.7486969056434284, + 0.7465158956368789, + 0.7394522618151603, + 0.7324985655561136, + 0.7264758484984223, + 0.7170356146942368, + 0.712241115433273, + 0.7073157381215093, + 0.7040926479904809 + ], + "peak_layer": 1, + "peak_normalized": 1.7806348537056282, + "population_cv": 0.31924035304186044, + "rest_mean": 2.6128873421974412e-05, + "spike_contrast": 0.7276668415955134, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9013114797417075e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.454620648175478e-05, + 4.242533032083884e-05, + 4.039026680402458e-05, + 3.8520061934832484e-05, + 3.587835090002045e-05, + 3.403794107725844e-05, + 3.22822124871891e-05, + 3.073487459914759e-05, + 2.8958633265574463e-05, + 2.73076457233401e-05, + 2.60341475950554e-05, + 2.4907007173169404e-05, + 2.4117609427776188e-05, + 2.331836731173098e-05, + 2.2451778932008892e-05, + 2.1744624973507598e-05, + 2.1241052309051156e-05, + 2.0702980691567063e-05, + 2.013668927247636e-05, + 1.968015931197442e-05, + 1.9517588953021914e-05, + 1.918386078614276e-05, + 1.8958331565954722e-05, + 1.873017754405737e-05, + 1.8675615137908608e-05, + 1.8498903955332935e-05, + 1.8324943084735423e-05, + 1.8174272554460913e-05, + 1.793810588424094e-05, + 1.7818161722971126e-05, + 1.769494338077493e-05, + 1.761431121849455e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010834934892045567, + "first_to_last_ratio": 4.772634589412918, + "imbalance_abs_log_ratio": 1.5628984772495462, + "last_quartile_mean": 2.270220920763677e-05, + "mean": 5.5645747750077135e-05, + "normalized": [ + 2.4035622873585307, + 2.266667857217833, + 2.121353069055484, + 2.022827767132241, + 1.842912339502741, + 1.7402381826456954, + 1.6341450450508916, + 1.545311039376737, + 1.3884783081507637, + 1.284206979436914, + 1.2074118445936788, + 1.1168610262727916, + 1.0022982360357862, + 0.9246149481154046, + 0.8572930756226724, + 0.7854058282657367, + 0.7196284333802556, + 0.6628520139237241, + 0.6164320412062984, + 0.5776386785157045, + 0.5436680818313624, + 0.5148121289164611, + 0.490108136365702, + 0.46745320481928015, + 0.44716915492660686, + 0.43065012877775255, + 0.41644951885427656, + 0.4076017882404048, + 0.3988567002714443, + 0.391504708352861, + 0.3871572366322811, + 0.3844302111516827 + ], + "peak_layer": 1, + "peak_normalized": 2.4035622873585307, + "population_cv": 0.6252223005215759, + "rest_mean": 6.087395282789927e-05, + "spike_contrast": 0.4503312017101952, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.741344032983761e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001337480207439512, + 0.0001261304278159514, + 0.00011804427776951343, + 0.00011256176367169246, + 0.00010255023516947404, + 9.683685493655503e-05, + 9.093322296394035e-05, + 8.598998829256743e-05, + 7.726291369181126e-05, + 7.1460657636635e-05, + 6.718733493471518e-05, + 6.214856693986803e-05, + 5.5773634812794626e-05, + 5.145089016878046e-05, + 4.770471423398703e-05, + 4.3704494601115584e-05, + 4.004426227766089e-05, + 3.6884895962430164e-05, + 3.4301821870030835e-05, + 3.214313619537279e-05, + 3.0252816941356286e-05, + 2.8647105864365585e-05, + 2.727243372646626e-05, + 2.6011783120338805e-05, + 2.4883061996661127e-05, + 2.3963848434505053e-05, + 2.3173644876806065e-05, + 2.2681306290905923e-05, + 2.2194679331732914e-05, + 2.178557224397082e-05, + 2.1543653929256834e-05, + 2.1391906557255425e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ae4066fc37f2b8857b68206f2c50d90301cff538a7afaed7e0b4959067354e04", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.002723898272961378, + 0.006246842443943024, + 0.009986688382923603, + 0.014312135986983776, + 0.0027809101156890392, + 0.007167908828705549, + 0.012327533215284348, + 0.017696699127554893, + 0.003559867851436138, + 0.008642485365271568, + 0.014099182561039925, + 0.020258057862520218, + 0.004887574352324009, + 0.011225050315260887, + 0.01857108250260353, + 0.026577936485409737, + 0.005625482182949781, + 0.013686468824744225, + 0.023031942546367645, + 0.03310495615005493, + 0.008495661430060863, + 0.017283262684941292, + 0.029314789921045303, + 0.04071469604969025, + 0.00961991585791111, + 0.023407738655805588, + 0.0400763601064682, + 0.05648425966501236, + 0.014384749345481396, + 0.03149676322937012, + 0.04818879812955856, + 0.06738673150539398 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4927425922905968e-05, + "first_to_last_ratio": 3.2580194125343316, + "imbalance_abs_log_ratio": 1.181119468582707, + "last_quartile_mean": 4.581748612508818e-06, + "mean": 8.434603628870718e-06, + "normalized": [ + 2.149249807253593, + 2.0535339039963048, + 1.9368405197515186, + 1.8272069144524385, + 1.6952778345494546, + 1.5885491419580562, + 1.4979415287755153, + 1.4096717556974747, + 1.2877403631156048, + 1.2069779670771674, + 1.1247537335636728, + 1.0538960762660219, + 0.9733880487902208, + 0.9100528645893027, + 0.8555679056241683, + 0.8085854713945597, + 0.7636821416331455, + 0.7224543782958864, + 0.6861960752222711, + 0.6566025735424599, + 0.6395340008907516, + 0.6187919105654988, + 0.6011578827453019, + 0.5866789847797121, + 0.5802252586132757, + 0.5655851563233361, + 0.5545362442810942, + 0.5449205411198527, + 0.5349741264784901, + 0.5278853954517888, + 0.5216927232447175, + 0.5158487699573434 + ], + "peak_layer": 1, + "peak_normalized": 2.149249807253593, + "population_cv": 0.5023647441278015, + "rest_mean": 9.05114565183693e-06, + "spike_contrast": 0.5640475693612389, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.105276704853167e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8128070223610848e-05, + 1.7320744518656284e-05, + 1.6336482076440006e-05, + 1.5411766071338207e-05, + 1.4298996575234924e-05, + 1.3398782357398886e-05, + 1.2634543054446112e-05, + 1.1890022506122477e-05, + 1.0861579539778177e-05, + 1.0180380741076078e-05, + 9.486851922702044e-06, + 8.889195669326e-06, + 8.210142368625384e-06, + 7.675935194129124e-06, + 7.2163761615229305e-06, + 6.8200979512766935e-06, + 6.441356163122691e-06, + 6.093616320868023e-06, + 5.787791906186612e-06, + 5.538182449527085e-06, + 5.394215804699343e-06, + 5.2192644943716004e-06, + 5.070528459327761e-06, + 4.9484046940051485e-06, + 4.893970071861986e-06, + 4.770486611960223e-06, + 4.677293418353656e-06, + 4.5961887735757045e-06, + 4.512294708547415e-06, + 4.452504072105512e-06, + 4.400271336635342e-06, + 4.350979907030705e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.967510560367373e-05, + "first_to_last_ratio": 3.2679096193643358, + "imbalance_abs_log_ratio": 1.184150520362052, + "last_quartile_mean": 1.8260941260450636e-05, + "mean": 3.383405919521465e-05, + "normalized": [ + 2.1128501814933904, + 2.028923058304855, + 1.9237119576664479, + 1.8241641545550669, + 1.696600823933448, + 1.5957782143766441, + 1.5065145595844665, + 1.4215241299739685, + 1.2975903849100163, + 1.2165984562514365, + 1.1353482545596325, + 1.0647386437786797, + 0.9824378235158376, + 0.9182368677416404, + 0.8622246832113106, + 0.8145243313263993, + 0.7677753532124987, + 0.7257958780424952, + 0.688282795208288, + 0.6577753415192462, + 0.6393623666755853, + 0.6177067857274526, + 0.5993015526233388, + 0.5844680033165289, + 0.5774972657614055, + 0.5625215157524979, + 0.5512919612531849, + 0.5414258131074452, + 0.5312052115670245, + 0.5240772736894307, + 0.5178563021046121, + 0.5118900552557252 + ], + "peak_layer": 1, + "peak_normalized": 2.1128501814933904, + "population_cv": 0.49917932069280174, + "rest_mean": 3.631730874918436e-05, + "spike_contrast": 0.5623905599623192, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0424511603778228e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.148629811126739e-05, + 6.864670285722241e-05, + 6.508698425022885e-05, + 6.171887798700482e-05, + 5.740289270761423e-05, + 5.399165456765331e-05, + 5.0971502787433565e-05, + 4.8095931560965255e-05, + 4.3902749894186854e-05, + 4.1162464185617864e-05, + 3.841344005195424e-05, + 3.602443030104041e-05, + 3.323985947645269e-05, + 3.1067680538399145e-05, + 2.9172560971346684e-05, + 2.7558664442040026e-05, + 2.5976956749218516e-05, + 2.4556620701332577e-05, + 2.3287400836125016e-05, + 2.225520984211471e-05, + 2.163222416129429e-05, + 2.0899527953588404e-05, + 2.0276804207242094e-05, + 1.9774925021920353e-05, + 1.9539076674846e-05, + 1.903238626255188e-05, + 1.865244485088624e-05, + 1.8318633010494523e-05, + 1.7972828572965227e-05, + 1.7731661500874907e-05, + 1.7521180780022405e-05, + 1.7319318430963904e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05292047979310155, + "first_to_last_ratio": 3.2679095977338717, + "imbalance_abs_log_ratio": 1.184150513743, + "last_quartile_mean": 0.016193985240533948, + "mean": 0.030004381958860904, + "normalized": [ + 2.112850138441349, + 2.0289230386859303, + 1.9237119939527503, + 1.8241640534063661, + 1.6966008064731326, + 1.5957782577626325, + 1.50651458990766, + 1.4215240751000642, + 1.2975903450635375, + 1.216598469505261, + 1.1353482207159766, + 1.0647387028334923, + 0.9824378272874627, + 0.9182369215521707, + 0.8622246841932674, + 0.8145243422322688, + 0.7677753625406581, + 0.7257958583332109, + 0.688282821503116, + 0.6577753494846466, + 0.6393623768313474, + 0.6177068263368124, + 0.5993015514924139, + 0.5844679978988628, + 0.577497259464603, + 0.562521544722804, + 0.5512919314350962, + 0.541425823588487, + 0.5312051819787147, + 0.5240772592530296, + 0.5178563122373452, + 0.51189007578553 + ], + "peak_layer": 1, + "peak_normalized": 2.112850138441349, + "population_cv": 0.4991793114170283, + "rest_mean": 0.0322065524480961, + "spike_contrast": 0.5623905677635356, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.018112661316990852, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06339476257562637, + 0.06087658181786537, + 0.05771978944540024, + 0.05473291501402855, + 0.05090545862913132, + 0.04788034036755562, + 0.04520203918218613, + 0.0426519513130188, + 0.038933396339416504, + 0.03650328516960144, + 0.03406542167067528, + 0.031946826726198196, + 0.02947743982076645, + 0.027551131322979927, + 0.025870518758893013, + 0.02443929947912693, + 0.023036625236272812, + 0.02177705615758896, + 0.020651500672101974, + 0.019736142829060555, + 0.019183672964572906, + 0.018533911556005478, + 0.017981672659516335, + 0.017536601051688194, + 0.01732744835317135, + 0.01687811128795147, + 0.016541173681616783, + 0.016245147213339806, + 0.01593848317861557, + 0.01572461426258087, + 0.015537958592176437, + 0.015358945354819298 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.549713025880919e-06, + "first_to_last_ratio": 3.680528123890709, + "imbalance_abs_log_ratio": 1.3030562538108112, + "last_quartile_mean": 1.236157658013326e-06, + "mean": 2.4387289627725295e-06, + "normalized": [ + 2.46688156577926, + 2.2376955014199336, + 2.0678263447651153, + 1.940166497412543, + 1.7430074161430915, + 1.5977933771333281, + 1.4874509058079846, + 1.3840448948125312, + 1.2636490405882963, + 1.179790668094215, + 1.1016966117962113, + 1.0305101131331023, + 0.950494959962906, + 0.8872551174370253, + 0.827006144436166, + 0.781561644601688, + 0.7317791765691539, + 0.6893042887937005, + 0.6521328590808986, + 0.6215289607815456, + 0.6024501053460986, + 0.584043470655544, + 0.565626393700991, + 0.5512156498280469, + 0.5440612538544417, + 0.5299479280364501, + 0.5180756798945998, + 0.5084212008038946, + 0.4992530328651117, + 0.49069438546998123, + 0.48492974295315366, + 0.4797050680429897 + ], + "peak_layer": 1, + "peak_normalized": 2.46688156577926, + "population_cv": 0.5676426036440108, + "rest_mean": 2.6331591698181e-06, + "spike_contrast": 0.5274295077355267, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3888058447264485e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.016055522195529e-06, + 5.45713282917859e-06, + 5.042867996962741e-06, + 4.731540229840903e-06, + 4.250722668075468e-06, + 3.896584985341178e-06, + 3.6274896046961658e-06, + 3.375310370756779e-06, + 3.081697514062398e-06, + 2.8771896722901147e-06, + 2.6867394353757845e-06, + 2.5131348593276925e-06, + 2.3179995878308546e-06, + 2.1637747522618156e-06, + 2.01684383682732e-06, + 1.9060170188822667e-06, + 1.7846110722530284e-06, + 1.6810263332445174e-06, + 1.5903752910162439e-06, + 1.515740677859867e-06, + 1.469212520532892e-06, + 1.4243237274058629e-06, + 1.379409468427184e-06, + 1.3442655699691386e-06, + 1.3268179372971645e-06, + 1.292399360863783e-06, + 1.2634461654670304e-06, + 1.239901507688046e-06, + 1.2175428310001735e-06, + 1.196670609715511e-06, + 1.1826122090496938e-06, + 1.169870643025206e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.834035416934057e-05, + "first_to_last_ratio": 2.733059908807899, + "imbalance_abs_log_ratio": 1.0054218272354634, + "last_quartile_mean": 1.7687264744381537e-05, + "mean": 2.9449753014887392e-05, + "normalized": [ + 1.9443940544381229, + 1.8698233871166146, + 1.7778950299739502, + 1.6896852248339365, + 1.5856582940221604, + 1.4966405913970182, + 1.4199046599899854, + 1.3476142647904932, + 1.2499731829995469, + 1.1821174532157463, + 1.1141837749131207, + 1.0564879118975747, + 0.9963758300960397, + 0.9433002389138315, + 0.8970939381604119, + 0.8560091392167688, + 0.8186039797822264, + 0.7814222892392128, + 0.7477601410889809, + 0.719324496715247, + 0.7043446073134325, + 0.6830500849305705, + 0.66442780083491, + 0.6491794152276021, + 0.642265097484849, + 0.626282254003255, + 0.613935752160955, + 0.6029174634318201, + 0.5913776237573929, + 0.5831511520257966, + 0.5759147253086592, + 0.5688861407197677 + ], + "peak_layer": 1, + "peak_normalized": 1.9443940544381229, + "population_cv": 0.4216757923764803, + "rest_mean": 3.125680403307477e-05, + "spike_contrast": 0.6299965119862723, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9691677516675553e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.726192466681823e-05, + 5.5065836932044476e-05, + 5.235856951912865e-05, + 4.97608125442639e-05, + 4.669724512496032e-05, + 4.407569576869719e-05, + 4.181584154139273e-05, + 3.968690725741908e-05, + 3.68114015145693e-05, + 3.4813067031791434e-05, + 3.2812436984386295e-05, + 3.111330806859769e-05, + 2.9343022106331773e-05, + 2.7779959054896608e-05, + 2.6419194909976795e-05, + 2.5209257728420198e-05, + 2.410768502159044e-05, + 2.3012693418422714e-05, + 2.2021351469447836e-05, + 2.1183928765822202e-05, + 2.0742774722748436e-05, + 2.011565629800316e-05, + 1.956723463081289e-05, + 1.911817344080191e-05, + 1.8914548491011374e-05, + 1.844385769800283e-05, + 1.8080256268149242e-05, + 1.7755770386429504e-05, + 1.7415924958186224e-05, + 1.717365739750676e-05, + 1.696054641797673e-05, + 1.6753556337789632e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.911596377292881e-05, + "first_to_last_ratio": 2.1441510473750993, + "imbalance_abs_log_ratio": 0.7627436919280587, + "last_quartile_mean": 1.8243100839754334e-05, + "mean": 2.586884403399381e-05, + "normalized": [ + 1.7871422338212197, + 1.7137406573133611, + 1.6331955543594734, + 1.556344845447187, + 1.4684323304243458, + 1.3786204440301357, + 1.3134601648053343, + 1.2457662653653259, + 1.1742886707715572, + 1.116694094274204, + 1.054651337969624, + 1.006236775170543, + 0.969530361649706, + 0.9312138561082258, + 0.901034861675357, + 0.8672035220710215, + 0.8443849784393078, + 0.8203617838993202, + 0.799640410086616, + 0.7782482231951742, + 0.770066975636356, + 0.7541424075474804, + 0.7413032280416163, + 0.732574782644356, + 0.7311844276122933, + 0.7218147017875606, + 0.7147312250657709, + 0.7081409436201098, + 0.6994441403492052, + 0.6933838996779177, + 0.688757539541391, + 0.6842643575989029 + ], + "peak_layer": 1, + "peak_normalized": 1.7871422338212197, + "population_cv": 0.33240131193143013, + "rest_mean": 2.7086335480648644e-05, + "spike_contrast": 0.7123292937076053, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.92943902220577e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.623130371328443e-05, + 4.433248977875337e-05, + 4.2248881072737277e-05, + 4.026084206998348e-05, + 3.798664693022147e-05, + 3.566331724869087e-05, + 3.3977696148213e-05, + 3.222653322154656e-05, + 3.037749047507532e-05, + 2.8887585358461365e-05, + 2.72826109721791e-05, + 2.603018219815567e-05, + 2.5080629711737856e-05, + 2.4089426005957648e-05, + 2.3308730305870995e-05, + 2.2433552658185363e-05, + 2.184326331189368e-05, + 2.1221811039140448e-05, + 2.068577305180952e-05, + 2.0132381905568764e-05, + 1.9920742488466203e-05, + 1.9508792320266366e-05, + 1.9176657588104717e-05, + 1.8950862795463763e-05, + 1.891489591798745e-05, + 1.867251194198616e-05, + 1.8489270587451756e-05, + 1.8318787624593824e-05, + 1.8093811377184466e-05, + 1.7937039956450462e-05, + 1.7817361367633566e-05, + 1.7701127944746986e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011420908140280517, + "first_to_last_ratio": 4.906424645343274, + "imbalance_abs_log_ratio": 1.5905454984181289, + "last_quartile_mean": 2.327745551156113e-05, + "mean": 5.893761050401736e-05, + "normalized": [ + 2.3409386034658266, + 2.26653318009987, + 2.128165373071052, + 2.001759557425238, + 1.8449035999101382, + 1.7381229263770657, + 1.6401158530503868, + 1.5418310130397193, + 1.392320688071593, + 1.3090601862719393, + 1.213617825327422, + 1.1417595841795705, + 1.019599771791136, + 0.9467828652279454, + 0.8718925846836042, + 0.8115322793787068, + 0.7365786680304162, + 0.6791875033995677, + 0.6260459221300255, + 0.5825401905799077, + 0.5456521576869235, + 0.5130954641492332, + 0.48578325250896687, + 0.46257467326905344, + 0.4440240870048044, + 0.4220562606793032, + 0.4066511043445411, + 0.39330858167914856, + 0.38230571865595697, + 0.3762268256398313, + 0.369497495638699, + 0.36553620323240776 + ], + "peak_layer": 1, + "peak_normalized": 2.3409386034658266, + "population_cv": 0.6231971411979633, + "rest_mean": 6.45014745343899e-05, + "spike_contrast": 0.4479392905134439, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.8892744740005583e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00013796932762488723, + 0.00013358404976315796, + 0.00012542898184619844, + 0.00011797892511822283, + 0.0001087342097889632, + 0.00010244081204291433, + 9.666450932854787e-05, + 9.087183570954949e-05, + 8.2060054410249e-05, + 7.715287938481197e-05, + 7.152773468988016e-05, + 6.729258166160434e-05, + 6.009277421981096e-05, + 5.5801119742682204e-05, + 5.1387265557423234e-05, + 4.782977339345962e-05, + 4.341218664194457e-05, + 4.002968853455968e-05, + 3.689765071612783e-05, + 3.433352685533464e-05, + 3.215943434042856e-05, + 3.024062061740551e-05, + 2.8630904125748202e-05, + 2.726304592215456e-05, + 2.6169718694291078e-05, + 2.4874987502698787e-05, + 2.3967044398887083e-05, + 2.3180667994893156e-05, + 2.2532185539603233e-05, + 2.2173910110723227e-05, + 2.1777299480163492e-05, + 2.1543830371228978e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "0a445d8d97762f2c9a1505daa03132d58d2d7cce48cebe69f58dcee8b68ddc3c", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0055193523876369, + 0.009546536952257156, + 0.013977172784507275, + 0.018744563683867455, + 0.00660830270498991, + 0.011765030212700367, + 0.017156247049570084, + 0.022267481312155724, + 0.007449587807059288, + 0.013175037689507008, + 0.019073888659477234, + 0.025712018832564354, + 0.008972279727458954, + 0.01624389924108982, + 0.023743286728858948, + 0.03214457631111145, + 0.00991375558078289, + 0.019281653687357903, + 0.02868141047656536, + 0.03889070823788643, + 0.0127727584913373, + 0.022841427475214005, + 0.034929387271404266, + 0.04634999856352806, + 0.014845813624560833, + 0.029589947313070297, + 0.046351123601198196, + 0.06254568696022034, + 0.01947367936372757, + 0.0370744951069355, + 0.05507569760084152, + 0.07403328269720078 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4170829444992705e-05, + "first_to_last_ratio": 3.140130857329134, + "imbalance_abs_log_ratio": 1.1442644733604534, + "last_quartile_mean": 4.512814939516829e-06, + "mean": 8.044093192438595e-06, + "normalized": [ + 2.1698218591421137, + 2.0423579473697657, + 1.9244829654882192, + 1.829554142460677, + 1.6732305026788175, + 1.5731474461065056, + 1.48081501720716, + 1.399743051502537, + 1.2679172563879277, + 1.1809283296176842, + 1.10654046198268, + 1.035952438184134, + 0.9549549814383372, + 0.8983396486930857, + 0.8480063030735002, + 0.799768196862845, + 0.7579745502722912, + 0.7199744159526508, + 0.6886643720771628, + 0.663306678196146, + 0.648790207419826, + 0.630529124410062, + 0.6153188384200302, + 0.6018030376590212, + 0.5932194699258551, + 0.5815579133602139, + 0.5714511518587554, + 0.5636597082007567, + 0.5536559477030776, + 0.5471005160904301, + 0.5409418248973891, + 0.5364916953603418 + ], + "peak_layer": 1, + "peak_normalized": 2.1698218591421137, + "population_cv": 0.49438916638334635, + "rest_mean": 8.613239397500703e-06, + "spike_contrast": 0.577100374865415, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.970703685103217e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.7454249245929532e-05, + 1.6428917660959996e-05, + 1.5480720321647823e-05, + 1.4717104022565763e-05, + 1.3459622095979284e-05, + 1.2654544661927503e-05, + 1.1911813999176957e-05, + 1.1259663551754784e-05, + 1.019924457068555e-05, + 9.499497537035495e-06, + 8.901114597392734e-06, + 8.333297955687158e-06, + 7.681746865273453e-06, + 7.22632785254973e-06, + 6.821441729698563e-06, + 6.433409907913301e-06, + 6.0972179198870435e-06, + 5.791541298094671e-06, + 5.539680387300905e-06, + 5.335700734576676e-06, + 5.218928890826646e-06, + 5.072035037301248e-06, + 4.9496820793137886e-06, + 4.8409597184218e-06, + 4.771912699652603e-06, + 4.67810605186969e-06, + 4.596806320478208e-06, + 4.534131221589632e-06, + 4.453660039871465e-06, + 4.400927537062671e-06, + 4.351386451162398e-06, + 4.315589194447966e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.6727842547843466e-05, + "first_to_last_ratio": 3.155359202626439, + "imbalance_abs_log_ratio": 1.1491023413751247, + "last_quartile_mean": 1.79782518898719e-05, + "mean": 3.227324236831919e-05, + "normalized": [ + 2.133789250541258, + 2.0225341944070556, + 1.9170292050567193, + 1.8285814320482991, + 1.67664620337743, + 1.581315371598383, + 1.4920577871848402, + 1.409935200362978, + 1.2768241415105412, + 1.1911541208190792, + 1.1168170635514478, + 1.0462850151897058, + 0.9632588303411923, + 0.904875136661487, + 0.8541193275958588, + 0.8042725273235084, + 0.7611969633115008, + 0.7217542218299198, + 0.6897417592562383, + 0.6635457580819046, + 0.647670850610065, + 0.628363657145899, + 0.6128288389014056, + 0.5988935221787641, + 0.5898175247840765, + 0.5780479801280578, + 0.5676825294116558, + 0.559690432569022, + 0.5494813304632692, + 0.5429417990112825, + 0.5366821620079203, + 0.5321658627392354 + ], + "peak_layer": 1, + "peak_normalized": 2.133789250541258, + "population_cv": 0.49221598973047537, + "rest_mean": 3.45711278323843e-05, + "spike_contrast": 0.5746026267549044, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.98646608623676e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.886429764563218e-05, + 6.52737362543121e-05, + 6.186874816194177e-05, + 5.901425174670294e-05, + 5.411080928752199e-05, + 5.1034174248343334e-05, + 4.815354259335436e-05, + 4.550318044493906e-05, + 4.120725498069078e-05, + 3.8442405639216304e-05, + 3.604330777307041e-05, + 3.37670098815579e-05, + 3.108748569502495e-05, + 2.9203254598542117e-05, + 2.7565200070966966e-05, + 2.5956482204492204e-05, + 2.4566294086980633e-05, + 2.329334893147461e-05, + 2.2260202968027443e-05, + 2.14147730730474e-05, + 2.0902438336634077e-05, + 2.0279332602513023e-05, + 1.9777973648160696e-05, + 1.9328235794091597e-05, + 1.9035323930438608e-05, + 1.8655482563190162e-05, + 1.832095585996285e-05, + 1.8063024981529452e-05, + 1.7733544154907577e-05, + 1.752249227138236e-05, + 1.7320473489235155e-05, + 1.7174717868329026e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05030681798234582, + "first_to_last_ratio": 3.1553591948948423, + "imbalance_abs_log_ratio": 1.1491023389248185, + "last_quartile_mean": 0.01594329357612878, + "mean": 0.028620234137633815, + "normalized": [ + 2.1337891266389915, + 2.022534150927612, + 1.9170292478198825, + 1.8285814009026793, + 1.6766462495068444, + 1.5813153928268515, + 1.4920578030251115, + 1.409935218199867, + 1.2768242133981067, + 1.1911540409956152, + 1.1168171450690334, + 1.046284973364676, + 0.9632588633452485, + 0.9048751127582434, + 0.8541193588171093, + 0.8042725318270637, + 0.7611969698214163, + 0.7217542520381491, + 0.6897417485003263, + 0.6635457656842302, + 0.6476708481976686, + 0.6283636673897187, + 0.6128288002625611, + 0.5988935039934771, + 0.5898175110899254, + 0.5780479939328508, + 0.567682478001312, + 0.559690415956301, + 0.5494813557307923, + 0.5429418131013527, + 0.5366821530691603, + 0.532165893807823 + ], + "peak_layer": 1, + "peak_normalized": 2.1337891266389915, + "population_cv": 0.4922159837762602, + "rest_mean": 0.030658022017666587, + "spike_contrast": 0.5746026138054693, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01761617958545685, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0610695444047451, + 0.05788540095090866, + 0.05486582592129707, + 0.05233442783355713, + 0.04798600822687149, + 0.04525761678814888, + 0.042703043669462204, + 0.040352676063776016, + 0.03654300794005394, + 0.03409110754728317, + 0.031963568180799484, + 0.029944920912384987, + 0.027568694204092026, + 0.025897737592458725, + 0.024445096030831337, + 0.02301846817135811, + 0.021785635501146317, + 0.020656775683164597, + 0.019740570336580276, + 0.018990835174918175, + 0.018536491319537163, + 0.01798391528427601, + 0.01753930374979973, + 0.017140472307801247, + 0.016880715265870094, + 0.01654386892914772, + 0.016247205436229706, + 0.016018470749258995, + 0.015726285055279732, + 0.015539121814072132, + 0.0153599688783288, + 0.015230712480843067 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 4.399315798764292e-06, + "first_to_last_ratio": 3.6121892887675737, + "imbalance_abs_log_ratio": 1.2843140397097386, + "last_quartile_mean": 1.2179084336594315e-06, + "mean": 2.3491519840490582e-06, + "normalized": [ + 2.495529360167265, + 2.243705447822421, + 2.0875170981243216, + 1.9621829763583638, + 1.7419047574397617, + 1.5859991304051935, + 1.4811645795131025, + 1.3837971590715752, + 1.247616882237959, + 1.1571559864040741, + 1.082652031260043, + 1.012746166788284, + 0.9261497926205429, + 0.8680129728421762, + 0.8137647753609764, + 0.7661118752622963, + 0.7198161600961848, + 0.6806799293737783, + 0.6465650990014783, + 0.6215149611896057, + 0.6055408403310509, + 0.5897405537428495, + 0.5729677728164664, + 0.559595745790972, + 0.5505845779251468, + 0.5392379718024432, + 0.5286539715907157, + 0.5213795493986563, + 0.5112658018609328, + 0.504179925971016, + 0.49809098381419403, + 0.49417516361615327 + ], + "peak_layer": 1, + "peak_normalized": 2.495529360167265, + "population_cv": 0.5694475109215509, + "rest_mean": 2.533740560769123e-06, + "spike_contrast": 0.5337459133346284, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3523736697607091e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.862377747689607e-06, + 5.270805104373721e-06, + 4.903894932795083e-06, + 4.609466031979537e-06, + 4.09199901696411e-06, + 3.725753003891441e-06, + 3.479480710666394e-06, + 3.250749841754441e-06, + 2.9308416742424015e-06, + 2.718335281315376e-06, + 2.5433141672692727e-06, + 2.3790946670487756e-06, + 2.175666622861172e-06, + 2.0390943973325193e-06, + 1.911657136588474e-06, + 1.7997132317759679e-06, + 1.690957560640527e-06, + 1.5990206065907842e-06, + 1.5188796851361985e-06, + 1.4600331041947356e-06, + 1.4225074664864223e-06, + 1.385390191899205e-06, + 1.345988380307972e-06, + 1.3145754564902745e-06, + 1.2934068536196719e-06, + 1.2667519513342995e-06, + 1.2418885262377444e-06, + 1.2247998029124574e-06, + 1.2010410728180432e-06, + 1.1843952734125196e-06, + 1.1700914228640613e-06, + 1.1608925660766545e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.614924046109081e-05, + "first_to_last_ratio": 2.650607057187456, + "imbalance_abs_log_ratio": 0.9747886919476757, + "last_quartile_mean": 1.7410819282304146e-05, + "mean": 2.826828892921185e-05, + "normalized": [ + 1.9567413134368976, + 1.8597988174558218, + 1.7664389243795566, + 1.6893991915938202, + 1.5636317866388034, + 1.481959302241336, + 1.406542023468853, + 1.33584488000565, + 1.2332974726907746, + 1.1620116796950406, + 1.1014492434217877, + 1.0446845875825088, + 0.983522176643611, + 0.9354772941758706, + 0.8921357523597812, + 0.8494665759749792, + 0.814434979275871, + 0.7792464393968499, + 0.7495682845985361, + 0.7246339449332857, + 0.7117101663246612, + 0.6923010116299827, + 0.676425813245603, + 0.6619707608147611, + 0.6525593178077636, + 0.6397085734382539, + 0.6282016590684582, + 0.6192136216062852, + 0.6075928767792258, + 0.600035924319621, + 0.59270322236514, + 0.5872923826306096 + ], + "peak_layer": 1, + "peak_normalized": 1.9567413134368976, + "population_cv": 0.41380204518157304, + "rest_mean": 2.9948716137032198e-05, + "spike_contrast": 0.6408949859205556, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9193982006981968e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.5313728807959706e-05, + 5.2573330322047696e-05, + 4.9934205890167505e-05, + 4.7756424464751035e-05, + 4.420119512360543e-05, + 4.189245373709127e-05, + 3.976053631049581e-05, + 3.776204903260805e-05, + 3.486320929368958e-05, + 3.284808190073818e-05, + 3.113608545390889e-05, + 2.9531445761676878e-05, + 2.7802489057648927e-05, + 2.6444342438480817e-05, + 2.5219151211786084e-05, + 2.4012966605369e-05, + 2.3022683308226988e-05, + 2.2027963495929725e-05, + 2.1189012841205113e-05, + 2.0484161723288707e-05, + 2.0118828615522943e-05, + 1.9570165022742003e-05, + 1.91214003280038e-05, + 1.8712780729401857e-05, + 1.844673533923924e-05, + 1.80834667844465e-05, + 1.7758186004357412e-05, + 1.7504109564470127e-05, + 1.7175610992126167e-05, + 1.696198887657374e-05, + 1.6754705939092673e-05, + 1.660175075812731e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.735129757842515e-05, + "first_to_last_ratio": 2.0645564856398715, + "imbalance_abs_log_ratio": 0.724915426415284, + "last_quartile_mean": 1.8091681113219238e-05, + "mean": 2.5016603217409283e-05, + "normalized": [ + 1.7802727363091704, + 1.6959341879966399, + 1.614090945613871, + 1.5398204382964547, + 1.4345125366505216, + 1.3604640896020457, + 1.2905915971592592, + 1.2287960198423467, + 1.1574098238374129, + 1.0917198300160997, + 1.0408054967224047, + 0.9954414340603451, + 0.9640031824094093, + 0.9321987567197101, + 0.8975491383896601, + 0.8692109324382222, + 0.8490852481619621, + 0.8275443114054619, + 0.8048684241781405, + 0.7865739736521292, + 0.7803940959907774, + 0.7668844843835221, + 0.7577356594722136, + 0.7485970141410137, + 0.7465552812163766, + 0.7393132370819799, + 0.7324712504458508, + 0.7265002723105846, + 0.7169511717671491, + 0.7123002670381074, + 0.7073969087223727, + 0.7040072539687854 + ], + "peak_layer": 1, + "peak_normalized": 1.7802727363091704, + "population_cv": 0.3192203007098056, + "rest_mean": 2.612829794762311e-05, + "spike_contrast": 0.7276957615979828, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.901345167425461e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.453637666301802e-05, + 4.2426512663951144e-05, + 4.0379072743235156e-05, + 3.852107693091966e-05, + 3.588663093978539e-05, + 3.4034190321108326e-05, + 3.228621790185571e-05, + 3.074030246352777e-05, + 2.8954462322872132e-05, + 2.7311121812090278e-05, + 2.6037418138002977e-05, + 2.4902563382056542e-05, + 2.4116085114656016e-05, + 2.3320446416619234e-05, + 2.2453630663221702e-05, + 2.1744705009041354e-05, + 2.12412287510233e-05, + 2.070234768325463e-05, + 2.013507400988601e-05, + 1.967740899999626e-05, + 1.952280945261009e-05, + 1.9184844859410077e-05, + 1.8955972336698323e-05, + 1.8727354472503066e-05, + 1.86762772500515e-05, + 1.849510590545833e-05, + 1.8323942640563473e-05, + 1.817456904973369e-05, + 1.7935682990355417e-05, + 1.781933315214701e-05, + 1.769666778272949e-05, + 1.761187013471499e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010835385364771355, + "first_to_last_ratio": 4.77251940630934, + "imbalance_abs_log_ratio": 1.5628743428869922, + "last_quartile_mean": 2.2703701006321353e-05, + "mean": 5.5646100065587234e-05, + "normalized": [ + 2.403350415322374, + 2.2674490142770183, + 2.122035773013255, + 2.0228671307801673, + 1.8424615989793292, + 1.7404558535507528, + 1.6337825778074309, + 1.5451642251234987, + 1.388568890335992, + 1.2841278491651367, + 1.2075395305627292, + 1.1167532743401667, + 1.0024609552195234, + 0.9243915845434088, + 0.8572737878531238, + 0.7853215531962799, + 0.7194510201884433, + 0.662862527008837, + 0.6162760059182385, + 0.5774974679283221, + 0.5437012181417744, + 0.5145768135563884, + 0.4900744695357144, + 0.4675432113937477, + 0.44713840772999236, + 0.4307423296777627, + 0.4166278785720478, + 0.407689035656545, + 0.39881798875942465, + 0.3914264902634726, + 0.3871640362552815, + 0.38440708534382073 + ], + "peak_layer": 1, + "peak_normalized": 2.403350415322374, + "population_cv": 0.6252801969720955, + "rest_mean": 6.087470218416355e-05, + "spike_contrast": 0.4502962255543685, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.741164862527512e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00013373707770369947, + 0.0001261746947420761, + 0.00011808301496785134, + 0.00011256466677878052, + 0.00010252580250380561, + 9.684958058642223e-05, + 9.091362881008536e-05, + 8.598236308898777e-05, + 7.726844341959804e-05, + 7.14567067916505e-05, + 6.719486555084586e-05, + 6.214296445250511e-05, + 5.5783042625989765e-05, + 5.143878661328927e-05, + 4.770394298247993e-05, + 4.370008173282258e-05, + 4.003464346169494e-05, + 3.688571450766176e-05, + 3.429335629334673e-05, + 3.213548188796267e-05, + 3.0254852390498854e-05, + 2.8634192858589813e-05, + 2.7270732971373945e-05, + 2.601695632620249e-05, + 2.48815085797105e-05, + 2.396913077973295e-05, + 2.31837166211335e-05, + 2.2686304873786867e-05, + 2.219266571046319e-05, + 2.1781357645522803e-05, + 2.1544168703258038e-05, + 2.139075513696298e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bbef604712053bff665de420d5353433a19666047adbe681202c1aa4c9b40d0a", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.028617795556783676, + 0.015273182652890682, + 0.015790818259119987, + 0.01658804342150688, + 0.017823126167058945, + 0.012884888797998428, + 0.013985250145196915, + 0.015267650596797466, + 0.016450626775622368, + 0.013461960479617119, + 0.014477741904556751, + 0.01562186423689127, + 0.016906939446926117, + 0.014646334573626518, + 0.015772713348269463, + 0.016963548958301544, + 0.018231164664030075, + 0.016293151304125786, + 0.017603756859898567, + 0.018859725445508957, + 0.020210443064570427, + 0.018590020015835762, + 0.019812360405921936, + 0.021170100197196007, + 0.022594628855586052, + 0.021153371781110764, + 0.022732628509402275, + 0.024445844814181328, + 0.02632913924753666, + 0.02505173534154892, + 0.026562821120023727, + 0.028511708602309227 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.69130083552227e-06, + "first_to_last_ratio": 6.293146230018865, + "imbalance_abs_log_ratio": 1.839461141207743, + "last_quartile_mean": 4.2765585561710395e-07, + "mean": 1.3660126239045667e-06, + "normalized": [ + 1.7482498804289475, + 2.815312168363761, + 2.2948186868978717, + 1.7697692849656192, + 1.588517521198396, + 2.217468575156241, + 1.8408222360962587, + 1.4865407043733083, + 1.34895693309848, + 1.581801071425344, + 1.3164943144629158, + 1.2058085504791274, + 0.9210663509923345, + 1.0910843610686138, + 0.9814732856417313, + 0.7613599580616169, + 0.7590642708352797, + 0.7736743925325051, + 0.5757114422524934, + 0.5783657057699658, + 0.4762344643636547, + 0.48455133703992814, + 0.46520203272174326, + 0.4131025246748022, + 0.35882542613057594, + 0.3804709170270179, + 0.35152785556076144, + 0.33882867043560416, + 0.26176283558069635, + 0.288666522999825, + 0.28234383302193, + 0.24212388634264995 + ], + "peak_layer": 2, + "peak_normalized": 2.815312168363761, + "population_cv": 0.6846932882794533, + "rest_mean": 1.5077786391278398e-06, + "spike_contrast": 0.39825218776559335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.004761416988913e-07, + "top_five_layers": [ + 2, + 3, + 6, + 7, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.3881314064055914e-06, + 3.845751962217037e-06, + 3.1347512958745938e-06, + 2.4175271846615942e-06, + 2.169934987250599e-06, + 3.0290900667750975e-06, + 2.514586412871722e-06, + 2.0306333681219257e-06, + 1.8426921997161116e-06, + 2.1607602320727892e-06, + 1.7983478528549313e-06, + 1.6471497019665549e-06, + 1.2581882629092433e-06, + 1.4904350109645748e-06, + 1.3407048982116976e-06, + 1.0400273140476202e-06, + 1.0368913763159071e-06, + 1.056848986991099e-06, + 7.864290978432109e-07, + 7.900548553152476e-07, + 6.505422902591818e-07, + 6.619032433263783e-07, + 6.354718493639666e-07, + 5.643032636726275e-07, + 4.901600618723023e-07, + 5.197280756874534e-07, + 4.801914883501013e-07, + 4.6284424115583533e-07, + 3.575713378722867e-07, + 3.9432211451639887e-07, + 3.856852401895594e-07, + 3.3074428529289435e-07 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.969959031266626e-06, + "first_to_last_ratio": 5.9310438749867265, + "imbalance_abs_log_ratio": 1.7802002303990685, + "last_quartile_mean": 1.1751656501246543e-06, + "mean": 3.538651533574466e-06, + "normalized": [ + 1.643447719443132, + 2.664460681731711, + 2.26682328846465, + 1.9117870285326903, + 1.6953623890774276, + 2.1430603290971333, + 1.8420439675939804, + 1.5903367627347242, + 1.322590649598985, + 1.514836257012002, + 1.3201556675944437, + 1.1519184327291594, + 0.9433654624583057, + 1.1090475597261495, + 0.9445312929810626, + 0.7800763891189801, + 0.6931905948073118, + 0.7356932905122949, + 0.6107506859415541, + 0.565158192377895, + 0.4854051424814719, + 0.5215479448254272, + 0.4630492306077362, + 0.4246074405439294, + 0.3827682671544914, + 0.3985910238724124, + 0.3689537794505018, + 0.34747432873448575, + 0.29323390945282946, + 0.31234102613386355, + 0.2913380855124665, + 0.2620531796967914 + ], + "peak_layer": 2, + "peak_normalized": 2.664460681731711, + "population_cv": 0.6669213577125401, + "rest_mean": 3.8954815493154995e-06, + "spike_contrast": 0.4137535830085245, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6117694485728862e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.815588792756898e-06, + 9.428597877558786e-06, + 8.021497706067748e-06, + 6.765148100384977e-06, + 5.99929671807331e-06, + 7.5835437201021705e-06, + 6.518351710838033e-06, + 5.627647624351084e-06, + 4.680187430494698e-06, + 5.360477643989725e-06, + 4.671570877690101e-06, + 4.076237928529736e-06, + 3.3382416404492687e-06, + 3.9245328480319586e-06, + 3.3423671084165107e-06, + 2.7604185106611112e-06, + 2.4529599613742903e-06, + 2.6033621907117777e-06, + 2.1612338514387375e-06, + 1.9998979041702114e-06, + 1.7176796518469928e-06, + 1.845576434789109e-06, + 1.6385698700105422e-06, + 1.5025377706479048e-06, + 1.354483515569882e-06, + 1.4104747378951288e-06, + 1.3055988574706134e-06, + 1.2295905662540463e-06, + 1.037652623381291e-06, + 1.1052660511268186e-06, + 1.0309439630873385e-06, + 9.273148862121161e-07 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.006181029370054603, + "first_to_last_ratio": 5.931043870000395, + "imbalance_abs_log_ratio": 1.7802002295583512, + "last_quartile_mean": 0.00104214865132235, + "mean": 0.003138111591397319, + "normalized": [ + 1.6434476617442173, + 2.664460684658614, + 2.2668230869277246, + 1.9117870850052245, + 1.6953623005090255, + 2.143060416265084, + 1.8420439512820037, + 1.5903368591312919, + 1.3225906951513633, + 1.514836380023738, + 1.3201556314188725, + 1.1519184729978353, + 0.9433654289575268, + 1.1090475511276263, + 0.9445313214466298, + 0.7800763764287361, + 0.6931906017734022, + 0.735693296303442, + 0.6107506732925484, + 0.5651581485023144, + 0.4854051370092815, + 0.5215479525604865, + 0.463049256112769, + 0.42460744955562396, + 0.3827682394079058, + 0.39859099715083124, + 0.3689537785909722, + 0.3474743574102401, + 0.293233887776444, + 0.3123410323265631, + 0.2913381068209067, + 0.2620531823307552 + ], + "peak_layer": 2, + "peak_normalized": 2.664460684658614, + "population_cv": 0.6669213544088491, + "rest_mean": 0.0034545520191184348, + "spike_contrast": 0.41375358477538404, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0014293332817032934, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.005157322157174349, + 0.008361374959349632, + 0.007113543804734945, + 0.005999401211738586, + 0.005320236086845398, + 0.006725162733346224, + 0.005780539475381374, + 0.004990654531866312, + 0.004150437191128731, + 0.004753725603222847, + 0.004142795689404011, + 0.0036148487124592066, + 0.002960385987535119, + 0.003480314975604415, + 0.002964044688269496, + 0.002447966719046235, + 0.0021753094624727964, + 0.002308687660843134, + 0.0019166037673130631, + 0.0017735293367877603, + 0.00152325548697263, + 0.0016366756753996015, + 0.0014531002379953861, + 0.0013324655592441559, + 0.0012011694489046931, + 0.0012508230283856392, + 0.00115781812928617, + 0.0010904133087024093, + 0.0009202006622217596, + 0.0009801610140129924, + 0.0009142514900304377, + 0.000822352129034698 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.7807550679881388e-06, + "first_to_last_ratio": 5.412775845237589, + "imbalance_abs_log_ratio": 1.68876205650804, + "last_quartile_mean": 5.137391880793984e-07, + "mean": 1.481595617747189e-06, + "normalized": [ + 1.4558359312069442, + 2.4048173635519636, + 2.145458829580914, + 1.8401177624478837, + 1.622434601507838, + 2.1139236833600807, + 1.8608213201971038, + 1.571511096081306, + 1.3844269423754563, + 1.5890147466157112, + 1.3681012923101372, + 1.2125713912683727, + 0.9918873677472284, + 1.167668026852955, + 0.9903155750421241, + 0.8104020925085065, + 0.7254100339117308, + 0.7742380459295685, + 0.633010046622503, + 0.5955680935941017, + 0.5095544541449822, + 0.535113273728584, + 0.48051365792731365, + 0.4433064686062578, + 0.3985779012901482, + 0.41577980046677077, + 0.38175667305719, + 0.36725377000482734, + 0.30148202599237856, + 0.322715116130679, + 0.30949829156073677, + 0.2769143243777028 + ], + "peak_layer": 2, + "peak_normalized": 2.4048173635519636, + "population_cv": 0.6256905046703491, + "rest_mean": 1.6260750191610929e-06, + "spike_contrast": 0.4313496252306795, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.014068501121074e-07, + "top_five_layers": [ + 2, + 3, + 6, + 7, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1569601358351065e-06, + 3.562966867320938e-06, + 3.178702399964095e-06, + 2.7263104129815474e-06, + 2.4037919956754195e-06, + 3.1319800655182917e-06, + 2.7569847134145675e-06, + 2.3283439531951444e-06, + 2.051160890914616e-06, + 2.3542772851214977e-06, + 2.026972879320965e-06, + 1.796540459508833e-06, + 1.469575977353088e-06, + 1.730011831568845e-06, + 1.4672472161691985e-06, + 1.2006881888737553e-06, + 1.07476432731346e-06, + 1.1471076959423954e-06, + 9.378649110658444e-07, + 8.823910775390686e-07, + 7.549536462647666e-07, + 7.928214813546219e-07, + 7.119269298527797e-07, + 6.568009212060133e-07, + 5.905312718823552e-07, + 6.160175303193682e-07, + 5.656090138472791e-07, + 5.441215762402862e-07, + 4.4667444853985216e-07, + 4.781333018399891e-07, + 4.585513124766294e-07, + 4.1027504948942806e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.0954280109417596e-06, + "first_to_last_ratio": 5.491512307311366, + "imbalance_abs_log_ratio": 1.7032036834003335, + "last_quartile_mean": 7.457741659777639e-07, + "mean": 2.1618307570747675e-06, + "normalized": [ + 1.4068765300276402, + 2.3612070010793174, + 2.1657643431477385, + 1.8665928880193514, + 1.6616002088094273, + 2.1771057296508944, + 1.869426341469116, + 1.6468335409960086, + 1.3542408725522084, + 1.5667520899899137, + 1.3594800268466822, + 1.1819651148069434, + 1.009334596910006, + 1.1450136844580612, + 0.9957592626981273, + 0.817213831842472, + 0.712537815574818, + 0.75485624218804, + 0.6404857973819618, + 0.5842511071078107, + 0.5073510858223989, + 0.539798913350487, + 0.47838870053156934, + 0.4373768546103689, + 0.39708015593236357, + 0.4165380876399892, + 0.3871856292058109, + 0.3596150701163213, + 0.30484043926272086, + 0.32078253274025254, + 0.30118994888789563, + 0.2725555563432833 + ], + "peak_layer": 2, + "peak_normalized": 2.3612070010793174, + "population_cv": 0.6307038510118401, + "rest_mean": 2.3732101079042842e-06, + "spike_contrast": 0.4299586704088529, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0203822625953763e-06, + "top_five_layers": [ + 2, + 6, + 3, + 7, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.0414289540203754e-06, + 5.104529918753542e-06, + 4.682015969592612e-06, + 4.035257916257251e-06, + 3.592098437366076e-06, + 4.706534127763007e-06, + 4.041383363073692e-06, + 3.5601754007075215e-06, + 2.9276395707711345e-06, + 3.387052856851369e-06, + 2.938965735665988e-06, + 2.555208538979059e-06, + 2.1820105757797137e-06, + 2.4753258003329393e-06, + 2.152663000742905e-06, + 1.766677996783983e-06, + 1.5403861652885098e-06, + 1.6318714415319846e-06, + 1.3846218962498824e-06, + 1.2630520132006495e-06, + 1.0968071819661418e-06, + 1.1669538935166202e-06, + 1.0341954066461767e-06, + 9.455347367293143e-07, + 8.584200941186282e-07, + 9.004848493532336e-07, + 8.370298019144684e-07, + 7.774269192850625e-07, + 6.590134375983325e-07, + 6.934775456102216e-07, + 6.5112169522763e-07, + 5.892189847145346e-07 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.600380369699451e-06, + "first_to_last_ratio": 5.360934028391998, + "imbalance_abs_log_ratio": 1.6791382189297277, + "last_quartile_mean": 4.850610650919407e-07, + "mean": 1.407810660936093e-06, + "normalized": [ + 1.2424006186409213, + 2.116197835877747, + 2.1289350583166167, + 1.8206110660116281, + 1.672772135624017, + 2.180903391012272, + 1.9110341366235388, + 1.7040213242162747, + 1.3715763299737698, + 1.620181348382822, + 1.418229895744138, + 1.2018589460048466, + 1.0672724508221327, + 1.18033346969631, + 1.0372483057503512, + 0.8599070649138534, + 0.7226324996147198, + 0.7616597051326, + 0.6571229137740889, + 0.5957180716477494, + 0.5220566569858939, + 0.5404710315048392, + 0.4712153724045113, + 0.439240928976565, + 0.4052289724922489, + 0.41792517172028115, + 0.3921276690571982, + 0.35367305227247126, + 0.30647629235309304, + 0.31562180347619373, + 0.2957780769261062, + 0.2695684040502012 + ], + "peak_layer": 6, + "peak_normalized": 2.180903391012272, + "population_cv": 0.6156495501618916, + "rest_mean": 1.5445136143541646e-06, + "spike_contrast": 0.43354406607707613, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.696147124785057e-07, + "top_five_layers": [ + 6, + 3, + 2, + 7, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.7490648360762862e-06, + 2.9792058739985805e-06, + 2.9971374715387356e-06, + 2.563075668149395e-06, + 2.354946445848327e-06, + 3.0702990443387534e-06, + 2.69037423095142e-06, + 2.39893938669411e-06, + 1.9309197796246735e-06, + 2.280908574903151e-06, + 1.996599166886881e-06, + 1.6919898371270392e-06, + 1.5025175343907904e-06, + 1.6616860420981538e-06, + 1.4602492228732444e-06, + 1.2105863333999878e-06, + 1.0173297368964995e-06, + 1.0722726528911153e-06, + 9.251046435565513e-07, + 8.386582521779928e-07, + 7.349569273173984e-07, + 7.608808800796396e-07, + 6.633820248680422e-07, + 6.183680625326815e-07, + 5.704856675947667e-07, + 5.883595122213592e-07, + 5.520415129467438e-07, + 4.979046934749931e-07, + 4.314605916988512e-07, + 4.4433573975766194e-07, + 4.163995299677481e-07, + 3.7950127307340153e-07 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 1.2294863836359582e-05, + "first_to_last_ratio": 5.599817429618977, + "imbalance_abs_log_ratio": 1.7227339953558973, + "last_quartile_mean": 2.195582979425126e-06, + "mean": 6.332394622887705e-06, + "normalized": [ + 1.5057250244887312, + 2.556447232676371, + 2.2172679267612243, + 1.8985687859696891, + 1.6555718867405833, + 2.216307645234537, + 1.8634142379769214, + 1.619353790332452, + 1.3446938842284024, + 1.5146763535149452, + 1.3286277681864072, + 1.142126618231696, + 0.9485689933012624, + 1.1114498891377516, + 0.9384618363381629, + 0.7856483849547331, + 0.6834110772125562, + 0.720452579841522, + 0.6239316473465666, + 0.58489030026986, + 0.4861726124726784, + 0.5510752056898781, + 0.49322129575339285, + 0.4361559268955639, + 0.38962144676483523, + 0.4232163990310226, + 0.3815927693281277, + 0.36416773858904095, + 0.3092119801312069, + 0.32838130926563275, + 0.29808727088779047, + 0.27950018244645464 + ], + "peak_layer": 2, + "peak_normalized": 2.556447232676371, + "population_cv": 0.6506449880493771, + "rest_mean": 6.95244242015852e-06, + "spike_contrast": 0.4292213206934063, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.984136517625302e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.5348450486199e-06, + 1.6188432709896006e-05, + 1.4040615496924147e-05, + 1.2022486771456897e-05, + 1.0483734513400123e-05, + 1.4034534615348093e-05, + 1.1799874300777446e-05, + 1.0254387234454043e-05, + 8.515132321917918e-06, + 9.591528396413196e-06, + 8.413395335082896e-06, + 7.23239645594731e-06, + 6.006713192618918e-06, + 7.0381393015850335e-06, + 5.942710686213104e-06, + 4.975035608367762e-06, + 4.3276286305626854e-06, + 4.562190042634029e-06, + 3.950981408706866e-06, + 3.7037561924080364e-06, + 3.0786368370172568e-06, + 3.48962566931732e-06, + 3.1232718811224913e-06, + 2.7619114462140715e-06, + 2.467236754455371e-06, + 2.679973249541945e-06, + 2.416396000626264e-06, + 2.306053829670418e-06, + 1.9580522803153144e-06, + 2.0794400370505173e-06, + 1.887606231321115e-06, + 1.769905452420062e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4e4a33d69f29ecf2b2276b1faef8a4abb641becd86149078c17228416d2e6c90", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014952295459806919, + 0.014778315089643002, + 0.01543855108320713, + 0.016689632087945938, + 0.01178102195262909, + 0.012982882559299469, + 0.014135614968836308, + 0.015412751585245132, + 0.012501097284257412, + 0.013585822656750679, + 0.014657454565167427, + 0.01575874350965023, + 0.013761760666966438, + 0.014892088249325752, + 0.016054842621088028, + 0.017220331355929375, + 0.015453077852725983, + 0.016655873507261276, + 0.017766626551747322, + 0.018939858302474022, + 0.017597980797290802, + 0.018625281751155853, + 0.01988193951547146, + 0.0208931565284729, + 0.019704710692167282, + 0.021027294918894768, + 0.02250491827726364, + 0.023921677842736244, + 0.022959763184189796, + 0.02448389306664467, + 0.025734128430485725, + 0.027005694806575775 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.67974108839553e-06, + "first_to_last_ratio": 3.8051990936026456, + "imbalance_abs_log_ratio": 1.3363683141504743, + "last_quartile_mean": 2.0182231992293964e-06, + "mean": 4.804195157248614e-06, + "normalized": [ + 1.5877510011530913, + 1.6138570201166533, + 1.5124622299167279, + 1.3859304173236653, + 1.9450590445309985, + 1.723923747485618, + 1.5835986184675985, + 1.4358094613589012, + 1.674997608449353, + 1.5338764199258852, + 1.3688100265161884, + 1.2248142207369597, + 1.332209841900902, + 1.1411212950118081, + 0.9749533862079496, + 0.9261620377466854, + 0.9577260337784566, + 0.8878246260085111, + 0.7898909908438367, + 0.6795963310980445, + 0.6887310902589849, + 0.63469625591136, + 0.534405205528709, + 0.5010250390176237, + 0.5713704795033905, + 0.4976710822451457, + 0.45953689758880223, + 0.3909391200832928, + 0.4202843960445734, + 0.3737497958233971, + 0.34227896525752816, + 0.30493731415935826 + ], + "peak_layer": 5, + "peak_normalized": 1.9450590445309985, + "population_cv": 0.4919814241796063, + "rest_mean": 5.172476204717424e-06, + "spike_contrast": 0.5443190822896888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.8154775009170407e-06, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.62786567065632e-06, + 7.753284080536105e-06, + 7.266163720487384e-06, + 6.658280199189903e-06, + 9.344443242298439e-06, + 8.282066119136289e-06, + 7.607916813867632e-06, + 6.897908860992175e-06, + 8.047015398915391e-06, + 7.36904166842578e-06, + 6.576030500582419e-06, + 5.884246547793737e-06, + 6.4001960708992556e-06, + 5.4821693993289955e-06, + 4.683866336563369e-06, + 4.449463176570134e-06, + 4.601102773449384e-06, + 4.265282768756151e-06, + 3.7947904729662696e-06, + 3.264913402745151e-06, + 3.3087985684687737e-06, + 3.049204678973183e-06, + 2.567386900409474e-06, + 2.407022066108766e-06, + 2.744975290625007e-06, + 2.390909003224806e-06, + 2.207704937973176e-06, + 1.8781478274831898e-06, + 2.019128260144498e-06, + 1.7955669591174228e-06, + 1.6443749473182834e-06, + 1.4649783679487882e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.071156265832542e-05, + "first_to_last_ratio": 3.7446231103146244, + "imbalance_abs_log_ratio": 1.3203209738045956, + "last_quartile_mean": 8.20150967228983e-06, + "mean": 1.9372753484958594e-05, + "normalized": [ + 1.5605669500883477, + 1.6008805234793573, + 1.4979441121615527, + 1.3667092062703194, + 1.935846243370447, + 1.7068521590622452, + 1.5728591107226166, + 1.4407154966572362, + 1.6735980219259619, + 1.5302773348762577, + 1.3683676597193748, + 1.2238142355733308, + 1.331126586595219, + 1.1475793660881108, + 0.9834211123659795, + 0.934717997327087, + 0.9606142085118456, + 0.8999286898408035, + 0.7947106491954754, + 0.6878868767005736, + 0.6987226911749845, + 0.6477891191535685, + 0.5408853487908869, + 0.5073638009416495, + 0.5749202632388171, + 0.5019822542376574, + 0.4637585312481849, + 0.3931847315331312, + 0.422636716488808, + 0.3778631680748615, + 0.346655525688182, + 0.3058213088971263 + ], + "peak_layer": 5, + "peak_normalized": 1.935846243370447, + "population_cv": 0.485582780500018, + "rest_mean": 2.0829526268547244e-05, + "spike_contrast": 0.552397606418649, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1506180453579872e-05, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.023247882083524e-05, + 3.101346374023706e-05, + 2.9019302019150928e-05, + 2.6476920538698323e-05, + 3.750267205759883e-05, + 3.306642611278221e-05, + 3.0470611818600446e-05, + 2.7910626158700325e-05, + 3.242220191168599e-05, + 2.9645685572177172e-05, + 2.6509049348533154e-05, + 2.3708651497145183e-05, + 2.5787587219383568e-05, + 2.223177216365002e-05, + 1.9051574781769887e-05, + 1.8108061340171844e-05, + 1.8609742255648598e-05, + 1.7434096662327647e-05, + 1.5395733498735353e-05, + 1.332626288785832e-05, + 1.3536182450479828e-05, + 1.2549458915600553e-05, + 1.0478438525751699e-05, + 9.829033842834178e-06, + 1.1137788533233106e-05, + 9.724778465169948e-06, + 8.984279702417552e-06, + 7.617070878040977e-06, + 8.187636922230013e-06, + 7.3202500061597675e-06, + 6.71567204335588e-06, + 5.9246008277114015e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.027235321002081037, + "first_to_last_ratio": 3.744623083996792, + "imbalance_abs_log_ratio": 1.3203209667764297, + "last_quartile_mean": 0.00727318087592721, + "mean": 0.017179951537400484, + "normalized": [ + 1.5605669970711773, + 1.600880476363625, + 1.497944105038087, + 1.366709216880459, + 1.935846276047663, + 1.7068521655411573, + 1.5728590802784164, + 1.4407155266358058, + 1.6735980011744565, + 1.530277352493985, + 1.3683676045738475, + 1.2238142239324827, + 1.331126522669949, + 1.1475793472944473, + 0.983421191686974, + 0.9347179793604132, + 0.9606142409543752, + 0.8999287004964132, + 0.7947106908462469, + 0.6878868770559932, + 0.6987226668860359, + 0.6477890987517776, + 0.540885325434597, + 0.507363798093792, + 0.5749202742530459, + 0.5019822779872913, + 0.46375853415484747, + 0.39318471070846445, + 0.4226367318074797, + 0.3778631643128976, + 0.34665553015673556, + 0.3058213110570609 + ], + "peak_layer": 5, + "peak_normalized": 1.935846276047663, + "population_cv": 0.4855827790155644, + "rest_mean": 0.018471832249175618, + "spike_contrast": 0.5523975941406755, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.010203795693814754, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.02681046538054943, + 0.02750304900109768, + 0.025734607130289078, + 0.023479998111724854, + 0.03325774520635605, + 0.02932363748550415, + 0.02702164277434349, + 0.02475142292678356, + 0.02875233255326748, + 0.02629009075462818, + 0.02350848913192749, + 0.021025069057941437, + 0.02286868914961815, + 0.019715357571840286, + 0.016895128414034843, + 0.016058409586548805, + 0.016503306105732918, + 0.015460731461644173, + 0.01365309115499258, + 0.011817863211035728, + 0.012004021555185318, + 0.011128985323011875, + 0.009292383678257465, + 0.00871648546308279, + 0.009877102449536324, + 0.008624031208455563, + 0.007967349141836166, + 0.006754894275218248, + 0.007260878570377827, + 0.006491670850664377, + 0.005955525208264589, + 0.005253995303064585 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0114431578122094e-06, + "first_to_last_ratio": 3.7338115900101996, + "imbalance_abs_log_ratio": 1.3174295858952805, + "last_quartile_mean": 5.387104060616821e-07, + "mean": 1.2825281023864932e-06, + "normalized": [ + 1.5532146686344612, + 1.5705007170330607, + 1.4609122866463597, + 1.3427289650819811, + 1.9488030425764962, + 1.6741560212020503, + 1.555147967348224, + 1.4412752548233487, + 1.6499717927986355, + 1.4946710860708503, + 1.3520299845289774, + 1.2293314664225108, + 1.3640908964772576, + 1.20475774170785, + 0.9983558990009345, + 0.9578823214532782, + 0.9984385140583039, + 0.9052889731611042, + 0.8076018091021662, + 0.7077176784237299, + 0.6965311830349138, + 0.6602340537013012, + 0.5446056382431776, + 0.5214488681337209, + 0.5862688460241238, + 0.5169358876607686, + 0.44543689900304595, + 0.3852469543573297, + 0.4223793644694745, + 0.37423357828937454, + 0.3347806778641219, + 0.2950209626670668 + ], + "peak_layer": 5, + "peak_normalized": 1.9488030425764962, + "population_cv": 0.4798785088756173, + "rest_mean": 1.3770984666010648e-06, + "spike_contrast": 0.5604887045825205, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.718481356278062e-07, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.9920414615626214e-06, + 2.0142113044130383e-06, + 1.8736610627456685e-06, + 1.722087631605973e-06, + 2.499394668120658e-06, + 2.1471521449711872e-06, + 1.99452097149333e-06, + 1.8484760175851989e-06, + 2.116135192409274e-06, + 1.9169576717104064e-06, + 1.734016450427589e-06, + 1.5766521528348676e-06, + 1.7494849089416675e-06, + 1.5451356603080058e-06, + 1.28041949665203e-06, + 1.2285109960430418e-06, + 1.2805254527847865e-06, + 1.161058548859728e-06, + 1.0357720157117e-06, + 9.076678111341607e-07, + 8.933208164307871e-07, + 8.467687280244718e-07, + 6.984720357650076e-07, + 6.687728273391258e-07, + 7.519062705796387e-07, + 6.629848030570429e-07, + 5.712853408113006e-07, + 4.94090045322082e-07, + 5.417134048002481e-07, + 4.799650810127787e-07, + 4.293656274967361e-07, + 3.783726754136296e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.3353832602879265e-05, + "first_to_last_ratio": 3.2227511873122405, + "imbalance_abs_log_ratio": 1.1702354007568778, + "last_quartile_mean": 7.246551547268609e-06, + "mean": 1.5164828951697018e-05, + "normalized": [ + 1.5418197968348877, + 1.5709075226993274, + 1.4767167540770285, + 1.3358674513994462, + 1.888773418938626, + 1.6450250100697903, + 1.500898730733707, + 1.3599892142097643, + 1.6017342532341532, + 1.4490627654746335, + 1.298243393256253, + 1.1640713503380204, + 1.2947764191223834, + 1.1168088786006158, + 0.9842365208656229, + 0.9219899773989011, + 0.9795395409797548, + 0.9070325940775823, + 0.8195392964359715, + 0.7255652736709899, + 0.7453740698008642, + 0.6952593569228117, + 0.5937884104162011, + 0.5601599368070976, + 0.6362245384563836, + 0.5594724554118488, + 0.5161812187846071, + 0.44522117998918276, + 0.483203222643258, + 0.4278241163331062, + 0.3991997791976546, + 0.3554935528195254 + ], + "peak_layer": 5, + "peak_normalized": 1.888773418938626, + "population_cv": 0.44033722604682474, + "rest_mean": 1.615851486483869e-05, + "spike_contrast": 0.6064248541835164, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.798925020731986e-06, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.338143349334132e-05, + 2.38225438806694e-05, + 2.2394156985683367e-05, + 2.025820140261203e-05, + 2.8642925826716237e-05, + 2.4946522898972034e-05, + 2.2760872525395826e-05, + 2.062400380964391e-05, + 2.429002597637009e-05, + 2.197478897869587e-05, + 1.9687638996401802e-05, + 1.7652942915447056e-05, + 1.9635062926681712e-05, + 1.69362156157149e-05, + 1.4925778486940544e-05, + 1.3981820302433334e-05, + 1.4854549590381794e-05, + 1.375499414280057e-05, + 1.2428173249645624e-05, + 1.1003073268511798e-05, + 1.1303470273560379e-05, + 1.0543489224801306e-05, + 9.004699677461758e-06, + 8.494729627273045e-06, + 9.64823630056344e-06, + 8.484304089506622e-06, + 7.827799890947063e-06, + 6.7517030402086675e-06, + 7.327694220293779e-06, + 6.487879545602482e-06, + 6.0537963690876495e-06, + 5.390998921939172e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7634835330682108e-05, + "first_to_last_ratio": 2.77858968586119, + "imbalance_abs_log_ratio": 1.0219434917345125, + "last_quartile_mean": 6.346685665903351e-06, + "mean": 1.1861588987471805e-05, + "normalized": [ + 1.541845373328513, + 1.5495572539383986, + 1.4327425587355114, + 1.2888012556294268, + 1.830222537296193, + 1.5687089838516481, + 1.401198821657203, + 1.280665512186601, + 1.5188285787114204, + 1.4028344660045762, + 1.232338325101161, + 1.090834003257619, + 1.2372257058599228, + 1.094748447061392, + 1.0061819707458282, + 0.9121581837805397, + 1.0091865816307033, + 0.8969224311834834, + 0.8201245892341719, + 0.7702972436236826, + 0.7896771601850562, + 0.7574046980046742, + 0.6630426948751132, + 0.62395653030935, + 0.7012582077643145, + 0.6230517196436284, + 0.5687327928300282, + 0.4928440937190253, + 0.5522704219858736, + 0.47317300148686847, + 0.45736835411973276, + 0.41179750225833966 + ], + "peak_layer": 5, + "peak_normalized": 1.830222537296193, + "population_cv": 0.3925976431843591, + "rest_mean": 1.2505040962827883e-05, + "spike_contrast": 0.6706853936328379, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.38694832054898e-06, + "top_five_layers": [ + 5, + 6, + 2, + 1, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8288736100657843e-05, + 1.838021125877276e-05, + 1.699460335657932e-05, + 1.5287230780813843e-05, + 2.170934749301523e-05, + 1.8607381207402796e-05, + 1.662044451222755e-05, + 1.5190727935987525e-05, + 1.801572034310084e-05, + 1.663984585320577e-05, + 1.4617490705859382e-05, + 1.2939024600200355e-05, + 1.467546280764509e-05, + 1.2985456123715267e-05, + 1.1934916983591393e-05, + 1.0819645467563532e-05, + 1.1970556442975067e-05, + 1.0638925232342444e-05, + 9.72798079601489e-06, + 9.13694930204656e-06, + 9.366825906909071e-06, + 8.984023224911653e-06, + 7.864739927754272e-06, + 7.401115908578504e-06, + 8.318036634591408e-06, + 7.3903834163502324e-06, + 6.746074632246746e-06, + 5.845914074598113e-06, + 6.550804755534045e-06, + 5.612583663605619e-06, + 5.425115432444727e-06, + 4.884572717855917e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.088671580073424e-05, + "first_to_last_ratio": 4.047770176202506, + "imbalance_abs_log_ratio": 1.3981661557198917, + "last_quartile_mean": 1.5042038739920827e-05, + "mean": 3.823578526862548e-05, + "normalized": [ + 1.5715861755410512, + 1.6089836583440513, + 1.4748513229105977, + 1.3543048727993656, + 1.9303102550135327, + 1.7240508454541201, + 1.6068360247318012, + 1.468287586760736, + 1.6942393465207237, + 1.561867876551647, + 1.3928243162419038, + 1.2823569584329257, + 1.3510573524000515, + 1.153964680955942, + 0.9791255926207968, + 0.9554361624959514, + 0.9535104091594309, + 0.9404174748271559, + 0.8183623910744282, + 0.6821935048854633, + 0.6986677823476758, + 0.637767260268953, + 0.5187138398690193, + 0.4930674009743013, + 0.5397023628810284, + 0.4874058383983964, + 0.45262415468651734, + 0.3647526912052027, + 0.37717677353562973, + 0.34769267169862494, + 0.31586324682274186, + 0.26199916959023306 + ], + "peak_layer": 5, + "peak_normalized": 1.9303102550135327, + "population_cv": 0.5019072738596447, + "rest_mean": 4.1226788568935636e-05, + "spike_contrast": 0.535680032656999, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.2084367446950637e-05, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.0090831539127976e-05, + 6.15207536611706e-05, + 5.639209848595783e-05, + 5.1782910304609686e-05, + 7.380692841252312e-05, + 6.592043791897595e-05, + 6.143863720353693e-05, + 5.61411288799718e-05, + 6.478057184722275e-05, + 5.971924474579282e-05, + 5.325573147274554e-05, + 4.903192530036904e-05, + 5.165873881196603e-05, + 4.4122745748609304e-05, + 3.743763591046445e-05, + 3.6531851947074756e-05, + 3.645821925601922e-05, + 3.5957600630354136e-05, + 3.1290728657040745e-05, + 2.608420436445158e-05, + 2.6714111299952492e-05, + 2.4385532015003264e-05, + 1.9833430997096002e-05, + 1.885281926661264e-05, + 2.0635943656088784e-05, + 1.8636344975675456e-05, + 1.73064399859868e-05, + 1.3946605577075388e-05, + 1.4421650121221319e-05, + 1.3294302334543318e-05, + 1.2077279279765207e-05, + 1.0017743989010341e-05 + ] + } + } + } + } + } + }, + "step": 100 + }, + { + "modes": { + "learned": { + "bits_per_byte": 3.0745070201808953, + "forward": { + "activation_sha256": { + "attention_branch_output": "fe2bfa0ef8e4259f3ed7359d56fb5bbe56e2b0eafc30ccf1c1e3233cb3ec1708", + "mlp_branch_output": "ace6376bf838ff2894b72bcf39de9a912ced4cfce16da0bea85a55c0a2d375f6", + "post_attention_state": "7b7c9c05d1cbde4434ba686237dd1c945f87f81120c1f3f234a3adcdf1d68140", + "post_mlp_state": "4c56d08d88ae21cddacd7e29fc552875e42d5e83a4e1d9907d475a90abe4ea95", + "pre_attention_input": "787f9e4002b423fdbb7f8faf6fa82a8ff8c9854d40d44697d8afa4d95a3e169c", + "pre_mlp_input": "ad0dc0ee191d8145bd7a67f8b576436d75a4c4558d60d541ecf9bca80ffb9f93" + }, + "logits_sha256": "7b3859e28e8af95527b68a3a79f7e5a03998e27718248cca6c3d4328f46ef140", + "loss_nats": 2.1310858726501465, + "mixer_summary_sha256": "9d8f4f3394602c9d71988fb3e74693f57cf5a16485049d9aa7c2782046516cad" + }, + "loss_nats": 2.1310858726501465, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6506600379943848, + "group": 1, + "latest_source_mass": 0.6252298355102539, + "layer": 1, + "max_source_mass": 0.6252298355102539, + "mixer_index": 1, + "normalized_entropy": 0.9387040101191234, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3747701644897461, + "median": 0.35756751894950867, + "p05": 0.2806163728237152, + "p95": 0.5326682329177856 + }, + { + "label": "current_group_partial", + "mean": 0.6252298355102539, + "median": 0.642432451248169, + "p05": 0.46733176708221436, + "p95": 0.7193835973739624 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6806463003158569, + "group": 1, + "latest_source_mass": 0.5466031432151794, + "layer": 2, + "max_source_mass": 0.5466031432151794, + "mixer_index": 2, + "normalized_entropy": 0.9819650420651069, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.45339685678482056, + "median": 0.43950676918029785, + "p05": 0.37430456280708313, + "p95": 0.5817784070968628 + }, + { + "label": "current_group_partial", + "mean": 0.5466031432151794, + "median": 0.5604932308197021, + "p05": 0.418221652507782, + "p95": 0.6256954669952393 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6869370937347412, + "group": 1, + "latest_source_mass": 0.5331428050994873, + "layer": 2, + "max_source_mass": 0.5331428050994873, + "mixer_index": 3, + "normalized_entropy": 0.9910407385337882, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4668571949005127, + "median": 0.45914435386657715, + "p05": 0.40756916999816895, + "p95": 0.5482956171035767 + }, + { + "label": "current_group_partial", + "mean": 0.5331428050994873, + "median": 0.5408556461334229, + "p05": 0.45170438289642334, + "p95": 0.5924307703971863 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6791321635246277, + "group": 1, + "latest_source_mass": 0.5272461175918579, + "layer": 3, + "max_source_mass": 0.5272461175918579, + "mixer_index": 4, + "normalized_entropy": 0.979780604425173, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4727538824081421, + "median": 0.45147764682769775, + "p05": 0.37644311785697937, + "p95": 0.6218997836112976 + }, + { + "label": "current_group_partial", + "mean": 0.5272461175918579, + "median": 0.5485223531723022, + "p05": 0.3781002163887024, + "p95": 0.6235568523406982 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6804035902023315, + "group": 1, + "latest_source_mass": 0.5512661933898926, + "layer": 3, + "max_source_mass": 0.5512661933898926, + "mixer_index": 5, + "normalized_entropy": 0.9816148853879503, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4487338662147522, + "median": 0.4356163740158081, + "p05": 0.36789458990097046, + "p95": 0.5608916282653809 + }, + { + "label": "current_group_partial", + "mean": 0.5512661933898926, + "median": 0.5643836259841919, + "p05": 0.43910837173461914, + "p95": 0.6321054100990295 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.682766318321228, + "group": 1, + "latest_source_mass": 0.5353820323944092, + "layer": 4, + "max_source_mass": 0.5353820323944092, + "mixer_index": 6, + "normalized_entropy": 0.9850235815280511, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46461793780326843, + "median": 0.44794324040412903, + "p05": 0.38936740159988403, + "p95": 0.5920549631118774 + }, + { + "label": "current_group_partial", + "mean": 0.5353820323944092, + "median": 0.5520567893981934, + "p05": 0.40794503688812256, + "p95": 0.610632598400116 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6873732805252075, + "group": 1, + "latest_source_mass": 0.5116249322891235, + "layer": 4, + "max_source_mass": 0.5116249322891235, + "mixer_index": 7, + "normalized_entropy": 0.9916700230532952, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4883750379085541, + "median": 0.48108816146850586, + "p05": 0.41739019751548767, + "p95": 0.5840339660644531 + }, + { + "label": "current_group_partial", + "mean": 0.5116249322891235, + "median": 0.5189118981361389, + "p05": 0.41596606373786926, + "p95": 0.5826097726821899 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6835889220237732, + "group": 2, + "latest_source_mass": 0.5333441495895386, + "layer": 5, + "max_source_mass": 0.5333441495895386, + "mixer_index": 8, + "normalized_entropy": 0.9862103478103299, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46665582060813904, + "median": 0.45750027894973755, + "p05": 0.3836124539375305, + "p95": 0.5830957889556885 + }, + { + "label": "completed_group_1", + "mean": 0.5333441495895386, + "median": 0.5424997806549072, + "p05": 0.4169042110443115, + "p95": 0.6163875460624695 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0856965780258179, + "group": 2, + "latest_source_mass": 0.37292569875717163, + "layer": 5, + "max_source_mass": 0.37292569875717163, + "mixer_index": 9, + "normalized_entropy": 0.9882436135336242, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27463093400001526, + "median": 0.2607242465019226, + "p05": 0.2352934032678604, + "p95": 0.3491206765174866 + }, + { + "label": "completed_group_1", + "mean": 0.3524433374404907, + "median": 0.3531361222267151, + "p05": 0.3015327453613281, + "p95": 0.39581894874572754 + }, + { + "label": "current_group_partial", + "mean": 0.37292569875717163, + "median": 0.37286806106567383, + "p05": 0.33192169666290283, + "p95": 0.41239237785339355 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.077350378036499, + "group": 2, + "latest_source_mass": 0.3963770568370819, + "layer": 6, + "max_source_mass": 0.3963770568370819, + "mixer_index": 10, + "normalized_entropy": 0.9806465749100737, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2904614210128784, + "median": 0.2787848711013794, + "p05": 0.2332231104373932, + "p95": 0.39111974835395813 + }, + { + "label": "completed_group_1", + "mean": 0.3131615221500397, + "median": 0.3065500855445862, + "p05": 0.2717450261116028, + "p95": 0.37716877460479736 + }, + { + "label": "current_group_partial", + "mean": 0.3963770568370819, + "median": 0.41992655396461487, + "p05": 0.24805063009262085, + "p95": 0.4772615134716034 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0850789546966553, + "group": 2, + "latest_source_mass": 0.3814815580844879, + "layer": 6, + "max_source_mass": 0.3814815580844879, + "mixer_index": 11, + "normalized_entropy": 0.9876814285521406, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27783387899398804, + "median": 0.2646753787994385, + "p05": 0.2372770756483078, + "p95": 0.3592980206012726 + }, + { + "label": "completed_group_1", + "mean": 0.34068456292152405, + "median": 0.3382711410522461, + "p05": 0.30354392528533936, + "p95": 0.38547736406326294 + }, + { + "label": "current_group_partial", + "mean": 0.3814815580844879, + "median": 0.38342589139938354, + "p05": 0.31723207235336304, + "p95": 0.4350103437900543 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.062077522277832, + "group": 2, + "latest_source_mass": 0.42493611574172974, + "layer": 7, + "max_source_mass": 0.42493611574172974, + "mixer_index": 12, + "normalized_entropy": 0.9667446224959214, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2839078903198242, + "median": 0.2714773416519165, + "p05": 0.22162550687789917, + "p95": 0.3881691098213196 + }, + { + "label": "completed_group_1", + "mean": 0.29115602374076843, + "median": 0.27455592155456543, + "p05": 0.2443995624780655, + "p95": 0.3912811577320099 + }, + { + "label": "current_group_partial", + "mean": 0.42493611574172974, + "median": 0.4549126625061035, + "p05": 0.2369934469461441, + "p95": 0.5239657163619995 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0936429500579834, + "group": 2, + "latest_source_mass": 0.36200207471847534, + "layer": 7, + "max_source_mass": 0.36200207471847534, + "mixer_index": 13, + "normalized_entropy": 0.9954767130666716, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30465999245643616, + "median": 0.29826998710632324, + "p05": 0.2748320400714874, + "p95": 0.3600652515888214 + }, + { + "label": "completed_group_1", + "mean": 0.3333379328250885, + "median": 0.33137744665145874, + "p05": 0.30286699533462524, + "p95": 0.36546915769577026 + }, + { + "label": "current_group_partial", + "mean": 0.36200207471847534, + "median": 0.3605042099952698, + "p05": 0.32377851009368896, + "p95": 0.403035044670105 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0627415180206299, + "group": 2, + "latest_source_mass": 0.4155862033367157, + "layer": 8, + "max_source_mass": 0.4155862033367157, + "mixer_index": 14, + "normalized_entropy": 0.9673490174673293, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2850727438926697, + "median": 0.2791768014431, + "p05": 0.22466334700584412, + "p95": 0.36319249868392944 + }, + { + "label": "completed_group_1", + "mean": 0.2993410527706146, + "median": 0.27798235416412354, + "p05": 0.22940492630004883, + "p95": 0.4312986731529236 + }, + { + "label": "current_group_partial", + "mean": 0.4155862033367157, + "median": 0.4446514844894409, + "p05": 0.22757133841514587, + "p95": 0.5349330306053162 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.087097406387329, + "group": 2, + "latest_source_mass": 0.3793410062789917, + "layer": 8, + "max_source_mass": 0.3793410062789917, + "mixer_index": 15, + "normalized_entropy": 0.9895187024580431, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28865373134613037, + "median": 0.2799619734287262, + "p05": 0.24506454169750214, + "p95": 0.37004515528678894 + }, + { + "label": "completed_group_1", + "mean": 0.3320052921772003, + "median": 0.33097362518310547, + "p05": 0.2870577275753021, + "p95": 0.3873226046562195 + }, + { + "label": "current_group_partial", + "mean": 0.3793410062789917, + "median": 0.3775429427623749, + "p05": 0.3292272984981537, + "p95": 0.4380772113800049 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0500720739364624, + "group": 3, + "latest_source_mass": 0.4391520023345947, + "layer": 9, + "max_source_mass": 0.4391520023345947, + "mixer_index": 16, + "normalized_entropy": 0.9558167924823646, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27458876371383667, + "median": 0.2675239145755768, + "p05": 0.2081141322851181, + "p95": 0.3727893829345703 + }, + { + "label": "completed_group_1", + "mean": 0.286259263753891, + "median": 0.2620168924331665, + "p05": 0.2130153328180313, + "p95": 0.42261889576911926 + }, + { + "label": "completed_group_2", + "mean": 0.4391520023345947, + "median": 0.4622455835342407, + "p05": 0.24257506430149078, + "p95": 0.5693691968917847 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3743348121643066, + "group": 3, + "latest_source_mass": 0.30083364248275757, + "layer": 9, + "max_source_mass": 0.30083364248275757, + "mixer_index": 17, + "normalized_entropy": 0.9913730090152552, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21668720245361328, + "median": 0.21049442887306213, + "p05": 0.1911841183900833, + "p95": 0.26713764667510986 + }, + { + "label": "completed_group_1", + "mean": 0.23293715715408325, + "median": 0.2271968424320221, + "p05": 0.20331373810768127, + "p95": 0.27224695682525635 + }, + { + "label": "completed_group_2", + "mean": 0.2495419979095459, + "median": 0.2486393302679062, + "p05": 0.22275272011756897, + "p95": 0.2802416682243347 + }, + { + "label": "current_group_partial", + "mean": 0.30083364248275757, + "median": 0.3044033646583557, + "p05": 0.24597351253032684, + "p95": 0.34015560150146484 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3609715700149536, + "group": 3, + "latest_source_mass": 0.3002921938896179, + "layer": 10, + "max_source_mass": 0.3002921938896179, + "mixer_index": 18, + "normalized_entropy": 0.9817334674257201, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23144879937171936, + "median": 0.22791123390197754, + "p05": 0.20063085854053497, + "p95": 0.2748033404350281 + }, + { + "label": "completed_group_1", + "mean": 0.21839870512485504, + "median": 0.19323360919952393, + "p05": 0.15541185438632965, + "p95": 0.3410910665988922 + }, + { + "label": "completed_group_2", + "mean": 0.24986030161380768, + "median": 0.24544566869735718, + "p05": 0.21982765197753906, + "p95": 0.28961730003356934 + }, + { + "label": "current_group_partial", + "mean": 0.3002921938896179, + "median": 0.32308897376060486, + "p05": 0.16778458654880524, + "p95": 0.376542329788208 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3810515403747559, + "group": 3, + "latest_source_mass": 0.27817302942276, + "layer": 10, + "max_source_mass": 0.27817302942276, + "mixer_index": 19, + "normalized_entropy": 0.9962181042553622, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2385442852973938, + "median": 0.23833481967449188, + "p05": 0.2114236056804657, + "p95": 0.2653840482234955 + }, + { + "label": "completed_group_1", + "mean": 0.23454181849956512, + "median": 0.23530863225460052, + "p05": 0.2103835791349411, + "p95": 0.25546443462371826 + }, + { + "label": "completed_group_2", + "mean": 0.24874086678028107, + "median": 0.24718257784843445, + "p05": 0.22418582439422607, + "p95": 0.2786770462989807 + }, + { + "label": "current_group_partial", + "mean": 0.27817302942276, + "median": 0.28068703413009644, + "p05": 0.22970353066921234, + "p95": 0.3196732997894287 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.348030686378479, + "group": 3, + "latest_source_mass": 0.32103732228279114, + "layer": 11, + "max_source_mass": 0.32103732228279114, + "mixer_index": 20, + "normalized_entropy": 0.9723985931021886, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23874624073505402, + "median": 0.23695378005504608, + "p05": 0.20177650451660156, + "p95": 0.2892652153968811 + }, + { + "label": "completed_group_1", + "mean": 0.21300309896469116, + "median": 0.186348557472229, + "p05": 0.1455306112766266, + "p95": 0.34363847970962524 + }, + { + "label": "completed_group_2", + "mean": 0.22721335291862488, + "median": 0.22495108842849731, + "p05": 0.20071068406105042, + "p95": 0.2614164650440216 + }, + { + "label": "current_group_partial", + "mean": 0.32103732228279114, + "median": 0.352428138256073, + "p05": 0.1508321464061737, + "p95": 0.4218788743019104 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3825427293777466, + "group": 3, + "latest_source_mass": 0.26641666889190674, + "layer": 11, + "max_source_mass": 0.26641666889190674, + "mixer_index": 21, + "normalized_entropy": 0.9972937697451836, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24534732103347778, + "median": 0.24462050199508667, + "p05": 0.21986223757266998, + "p95": 0.27343857288360596 + }, + { + "label": "completed_group_1", + "mean": 0.2375141680240631, + "median": 0.2376786768436432, + "p05": 0.20753934979438782, + "p95": 0.27001917362213135 + }, + { + "label": "completed_group_2", + "mean": 0.25072187185287476, + "median": 0.25058382749557495, + "p05": 0.2306913286447525, + "p95": 0.2714776396751404 + }, + { + "label": "current_group_partial", + "mean": 0.26641666889190674, + "median": 0.2650451064109802, + "p05": 0.22704681754112244, + "p95": 0.31099480390548706 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3499886989593506, + "group": 3, + "latest_source_mass": 0.30865931510925293, + "layer": 12, + "max_source_mass": 0.30865931510925293, + "mixer_index": 22, + "normalized_entropy": 0.9738110006223994, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24629786610603333, + "median": 0.24514088034629822, + "p05": 0.21092942357063293, + "p95": 0.2848513722419739 + }, + { + "label": "completed_group_1", + "mean": 0.22150886058807373, + "median": 0.19684141874313354, + "p05": 0.15013524889945984, + "p95": 0.34530556201934814 + }, + { + "label": "completed_group_2", + "mean": 0.2235339879989624, + "median": 0.2229708731174469, + "p05": 0.19156691431999207, + "p95": 0.25894880294799805 + }, + { + "label": "current_group_partial", + "mean": 0.30865931510925293, + "median": 0.34268343448638916, + "p05": 0.13754047453403473, + "p95": 0.424306720495224 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3807358741760254, + "group": 3, + "latest_source_mass": 0.2846421003341675, + "layer": 12, + "max_source_mass": 0.2846421003341675, + "mixer_index": 23, + "normalized_entropy": 0.9959903992256198, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24433431029319763, + "median": 0.24396280944347382, + "p05": 0.22202064096927643, + "p95": 0.26817846298217773 + }, + { + "label": "completed_group_1", + "mean": 0.23110222816467285, + "median": 0.23034469783306122, + "p05": 0.20346714556217194, + "p95": 0.2573610544204712 + }, + { + "label": "completed_group_2", + "mean": 0.23992133140563965, + "median": 0.23892486095428467, + "p05": 0.2228689044713974, + "p95": 0.2597067654132843 + }, + { + "label": "current_group_partial", + "mean": 0.2846421003341675, + "median": 0.28294801712036133, + "p05": 0.24798741936683655, + "p95": 0.32731086015701294 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3399771451950073, + "group": 4, + "latest_source_mass": 0.3220147490501404, + "layer": 13, + "max_source_mass": 0.3220147490501404, + "mixer_index": 24, + "normalized_entropy": 0.9665891911386938, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24593281745910645, + "median": 0.24438412487506866, + "p05": 0.20850856602191925, + "p95": 0.2918635606765747 + }, + { + "label": "completed_group_1", + "mean": 0.21928200125694275, + "median": 0.1978234350681305, + "p05": 0.14302152395248413, + "p95": 0.34290891885757446 + }, + { + "label": "completed_group_2", + "mean": 0.21277041733264923, + "median": 0.2074536681175232, + "p05": 0.17290188372135162, + "p95": 0.267916738986969 + }, + { + "label": "completed_group_3", + "mean": 0.3220147490501404, + "median": 0.3550489544868469, + "p05": 0.1336003690958023, + "p95": 0.4569035768508911 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6075894832611084, + "group": 4, + "latest_source_mass": 0.20682592689990997, + "layer": 13, + "max_source_mass": 0.21128275990486145, + "mixer_index": 25, + "normalized_entropy": 0.998851506380761, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19650748372077942, + "median": 0.19601407647132874, + "p05": 0.18472915887832642, + "p95": 0.20989641547203064 + }, + { + "label": "completed_group_1", + "mean": 0.1956745684146881, + "median": 0.1953330934047699, + "p05": 0.18065369129180908, + "p95": 0.2107330560684204 + }, + { + "label": "completed_group_2", + "mean": 0.18970930576324463, + "median": 0.1893819272518158, + "p05": 0.17753958702087402, + "p95": 0.20207379758358002 + }, + { + "label": "completed_group_3", + "mean": 0.21128275990486145, + "median": 0.21213039755821228, + "p05": 0.19407641887664795, + "p95": 0.22417660057544708 + }, + { + "label": "current_group_partial", + "mean": 0.20682592689990997, + "median": 0.20550638437271118, + "p05": 0.1884995847940445, + "p95": 0.22830142080783844 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5706448554992676, + "group": 4, + "latest_source_mass": 0.2440074235200882, + "layer": 14, + "max_source_mass": 0.2440074235200882, + "mixer_index": 26, + "normalized_entropy": 0.9758965185080284, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1975565403699875, + "median": 0.19368210434913635, + "p05": 0.17189300060272217, + "p95": 0.23318083584308624 + }, + { + "label": "completed_group_1", + "mean": 0.1772933304309845, + "median": 0.15165507793426514, + "p05": 0.12244623899459839, + "p95": 0.29638439416885376 + }, + { + "label": "completed_group_2", + "mean": 0.16386015713214874, + "median": 0.1534792184829712, + "p05": 0.1269839107990265, + "p95": 0.23500467836856842 + }, + { + "label": "completed_group_3", + "mean": 0.21728253364562988, + "median": 0.21826648712158203, + "p05": 0.14061616361141205, + "p95": 0.29020458459854126 + }, + { + "label": "current_group_partial", + "mean": 0.2440074235200882, + "median": 0.258046954870224, + "p05": 0.10537702590227127, + "p95": 0.330468088388443 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6042816638946533, + "group": 4, + "latest_source_mass": 0.2172817587852478, + "layer": 14, + "max_source_mass": 0.2172817587852478, + "mixer_index": 27, + "normalized_entropy": 0.9967962426511696, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21478912234306335, + "median": 0.21512055397033691, + "p05": 0.19715391099452972, + "p95": 0.2305220067501068 + }, + { + "label": "completed_group_1", + "mean": 0.18552109599113464, + "median": 0.18529027700424194, + "p05": 0.1659296154975891, + "p95": 0.20769654214382172 + }, + { + "label": "completed_group_2", + "mean": 0.17930489778518677, + "median": 0.17780452966690063, + "p05": 0.16197741031646729, + "p95": 0.20606350898742676 + }, + { + "label": "completed_group_3", + "mean": 0.20310312509536743, + "median": 0.20250298082828522, + "p05": 0.19248157739639282, + "p95": 0.21611490845680237 + }, + { + "label": "current_group_partial", + "mean": 0.2172817587852478, + "median": 0.21857355535030365, + "p05": 0.17562012374401093, + "p95": 0.24863092601299286 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5764570236206055, + "group": 4, + "latest_source_mass": 0.25344932079315186, + "layer": 15, + "max_source_mass": 0.25344932079315186, + "mixer_index": 28, + "normalized_entropy": 0.9795078216073494, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20157726109027863, + "median": 0.20011398196220398, + "p05": 0.17114980518817902, + "p95": 0.23868146538734436 + }, + { + "label": "completed_group_1", + "mean": 0.17631688714027405, + "median": 0.156215101480484, + "p05": 0.1293901652097702, + "p95": 0.2730254530906677 + }, + { + "label": "completed_group_2", + "mean": 0.1703747808933258, + "median": 0.1609533429145813, + "p05": 0.1380072385072708, + "p95": 0.2378104031085968 + }, + { + "label": "completed_group_3", + "mean": 0.19828173518180847, + "median": 0.19690614938735962, + "p05": 0.15977059304714203, + "p95": 0.2402237504720688 + }, + { + "label": "current_group_partial", + "mean": 0.25344932079315186, + "median": 0.27549347281455994, + "p05": 0.10519556701183319, + "p95": 0.33760181069374084 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6011263132095337, + "group": 4, + "latest_source_mass": 0.22829055786132812, + "layer": 15, + "max_source_mass": 0.22829055786132812, + "mixer_index": 29, + "normalized_entropy": 0.9948357130397182, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20813238620758057, + "median": 0.20579253137111664, + "p05": 0.18812841176986694, + "p95": 0.23387062549591064 + }, + { + "label": "completed_group_1", + "mean": 0.18490281701087952, + "median": 0.1840219497680664, + "p05": 0.16537097096443176, + "p95": 0.20639142394065857 + }, + { + "label": "completed_group_2", + "mean": 0.17333751916885376, + "median": 0.16917650401592255, + "p05": 0.15315541625022888, + "p95": 0.2153732180595398 + }, + { + "label": "completed_group_3", + "mean": 0.20533671975135803, + "median": 0.20556354522705078, + "p05": 0.18881049752235413, + "p95": 0.21959951519966125 + }, + { + "label": "current_group_partial", + "mean": 0.22829055786132812, + "median": 0.23129136860370636, + "p05": 0.17272692918777466, + "p95": 0.2663574516773224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5817476511001587, + "group": 4, + "latest_source_mass": 0.24933090806007385, + "layer": 16, + "max_source_mass": 0.24933090806007385, + "mixer_index": 30, + "normalized_entropy": 0.9827950732861368, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20727969706058502, + "median": 0.20713281631469727, + "p05": 0.17457211017608643, + "p95": 0.2406192123889923 + }, + { + "label": "completed_group_1", + "mean": 0.18177060782909393, + "median": 0.1663498431444168, + "p05": 0.14110450446605682, + "p95": 0.2629965543746948 + }, + { + "label": "completed_group_2", + "mean": 0.17236655950546265, + "median": 0.16402167081832886, + "p05": 0.14090250432491302, + "p95": 0.23658250272274017 + }, + { + "label": "completed_group_3", + "mean": 0.18925222754478455, + "median": 0.18786820769309998, + "p05": 0.16428329050540924, + "p95": 0.21854622662067413 + }, + { + "label": "current_group_partial", + "mean": 0.24933090806007385, + "median": 0.267303466796875, + "p05": 0.10528035461902618, + "p95": 0.33821040391921997 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.603710412979126, + "group": 4, + "latest_source_mass": 0.22391749918460846, + "layer": 16, + "max_source_mass": 0.22391749918460846, + "mixer_index": 31, + "normalized_entropy": 0.9964413045009534, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2016504406929016, + "median": 0.1985834836959839, + "p05": 0.18258583545684814, + "p95": 0.2268678992986679 + }, + { + "label": "completed_group_1", + "mean": 0.1979413628578186, + "median": 0.19931256771087646, + "p05": 0.1786929965019226, + "p95": 0.2149808406829834 + }, + { + "label": "completed_group_2", + "mean": 0.17889465391635895, + "median": 0.17638051509857178, + "p05": 0.15777523815631866, + "p95": 0.20938827097415924 + }, + { + "label": "completed_group_3", + "mean": 0.19759604334831238, + "median": 0.19828101992607117, + "p05": 0.1828460842370987, + "p95": 0.21084150671958923 + }, + { + "label": "current_group_partial", + "mean": 0.22391749918460846, + "median": 0.22658780217170715, + "p05": 0.17493478953838348, + "p95": 0.26284241676330566 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5885388851165771, + "group": 5, + "latest_source_mass": 0.238150954246521, + "layer": 17, + "max_source_mass": 0.238150954246521, + "mixer_index": 32, + "normalized_entropy": 0.9870147042293073, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20988407731056213, + "median": 0.2089688777923584, + "p05": 0.18096143007278442, + "p95": 0.24111014604568481 + }, + { + "label": "completed_group_1", + "mean": 0.18461555242538452, + "median": 0.17398875951766968, + "p05": 0.1473308950662613, + "p95": 0.2504626214504242 + }, + { + "label": "completed_group_2", + "mean": 0.17252586781978607, + "median": 0.1655999720096588, + "p05": 0.14288564026355743, + "p95": 0.23261940479278564 + }, + { + "label": "completed_group_3", + "mean": 0.19482356309890747, + "median": 0.19468602538108826, + "p05": 0.17603687942028046, + "p95": 0.21431377530097961 + }, + { + "label": "completed_group_4", + "mean": 0.238150954246521, + "median": 0.251210480928421, + "p05": 0.11023568361997604, + "p95": 0.3212902545928955 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.783994197845459, + "group": 5, + "latest_source_mass": 0.19404590129852295, + "layer": 17, + "max_source_mass": 0.19404590129852295, + "mixer_index": 33, + "normalized_entropy": 0.9956661195233188, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16140902042388916, + "median": 0.1617078185081482, + "p05": 0.13892026245594025, + "p95": 0.18873091042041779 + }, + { + "label": "completed_group_1", + "mean": 0.16211673617362976, + "median": 0.16129055619239807, + "p05": 0.14639844000339508, + "p95": 0.18146981298923492 + }, + { + "label": "completed_group_2", + "mean": 0.1512581706047058, + "median": 0.14888465404510498, + "p05": 0.1341290920972824, + "p95": 0.18040500581264496 + }, + { + "label": "completed_group_3", + "mean": 0.1585075855255127, + "median": 0.15648186206817627, + "p05": 0.14301685988903046, + "p95": 0.18330132961273193 + }, + { + "label": "completed_group_4", + "mean": 0.17266258597373962, + "median": 0.1729961335659027, + "p05": 0.1528140753507614, + "p95": 0.19015821814537048 + }, + { + "label": "current_group_partial", + "mean": 0.19404590129852295, + "median": 0.19937825202941895, + "p05": 0.1327856332063675, + "p95": 0.2279757857322693 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.765566349029541, + "group": 5, + "latest_source_mass": 0.22537899017333984, + "layer": 18, + "max_source_mass": 0.22537899017333984, + "mixer_index": 34, + "normalized_entropy": 0.9853813412746752, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1630604863166809, + "median": 0.16281193494796753, + "p05": 0.1371261477470398, + "p95": 0.19141645729541779 + }, + { + "label": "completed_group_1", + "mean": 0.1509958803653717, + "median": 0.14023837447166443, + "p05": 0.12240280210971832, + "p95": 0.21479898691177368 + }, + { + "label": "completed_group_2", + "mean": 0.14303798973560333, + "median": 0.13656893372535706, + "p05": 0.12068455666303635, + "p95": 0.19587433338165283 + }, + { + "label": "completed_group_3", + "mean": 0.14991191029548645, + "median": 0.1466745138168335, + "p05": 0.1285857856273651, + "p95": 0.18114891648292542 + }, + { + "label": "completed_group_4", + "mean": 0.16761471331119537, + "median": 0.17043620347976685, + "p05": 0.11439520120620728, + "p95": 0.20673808455467224 + }, + { + "label": "current_group_partial", + "mean": 0.22537899017333984, + "median": 0.2385822832584381, + "p05": 0.11417834460735321, + "p95": 0.2751238942146301 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7848252058029175, + "group": 5, + "latest_source_mass": 0.19710269570350647, + "layer": 18, + "max_source_mass": 0.19710269570350647, + "mixer_index": 35, + "normalized_entropy": 0.9961299138951252, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16126418113708496, + "median": 0.16255977749824524, + "p05": 0.1384611874818802, + "p95": 0.1816936731338501 + }, + { + "label": "completed_group_1", + "mean": 0.1664336919784546, + "median": 0.1663765162229538, + "p05": 0.1537502110004425, + "p95": 0.1792917549610138 + }, + { + "label": "completed_group_2", + "mean": 0.15361233055591583, + "median": 0.15213868021965027, + "p05": 0.13601605594158173, + "p95": 0.17843274772167206 + }, + { + "label": "completed_group_3", + "mean": 0.1546509563922882, + "median": 0.15342193841934204, + "p05": 0.13999418914318085, + "p95": 0.17627598345279694 + }, + { + "label": "completed_group_4", + "mean": 0.16693615913391113, + "median": 0.16708451509475708, + "p05": 0.1510891318321228, + "p95": 0.18256354331970215 + }, + { + "label": "current_group_partial", + "mean": 0.19710269570350647, + "median": 0.20123803615570068, + "p05": 0.14279718697071075, + "p95": 0.22455936670303345 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7731435298919678, + "group": 5, + "latest_source_mass": 0.2158534973859787, + "layer": 19, + "max_source_mass": 0.2158534973859787, + "mixer_index": 36, + "normalized_entropy": 0.9896102464332964, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16123875975608826, + "median": 0.16278928518295288, + "p05": 0.131410151720047, + "p95": 0.18839234113693237 + }, + { + "label": "completed_group_1", + "mean": 0.15981441736221313, + "median": 0.15127545595169067, + "p05": 0.13407617807388306, + "p95": 0.21254204213619232 + }, + { + "label": "completed_group_2", + "mean": 0.1509810984134674, + "median": 0.1480858325958252, + "p05": 0.12804624438285828, + "p95": 0.18798498809337616 + }, + { + "label": "completed_group_3", + "mean": 0.14913737773895264, + "median": 0.14716270565986633, + "p05": 0.12724824249744415, + "p95": 0.17841869592666626 + }, + { + "label": "completed_group_4", + "mean": 0.16297483444213867, + "median": 0.16397103667259216, + "p05": 0.13277193903923035, + "p95": 0.18706153333187103 + }, + { + "label": "current_group_partial", + "mean": 0.2158534973859787, + "median": 0.2266644686460495, + "p05": 0.1148078441619873, + "p95": 0.2620621919631958 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7815005779266357, + "group": 5, + "latest_source_mass": 0.19993525743484497, + "layer": 19, + "max_source_mass": 0.19993525743484497, + "mixer_index": 37, + "normalized_entropy": 0.9942744037480438, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1552722007036209, + "median": 0.15699897706508636, + "p05": 0.12162899225950241, + "p95": 0.19721786677837372 + }, + { + "label": "completed_group_1", + "mean": 0.17254950106143951, + "median": 0.17343363165855408, + "p05": 0.15400823950767517, + "p95": 0.18830114603042603 + }, + { + "label": "completed_group_2", + "mean": 0.15193141996860504, + "median": 0.14969776570796967, + "p05": 0.13268329203128815, + "p95": 0.1832970529794693 + }, + { + "label": "completed_group_3", + "mean": 0.15522463619709015, + "median": 0.1547158658504486, + "p05": 0.13487689197063446, + "p95": 0.17792025208473206 + }, + { + "label": "completed_group_4", + "mean": 0.1650869995355606, + "median": 0.16515228152275085, + "p05": 0.14664505422115326, + "p95": 0.18368470668792725 + }, + { + "label": "current_group_partial", + "mean": 0.19993525743484497, + "median": 0.20347005128860474, + "p05": 0.14352381229400635, + "p95": 0.2376205176115036 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7801456451416016, + "group": 5, + "latest_source_mass": 0.2048441767692566, + "layer": 20, + "max_source_mass": 0.2048441767692566, + "mixer_index": 38, + "normalized_entropy": 0.9935182013624535, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1661754846572876, + "median": 0.17102378606796265, + "p05": 0.1277879923582077, + "p95": 0.1914365589618683 + }, + { + "label": "completed_group_1", + "mean": 0.16477233171463013, + "median": 0.16166681051254272, + "p05": 0.14654575288295746, + "p95": 0.19397443532943726 + }, + { + "label": "completed_group_2", + "mean": 0.15300315618515015, + "median": 0.14975905418395996, + "p05": 0.13582628965377808, + "p95": 0.1866467446088791 + }, + { + "label": "completed_group_3", + "mean": 0.15292593836784363, + "median": 0.15278922021389008, + "p05": 0.13439597189426422, + "p95": 0.17322586476802826 + }, + { + "label": "completed_group_4", + "mean": 0.1582789272069931, + "median": 0.15707442164421082, + "p05": 0.1421792358160019, + "p95": 0.17823022603988647 + }, + { + "label": "current_group_partial", + "mean": 0.2048441767692566, + "median": 0.2113300859928131, + "p05": 0.1285371482372284, + "p95": 0.2448955774307251 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7743862867355347, + "group": 5, + "latest_source_mass": 0.20740380883216858, + "layer": 20, + "max_source_mass": 0.20740380883216858, + "mixer_index": 39, + "normalized_entropy": 0.9903038422339103, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15205824375152588, + "median": 0.15292710065841675, + "p05": 0.09222303330898285, + "p95": 0.21958938241004944 + }, + { + "label": "completed_group_1", + "mean": 0.1771804541349411, + "median": 0.176810160279274, + "p05": 0.1583203673362732, + "p95": 0.19582124054431915 + }, + { + "label": "completed_group_2", + "mean": 0.15399661660194397, + "median": 0.15262216329574585, + "p05": 0.13103288412094116, + "p95": 0.18263062834739685 + }, + { + "label": "completed_group_3", + "mean": 0.15122172236442566, + "median": 0.14942464232444763, + "p05": 0.12676094472408295, + "p95": 0.18100851774215698 + }, + { + "label": "completed_group_4", + "mean": 0.15813913941383362, + "median": 0.1581244319677353, + "p05": 0.1325223445892334, + "p95": 0.18428882956504822 + }, + { + "label": "current_group_partial", + "mean": 0.20740380883216858, + "median": 0.21059799194335938, + "p05": 0.1396750807762146, + "p95": 0.2541891634464264 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.781898856163025, + "group": 6, + "latest_source_mass": 0.19959214329719543, + "layer": 21, + "max_source_mass": 0.19959214329719543, + "mixer_index": 40, + "normalized_entropy": 0.9944966870640967, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17058727145195007, + "median": 0.1770990937948227, + "p05": 0.12852564454078674, + "p95": 0.19631126523017883 + }, + { + "label": "completed_group_1", + "mean": 0.1644459068775177, + "median": 0.16324234008789062, + "p05": 0.1497141271829605, + "p95": 0.1824047863483429 + }, + { + "label": "completed_group_2", + "mean": 0.15141303837299347, + "median": 0.14833912253379822, + "p05": 0.1360013633966446, + "p95": 0.1818353235721588 + }, + { + "label": "completed_group_3", + "mean": 0.15486884117126465, + "median": 0.15303462743759155, + "p05": 0.13701528310775757, + "p95": 0.17965000867843628 + }, + { + "label": "completed_group_4", + "mean": 0.15909278392791748, + "median": 0.158164381980896, + "p05": 0.14224998652935028, + "p95": 0.17945128679275513 + }, + { + "label": "completed_group_5", + "mean": 0.19959214329719543, + "median": 0.20485615730285645, + "p05": 0.1373821198940277, + "p95": 0.23710061609745026 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9389331340789795, + "group": 6, + "latest_source_mass": 0.15687686204910278, + "layer": 21, + "max_source_mass": 0.15703952312469482, + "mixer_index": 41, + "normalized_entropy": 0.9964145235689732, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12895603477954865, + "median": 0.1290992647409439, + "p05": 0.10673296451568604, + "p95": 0.16424264013767242 + }, + { + "label": "completed_group_1", + "mean": 0.15703952312469482, + "median": 0.1581842303276062, + "p05": 0.13781607151031494, + "p95": 0.1730874478816986 + }, + { + "label": "completed_group_2", + "mean": 0.13832643628120422, + "median": 0.1382940709590912, + "p05": 0.12425417453050613, + "p95": 0.15265563130378723 + }, + { + "label": "completed_group_3", + "mean": 0.13419196009635925, + "median": 0.13376033306121826, + "p05": 0.12005055695772171, + "p95": 0.15059247612953186 + }, + { + "label": "completed_group_4", + "mean": 0.1312965601682663, + "median": 0.12827834486961365, + "p05": 0.11234064400196075, + "p95": 0.1575525403022766 + }, + { + "label": "completed_group_5", + "mean": 0.15331262350082397, + "median": 0.15450122952461243, + "p05": 0.12799498438835144, + "p95": 0.1745292842388153 + }, + { + "label": "current_group_partial", + "mean": 0.15687686204910278, + "median": 0.15919944643974304, + "p05": 0.13485465943813324, + "p95": 0.171228289604187 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.941878080368042, + "group": 6, + "latest_source_mass": 0.1529344916343689, + "layer": 22, + "max_source_mass": 0.1529344916343689, + "mixer_index": 42, + "normalized_entropy": 0.9979279265852903, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14007402956485748, + "median": 0.1409616768360138, + "p05": 0.12191025912761688, + "p95": 0.15652990341186523 + }, + { + "label": "completed_group_1", + "mean": 0.15248695015907288, + "median": 0.15473851561546326, + "p05": 0.1278151273727417, + "p95": 0.17166844010353088 + }, + { + "label": "completed_group_2", + "mean": 0.14026451110839844, + "median": 0.13944703340530396, + "p05": 0.12999942898750305, + "p95": 0.1532590687274933 + }, + { + "label": "completed_group_3", + "mean": 0.13737857341766357, + "median": 0.13697952032089233, + "p05": 0.12482102960348129, + "p95": 0.15122388303279877 + }, + { + "label": "completed_group_4", + "mean": 0.1305495798587799, + "median": 0.12951749563217163, + "p05": 0.11473267525434494, + "p95": 0.14860308170318604 + }, + { + "label": "completed_group_5", + "mean": 0.14631186425685883, + "median": 0.14591258764266968, + "p05": 0.13049930334091187, + "p95": 0.1627071499824524 + }, + { + "label": "current_group_partial", + "mean": 0.1529344916343689, + "median": 0.15251776576042175, + "p05": 0.13440527021884918, + "p95": 0.17216354608535767 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.936349630355835, + "group": 6, + "latest_source_mass": 0.16187208890914917, + "layer": 22, + "max_source_mass": 0.16187208890914917, + "mixer_index": 43, + "normalized_entropy": 0.9950868652881432, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12599323689937592, + "median": 0.12448512017726898, + "p05": 0.09913606941699982, + "p95": 0.1710376739501953 + }, + { + "label": "completed_group_1", + "mean": 0.16029544174671173, + "median": 0.1607722043991089, + "p05": 0.13700781762599945, + "p95": 0.1817699670791626 + }, + { + "label": "completed_group_2", + "mean": 0.1399340182542801, + "median": 0.13928979635238647, + "p05": 0.12660659849643707, + "p95": 0.1542026251554489 + }, + { + "label": "completed_group_3", + "mean": 0.1333811730146408, + "median": 0.1331656277179718, + "p05": 0.11817716062068939, + "p95": 0.15023785829544067 + }, + { + "label": "completed_group_4", + "mean": 0.12978437542915344, + "median": 0.12865802645683289, + "p05": 0.11021189391613007, + "p95": 0.15201398730278015 + }, + { + "label": "completed_group_5", + "mean": 0.14873966574668884, + "median": 0.14983436465263367, + "p05": 0.12514030933380127, + "p95": 0.16848590970039368 + }, + { + "label": "current_group_partial", + "mean": 0.16187208890914917, + "median": 0.16017910838127136, + "p05": 0.13439762592315674, + "p95": 0.1932365447282791 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9385986328125, + "group": 6, + "latest_source_mass": 0.16527059674263, + "layer": 23, + "max_source_mass": 0.16527059674263, + "mixer_index": 44, + "normalized_entropy": 0.9962426239226088, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1397915780544281, + "median": 0.14002375304698944, + "p05": 0.12451328337192535, + "p95": 0.15780359506607056 + }, + { + "label": "completed_group_1", + "mean": 0.15300613641738892, + "median": 0.15646013617515564, + "p05": 0.12481506168842316, + "p95": 0.17462868988513947 + }, + { + "label": "completed_group_2", + "mean": 0.14345945417881012, + "median": 0.14333182573318481, + "p05": 0.12926152348518372, + "p95": 0.158186674118042 + }, + { + "label": "completed_group_3", + "mean": 0.13018865883350372, + "median": 0.1291605830192566, + "p05": 0.11751946061849594, + "p95": 0.14813899993896484 + }, + { + "label": "completed_group_4", + "mean": 0.1257915198802948, + "median": 0.1252322494983673, + "p05": 0.11126883327960968, + "p95": 0.1417967826128006 + }, + { + "label": "completed_group_5", + "mean": 0.14249205589294434, + "median": 0.1427975594997406, + "p05": 0.12757454812526703, + "p95": 0.15669940412044525 + }, + { + "label": "current_group_partial", + "mean": 0.16527059674263, + "median": 0.16401387751102448, + "p05": 0.13626056909561157, + "p95": 0.20071014761924744 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.939380407333374, + "group": 6, + "latest_source_mass": 0.15541282296180725, + "layer": 23, + "max_source_mass": 0.15541282296180725, + "mixer_index": 45, + "normalized_entropy": 0.9966443765529929, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.128556489944458, + "median": 0.12622612714767456, + "p05": 0.1073562428355217, + "p95": 0.16966179013252258 + }, + { + "label": "completed_group_1", + "mean": 0.15473899245262146, + "median": 0.1559004783630371, + "p05": 0.13596564531326294, + "p95": 0.1694478690624237 + }, + { + "label": "completed_group_2", + "mean": 0.1372848004102707, + "median": 0.13732792437076569, + "p05": 0.12097055464982986, + "p95": 0.15409716963768005 + }, + { + "label": "completed_group_3", + "mean": 0.13797816634178162, + "median": 0.13845494389533997, + "p05": 0.12182354182004929, + "p95": 0.15258952975273132 + }, + { + "label": "completed_group_4", + "mean": 0.13512185215950012, + "median": 0.1341967135667801, + "p05": 0.11925841122865677, + "p95": 0.15510126948356628 + }, + { + "label": "completed_group_5", + "mean": 0.15090686082839966, + "median": 0.1500159651041031, + "p05": 0.12717851996421814, + "p95": 0.1754632294178009 + }, + { + "label": "current_group_partial", + "mean": 0.15541282296180725, + "median": 0.15449640154838562, + "p05": 0.13319969177246094, + "p95": 0.17916548252105713 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9388402700424194, + "group": 6, + "latest_source_mass": 0.16304057836532593, + "layer": 24, + "max_source_mass": 0.16304057836532593, + "mixer_index": 46, + "normalized_entropy": 0.9963668008945192, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14282521605491638, + "median": 0.14215797185897827, + "p05": 0.12491188943386078, + "p95": 0.16037693619728088 + }, + { + "label": "completed_group_1", + "mean": 0.15391701459884644, + "median": 0.15735432505607605, + "p05": 0.1199636161327362, + "p95": 0.1801576018333435 + }, + { + "label": "completed_group_2", + "mean": 0.1462559700012207, + "median": 0.14565740525722504, + "p05": 0.13299958407878876, + "p95": 0.1612643003463745 + }, + { + "label": "completed_group_3", + "mean": 0.13105326890945435, + "median": 0.13004028797149658, + "p05": 0.11590667068958282, + "p95": 0.15094627439975739 + }, + { + "label": "completed_group_4", + "mean": 0.12675246596336365, + "median": 0.1260158270597458, + "p05": 0.11388987302780151, + "p95": 0.1422860026359558 + }, + { + "label": "completed_group_5", + "mean": 0.13615547120571136, + "median": 0.13547730445861816, + "p05": 0.12139737606048584, + "p95": 0.15295183658599854 + }, + { + "label": "current_group_partial", + "mean": 0.16304057836532593, + "median": 0.16385850310325623, + "p05": 0.13662296533584595, + "p95": 0.18634286522865295 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9313902854919434, + "group": 6, + "latest_source_mass": 0.14112016558647156, + "layer": 24, + "max_source_mass": 0.17515115439891815, + "mixer_index": 47, + "normalized_entropy": 0.9925382661833493, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11694865673780441, + "median": 0.11002877354621887, + "p05": 0.09029232710599899, + "p95": 0.1822434961795807 + }, + { + "label": "completed_group_1", + "mean": 0.17515115439891815, + "median": 0.17486894130706787, + "p05": 0.14681096374988556, + "p95": 0.20170748233795166 + }, + { + "label": "completed_group_2", + "mean": 0.15024805068969727, + "median": 0.14959201216697693, + "p05": 0.13276571035385132, + "p95": 0.17043110728263855 + }, + { + "label": "completed_group_3", + "mean": 0.1394161581993103, + "median": 0.1390470266342163, + "p05": 0.12232993543148041, + "p95": 0.15828141570091248 + }, + { + "label": "completed_group_4", + "mean": 0.13194763660430908, + "median": 0.13136030733585358, + "p05": 0.11043333262205124, + "p95": 0.15552270412445068 + }, + { + "label": "completed_group_5", + "mean": 0.1451682150363922, + "median": 0.14562314748764038, + "p05": 0.11801350116729736, + "p95": 0.1739269196987152 + }, + { + "label": "current_group_partial", + "mean": 0.14112016558647156, + "median": 0.137407124042511, + "p05": 0.10931363701820374, + "p95": 0.1778358519077301 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9366493225097656, + "group": 7, + "latest_source_mass": 0.16003848612308502, + "layer": 25, + "max_source_mass": 0.16076743602752686, + "mixer_index": 48, + "normalized_entropy": 0.9952408765892693, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14114168286323547, + "median": 0.1450381577014923, + "p05": 0.1118045300245285, + "p95": 0.16293637454509735 + }, + { + "label": "completed_group_1", + "mean": 0.16076743602752686, + "median": 0.1661323606967926, + "p05": 0.12472015619277954, + "p95": 0.18624728918075562 + }, + { + "label": "completed_group_2", + "mean": 0.14226865768432617, + "median": 0.14061585068702698, + "p05": 0.12671124935150146, + "p95": 0.16414466500282288 + }, + { + "label": "completed_group_3", + "mean": 0.12737974524497986, + "median": 0.127046599984169, + "p05": 0.11332333087921143, + "p95": 0.1442527025938034 + }, + { + "label": "completed_group_4", + "mean": 0.12711969017982483, + "median": 0.12545642256736755, + "p05": 0.11273720115423203, + "p95": 0.14693106710910797 + }, + { + "label": "completed_group_5", + "mean": 0.1412842869758606, + "median": 0.14015623927116394, + "p05": 0.12225915491580963, + "p95": 0.16319061815738678 + }, + { + "label": "completed_group_6", + "mean": 0.16003848612308502, + "median": 0.1578693687915802, + "p05": 0.12668105959892273, + "p95": 0.19828300178050995 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.064785957336426, + "group": 7, + "latest_source_mass": 0.11369390040636063, + "layer": 25, + "max_source_mass": 0.15484434366226196, + "mixer_index": 49, + "normalized_entropy": 0.9929521537154775, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1061047688126564, + "median": 0.09920068830251694, + "p05": 0.08633361011743546, + "p95": 0.15746362507343292 + }, + { + "label": "completed_group_1", + "mean": 0.15484434366226196, + "median": 0.1575447916984558, + "p05": 0.12716035544872284, + "p95": 0.17626896500587463 + }, + { + "label": "completed_group_2", + "mean": 0.13487765192985535, + "median": 0.1332845687866211, + "p05": 0.11492810398340225, + "p95": 0.15705637633800507 + }, + { + "label": "completed_group_3", + "mean": 0.12357068061828613, + "median": 0.12331797182559967, + "p05": 0.10956361889839172, + "p95": 0.13827478885650635 + }, + { + "label": "completed_group_4", + "mean": 0.12023378908634186, + "median": 0.11931760609149933, + "p05": 0.10679896175861359, + "p95": 0.13665954768657684 + }, + { + "label": "completed_group_5", + "mean": 0.12704278528690338, + "median": 0.12507274746894836, + "p05": 0.10655903816223145, + "p95": 0.15118366479873657 + }, + { + "label": "completed_group_6", + "mean": 0.11963208019733429, + "median": 0.11901399493217468, + "p05": 0.09560881555080414, + "p95": 0.14494498074054718 + }, + { + "label": "current_group_partial", + "mean": 0.11369390040636063, + "median": 0.10261981189250946, + "p05": 0.08823835849761963, + "p95": 0.17856189608573914 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.062044143676758, + "group": 7, + "latest_source_mass": 0.12150939553976059, + "layer": 26, + "max_source_mass": 0.14862585067749023, + "mixer_index": 50, + "normalized_entropy": 0.9916336200588627, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11719765514135361, + "median": 0.12396101653575897, + "p05": 0.05985046178102493, + "p95": 0.15798619389533997 + }, + { + "label": "completed_group_1", + "mean": 0.14862585067749023, + "median": 0.15304353833198547, + "p05": 0.10433162748813629, + "p95": 0.18226584792137146 + }, + { + "label": "completed_group_2", + "mean": 0.12573553621768951, + "median": 0.12414389848709106, + "p05": 0.11025667190551758, + "p95": 0.1469687968492508 + }, + { + "label": "completed_group_3", + "mean": 0.12179906666278839, + "median": 0.1216571182012558, + "p05": 0.10140006989240646, + "p95": 0.14223159849643707 + }, + { + "label": "completed_group_4", + "mean": 0.11081404983997345, + "median": 0.11002233624458313, + "p05": 0.093814417719841, + "p95": 0.13098381459712982 + }, + { + "label": "completed_group_5", + "mean": 0.1260838657617569, + "median": 0.1261553317308426, + "p05": 0.10700763761997223, + "p95": 0.14507941901683807 + }, + { + "label": "completed_group_6", + "mean": 0.1282345950603485, + "median": 0.12765415012836456, + "p05": 0.09908819198608398, + "p95": 0.15842299163341522 + }, + { + "label": "current_group_partial", + "mean": 0.12150939553976059, + "median": 0.11118246614933014, + "p05": 0.0877910628914833, + "p95": 0.18945831060409546 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.054962158203125, + "group": 7, + "latest_source_mass": 0.09995201230049133, + "layer": 26, + "max_source_mass": 0.16456037759780884, + "mixer_index": 51, + "normalized_entropy": 0.9882279049513767, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10514847934246063, + "median": 0.09734492748975754, + "p05": 0.08340487629175186, + "p95": 0.16154851019382477 + }, + { + "label": "completed_group_1", + "mean": 0.16456037759780884, + "median": 0.16979515552520752, + "p05": 0.12305554002523422, + "p95": 0.1964556723833084 + }, + { + "label": "completed_group_2", + "mean": 0.14191412925720215, + "median": 0.141158789396286, + "p05": 0.11155636608600616, + "p95": 0.17160338163375854 + }, + { + "label": "completed_group_3", + "mean": 0.1262327879667282, + "median": 0.12533226609230042, + "p05": 0.11042644083499908, + "p95": 0.14479395747184753 + }, + { + "label": "completed_group_4", + "mean": 0.1181877925992012, + "median": 0.11585173010826111, + "p05": 0.10073923319578171, + "p95": 0.14371660351753235 + }, + { + "label": "completed_group_5", + "mean": 0.12522782385349274, + "median": 0.12385813146829605, + "p05": 0.10391169786453247, + "p95": 0.14975282549858093 + }, + { + "label": "completed_group_6", + "mean": 0.11877657473087311, + "median": 0.11736974120140076, + "p05": 0.09251725673675537, + "p95": 0.1482948511838913 + }, + { + "label": "current_group_partial", + "mean": 0.09995201230049133, + "median": 0.08531178534030914, + "p05": 0.07465144991874695, + "p95": 0.17865940928459167 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0610406398773193, + "group": 7, + "latest_source_mass": 0.10865749418735504, + "layer": 27, + "max_source_mass": 0.15103808045387268, + "mixer_index": 52, + "normalized_entropy": 0.9911510367405416, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11198848485946655, + "median": 0.11365711688995361, + "p05": 0.06384186446666718, + "p95": 0.16414028406143188 + }, + { + "label": "completed_group_1", + "mean": 0.15103808045387268, + "median": 0.15550360083580017, + "p05": 0.10918902605772018, + "p95": 0.18123289942741394 + }, + { + "label": "completed_group_2", + "mean": 0.13588255643844604, + "median": 0.1351599097251892, + "p05": 0.118770532310009, + "p95": 0.1558580845594406 + }, + { + "label": "completed_group_3", + "mean": 0.12388212233781815, + "median": 0.12342942506074905, + "p05": 0.10572439432144165, + "p95": 0.14399279654026031 + }, + { + "label": "completed_group_4", + "mean": 0.11551117897033691, + "median": 0.11412014812231064, + "p05": 0.09702129662036896, + "p95": 0.13879118859767914 + }, + { + "label": "completed_group_5", + "mean": 0.12874530255794525, + "median": 0.12885355949401855, + "p05": 0.10612692683935165, + "p95": 0.15165534615516663 + }, + { + "label": "completed_group_6", + "mean": 0.12429477274417877, + "median": 0.12331919372081757, + "p05": 0.09724370390176773, + "p95": 0.1537407487630844 + }, + { + "label": "current_group_partial", + "mean": 0.10865749418735504, + "median": 0.0960599035024643, + "p05": 0.08229582756757736, + "p95": 0.18050634860992432 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0338308811187744, + "group": 7, + "latest_source_mass": 0.08955829590559006, + "layer": 27, + "max_source_mass": 0.17635446786880493, + "mixer_index": 53, + "normalized_entropy": 0.9780659087322957, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0961676836013794, + "median": 0.08641783893108368, + "p05": 0.06882427632808685, + "p95": 0.17455565929412842 + }, + { + "label": "completed_group_1", + "mean": 0.17635446786880493, + "median": 0.1809459924697876, + "p05": 0.12560009956359863, + "p95": 0.21768227219581604 + }, + { + "label": "completed_group_2", + "mean": 0.14848342537879944, + "median": 0.14817440509796143, + "p05": 0.1071014404296875, + "p95": 0.18990638852119446 + }, + { + "label": "completed_group_3", + "mean": 0.13749773800373077, + "median": 0.13687211275100708, + "p05": 0.11250527948141098, + "p95": 0.16500277817249298 + }, + { + "label": "completed_group_4", + "mean": 0.12027013301849365, + "median": 0.11871654540300369, + "p05": 0.09309259057044983, + "p95": 0.15373988449573517 + }, + { + "label": "completed_group_5", + "mean": 0.12292921543121338, + "median": 0.1185426190495491, + "p05": 0.09395024180412292, + "p95": 0.1612403392791748 + }, + { + "label": "completed_group_6", + "mean": 0.10873906314373016, + "median": 0.1079384982585907, + "p05": 0.07837513834238052, + "p95": 0.14415065944194794 + }, + { + "label": "current_group_partial", + "mean": 0.08955829590559006, + "median": 0.07020936906337738, + "p05": 0.059403665363788605, + "p95": 0.19719363749027252 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.068699598312378, + "group": 7, + "latest_source_mass": 0.1132618635892868, + "layer": 28, + "max_source_mass": 0.14292436838150024, + "mixer_index": 54, + "normalized_entropy": 0.9948342171914195, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11418520659208298, + "median": 0.10965901613235474, + "p05": 0.09243083000183105, + "p95": 0.14435738325119019 + }, + { + "label": "completed_group_1", + "mean": 0.14292436838150024, + "median": 0.14914286136627197, + "p05": 0.10227109491825104, + "p95": 0.16936004161834717 + }, + { + "label": "completed_group_2", + "mean": 0.1359846293926239, + "median": 0.1383761763572693, + "p05": 0.10866174101829529, + "p95": 0.15511028468608856 + }, + { + "label": "completed_group_3", + "mean": 0.13056153059005737, + "median": 0.1301957666873932, + "p05": 0.11531321704387665, + "p95": 0.1473139524459839 + }, + { + "label": "completed_group_4", + "mean": 0.12100572884082794, + "median": 0.11918226629495621, + "p05": 0.1037902981042862, + "p95": 0.14383943378925323 + }, + { + "label": "completed_group_5", + "mean": 0.11985942721366882, + "median": 0.11902119219303131, + "p05": 0.10603872686624527, + "p95": 0.1366841346025467 + }, + { + "label": "completed_group_6", + "mean": 0.12221725285053253, + "median": 0.12153760343790054, + "p05": 0.10227984189987183, + "p95": 0.14314256608486176 + }, + { + "label": "current_group_partial", + "mean": 0.1132618635892868, + "median": 0.10621161758899689, + "p05": 0.09176099300384521, + "p95": 0.16215074062347412 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.036879539489746, + "group": 7, + "latest_source_mass": 0.08889776468276978, + "layer": 28, + "max_source_mass": 0.17786666750907898, + "mixer_index": 55, + "normalized_entropy": 0.9795320035033508, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09746268391609192, + "median": 0.08779538422822952, + "p05": 0.0718376636505127, + "p95": 0.17390751838684082 + }, + { + "label": "completed_group_1", + "mean": 0.17786666750907898, + "median": 0.18242985010147095, + "p05": 0.12422755360603333, + "p95": 0.21792855858802795 + }, + { + "label": "completed_group_2", + "mean": 0.15074898302555084, + "median": 0.15042778849601746, + "p05": 0.10790883004665375, + "p95": 0.19122514128684998 + }, + { + "label": "completed_group_3", + "mean": 0.13200223445892334, + "median": 0.13159450888633728, + "p05": 0.1103091835975647, + "p95": 0.15569928288459778 + }, + { + "label": "completed_group_4", + "mean": 0.11842608451843262, + "median": 0.11631985008716583, + "p05": 0.09421451389789581, + "p95": 0.1504896730184555 + }, + { + "label": "completed_group_5", + "mean": 0.12164927273988724, + "median": 0.11936528235673904, + "p05": 0.09542038291692734, + "p95": 0.15253053605556488 + }, + { + "label": "completed_group_6", + "mean": 0.1129462867975235, + "median": 0.11048559844493866, + "p05": 0.08038562536239624, + "p95": 0.15222208201885223 + }, + { + "label": "current_group_partial", + "mean": 0.08889776468276978, + "median": 0.07258449494838715, + "p05": 0.06192244961857796, + "p95": 0.18029186129570007 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.06752347946167, + "group": 8, + "latest_source_mass": 0.1112038791179657, + "layer": 29, + "max_source_mass": 0.14705273509025574, + "mixer_index": 56, + "normalized_entropy": 0.994268623580282, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11505837738513947, + "median": 0.11106020212173462, + "p05": 0.08959393203258514, + "p95": 0.14650048315525055 + }, + { + "label": "completed_group_1", + "mean": 0.14705273509025574, + "median": 0.1517227590084076, + "p05": 0.10864706337451935, + "p95": 0.17754200100898743 + }, + { + "label": "completed_group_2", + "mean": 0.1377231776714325, + "median": 0.13775423169136047, + "p05": 0.11708228290081024, + "p95": 0.1583956629037857 + }, + { + "label": "completed_group_3", + "mean": 0.12483140826225281, + "median": 0.12468798458576202, + "p05": 0.10894858092069626, + "p95": 0.1416374146938324 + }, + { + "label": "completed_group_4", + "mean": 0.11869609355926514, + "median": 0.11688373982906342, + "p05": 0.10011905431747437, + "p95": 0.14243903756141663 + }, + { + "label": "completed_group_5", + "mean": 0.1215740367770195, + "median": 0.12072279304265976, + "p05": 0.1052989736199379, + "p95": 0.13997763395309448 + }, + { + "label": "completed_group_6", + "mean": 0.12386028468608856, + "median": 0.12289520353078842, + "p05": 0.10213380306959152, + "p95": 0.14759738743305206 + }, + { + "label": "completed_group_7", + "mean": 0.1112038791179657, + "median": 0.10434558987617493, + "p05": 0.09146858006715775, + "p95": 0.15371212363243103 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1517844200134277, + "group": 8, + "latest_source_mass": 0.0917041003704071, + "layer": 29, + "max_source_mass": 0.16287443041801453, + "mixer_index": 57, + "normalized_entropy": 0.97931929317035, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08786985278129578, + "median": 0.07908640056848526, + "p05": 0.06481664627790451, + "p95": 0.1606782078742981 + }, + { + "label": "completed_group_1", + "mean": 0.16287443041801453, + "median": 0.16780570149421692, + "p05": 0.11163335293531418, + "p95": 0.20008432865142822 + }, + { + "label": "completed_group_2", + "mean": 0.13886377215385437, + "median": 0.13884207606315613, + "p05": 0.09857554733753204, + "p95": 0.17596080899238586 + }, + { + "label": "completed_group_3", + "mean": 0.12039706110954285, + "median": 0.11977612972259521, + "p05": 0.09957708418369293, + "p95": 0.14385205507278442 + }, + { + "label": "completed_group_4", + "mean": 0.10935483872890472, + "median": 0.10729463398456573, + "p05": 0.08751945197582245, + "p95": 0.1386163830757141 + }, + { + "label": "completed_group_5", + "mean": 0.1119438111782074, + "median": 0.10884027928113937, + "p05": 0.08701130002737045, + "p95": 0.14368495345115662 + }, + { + "label": "completed_group_6", + "mean": 0.10067413747310638, + "median": 0.09888175874948502, + "p05": 0.07253709435462952, + "p95": 0.13528487086296082 + }, + { + "label": "completed_group_7", + "mean": 0.07631799578666687, + "median": 0.06552867591381073, + "p05": 0.05597877502441406, + "p95": 0.13834024965763092 + }, + { + "label": "current_group_partial", + "mean": 0.0917041003704071, + "median": 0.08405104279518127, + "p05": 0.06457734853029251, + "p95": 0.149135559797287 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.177518129348755, + "group": 8, + "latest_source_mass": 0.09682197868824005, + "layer": 30, + "max_source_mass": 0.13366679847240448, + "mixer_index": 58, + "normalized_entropy": 0.9910312090121641, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10418819636106491, + "median": 0.10777237266302109, + "p05": 0.05029530078172684, + "p95": 0.157025545835495 + }, + { + "label": "completed_group_1", + "mean": 0.13366679847240448, + "median": 0.13715378940105438, + "p05": 0.09650824964046478, + "p95": 0.16189607977867126 + }, + { + "label": "completed_group_2", + "mean": 0.12014994770288467, + "median": 0.1188649833202362, + "p05": 0.10183564573526382, + "p95": 0.14117459952831268 + }, + { + "label": "completed_group_3", + "mean": 0.11398349702358246, + "median": 0.11387990415096283, + "p05": 0.09552477300167084, + "p95": 0.13391552865505219 + }, + { + "label": "completed_group_4", + "mean": 0.10732515156269073, + "median": 0.10627084970474243, + "p05": 0.08923279494047165, + "p95": 0.12864264845848083 + }, + { + "label": "completed_group_5", + "mean": 0.12029910832643509, + "median": 0.12015098333358765, + "p05": 0.09821736067533493, + "p95": 0.1429397165775299 + }, + { + "label": "completed_group_6", + "mean": 0.11035183072090149, + "median": 0.1095365658402443, + "p05": 0.0866188108921051, + "p95": 0.1372661590576172 + }, + { + "label": "completed_group_7", + "mean": 0.09321349859237671, + "median": 0.08730670809745789, + "p05": 0.07530548423528671, + "p95": 0.13010826706886292 + }, + { + "label": "current_group_partial", + "mean": 0.09682197868824005, + "median": 0.08937598764896393, + "p05": 0.07608069479465485, + "p95": 0.14057159423828125 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1535391807556152, + "group": 8, + "latest_source_mass": 0.08107093721628189, + "layer": 30, + "max_source_mass": 0.1632661372423172, + "mixer_index": 59, + "normalized_entropy": 0.980117919200792, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08976081013679504, + "median": 0.08317285776138306, + "p05": 0.06678152084350586, + "p95": 0.15141215920448303 + }, + { + "label": "completed_group_1", + "mean": 0.1632661372423172, + "median": 0.16861677169799805, + "p05": 0.10940256714820862, + "p95": 0.20118498802185059 + }, + { + "label": "completed_group_2", + "mean": 0.13892531394958496, + "median": 0.13932356238365173, + "p05": 0.09704238921403885, + "p95": 0.1767924726009369 + }, + { + "label": "completed_group_3", + "mean": 0.12143255770206451, + "median": 0.12173299491405487, + "p05": 0.09844215214252472, + "p95": 0.14377053081989288 + }, + { + "label": "completed_group_4", + "mean": 0.10986582189798355, + "median": 0.10805591940879822, + "p05": 0.08902113884687424, + "p95": 0.13708043098449707 + }, + { + "label": "completed_group_5", + "mean": 0.11029069125652313, + "median": 0.10775443911552429, + "p05": 0.08701202273368835, + "p95": 0.13905951380729675 + }, + { + "label": "completed_group_6", + "mean": 0.10418520867824554, + "median": 0.10218185186386108, + "p05": 0.07623609900474548, + "p95": 0.14061832427978516 + }, + { + "label": "completed_group_7", + "mean": 0.08120252937078476, + "median": 0.06974737346172333, + "p05": 0.059275463223457336, + "p95": 0.14717024564743042 + }, + { + "label": "current_group_partial", + "mean": 0.08107093721628189, + "median": 0.07502967119216919, + "p05": 0.06442901492118835, + "p95": 0.11666787415742874 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.191593647003174, + "group": 8, + "latest_source_mass": 0.10164810717105865, + "layer": 31, + "max_source_mass": 0.12338979542255402, + "mixer_index": 60, + "normalized_entropy": 0.9974372531642294, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10739251971244812, + "median": 0.10964220017194748, + "p05": 0.08550600707530975, + "p95": 0.12537474930286407 + }, + { + "label": "completed_group_1", + "mean": 0.12338979542255402, + "median": 0.1267070472240448, + "p05": 0.09627094119787216, + "p95": 0.13979919254779816 + }, + { + "label": "completed_group_2", + "mean": 0.11759915947914124, + "median": 0.11828742921352386, + "p05": 0.10300115495920181, + "p95": 0.12881097197532654 + }, + { + "label": "completed_group_3", + "mean": 0.1066875234246254, + "median": 0.10600249469280243, + "p05": 0.09917435050010681, + "p95": 0.11636147648096085 + }, + { + "label": "completed_group_4", + "mean": 0.1060015857219696, + "median": 0.10592096298933029, + "p05": 0.0979120284318924, + "p95": 0.11414148658514023 + }, + { + "label": "completed_group_5", + "mean": 0.1124890148639679, + "median": 0.11278317868709564, + "p05": 0.101162388920784, + "p95": 0.12259824573993683 + }, + { + "label": "completed_group_6", + "mean": 0.11740899085998535, + "median": 0.11669919639825821, + "p05": 0.10286425054073334, + "p95": 0.13360337913036346 + }, + { + "label": "completed_group_7", + "mean": 0.10738331079483032, + "median": 0.10325507819652557, + "p05": 0.09494076669216156, + "p95": 0.13414672017097473 + }, + { + "label": "current_group_partial", + "mean": 0.10164810717105865, + "median": 0.09851109236478806, + "p05": 0.08805661648511887, + "p95": 0.12722453474998474 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1465134620666504, + "group": 8, + "latest_source_mass": 0.07361725717782974, + "layer": 31, + "max_source_mass": 0.165542334318161, + "mixer_index": 61, + "normalized_entropy": 0.9769203768278215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08895281702280045, + "median": 0.08148864656686783, + "p05": 0.0638735294342041, + "p95": 0.16049745678901672 + }, + { + "label": "completed_group_1", + "mean": 0.165542334318161, + "median": 0.1687086820602417, + "p05": 0.11728888005018234, + "p95": 0.20459574460983276 + }, + { + "label": "completed_group_2", + "mean": 0.14156755805015564, + "median": 0.14117418229579926, + "p05": 0.09794186800718307, + "p95": 0.18480409681797028 + }, + { + "label": "completed_group_3", + "mean": 0.12290289998054504, + "median": 0.12261313199996948, + "p05": 0.09803362190723419, + "p95": 0.14901992678642273 + }, + { + "label": "completed_group_4", + "mean": 0.11169517040252686, + "median": 0.10987502336502075, + "p05": 0.09000048041343689, + "p95": 0.14066611230373383 + }, + { + "label": "completed_group_5", + "mean": 0.11284246295690536, + "median": 0.11086639761924744, + "p05": 0.08716568350791931, + "p95": 0.14374534785747528 + }, + { + "label": "completed_group_6", + "mean": 0.10374521464109421, + "median": 0.10118988156318665, + "p05": 0.07450100779533386, + "p95": 0.14089089632034302 + }, + { + "label": "completed_group_7", + "mean": 0.0791342556476593, + "median": 0.06692048162221909, + "p05": 0.055950865149497986, + "p95": 0.15064169466495514 + }, + { + "label": "current_group_partial", + "mean": 0.07361725717782974, + "median": 0.06859230995178223, + "p05": 0.05816151574254036, + "p95": 0.10351020097732544 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1857833862304688, + "group": 8, + "latest_source_mass": 0.09919881820678711, + "layer": 32, + "max_source_mass": 0.12139561772346497, + "mixer_index": 62, + "normalized_entropy": 0.9947928895281057, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10998444259166718, + "median": 0.11795933544635773, + "p05": 0.05861174687743187, + "p95": 0.14438092708587646 + }, + { + "label": "completed_group_1", + "mean": 0.12038751691579819, + "median": 0.12215788662433624, + "p05": 0.09455405175685883, + "p95": 0.141767680644989 + }, + { + "label": "completed_group_2", + "mean": 0.11211743205785751, + "median": 0.111464723944664, + "p05": 0.09543715417385101, + "p95": 0.130376935005188 + }, + { + "label": "completed_group_3", + "mean": 0.11047163605690002, + "median": 0.11037711799144745, + "p05": 0.09404903650283813, + "p95": 0.127632275223732 + }, + { + "label": "completed_group_4", + "mean": 0.10957512259483337, + "median": 0.10831746459007263, + "p05": 0.09309716522693634, + "p95": 0.12977787852287292 + }, + { + "label": "completed_group_5", + "mean": 0.12139561772346497, + "median": 0.12195655703544617, + "p05": 0.10015495121479034, + "p95": 0.1401825100183487 + }, + { + "label": "completed_group_6", + "mean": 0.11392621695995331, + "median": 0.11355376243591309, + "p05": 0.09452947974205017, + "p95": 0.1352669596672058 + }, + { + "label": "completed_group_7", + "mean": 0.10294321179389954, + "median": 0.09949207305908203, + "p05": 0.08735473453998566, + "p95": 0.13111063838005066 + }, + { + "label": "current_group_partial", + "mean": 0.09919881820678711, + "median": 0.09456383436918259, + "p05": 0.0812632143497467, + "p95": 0.13172483444213867 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1400556564331055, + "group": 8, + "latest_source_mass": 0.0691327303647995, + "layer": 32, + "max_source_mass": 0.16802933812141418, + "mixer_index": 63, + "normalized_entropy": 0.9739813028250293, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08778932690620422, + "median": 0.0795741081237793, + "p05": 0.06344711035490036, + "p95": 0.16418468952178955 + }, + { + "label": "completed_group_1", + "mean": 0.16802933812141418, + "median": 0.16981908679008484, + "p05": 0.12191798537969589, + "p95": 0.20580793917179108 + }, + { + "label": "completed_group_2", + "mean": 0.14664983749389648, + "median": 0.14726951718330383, + "p05": 0.09816566854715347, + "p95": 0.1929507553577423 + }, + { + "label": "completed_group_3", + "mean": 0.12668848037719727, + "median": 0.1260564625263214, + "p05": 0.10192481428384781, + "p95": 0.1541551649570465 + }, + { + "label": "completed_group_4", + "mean": 0.11206841468811035, + "median": 0.10879923403263092, + "p05": 0.08885316550731659, + "p95": 0.1473042368888855 + }, + { + "label": "completed_group_5", + "mean": 0.11208956688642502, + "median": 0.10941921174526215, + "p05": 0.08551695197820663, + "p95": 0.1449490785598755 + }, + { + "label": "completed_group_6", + "mean": 0.10068599134683609, + "median": 0.09787309914827347, + "p05": 0.07257338613271713, + "p95": 0.1365347057580948 + }, + { + "label": "completed_group_7", + "mean": 0.07686632871627808, + "median": 0.06485281884670258, + "p05": 0.05413344129920006, + "p95": 0.14826452732086182 + }, + { + "label": "current_group_partial", + "mean": 0.0691327303647995, + "median": 0.06579415500164032, + "p05": 0.055686283856630325, + "p95": 0.0925217717885971 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1613364219665527, + "group": null, + "latest_source_mass": 0.12122741341590881, + "layer": null, + "max_source_mass": 0.15537577867507935, + "mixer_index": null, + "normalized_entropy": 0.9836665966056254, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1255332976579666, + "median": 0.12688711285591125, + "p05": 0.08593897521495819, + "p95": 0.16157466173171997 + }, + { + "label": "completed_group_1", + "mean": 0.08712290227413177, + "median": 0.08774389326572418, + "p05": 0.06929010152816772, + "p95": 0.10448890924453735 + }, + { + "label": "completed_group_2", + "mean": 0.09080197662115097, + "median": 0.09027823060750961, + "p05": 0.07495184242725372, + "p95": 0.10968034714460373 + }, + { + "label": "completed_group_3", + "mean": 0.08838342130184174, + "median": 0.0878291055560112, + "p05": 0.07189641892910004, + "p95": 0.10652950406074524 + }, + { + "label": "completed_group_4", + "mean": 0.09530800580978394, + "median": 0.09452416002750397, + "p05": 0.07556769251823425, + "p95": 0.11846894770860672 + }, + { + "label": "completed_group_5", + "mean": 0.09807588160037994, + "median": 0.0959044098854065, + "p05": 0.07471995055675507, + "p95": 0.12936776876449585 + }, + { + "label": "completed_group_6", + "mean": 0.13817133009433746, + "median": 0.13675633072853088, + "p05": 0.10469131171703339, + "p95": 0.17332828044891357 + }, + { + "label": "completed_group_7", + "mean": 0.15537577867507935, + "median": 0.1563301980495453, + "p05": 0.12374594062566757, + "p95": 0.18756605684757233 + }, + { + "label": "completed_group_8", + "mean": 0.12122741341590881, + "median": 0.12231454998254776, + "p05": 0.06295841932296753, + "p95": 0.17872877418994904 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "83aa24720e9c1fc671d5681102b923705e3423225e358f5b850e1a9ed514d9b6", + "optimizer_state_before": "83aa24720e9c1fc671d5681102b923705e3423225e358f5b850e1a9ed514d9b6", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "fe2bfa0ef8e4259f3ed7359d56fb5bbe56e2b0eafc30ccf1c1e3233cb3ec1708", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.01049753837287426, + 0.013886291533708572, + 0.01245343592017889, + 0.01395952794700861, + 0.014544809237122536, + 0.014345151372253895, + 0.015135552734136581, + 0.014776745811104774, + 0.01795734278857708, + 0.013709688559174538, + 0.016144096851348877, + 0.018440017476677895, + 0.018456056714057922, + 0.012218938209116459, + 0.01701919361948967, + 0.01305371057242155, + 0.017418840900063515, + 0.026864025741815567, + 0.035265542566776276, + 0.027374446392059326, + 0.01969025284051895, + 0.018602686002850533, + 0.0133047541603446, + 0.015662170946598053, + 0.012774706818163395, + 0.016374029219150543, + 0.010044055990874767, + 0.006929588038474321, + 0.005331806372851133, + 0.01223068032413721, + 0.00525518087670207, + 0.010387555696070194 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.7470343841050635e-05, + "first_to_last_ratio": 5.543186856446801, + "imbalance_abs_log_ratio": 1.7125695802152925, + "last_quartile_mean": 6.759711482118291e-06, + "mean": 1.9353574231217863e-05, + "normalized": [ + 2.3427102645191087, + 2.192158088708691, + 2.1504138353757374, + 2.019465522152087, + 1.9995945477738173, + 1.7731063055219731, + 1.5680823096714636, + 1.4432231490291196, + 1.4193844085037457, + 1.2773784002125157, + 1.1654379763631353, + 1.0792980891066979, + 1.0342522513219847, + 0.9566360686297143, + 0.8835644518961069, + 0.8115541392967873, + 0.8495762151917138, + 0.7730101521498506, + 0.7009121964310462, + 0.6382247620634326, + 0.6098072442169371, + 0.5543251081308368, + 0.501441625782686, + 0.46224625993377594, + 0.4400253690998266, + 0.4103016633144128, + 0.39090671090870194, + 0.3772935492489135, + 0.30325680084430284, + 0.29393754112208775, + 0.2900082627577495, + 0.28846673072104 + ], + "peak_layer": 1, + "peak_normalized": 2.3427102645191087, + "population_cv": 0.6284664540855158, + "rest_mean": 2.109694017875612e-05, + "spike_contrast": 0.4711298430148607, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.939398114511278e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.533981700660661e-05, + 4.242609429638833e-05, + 4.161819379078224e-05, + 3.908387589035556e-05, + 3.8699301512679085e-05, + 3.4315944503759965e-05, + 3.0347997380886227e-05, + 2.7931526346947066e-05, + 2.7470161512610503e-05, + 2.4721837689867243e-05, + 2.2555390387424268e-05, + 2.088827568513807e-05, + 2.0016477719764225e-05, + 1.85143271664856e-05, + 1.7100130207836628e-05, + 1.5706473277532496e-05, + 1.6442336345789954e-05, + 1.4960509361117147e-05, + 1.3565156223194208e-05, + 1.2351930308796e-05, + 1.1801949767686892e-05, + 1.0728172128438018e-05, + 9.704687727207784e-06, + 8.94611730473116e-06, + 8.516063644492533e-06, + 7.940803698147647e-06, + 7.565442047052784e-06, + 7.3019787123485e-06, + 5.8691030062618665e-06, + 5.688742021447979e-06, + 5.612696440948639e-06, + 5.582862286246382e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001431262517144205, + "first_to_last_ratio": 5.283860582769421, + "imbalance_abs_log_ratio": 1.6646570015219528, + "last_quartile_mean": 2.7087439093520516e-05, + "mean": 7.517440684523535e-05, + "normalized": [ + 2.3167821007513925, + 2.1576627418693985, + 2.0809366190077223, + 1.9632050338348952, + 1.9581380043895957, + 1.7565617549117303, + 1.5601488386363862, + 1.4379456619222077, + 1.4181936161344835, + 1.2807776397732267, + 1.1719405534721656, + 1.0858056019237723, + 1.04268310234457, + 0.9685159493695125, + 0.894804763228899, + 0.8229797757020919, + 0.8654572820771477, + 0.7870405201408273, + 0.7143484600420894, + 0.6515578296073351, + 0.6246802736621585, + 0.5677900631265306, + 0.5145973361420437, + 0.47482296492635234, + 0.451578528714676, + 0.4221160637804328, + 0.4032803071079029, + 0.38978326294250304, + 0.3131724384233296, + 0.3039367871211443, + 0.2999847761272356, + 0.29877134878624156 + ], + "peak_layer": 1, + "peak_normalized": 2.3167821007513925, + "population_cv": 0.6100414888257124, + "rest_mean": 8.17633902409364e-05, + "spike_contrast": 0.48424969160129383, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.959389650844969e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001741627202136442, + 0.0001622010167920962, + 0.00015643317601643503, + 0.00014758277393411845, + 0.00014720186300110072, + 0.000132048488012515, + 0.00011728326353477314, + 0.00010809671221068129, + 0.00010661186388460919, + 9.628169937059283e-05, + 8.809993596514687e-05, + 8.162479207385331e-05, + 7.838308374630287e-05, + 7.28076120140031e-05, + 6.726641731802374e-05, + 6.186701648402959e-05, + 6.506023783003911e-05, + 5.9165304264752194e-05, + 5.370072176447138e-05, + 4.898047336610034e-05, + 4.695996904047206e-05, + 4.268328120815568e-05, + 3.868454950861633e-05, + 3.569453474483453e-05, + 3.394714804016985e-05, + 3.173232471453957e-05, + 3.031635787920095e-05, + 2.930172558990307e-05, + 2.3542552298749797e-05, + 2.284826769027859e-05, + 2.2551177607965656e-05, + 2.2459958927356638e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12692579254508018, + "first_to_last_ratio": 5.283860704622959, + "imbalance_abs_log_ratio": 1.6646570245834122, + "last_quartile_mean": 0.02402141154743731, + "mean": 0.06666541617596522, + "normalized": [ + 2.316782047314218, + 2.1576628243019496, + 2.0809365727315674, + 1.9632050581780958, + 1.9581380467877862, + 1.7565616761708855, + 1.5601488782800663, + 1.4379457021460735, + 1.4181936562228672, + 1.2807775992725852, + 1.1719405684082436, + 1.085805621683585, + 1.042683162005052, + 0.9685160093542362, + 0.8948047849086074, + 0.8229797475738845, + 0.8654572915254822, + 0.7870405460142597, + 0.714348412471682, + 0.651557783354314, + 0.6246802596485354, + 0.5677900422465361, + 0.5145973193268033, + 0.4748229339727928, + 0.4515785025852313, + 0.4221160632138638, + 0.40328031619304167, + 0.38978324313999146, + 0.31317244048147536, + 0.3039367789939916, + 0.29998477721787487, + 0.2987713342744216 + ], + "peak_layer": 1, + "peak_normalized": 2.316782047314218, + "population_cv": 0.6100414954327734, + "rest_mean": 0.07250859284842456, + "spike_contrast": 0.48424966980237977, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03511226214468479, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.1544492393732071, + 0.14384149014949799, + 0.13872650265693665, + 0.13087788224220276, + 0.13054008781909943, + 0.11710191518068314, + 0.10400797426700592, + 0.09586124867200851, + 0.09454447031021118, + 0.0853835716843605, + 0.0781279057264328, + 0.07238568365573883, + 0.06951090693473816, + 0.06456652283668518, + 0.05965253338217735, + 0.05486428737640381, + 0.05769607052206993, + 0.05246838554739952, + 0.04762233421206474, + 0.04343637079000473, + 0.04164456948637962, + 0.037851959466934204, + 0.03430584445595741, + 0.03165426850318909, + 0.03010466881096363, + 0.028140543028712273, + 0.02688485011458397, + 0.02598506212234497, + 0.020877771079540253, + 0.02026207186281681, + 0.019998610019683838, + 0.019917715340852737 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.987575202150765e-06, + "first_to_last_ratio": 4.909869812054598, + "imbalance_abs_log_ratio": 1.591247426598407, + "last_quartile_mean": 1.6268405289565635e-06, + "mean": 4.1047768952751085e-06, + "normalized": [ + 3.0399271535189927, + 2.272270352549729, + 2.3432241010161556, + 1.9571821071865307, + 1.8062987058242357, + 1.478417794211781, + 1.3560422313507472, + 1.3140126510883237, + 1.3159728793374756, + 1.1652738240702745, + 1.0556436223006813, + 1.0336446190031157, + 1.0145704481914697, + 0.9357289247102711, + 0.8598075807486498, + 0.8414029385582174, + 0.8707583924282057, + 0.7424610436170234, + 0.6796317067393598, + 0.6258542173489432, + 0.5891788165353382, + 0.546613535826505, + 0.5056719741938386, + 0.47978148465269643, + 0.4608872008433993, + 0.4431059730541786, + 0.4372858876540878, + 0.43183612852165015, + 0.35323279260322304, + 0.34612823922654523, + 0.3468995237902729, + 0.35125314929808227 + ], + "peak_layer": 1, + "peak_normalized": 3.0399271535189927, + "population_cv": 0.659872101277405, + "rest_mean": 4.472362248903934e-06, + "spike_contrast": 0.4739812805188056, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.119815985679452e-06, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.2478222743084189e-05, + 9.327162842964754e-06, + 9.618412150302902e-06, + 8.033795893425122e-06, + 7.414453193632653e-06, + 6.068575203244109e-06, + 5.56625082026585e-06, + 5.393728770286543e-06, + 5.401775069913128e-06, + 4.7831890697125345e-06, + 4.33318155046436e-06, + 4.242880550009431e-06, + 4.164585334365256e-06, + 3.840958470391342e-06, + 3.529318291839445e-06, + 3.453771341810352e-06, + 3.5742689306061948e-06, + 3.047636937481002e-06, + 2.7897365271201124e-06, + 2.568991931184428e-06, + 2.4184475932997884e-06, + 2.24372661250527e-06, + 2.0756706362590194e-06, + 1.9693959529831773e-06, + 1.891839133350004e-06, + 1.818851160351187e-06, + 1.7949610082723666e-06, + 1.772590962900722e-06, + 1.4499418057312141e-06, + 1.4207791991793783e-06, + 1.42394515023625e-06, + 1.4418158116313862e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001181762008855003, + "first_to_last_ratio": 4.901880355055182, + "imbalance_abs_log_ratio": 1.589618877434572, + "last_quartile_mean": 2.4108340539896744e-05, + "mean": 6.302894070131515e-05, + "normalized": [ + 2.3744696571174315, + 2.136630154802725, + 2.0361131254128098, + 1.9194114069249477, + 1.8802010360140171, + 1.7011133103712184, + 1.531914150175555, + 1.4197583943464076, + 1.391271208233108, + 1.2656343297221408, + 1.162210793277347, + 1.0817658397942223, + 1.0342567205447348, + 0.961623134639753, + 0.8926032889087109, + 0.8268877508247753, + 0.8608377110603633, + 0.7877810435620806, + 0.7225748772463415, + 0.6647855101906275, + 0.649524097104473, + 0.5935729705103481, + 0.541611739989691, + 0.5034768505696168, + 0.479620762906317, + 0.4516427176844916, + 0.43439036804465014, + 0.4220573087232059, + 0.32687806257013674, + 0.3181526251894958, + 0.31448965055456213, + 0.31273940298369435 + ], + "peak_layer": 1, + "peak_normalized": 2.3744696571174315, + "population_cv": 0.5918793634752633, + "rest_mean": 6.823978502426244e-05, + "spike_contrast": 0.5112909037593624, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.489038135739975e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001496603072155267, + 0.00013466953532770276, + 0.00012833405344281346, + 0.0001209784677485004, + 0.0001185070796054788, + 0.00010721936996560544, + 9.655492613092065e-05, + 8.948586764745414e-05, + 8.769035048317164e-05, + 7.977159111760557e-05, + 7.325291517190635e-05, + 6.818255496909842e-05, + 6.518810550915077e-05, + 6.061008753022179e-05, + 5.625983976642601e-05, + 5.211785901337862e-05, + 5.425768904387951e-05, + 4.9653004680294544e-05, + 4.5543129090219736e-05, + 4.19007265008986e-05, + 4.0938815800473094e-05, + 3.7412275560200214e-05, + 3.4137214242946357e-05, + 3.173361255903728e-05, + 3.0229988624341786e-05, + 2.846656207111664e-05, + 2.7379164748708718e-05, + 2.6601825084071606e-05, + 2.0602778022293933e-05, + 2.0052822947036475e-05, + 1.982194953598082e-05, + 1.9711633285623975e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.71794661381864e-05, + "first_to_last_ratio": 4.097119254175958, + "imbalance_abs_log_ratio": 1.410284105823798, + "last_quartile_mean": 2.3718974262010306e-05, + "mean": 5.355534244699811e-05, + "normalized": [ + 2.371829633380241, + 2.0724601623849908, + 1.9622627494202263, + 1.8551438939252523, + 1.7690721277799024, + 1.623771959149998, + 1.4821770640361394, + 1.379774949857855, + 1.3569338467664578, + 1.240304612630695, + 1.1353847655737628, + 1.0568513894692897, + 1.0172105466389505, + 0.9484210902295721, + 0.8824818807944864, + 0.8225686147527733, + 0.84894808898371, + 0.7863829975557285, + 0.7313823810333281, + 0.6780532337320588, + 0.6794732285544001, + 0.6281359052189119, + 0.5798405933708942, + 0.5480368937974985, + 0.5477339289626515, + 0.5240982224731324, + 0.5122994743538888, + 0.5044283664293616, + 0.36991721031610375, + 0.3633394769331656, + 0.3616997646762252, + 0.3595809468183485 + ], + "peak_layer": 1, + "peak_normalized": 2.371829633380241, + "population_cv": 0.5506248473620988, + "rest_mean": 5.755568741329221e-05, + "spike_contrast": 0.5551750151042494, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.195347962900996e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012702414824161679, + 0.0001109913137042895, + 0.00010508965351618826, + 9.935286652762443e-05, + 9.474326361669227e-05, + 8.696166332811117e-05, + 7.937850023154169e-05, + 7.389431993942708e-05, + 7.26710568415001e-05, + 6.64249382680282e-05, + 6.080591992940754e-05, + 5.660003807861358e-05, + 5.447705916594714e-05, + 5.079301627120003e-05, + 4.7261619329219684e-05, + 4.4052943849237636e-05, + 4.546570562524721e-05, + 4.2115010728593916e-05, + 3.916943387594074e-05, + 3.631337312981486e-05, + 3.638942143879831e-05, + 3.3640033507253975e-05, + 3.105356154264882e-05, + 2.9350303520914167e-05, + 2.9334078135434538e-05, + 2.806825978041161e-05, + 2.7436373784439638e-05, + 2.7014833904104307e-05, + 1.981104287551716e-05, + 1.9458770111668855e-05, + 1.937095476023387e-05, + 1.925748074427247e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002535142939450452, + "first_to_last_ratio": 5.8113823462408005, + "imbalance_abs_log_ratio": 1.759818467906725, + "last_quartile_mean": 4.3623750570986886e-05, + "mean": 0.0001317092663839503, + "normalized": [ + 2.263052967073855, + 2.19252412234975, + 2.11375794838107, + 1.992924119774411, + 1.9927385047584378, + 1.7937674940199644, + 1.5956161724502433, + 1.4540367722249958, + 1.4568245329887053, + 1.3084040040884366, + 1.1868508068097534, + 1.107565794677377, + 1.0648325776249805, + 0.9875608255051405, + 0.9068516600656852, + 0.8192947368434954, + 0.8729470886658927, + 0.7913471921200238, + 0.7163422081401906, + 0.6492365241541177, + 0.6063969651654865, + 0.5480260697153099, + 0.48690171913384017, + 0.4424994861082037, + 0.407070855223836, + 0.38050431936474255, + 0.3622742455318211, + 0.34702058581961753, + 0.2955446253020686, + 0.28773683461603417, + 0.28448542301034185, + 0.28506281829217234 + ], + "peak_layer": 1, + "peak_normalized": 2.263052967073855, + "population_cv": 0.6276890850082345, + "rest_mean": 0.0001439489836532933, + "spike_contrast": 0.45581977353542075, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.561479312949814e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00029806504608131945, + 0.0002887757436838001, + 0.00027840150869451463, + 0.00026248657377436757, + 0.0002624621265567839, + 0.00023625580070074648, + 0.0002101574355037883, + 0.00019151011656504124, + 0.0001918772904900834, + 0.0001723289315123111, + 0.00015631924907211214, + 0.00014587667828891426, + 0.000140248317620717, + 0.0001300709118368104, + 0.00011944076686631888, + 0.0001079087087418884, + 0.00011497522064018995, + 0.00010422775812912732, + 9.434890671400353e-05, + 8.551046630600467e-05, + 7.98680994194001e-05, + 7.218011160148308e-05, + 6.412946822820231e-05, + 5.828128269058652e-05, + 5.361500370781869e-05, + 5.011594475945458e-05, + 4.771487510879524e-05, + 4.570582677843049e-05, + 3.8925965782254934e-05, + 3.78976073989179e-05, + 3.7469366361619905e-05, + 3.754541467060335e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ace6376bf838ff2894b72bcf39de9a912ced4cfce16da0bea85a55c0a2d375f6", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.01706349104642868, + 0.012710319831967354, + 0.013922877609729767, + 0.011416541412472725, + 0.008325628004968166, + 0.00801781378686428, + 0.007913725450634956, + 0.008828459307551384, + 0.008378393016755581, + 0.008136541582643986, + 0.008199729025363922, + 0.009040302596986294, + 0.00936646107584238, + 0.009016151539981365, + 0.009497544728219509, + 0.009566655382514, + 0.011108046397566795, + 0.0114391278475523, + 0.012216469272971153, + 0.01392423827201128, + 0.015176027081906796, + 0.017532862722873688, + 0.018514130264520645, + 0.022848375141620636, + 0.0230264812707901, + 0.02426965720951557, + 0.025484448298811913, + 0.027965376153588295, + 0.02452327311038971, + 0.02779858186841011, + 0.03124845214188099, + 0.03395458683371544 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.540868192430935e-05, + "first_to_last_ratio": 5.391981827595804, + "imbalance_abs_log_ratio": 1.6849130033513229, + "last_quartile_mean": 6.566914180439198e-06, + "mean": 1.820804214958116e-05, + "normalized": [ + 2.169983600580502, + 2.229739182249934, + 2.1980957529981846, + 2.095076135992008, + 2.0397373476740466, + 1.7793673196646478, + 1.594082179228277, + 1.4512998336206846, + 1.4154497014096232, + 1.2782832528040422, + 1.1723161764872307, + 1.078459317768477, + 1.0399977909539722, + 0.9562526437490421, + 0.8686051898876579, + 0.800284903251136, + 0.826346801214095, + 0.7461827311573033, + 0.6773489568290562, + 0.6148904219045332, + 0.5952269402216567, + 0.5351818973485719, + 0.49310547712451824, + 0.45940569949796806, + 0.4422680977368002, + 0.4169925152644191, + 0.40092337249837284, + 0.3905686784090242, + 0.31321857584806834, + 0.3079679144493542, + 0.3062939345842967, + 0.307047657592496 + ], + "peak_layer": 2, + "peak_normalized": 2.229739182249934, + "population_cv": 0.6294018473328937, + "rest_mean": 1.9876985822554195e-05, + "spike_contrast": 0.46263283566326563, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.195746315526776e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.951115286326967e-05, + 4.059918501297943e-05, + 4.002302011940628e-05, + 3.814723459072411e-05, + 3.713962360052392e-05, + 3.239879515604116e-05, + 2.902511550928466e-05, + 2.6425328542245552e-05, + 2.577256782387849e-05, + 2.327503534615971e-05, + 2.1345582354115322e-05, + 1.9636632714536972e-05, + 1.893632361316122e-05, + 1.7411488443030976e-05, + 1.5815599908819422e-05, + 1.4571621250070166e-05, + 1.5046157386677805e-05, + 1.3586526620201766e-05, + 1.2333198355918285e-05, + 1.1195950719411485e-05, + 1.083791721612215e-05, + 9.744614544615615e-06, + 8.978485311672557e-06, + 8.36487834021682e-06, + 8.052836165006738e-06, + 7.592617293994408e-06, + 7.300029665202601e-06, + 7.111490958777722e-06, + 5.703097031073412e-06, + 5.607492767012445e-06, + 5.577012871071929e-06, + 5.590736691374332e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013440313705359586, + "first_to_last_ratio": 5.096756498301237, + "imbalance_abs_log_ratio": 1.6286043566835047, + "last_quartile_mean": 2.6370327304903185e-05, + "mean": 7.053613683183357e-05, + "normalized": [ + 2.118631459598408, + 2.1956122951818604, + 2.1120000110372854, + 2.0289234994136547, + 1.9870377238438375, + 1.7654736087105818, + 1.588703129633362, + 1.4472245249652265, + 1.4174081755766987, + 1.2876606578775502, + 1.181827430204035, + 1.0872648180837396, + 1.0499502322069625, + 0.9708212520819881, + 0.8822750951131371, + 0.8137021952597919, + 0.8431394603174627, + 0.7640069214416272, + 0.6933471494528902, + 0.6299834356426122, + 0.6115185731312205, + 0.5509443526589552, + 0.5080623331524264, + 0.4736371435182702, + 0.4570686826070521, + 0.4315058915031859, + 0.41528384146383185, + 0.4052875681754842, + 0.32485087604930224, + 0.31941934544156514, + 0.3181727511181896, + 0.31925556553780515 + ], + "peak_layer": 2, + "peak_normalized": 2.1956122951818604, + "population_cv": 0.6066332113433263, + "rest_mean": 7.68027995137446e-05, + "spike_contrast": 0.47779714518019456, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6696158349514005e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001494400785304606, + 0.00015487000928260386, + 0.00014897232176735997, + 0.00014311242557596415, + 0.00014015796477906406, + 0.0001245296880370006, + 0.00011206098133698106, + 0.00010208162711933255, + 9.99784970190376e-05, + 9.082660835701972e-05, + 8.336154132848606e-05, + 7.66914599807933e-05, + 7.405943324556574e-05, + 6.847798067610711e-05, + 6.223227683221921e-05, + 5.739540938520804e-05, + 5.9471800341270864e-05, + 5.389009675127454e-05, + 4.890602940577082e-05, + 4.4436597818275914e-05, + 4.3134157749591395e-05, + 3.886148624587804e-05, + 3.5836754250340164e-05, + 3.34085343638435e-05, + 3.223985913791694e-05, + 3.0436758606811054e-05, + 2.9292517865542322e-05, + 2.8587419365067035e-05, + 2.291372584295459e-05, + 2.2530606656800956e-05, + 2.244267670903355e-05, + 2.2519054255099036e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11919004376977682, + "first_to_last_ratio": 5.096756387349086, + "imbalance_abs_log_ratio": 1.628604334914336, + "last_quartile_mean": 0.0233854700345546, + "mean": 0.06255215115379542, + "normalized": [ + 2.118631411517831, + 2.195612379432328, + 2.1119998486113145, + 2.028923550569329, + 1.987037719744253, + 1.7654735296881534, + 1.5887031196057697, + 1.4472244970610906, + 1.4174081917206085, + 1.2876605843188982, + 1.1818275057210115, + 1.0872648442748674, + 1.049950213270653, + 0.9708212769375442, + 0.8822751544186832, + 0.8137022140448229, + 0.843139511676983, + 0.7640069424918614, + 0.6933471370791401, + 0.6299834087164704, + 0.6115185748160756, + 0.5509443408044579, + 0.5080623343135898, + 0.47363716064572287, + 0.4570686754958606, + 0.4315059038674738, + 0.415283849867723, + 0.4052875513382056, + 0.3248508776130179, + 0.31941937674712706, + 0.31817274273479984, + 0.31925557085433326 + ], + "peak_layer": 2, + "peak_normalized": 2.195612379432328, + "population_cv": 0.6066332011935822, + "rest_mean": 0.0681094902533072, + "spike_contrast": 0.47779714538168394, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.032542520016431806, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.13252495229244232, + 0.13734027743339539, + 0.132110133767128, + 0.12691353261470795, + 0.1242934837937355, + 0.1104341670870781, + 0.09937679767608643, + 0.0905270054936409, + 0.08866193145513535, + 0.08054593950510025, + 0.07392585277557373, + 0.06801075488328934, + 0.06567664444446564, + 0.06072695925831795, + 0.05518820881843567, + 0.050898823887109756, + 0.0527401901781559, + 0.0477902777493, + 0.04337035492062569, + 0.03940681740641594, + 0.03825180232524872, + 0.03446275368332863, + 0.031780391931533813, + 0.02962702326476574, + 0.028590628877282143, + 0.026991622522473335, + 0.02597689814865589, + 0.02535160817205906, + 0.020320121198892593, + 0.01998036913573742, + 0.019902389496564865, + 0.0199701227247715 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.877708355863433e-06, + "first_to_last_ratio": 4.956106826106702, + "imbalance_abs_log_ratio": 1.6006205184390925, + "last_quartile_mean": 1.5894952696271503e-06, + "mean": 3.999532470544409e-06, + "normalized": [ + 2.814716234830519, + 2.2772104806158655, + 2.4900910014868787, + 2.034929380837634, + 1.946817467216516, + 1.4831174218104588, + 1.3741082206996111, + 1.3362682498464256, + 1.335883261215519, + 1.175420204513858, + 1.0516337389632453, + 1.0321730525814614, + 1.0275683111275076, + 0.9193973073548298, + 0.8640024348029502, + 0.8181451837262547, + 0.853266641386687, + 0.7182658185841684, + 0.6535137098090048, + 0.6032328415810824, + 0.5571593897123995, + 0.5134489675885875, + 0.48188120740570606, + 0.4583873216846789, + 0.454112395874878, + 0.4406721596566101, + 0.43551085607971224, + 0.4317150626061903, + 0.3525239188448322, + 0.350593887609604, + 0.35373076042061513, + 0.3605031095257097 + ], + "peak_layer": 1, + "peak_normalized": 2.814716234830519, + "population_cv": 0.663747605253736, + "rest_mean": 4.375045717905677e-06, + "spike_contrast": 0.45068350411145924, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.971760934793565e-06, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1257548976573162e-05, + 9.107777259487193e-06, + 9.959199815057218e-06, + 8.138766133924946e-06, + 7.78635967435548e-06, + 5.931776286161039e-06, + 5.495790446730098e-06, + 5.344448254618328e-06, + 5.3429084800882265e-06, + 4.701131274487125e-06, + 4.206043286103522e-06, + 4.128209639020497e-06, + 4.109792826056946e-06, + 3.67715938409674e-06, + 3.455605792623828e-06, + 3.2721982279326767e-06, + 3.4126676382584265e-06, + 2.8727274639095413e-06, + 2.613749302327051e-06, + 2.4126493372023106e-06, + 2.2283770704234485e-06, + 2.0535558178380597e-06, + 1.9272995359642664e-06, + 1.8333349771637586e-06, + 1.8162372725782916e-06, + 1.7624826114115422e-06, + 1.741839810165402e-06, + 1.7266584109165706e-06, + 1.4099308600634686e-06, + 1.4022116374690086e-06, + 1.4147576621326152e-06, + 1.4418438922803034e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00011129347331007011, + "first_to_last_ratio": 4.7212247859996666, + "imbalance_abs_log_ratio": 1.5520682544953823, + "last_quartile_mean": 2.3573008775201743e-05, + "mean": 5.946740844819942e-05, + "normalized": [ + 2.156243192585657, + 2.162278083675196, + 2.065029760843484, + 1.9785360583158715, + 1.9104190175998155, + 1.7137610234833742, + 1.5574858033444396, + 1.4282764408526964, + 1.3933238942763717, + 1.272869675359645, + 1.1719605800561352, + 1.0826910772254577, + 1.0407375480008605, + 0.9652826895223989, + 0.8847702267230649, + 0.8215667819990273, + 0.8441113654099096, + 0.7717750152456593, + 0.7061028737915589, + 0.6469451209977515, + 0.637162587501422, + 0.5782978066221776, + 0.536179040993063, + 0.5029771071348699, + 0.48641749518891064, + 0.46322823897876675, + 0.44748104445301945, + 0.4384066844549108, + 0.33861230782778606, + 0.3333797707047228, + 0.3318178248031526, + 0.33187386202882385 + ], + "peak_layer": 2, + "peak_normalized": 2.162278083675196, + "population_cv": 0.5844863507438243, + "rest_mean": 6.444277332101828e-05, + "spike_contrast": 0.5058819857516098, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.260043813497759e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001282261946471408, + 0.00012858507398050278, + 0.00012280196824576706, + 0.00011765841190936044, + 0.0001136076680268161, + 0.0001019129267660901, + 9.261964441975579e-05, + 8.493589848512784e-05, + 8.285736112156883e-05, + 7.5694260885939e-05, + 6.96934584993869e-05, + 6.438483251258731e-05, + 6.188996485434473e-05, + 5.7402859965804964e-05, + 5.261499245534651e-05, + 4.885644739260897e-05, + 5.019711534259841e-05, + 4.5895460061728954e-05, + 4.1990108002210036e-05, + 3.8472149753943086e-05, + 3.7890407838858664e-05, + 3.4389871871098876e-05, + 3.188517803209834e-05, + 2.9910745070083067e-05, + 2.8925987862749025e-05, + 2.7546982892090455e-05, + 2.661053804331459e-05, + 2.6070909370901063e-05, + 2.013639641518239e-05, + 1.9825230992864817e-05, + 1.9732346117962152e-05, + 1.9735678506549448e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.178715026791906e-05, + "first_to_last_ratio": 3.9336058830578526, + "imbalance_abs_log_ratio": 1.3695565327057526, + "last_quartile_mean": 2.3334099296334898e-05, + "mean": 5.082815437162935e-05, + "normalized": [ + 2.1389396648642625, + 2.0813842206789825, + 1.9784383506454575, + 1.9084846997435907, + 1.8069300868376652, + 1.6378043812798742, + 1.5010746826985348, + 1.3936066186205618, + 1.3599265725007244, + 1.2387510684837115, + 1.136508771737783, + 1.057443309955879, + 1.023880001173994, + 0.9508578975343295, + 0.8771227728274181, + 0.8254997469475742, + 0.8391602342847156, + 0.7782386607285642, + 0.7170818947183901, + 0.6619181702664086, + 0.6688760305343471, + 0.6150432978276499, + 0.5789519206743307, + 0.5514510803162503, + 0.5572369864630182, + 0.5414044044824042, + 0.5311915694026624, + 0.5242909684053321, + 0.38287903611819507, + 0.38027305935168987, + 0.3789296852891365, + 0.37642015460656175 + ], + "peak_layer": 1, + "peak_normalized": 2.1389396648642625, + "population_cv": 0.5381630565291086, + "rest_mean": 5.464674312086707e-05, + "spike_contrast": 0.5527827167839162, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.0207775125745684e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010871835547732189, + 0.00010579291847534478, + 0.00010056036990135908, + 9.700475493445992e-05, + 9.184292139252648e-05, + 8.324657392222434e-05, + 7.629685569554567e-05, + 7.083445234457031e-05, + 6.912255776114762e-05, + 6.296343053691089e-05, + 5.776664329459891e-05, + 5.374789179768413e-05, + 5.204193075769581e-05, + 4.8330352001357824e-05, + 4.458253170014359e-05, + 4.195862857159227e-05, + 4.265296593075618e-05, + 3.955643478548154e-05, + 3.64479492418468e-05, + 3.364407893968746e-05, + 3.399773413548246e-05, + 3.12615156872198e-05, + 2.9427057597786188e-05, + 2.8029240638716146e-05, + 2.8323327569523826e-05, + 2.75185866485117e-05, + 2.699948709050659e-05, + 2.6648742277757265e-05, + 1.946103475347627e-05, + 1.9328577764099464e-05, + 1.926029653986916e-05, + 1.913274172693491e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00023838880770199466, + "first_to_last_ratio": 5.615601076105167, + "imbalance_abs_log_ratio": 1.7255486307434573, + "last_quartile_mean": 4.245116497259005e-05, + "mean": 0.0001231883321679561, + "normalized": [ + 2.1010282721933486, + 2.235453689609075, + 2.1351018794085928, + 2.0660271287710446, + 2.05547953479137, + 1.8078923375666518, + 1.6237057602904945, + 1.456570280035237, + 1.4451526778837593, + 1.3168179083075675, + 1.1987172226380065, + 1.1000538071149966, + 1.0715582930268015, + 0.9881627826227964, + 0.8804515183677988, + 0.8100379119774714, + 0.8454810290012423, + 0.7652053218316441, + 0.6893306865502836, + 0.6202261087486645, + 0.5903640380365041, + 0.5227167386885779, + 0.4731335967935112, + 0.4445012321327385, + 0.4104702393707161, + 0.39051495112024115, + 0.3718063497323823, + 0.3604546331284706, + 0.3081351294824555, + 0.30324722511926294, + 0.3046138998706107, + 0.307587815787682 + ], + "peak_layer": 2, + "peak_normalized": 2.235453689609075, + "population_cv": 0.6295499532649526, + "rest_mean": 0.0001348630006153447, + "spike_contrast": 0.4459720032746656, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.014512255205773e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00025882216868922114, + 0.00027538181166164577, + 0.0002630196395330131, + 0.00025451043620705605, + 0.00025321109569631517, + 0.00022271124180406332, + 0.00020002160454168916, + 0.0001794324634829536, + 0.0001780259481165558, + 0.00016221660189330578, + 0.00014766797539778054, + 0.00013551379379350692, + 0.00013200347893871367, + 0.00012173012510174885, + 0.0001084613541024737, + 9.978721936931834e-05, + 0.00010415339784231037, + 9.426436736248434e-05, + 8.491749758832157e-05, + 7.640461990376934e-05, + 7.272596121765673e-05, + 6.439260323531926e-05, + 5.828453868161887e-05, + 5.4757365433033556e-05, + 5.056514419266023e-05, + 4.810688551515341e-05, + 4.5802204112987965e-05, + 4.440380507730879e-05, + 3.79586526833009e-05, + 3.735651989700273e-05, + 3.752487828023732e-05, + 3.789123002206907e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7b7c9c05d1cbde4434ba686237dd1c945f87f81120c1f3f234a3adcdf1d68140", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.01049753837287426, + 0.028129518032073975, + 0.040982723236083984, + 0.05135732889175415, + 0.014544809237122536, + 0.023477500304579735, + 0.029818039387464523, + 0.03540094941854477, + 0.01795734278857708, + 0.024303091689944267, + 0.031844362616539, + 0.03825921565294266, + 0.018456056714057922, + 0.02560909278690815, + 0.0305727981030941, + 0.037264734506607056, + 0.017418840900063515, + 0.02998829260468483, + 0.03708711639046669, + 0.04352690279483795, + 0.01969025284051895, + 0.030230700969696045, + 0.041474223136901855, + 0.056570712476968765, + 0.012774706818163395, + 0.03524070605635643, + 0.05920406058430672, + 0.08430205285549164, + 0.005331806372851133, + 0.032364487648010254, + 0.059356335550546646, + 0.09272631257772446 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.746988363673154e-05, + "first_to_last_ratio": 5.5431540155630445, + "imbalance_abs_log_ratio": 1.7125636556484518, + "last_quartile_mean": 6.7596685084936325e-06, + "mean": 1.9353528273313714e-05, + "normalized": [ + 2.342683119997902, + 2.192172693070629, + 2.1504636799030705, + 2.019405278342902, + 1.9995383922272116, + 1.773068785590967, + 1.5680654500633198, + 1.443203173299378, + 1.419437780389259, + 1.2773938398900322, + 1.1654352925993345, + 1.0793229270979923, + 1.0342651399222493, + 0.9566605213538533, + 0.8835696516415633, + 0.8115626455796351, + 0.8495787965648868, + 0.7730098730619744, + 0.7009135788914922, + 0.6382359583361713, + 0.6098093032158669, + 0.5543264714540777, + 0.5014428165322925, + 0.4622493313451456, + 0.4400298915440255, + 0.4102977502891804, + 0.39090442010501514, + 0.37729033323761296, + 0.3032525161403676, + 0.2939411057398244, + 0.29000420505871766, + 0.2884652775140499 + ], + "peak_layer": 1, + "peak_normalized": 2.342683119997902, + "population_cv": 0.6284613723388149, + "rest_mean": 2.1096881331080414e-05, + "spike_contrast": 0.4711322780552658, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.939421761373524e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.5339183998294175e-05, + 4.242627619532868e-05, + 4.161905962973833e-05, + 3.9082617149688303e-05, + 3.869812280754559e-05, + 3.431513687246479e-05, + 3.034759902220685e-05, + 2.7931073418585584e-05, + 2.747112921497319e-05, + 2.472207779646851e-05, + 2.2555284886038862e-05, + 2.088870678562671e-05, + 2.001667962758802e-05, + 1.8514756447984837e-05, + 1.7100190234486945e-05, + 1.5706600606790744e-05, + 1.6442347259726375e-05, + 1.4960468433855567e-05, + 1.3565150766225997e-05, + 1.2352117664704565e-05, + 1.1801961591118015e-05, + 1.072817303793272e-05, + 9.704687727207784e-06, + 8.946155503508635e-06, + 8.516130947100464e-06, + 7.940709110698663e-06, + 7.5653797466657124e-06, + 7.301899131562095e-06, + 5.869006145076128e-06, + 5.688797500624787e-06, + 5.61260458198376e-06, + 5.582820904237451e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00014312549137684982, + "first_to_last_ratio": 5.283834952428555, + "imbalance_abs_log_ratio": 1.6646521508256433, + "last_quartile_mean": 2.7087426587968366e-05, + "mean": 7.517451274452469e-05, + "normalized": [ + 2.316758705258724, + 2.1576744140515562, + 2.080949754300773, + 1.9631709090738392, + 1.9581116297712768, + 1.756527727676563, + 1.5601436404205942, + 1.4379416037311736, + 1.4182272361289956, + 1.2807847399809855, + 1.1719416125963091, + 1.0858232362724045, + 1.0426842467677646, + 0.968528909567598, + 0.8948112457108309, + 0.8229911987427623, + 0.8654616765749427, + 0.7870364110125266, + 0.7143514220202891, + 0.6515701232514104, + 0.6246851525269175, + 0.5677858273151118, + 0.5145937075954704, + 0.474826748264951, + 0.45157784417649677, + 0.42211454965836737, + 0.4032790856857949, + 0.3897818669587336, + 0.3131725537818108, + 0.3039381495313863, + 0.2999843051410979, + 0.2987697664525418 + ], + "peak_layer": 1, + "peak_normalized": 2.316758705258724, + "population_cv": 0.6100374517355251, + "rest_mean": 8.176350510674874e-05, + "spike_contrast": 0.4842497143049554, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.959395398851484e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00017416120681446046, + 0.00016220212273765355, + 0.00015643438382539898, + 0.00014758041652385145, + 0.00014720008766744286, + 0.00013204611605033278, + 0.0001172830379800871, + 0.00010809655941557139, + 0.00010661454143701121, + 9.628236875869334e-05, + 8.810013969196007e-05, + 8.162623271346092e-05, + 7.838328019715846e-05, + 7.280868885573e-05, + 6.726699939463288e-05, + 6.186796235851943e-05, + 6.506065983558074e-05, + 5.9165078710066155e-05, + 5.3701020078733563e-05, + 4.898146653431468e-05, + 4.6960401959950104e-05, + 4.268302291166037e-05, + 3.86843312298879e-05, + 3.569486943888478e-05, + 3.394714440219104e-05, + 3.173225559294224e-05, + 3.0316308766487055e-05, + 2.9301661925273947e-05, + 2.354259413550608e-05, + 2.2848402295494452e-05, + 2.255117396998685e-05, + 2.2459871615865268e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12692511826753616, + "first_to_last_ratio": 5.2838350418377695, + "imbalance_abs_log_ratio": 1.6646521677469164, + "last_quartile_mean": 0.02402140060439706, + "mean": 0.06666550948284566, + "normalized": [ + 2.3167586877655544, + 2.1576745567833733, + 2.080949753728949, + 1.963171017440623, + 1.9581116128663651, + 1.7565277011358755, + 1.560143677115271, + 1.4379415661053954, + 1.418227211166364, + 1.280784747512944, + 1.1719416103835674, + 1.0858232130311325, + 1.0426842731319477, + 0.9685288473950006, + 0.8948111881196698, + 0.8229911687817946, + 0.8654616682400994, + 0.7870363710328231, + 0.7143513801530333, + 0.6515701150530128, + 0.6246851968826057, + 0.5677858388521877, + 0.5145936933053726, + 0.47482679570383546, + 0.4515778146632158, + 0.4221145224430977, + 0.40327910912708953, + 0.3897818593837784, + 0.31317256095995943, + 0.3039381697062612, + 0.29998430147060634, + 0.29876977055919474 + ], + "peak_layer": 1, + "peak_normalized": 2.3167586877655544, + "population_cv": 0.6100374622681133, + "rest_mean": 0.0725086936382232, + "spike_contrast": 0.4842497262327929, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03511231504380703, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.15444789826869965, + 0.14384247362613678, + 0.1387275755405426, + 0.13087579607963562, + 0.13053850829601288, + 0.1170998141169548, + 0.1040077731013298, + 0.09586110711097717, + 0.094546839594841, + 0.08538416773080826, + 0.07812808454036713, + 0.0723869577050209, + 0.06951107829809189, + 0.06456746906042099, + 0.05965304374694824, + 0.05486512556672096, + 0.05769644305109978, + 0.052468180656433105, + 0.047622598707675934, + 0.043437253683805466, + 0.04164495691657066, + 0.037851732224226, + 0.03430565074086189, + 0.03165457025170326, + 0.030104665085673332, + 0.0281404796987772, + 0.026884807273745537, + 0.025985006242990494, + 0.020877808332443237, + 0.02026219293475151, + 0.01999860629439354, + 0.01991763897240162 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.987624144334404e-06, + "first_to_last_ratio": 4.909876092861521, + "imbalance_abs_log_ratio": 1.591248705818253, + "last_quartile_mean": 1.6268484159809304e-06, + "mean": 4.104807253213494e-06, + "normalized": [ + 3.039754669455638, + 2.272302514080732, + 2.3431733144286278, + 1.9573395693607292, + 1.8063799565900491, + 1.4783923475555005, + 1.35608582196543, + 1.313927156720024, + 1.3158812773467614, + 1.1653219275062023, + 1.0555854083106069, + 1.0336141529504548, + 1.0145705888314611, + 0.9357245523705084, + 0.8598480281460422, + 0.8414154936952141, + 0.8707475211933563, + 0.7424554418110071, + 0.6796411377088122, + 0.625913455746514, + 0.5892207668944752, + 0.5466000211947281, + 0.5056892833715051, + 0.4797947755131972, + 0.4608548499137519, + 0.44312659764186396, + 0.43727941318247243, + 0.43184315462220935, + 0.3532260811866724, + 0.34613656390182396, + 0.34690003248276174, + 0.3512541243208678 + ], + "peak_layer": 1, + "peak_normalized": 3.039754669455638, + "population_cv": 0.6598595805076404, + "rest_mean": 4.472391268521918e-06, + "spike_contrast": 0.4739866087898418, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1198535705480025e-06, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.2477607015171088e-05, + 9.327363841293845e-06, + 9.618274816602934e-06, + 8.034501661313698e-06, + 7.41484154787031e-06, + 6.068515631341143e-06, + 5.566470917983679e-06, + 5.393417723098537e-06, + 5.401439011620823e-06, + 4.783421900356188e-06, + 4.332974640419707e-06, + 4.242786872055149e-06, + 4.164616711932467e-06, + 3.840968929580413e-06, + 3.5295104225951945e-06, + 3.4538484214863274e-06, + 3.5742507407121593e-06, + 3.047636482733651e-06, + 2.789795871649403e-06, + 2.569254093032214e-06, + 2.418637677692459e-06, + 2.2436877316067694e-06, + 2.075757038255688e-06, + 1.969465074580512e-06, + 1.8917203306045849e-06, + 1.8189492720921407e-06, + 1.7949477069123532e-06, + 1.7726329133438412e-06, + 1.4499249800792313e-06, + 1.420823878106603e-06, + 1.423957769475237e-06, + 1.4418304772334523e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00011817605991382152, + "first_to_last_ratio": 4.9018741377821335, + "imbalance_abs_log_ratio": 1.5896176090892555, + "last_quartile_mean": 2.4108342358886148e-05, + "mean": 6.302900811760992e-05, + "normalized": [ + 2.374456497049209, + 2.136637566259215, + 2.0361178738653583, + 1.9193980409609805, + 1.880195792666385, + 1.7010985617651888, + 1.5319147049553272, + 1.4197582610211454, + 1.3912798786840082, + 1.265638517025675, + 1.16221093542691, + 1.081772301653134, + 1.0342582693755369, + 0.961625165193809, + 0.8926056818824999, + 0.8268911375942922, + 0.8608363285490097, + 0.787779046563255, + 0.722577567522019, + 0.6647906287621113, + 0.6495296937526279, + 0.5935714121143937, + 0.5416093713850725, + 0.5034789094073873, + 0.4796213754232066, + 0.4516410802217136, + 0.4343902208726426, + 0.4220558183434384, + 0.3268775109220198, + 0.31815320839698785, + 0.3144898336461675, + 0.3127388087392738 + ], + "peak_layer": 1, + "peak_normalized": 2.374456497049209, + "population_cv": 0.5918778380779056, + "rest_mean": 6.823984788045184e-05, + "spike_contrast": 0.5112917815905388, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.489047339826357e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001496596378274262, + 0.00013467014650814235, + 0.0001283344900002703, + 0.00012097775470465422, + 0.0001185068758786656, + 0.00010721855505835265, + 9.655506437411532e-05, + 8.948595495894551e-05, + 8.769099076744169e-05, + 7.977194036357105e-05, + 7.325300248339772e-05, + 6.818303518230096e-05, + 6.51882728561759e-05, + 6.0610280343098566e-05, + 5.626005076919682e-05, + 5.2118128223810345e-05, + 5.425765994004905e-05, + 4.96529319207184e-05, + 4.554334736894816e-05, + 4.1901093936758116e-05, + 4.093921234016307e-05, + 3.74122173525393e-05, + 3.413710146560334e-05, + 3.17337762680836e-05, + 3.0230059564928524e-05, + 2.84664893115405e-05, + 2.7379184757592157e-05, + 2.660175960045308e-05, + 2.0602765289368108e-05, + 2.005288115469739e-05, + 1.9821982277790084e-05, + 1.9711616914719343e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.71798262980883e-05, + "first_to_last_ratio": 4.097156236797104, + "imbalance_abs_log_ratio": 1.4102931322767618, + "last_quartile_mean": 2.3718848069620435e-05, + "mean": 5.355637233606103e-05, + "normalized": [ + 2.3716408308104846, + 2.0723215416211813, + 1.962393612474412, + 1.8549484527834383, + 1.7689695012129536, + 1.624026031735852, + 1.4819536083276605, + 1.3800136078097518, + 1.3569801642067971, + 1.240510630022215, + 1.1354021945249317, + 1.0569709979561241, + 1.016947599552204, + 0.9485224054224415, + 0.8823796609912891, + 0.8224276054007224, + 0.8490865037613989, + 0.7863957259021926, + 0.731366006996846, + 0.6781886854392298, + 0.6795365133883927, + 0.6281649905506911, + 0.579800641551788, + 0.5480420803864836, + 0.5476558755782504, + 0.5241602836307675, + 0.5123889675770241, + 0.504358312217994, + 0.3698923675924519, + 0.36330593005557427, + 0.36164828236666086, + 0.35960038815179535 + ], + "peak_layer": 1, + "peak_normalized": 2.3716408308104846, + "population_cv": 0.5506073147861105, + "rest_mean": 5.755683479138598e-05, + "spike_contrast": 0.5551708184288228, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.1953875077306296e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012701647938229144, + 0.00011098602408310398, + 0.00010509868297958747, + 9.934431000147015e-05, + 9.473958925809711e-05, + 8.697694283910096e-05, + 7.936805923236534e-05, + 7.390852260868996e-05, + 7.267493492690846e-05, + 6.64372491883114e-05, + 6.0808022681158036e-05, + 5.660753231495619e-05, + 5.446402428788133e-05, + 5.079941911390051e-05, + 4.7257053665816784e-05, + 4.4046239054296166e-05, + 4.5473992940969765e-05, + 4.211650229990482e-05, + 3.91693101846613e-05, + 3.6321325751487166e-05, + 3.639351052697748e-05, + 3.364223812241107e-05, + 3.1052019039634615e-05, + 2.935114571300801e-05, + 2.933046198450029e-05, + 2.807212331390474e-05, + 2.744169432844501e-05, + 2.7011601559934206e-05, + 1.981009336304851e-05, + 1.9457347661955282e-05, + 1.9368570065125823e-05, + 1.9258892280049622e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002534907544031739, + "first_to_last_ratio": 5.810954685354278, + "imbalance_abs_log_ratio": 1.7597448749820743, + "last_quartile_mean": 4.362291019788245e-05, + "mean": 0.00013170268027806742, + "normalized": [ + 2.263139397683698, + 2.19239908258678, + 2.1138453104279673, + 1.9929654416215998, + 1.9922306787749795, + 1.793581963413166, + 1.5951409704801716, + 1.4544554310298363, + 1.4566936402003607, + 1.307837648386261, + 1.187117991027653, + 1.1077423893871614, + 1.0650861466665695, + 0.9874986152737254, + 0.9069117045975197, + 0.8193320614345609, + 0.8732068621805286, + 0.7912116928785183, + 0.7164957030827894, + 0.6492442961101281, + 0.6064825348153222, + 0.5480344706704015, + 0.48698153412089457, + 0.44258326813255405, + 0.4070688650357849, + 0.380575498977065, + 0.3622952070517462, + 0.3470762520438106, + 0.29549026518772864, + 0.28774337874464384, + 0.28440730681701687, + 0.28512439115905686 + ], + "peak_layer": 1, + "peak_normalized": 2.263139397683698, + "population_cv": 0.6276378428727017, + "rest_mean": 0.0001439405388265624, + "spike_contrast": 0.4558704910453311, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.561824411619455e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002980615245178342, + 0.00028874483541585505, + 0.0002783990930765867, + 0.000262478890363127, + 0.00026238212012685835, + 0.00023621955187991261, + 0.0002100843412335962, + 0.00019155567861162126, + 0.0001918504567584023, + 0.0001722457236610353, + 0.0001563466212246567, + 0.00014589264173991978, + 0.00014027470024302602, + 0.00013005621440242976, + 0.00011944270227104425, + 0.00010790822852868587, + 0.00011500368418637663, + 0.00010420470061944798, + 9.436440450372174e-05, + 8.550721395295113e-05, + 7.987537537701428e-05, + 7.21776086720638e-05, + 6.413677328964695e-05, + 5.828940265928395e-05, + 5.361206058296375e-05, + 5.012281326344237e-05, + 4.771524982061237e-05, + 4.571087265503593e-05, + 3.8916859921300784e-05, + 3.7896574212936684e-05, + 3.74572045984678e-05, + 3.75516465282999e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4c56d08d88ae21cddacd7e29fc552875e42d5e83a4e1d9907d475a90abe4ea95", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.021243074908852577, + 0.03624089062213898, + 0.04951142147183418, + 0.05851924046874046, + 0.017167888581752777, + 0.02674153819680214, + 0.03352004662156105, + 0.03946679085493088, + 0.020600929856300354, + 0.027865095064044, + 0.035815127193927765, + 0.04293550178408623, + 0.021564343944191933, + 0.029616132378578186, + 0.035480253398418427, + 0.04259813204407692, + 0.021436138078570366, + 0.03586751967668533, + 0.04331304505467415, + 0.051551222801208496, + 0.025608548894524574, + 0.04091007262468338, + 0.05466190353035927, + 0.07478146255016327, + 0.02793189324438572, + 0.05498877540230751, + 0.08125203847885132, + 0.10916247963905334, + 0.026290444657206535, + 0.05761713907122612, + 0.08842677623033524, + 0.12436132878065109 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.540896136655647e-05, + "first_to_last_ratio": 5.3920073915355005, + "imbalance_abs_log_ratio": 1.6849177444428016, + "last_quartile_mean": 6.566934871443664e-06, + "mean": 1.8208185622370365e-05, + "normalized": [ + 2.1699627058464013, + 2.229719814658003, + 2.1981171939571054, + 2.095121565395603, + 2.0397310655629486, + 1.7793850670583011, + 1.594066721449286, + 1.451277409043192, + 1.4154824041645198, + 1.278296457066803, + 1.1723320139174174, + 1.078457313436146, + 1.0400037819469097, + 0.956235318728193, + 0.8685919520760419, + 0.8002742017652755, + 0.826347582617928, + 0.746171556877762, + 0.6773531600124774, + 0.6148977645726124, + 0.5952208514919352, + 0.5351825254697238, + 0.4931013918631679, + 0.45940307857109275, + 0.44226511234925703, + 0.416991702053712, + 0.40092308550387123, + 0.3905686478293239, + 0.31321395997822676, + 0.3079637645255534, + 0.30629421840547977, + 0.3070466118057301 + ], + "peak_layer": 2, + "peak_normalized": 2.229719814658003, + "population_cv": 0.6294044766409223, + "rest_mean": 1.9877152664304487e-05, + "spike_contrast": 0.4626298218476693, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.19576359592611e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.951108374167234e-05, + 4.059915227117017e-05, + 4.002372588729486e-05, + 3.814836236415431e-05, + 3.7139801861485466e-05, + 3.239937359467149e-05, + 2.9025062758591957e-05, + 2.6425128453411162e-05, + 2.5773366360226646e-05, + 2.3275459170690738e-05, + 2.1346038920455612e-05, + 1.9636750948848203e-05, + 1.8936581909656525e-05, + 1.7411310182069428e-05, + 1.5815483493497595e-05, + 1.457154121453641e-05, + 1.5046290172904264e-05, + 1.3586430213763379e-05, + 1.2333372069406323e-05, + 1.1196172636118717e-05, + 1.08378917502705e-05, + 9.744702765601687e-06, + 8.97848167369375e-06, + 8.364896530110855e-06, + 8.052845259953756e-06, + 7.592662313982146e-06, + 7.300081961147953e-06, + 7.11154643795453e-06, + 5.703057922801236e-06, + 5.607461389445234e-06, + 5.577061983785825e-06, + 5.5907617024786305e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013440396742225857, + "first_to_last_ratio": 5.096772825661455, + "imbalance_abs_log_ratio": 1.6286075601589056, + "last_quartile_mean": 2.6370405748821213e-05, + "mean": 7.053641201082428e-05, + "normalized": [ + 2.1186363977530145, + 2.195594858531836, + 2.112032207136982, + 2.0289327073081966, + 1.9870506023056362, + 1.7654824002594274, + 1.5886840377574483, + 1.4472277500593902, + 1.4174334883240383, + 1.287660792002894, + 1.1818213754945048, + 1.0872652182409996, + 1.0499307664886113, + 0.9708194245660434, + 0.8822672176250175, + 0.8136967514782698, + 0.8431298271983457, + 0.7640029609304118, + 0.6933490863748281, + 0.6299871670381194, + 0.6115155169679979, + 0.5509413780821966, + 0.5080592679889337, + 0.47363704932987183, + 0.4570686014801849, + 0.43150580695093105, + 0.4152831239229507, + 0.4052856775987878, + 0.32485051130850606, + 0.319418692435047, + 0.31817385655546626, + 0.3192554805051111 + ], + "peak_layer": 2, + "peak_normalized": 2.195594858531836, + "population_cv": 0.606636113722199, + "rest_mean": 7.680312336122841e-05, + "spike_contrast": 0.47779529155407807, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.669617071864195e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001494410098530352, + 0.00015486938355024904, + 0.00014897517394274473, + 0.0001431136333849281, + 0.0001401594199705869, + 0.00012453079398255795, + 0.00011206007184227929, + 0.00010208225285168737, + 9.998067253036425e-05, + 9.082697215490043e-05, + 8.336143946507946e-05, + 7.669178739888594e-05, + 7.405834912788123e-05, + 6.847811891930178e-05, + 6.22319639660418e-05, + 5.739524931414053e-05, + 5.947135286987759e-05, + 5.3890027629677206e-05, + 4.890635682386346e-05, + 4.4437034375732765e-05, + 4.31341104558669e-05, + 3.886142803821713e-05, + 3.5836677852785215e-05, + 3.340865805512294e-05, + 3.223997919121757e-05, + 3.0436871384154074e-05, + 2.9292581530171447e-05, + 2.8587397537194192e-05, + 2.2913789507583715e-05, + 2.2530648493557237e-05, + 2.244284223706927e-05, + 2.2519136109622195e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11919078417122364, + "first_to_last_ratio": 5.096772875478614, + "imbalance_abs_log_ratio": 1.6286075699331606, + "last_quartile_mean": 0.023385539650917053, + "mean": 0.06255239609163254, + "normalized": [ + 2.118636455801519, + 2.195594967924862, + 2.112032314032563, + 2.0289327576260647, + 1.9870505449879214, + 1.7654823390288878, + 1.5886840348658555, + 1.447227644229518, + 1.4174334908870143, + 1.2876607830122906, + 1.18182132958956, + 1.0872652321152292, + 1.0499307368395738, + 0.9708194407763235, + 0.8822672330699993, + 0.8136967647333175, + 0.8431298378216533, + 0.7640029384223331, + 0.6933490673920089, + 0.629987135566792, + 0.6115155251770166, + 0.5509413496931483, + 0.5080592728974551, + 0.4736370628781186, + 0.4570685830511319, + 0.43150579241003445, + 0.41528311705209375, + 0.40528569634831185, + 0.3248505286851896, + 0.31941869175924303, + 0.31817384926992365, + 0.31925548205504595 + ], + "peak_layer": 2, + "peak_normalized": 2.195594967924862, + "population_cv": 0.6066361252561874, + "rest_mean": 0.06810977848039733, + "spike_contrast": 0.4777952875249589, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0325425311923027, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.13252578675746918, + 0.1373397260904312, + 0.13211268186569214, + 0.1269146054983139, + 0.12429477274417877, + 0.11043515056371689, + 0.09937599301338196, + 0.09052755683660507, + 0.08866386115550995, + 0.08054626733064651, + 0.07392575591802597, + 0.06801104545593262, + 0.06567568331956863, + 0.0607270821928978, + 0.055187929421663284, + 0.050898682326078415, + 0.052739791572093964, + 0.04779021441936493, + 0.04337064549326897, + 0.03940720483660698, + 0.038251761347055435, + 0.03446270152926445, + 0.03178032487630844, + 0.029627133160829544, + 0.02859073504805565, + 0.026991721242666245, + 0.02597695402801037, + 0.025351591408252716, + 0.02032017894089222, + 0.019980404525995255, + 0.019902536645531654, + 0.01997019536793232 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.877724669924646e-06, + "first_to_last_ratio": 4.955998917619468, + "imbalance_abs_log_ratio": 1.6005987453688633, + "last_quartile_mean": 1.5895331699766757e-06, + "mean": 3.999530946430241e-06, + "normalized": [ + 2.814718217044885, + 2.277240228243144, + 2.490109915020848, + 2.0349008216507296, + 1.9468032006740146, + 1.4832109937263445, + 1.3741164759470665, + 1.3361972416551804, + 1.3359113994263188, + 1.175339925313911, + 1.0516196997910439, + 1.032190500940567, + 1.0275201527334392, + 0.9193727005250851, + 0.8639824117206115, + 0.8181664731809545, + 0.8532322879917306, + 0.7182487530200276, + 0.6535170287501896, + 0.6032571758931335, + 0.5571458443102854, + 0.5134825911003963, + 0.4819078263273953, + 0.4583689632360092, + 0.45412774789772825, + 0.4406778136182034, + 0.43551940742882095, + 0.4317317420710209, + 0.35252700938669523, + 0.3506082337328722, + 0.35372873491469664, + 0.3605184827266523 + ], + "peak_layer": 1, + "peak_normalized": 2.814718217044885, + "population_cv": 0.6637468990729557, + "rest_mean": 4.37503757877022e-06, + "spike_contrast": 0.45069215893423337, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9717951317943514e-06, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1257552614551969e-05, + 9.107892765314318e-06, + 9.959271665138658e-06, + 8.138648809108417e-06, + 7.786299647705164e-06, + 5.932148269494064e-06, + 5.495821369549958e-06, + 5.3441622185346205e-06, + 5.343018983694492e-06, + 4.700808403867995e-06, + 4.20598553318996e-06, + 4.12827785112313e-06, + 4.1095986489381175e-06, + 3.6770595670532202e-06, + 3.4555243928480195e-06, + 3.2722821288189152e-06, + 3.412528940316406e-06, + 2.872658114938531e-06, + 2.6137615805055248e-06, + 2.4127457436406985e-06, + 2.228322045993991e-06, + 2.0536895135592204e-06, + 1.9274052647233475e-06, + 1.833260853345564e-06, + 1.816297981349635e-06, + 1.7625045529712224e-06, + 1.7418733477825299e-06, + 1.7267244629692868e-06, + 1.4099426834945916e-06, + 1.4022684808878694e-06, + 1.4147490219329484e-06, + 1.4419048284253222e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00011129383165098261, + "first_to_last_ratio": 4.721235661143692, + "imbalance_abs_log_ratio": 1.5520705579509457, + "last_quartile_mean": 2.357303037570091e-05, + "mean": 5.946752537511202e-05, + "normalized": [ + 2.1562533904228016, + 2.1622669804365335, + 2.0650538414267587, + 1.978536205665008, + 1.9104188094756904, + 1.7137689101965365, + 1.5574747881012931, + 1.4282752231039315, + 1.3933371827625038, + 1.272867417306948, + 1.1719582757105038, + 1.0826854002026853, + 1.0407336663933602, + 0.9652771209998985, + 0.884767569421755, + 0.821562719573055, + 0.844106891600729, + 0.7717762506724927, + 0.7061036265861128, + 0.6469448277682184, + 0.6371632923208572, + 0.5782976483691459, + 0.5361789043793784, + 0.502977158155649, + 0.48641626348807704, + 0.4632276952202235, + 0.44748102106422594, + 0.4384054248026578, + 0.33861167262559666, + 0.3333792069668021, + 0.33181827353856846, + 0.33187434124200266 + ], + "peak_layer": 2, + "peak_normalized": 2.1622669804365335, + "population_cv": 0.5844879671939917, + "rest_mean": 6.444290172819617e-05, + "spike_contrast": 0.5058818301813628, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.2600493068457584e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012822705321013927, + 0.0001285846665268764, + 0.00012280364171601832, + 0.0001176586520159617, + 0.00011360787902958691, + 0.0001019135961541906, + 9.261917148251086e-05, + 8.493599307257682e-05, + 8.285831427201629e-05, + 7.569427543785423e-05, + 6.96934584993869e-05, + 6.43846215098165e-05, + 6.188985571498051e-05, + 5.740264168707654e-05, + 5.2614937885664403e-05, + 4.885630187345669e-05, + 5.019694799557328e-05, + 4.5895623770775273e-05, + 4.1990235331468284e-05, + 3.8472207961604e-05, + 3.789052425418049e-05, + 3.438993007875979e-05, + 3.1885232601780444e-05, + 2.9910806915722787e-05, + 2.8925971491844393e-05, + 2.7547004719963297e-05, + 2.661058897501789e-05, + 2.6070885724038817e-05, + 2.0136398234171793e-05, + 1.9825236449833028e-05, + 1.973241160158068e-05, + 1.973574580915738e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.179371863865526e-05, + "first_to_last_ratio": 3.933988193388572, + "imbalance_abs_log_ratio": 1.3696537187901656, + "last_quartile_mean": 2.3333501303568482e-05, + "mean": 5.082899730268764e-05, + "normalized": [ + 2.1394359800622813, + 2.0814587808021265, + 1.9787622602269084, + 1.9084551972599093, + 1.8068306956003535, + 1.6376936233321315, + 1.5012763891845262, + 1.393543999273884, + 1.35980639449679, + 1.238717928620473, + 1.1365151179184183, + 1.05713604656592, + 1.023790231850539, + 0.9508624555295617, + 0.8771210384666132, + 0.8254319480230166, + 0.8391731577705832, + 0.7782341287088573, + 0.7170084501774274, + 0.6619086246953101, + 0.6688703776588982, + 0.6150814814369804, + 0.5789313330893375, + 0.5514835190919826, + 0.5572498614580683, + 0.5413707333622386, + 0.5311388145382818, + 0.5242249796321137, + 0.38285690475582773, + 0.3802104967302831, + 0.3789175322683099, + 0.37650151741204796 + ], + "peak_layer": 1, + "peak_normalized": 2.1394359800622813, + "population_cv": 0.5382143248139301, + "rest_mean": 5.464754158984525e-05, + "spike_contrast": 0.552794458326558, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.0208858152036556e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010874538565985858, + 0.00010579846275504678, + 0.00010057850158773363, + 9.700486407382414e-05, + 9.18393925530836e-05, + 8.324232476297766e-05, + 7.630837353644893e-05, + 7.08324441802688e-05, + 6.911759555805475e-05, + 6.296279025264084e-05, + 5.7767923863139004e-05, + 5.3733165259473026e-05, + 5.2038230933249e-05, + 4.833138518733904e-05, + 4.458318289835006e-05, + 4.195587825961411e-05, + 4.265433017280884e-05, + 3.955686042900197e-05, + 3.64448205800727e-05, + 3.36441516992636e-05, + 3.39980106218718e-05, + 3.126397496089339e-05, + 2.94264991680393e-05, + 2.8031354304403067e-05, + 2.8324451704975218e-05, + 2.7517331545823254e-05, + 2.6997253371519037e-05, + 2.6645830075722188e-05, + 1.9460232579149306e-05, + 1.932571831275709e-05, + 1.9259998225606978e-05, + 1.913719461299479e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002383605042268755, + "first_to_last_ratio": 5.6146902736182, + "imbalance_abs_log_ratio": 1.7253864261379666, + "last_quartile_mean": 4.245301033733995e-05, + "mean": 0.0001231808448665106, + "normalized": [ + 2.100827328644271, + 2.2345440763144846, + 2.136083407505072, + 2.0655627440949633, + 2.055234003018172, + 1.8075987971575753, + 1.623843438405382, + 1.4566679111096852, + 1.4453319547522105, + 1.317016319283833, + 1.1984433593272132, + 1.1000521536406067, + 1.071769085471528, + 0.9883476552878133, + 0.8803897945877148, + 0.8099451507686356, + 0.8457679211262041, + 0.7651810707046709, + 0.6892705769434541, + 0.6200948164612059, + 0.5902902932604684, + 0.5228157886141485, + 0.47313831489002084, + 0.4446663790889937, + 0.4105354433359864, + 0.3905597157431388, + 0.3717521617725102, + 0.3606343113512262, + 0.3081187728953204, + 0.3032816350953125, + 0.304634216769742, + 0.3076014025784363 + ], + "peak_layer": 2, + "peak_normalized": 2.2345440763144846, + "population_cv": 0.6294889602855173, + "rest_mean": 0.000134853615842682, + "spike_contrast": 0.4460234990313686, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.014788159518503e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00025878168526105583, + 0.0002752530272118747, + 0.00026312455884180963, + 0.0002544377639424056, + 0.000253165460890159, + 0.00022266154701355845, + 0.00020002640667371452, + 0.0001794335839804262, + 0.00017803721129894257, + 0.0001622311829123646, + 0.00014762526552658528, + 0.00013550535368267447, + 0.00013202142145019025, + 0.00012174549920018762, + 0.00010844715870916843, + 9.976972796721384e-05, + 0.00010418240708531812, + 9.425565076526254e-05, + 8.490493200952187e-05, + 7.638380338903517e-05, + 7.271245704032481e-05, + 6.440089055104181e-05, + 5.82815773668699e-05, + 5.4774380259914324e-05, + 5.0570102757774293e-05, + 4.810947575606406e-05, + 4.579274536808953e-05, + 4.442323916009627e-05, + 3.795433076447807e-05, + 3.7358488043537363e-05, + 3.752510019694455e-05, + 3.789060065173544e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "787f9e4002b423fdbb7f8faf6fa82a8ff8c9854d40d44697d8afa4d95a3e169c", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04377923160791397, + 0.023813940584659576, + 0.029579024761915207, + 0.034570932388305664, + 0.037983279675245285, + 0.02627667225897312, + 0.02788778953254223, + 0.03023744560778141, + 0.031768180429935455, + 0.02424551732838154, + 0.025113960728049278, + 0.026638666167855263, + 0.027698444202542305, + 0.022435806691646576, + 0.022681118920445442, + 0.023235918954014778, + 0.023750189691781998, + 0.019677624106407166, + 0.020757248625159264, + 0.02137788012623787, + 0.02207154594361782, + 0.019271831959486008, + 0.020465917885303497, + 0.021911408752202988, + 0.024193286895751953, + 0.02259150519967079, + 0.024306390434503555, + 0.02611992135643959, + 0.02851196378469467, + 0.027104467153549194, + 0.03068944811820984, + 0.03286270797252655 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.8306669605626666e-05, + "first_to_last_ratio": 5.668716287294882, + "imbalance_abs_log_ratio": 1.7349626877491615, + "last_quartile_mean": 4.993488502691434e-06, + "mean": 1.5805165769933183e-05, + "normalized": [ + 0.8783128680331268, + 2.1826435477445063, + 2.2956954594453394, + 2.767580901011819, + 2.1125287729616464, + 1.9101651305644987, + 1.2546030463288333, + 0.9262767734665253, + 0.8744163240087035, + 1.1038744529856963, + 1.057809216514194, + 0.8349821291845769, + 1.5598261518374243, + 1.5571623184207217, + 1.076399772260337, + 1.4673081766428846, + 1.1246096789003457, + 0.7371277404470634, + 0.6369864324222737, + 0.6349327395421065, + 0.6011227365145811, + 0.8094112388497428, + 0.6205992994135143, + 0.4481028505293834, + 0.5856560220554665, + 0.7623288521260666, + 0.3625047598024719, + 0.13435636426712916, + 0.1475298295833587, + 0.2669943453689258, + 0.11457117055614831, + 0.1535808982105884 + ], + "peak_layer": 4, + "peak_normalized": 2.767580901011819, + "population_cv": 0.670823858490635, + "rest_mean": 1.6937932451178818e-05, + "spike_contrast": 0.5719839608011006, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.688225691206754e-06, + "top_five_layers": [ + 4, + 3, + 2, + 5, + 6 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.3881880477129016e-05, + 3.449704308877699e-05, + 3.628384729381651e-05, + 4.374207492219284e-05, + 3.338886745041236e-05, + 3.0190476536517963e-05, + 1.9829209122690372e-05, + 1.4639957953477278e-05, + 1.3820294952893164e-05, + 1.7446918718633242e-05, + 1.6718850019969977e-05, + 1.3197030966694001e-05, + 2.465331090206746e-05, + 2.4611208573332988e-05, + 1.701267683529295e-05, + 2.3191048967419192e-05, + 1.7774642401491292e-05, + 1.1650426131382119e-05, + 1.0067676157632377e-05, + 1.0035217201220803e-05, + 9.50084449868882e-06, + 1.2792878806067165e-05, + 9.808674803934991e-06, + 7.082339834596496e-06, + 9.256390512746293e-06, + 1.2048733879055362e-05, + 5.729447821067879e-06, + 2.1235246094875038e-06, + 2.3317334125749767e-06, + 4.219889888190664e-06, + 1.8108163430952118e-06, + 2.4273715553135844e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.993926990617183e-05, + "first_to_last_ratio": 3.4893308817982382, + "imbalance_abs_log_ratio": 1.2497099934197489, + "last_quartile_mean": 2.004374829311928e-05, + "mean": 4.464254881497709e-05, + "normalized": [ + 0.8066620510828731, + 2.249059806280304, + 1.8293400677886618, + 1.6505903941799722, + 1.482941789801698, + 1.867443438314272, + 1.4782170879001166, + 1.1689490929675366, + 1.2145199991696674, + 1.4010197387507275, + 1.1581363399144262, + 1.0008260463099323, + 1.1290384086543988, + 1.2173363380864748, + 0.9961077007288228, + 1.0849452654707106, + 1.086573542898278, + 1.0033179684088052, + 0.8130676737841358, + 0.6422200474001272, + 0.6427230931215812, + 1.0345398071106366, + 0.8415727559232208, + 0.6089870444535808, + 0.8527565388110598, + 1.0992667074243434, + 0.5132675404314736, + 0.1873435614965349, + 0.20684501755119733, + 0.3674535488046901, + 0.15217388752900993, + 0.21275769945073061 + ], + "peak_layer": 2, + "peak_normalized": 2.249059806280304, + "population_cv": 0.5008866669948363, + "rest_mean": 4.632808737302117e-05, + "spike_contrast": 0.7671510441468363, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5540640601539054e-05, + "top_five_layers": [ + 2, + 6, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.601144999265671e-05, + 0.0001004037621896714, + 8.166640327544883e-05, + 7.368656224571168e-05, + 6.62023012409918e-05, + 8.336743485415354e-05, + 6.599137850571424e-05, + 5.218486694502644e-05, + 5.4219268349697813e-05, + 6.25450920779258e-05, + 5.170215808902867e-05, + 4.467942562769167e-05, + 5.040315227233805e-05, + 5.43449968972709e-05, + 4.446878665476106e-05, + 4.843472197535448e-05, + 4.850741242989898e-05, + 4.479067138163373e-05, + 3.629741331678815e-05, + 2.8670339816017076e-05, + 2.8692797059193254e-05, + 4.6184493839973584e-05, + 3.7569952837657183e-05, + 2.71867338597076e-05, + 3.806922541116364e-05, + 4.907406764687039e-05, + 2.291357122885529e-05, + 8.36349408928072e-06, + 9.234088793164119e-06, + 1.6404062989749946e-05, + 6.793430202378659e-06, + 9.498045983491465e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06202284339815378, + "first_to_last_ratio": 3.489330820237693, + "imbalance_abs_log_ratio": 1.2497099757772414, + "last_quartile_mean": 0.017774996580556035, + "mean": 0.039589458669070154, + "normalized": [ + 0.8066620927081755, + 2.249059736772125, + 1.8293399648688806, + 1.6505903951156085, + 1.4829417588396991, + 1.867443458890533, + 1.4782170964630472, + 1.1689491284714175, + 1.2145200544167176, + 1.4010197265931286, + 1.1581363231796216, + 1.0008260498990915, + 1.1290383875745214, + 1.2173363145296596, + 0.996107692090837, + 1.0849452712811893, + 1.086573543692374, + 1.0033179540826904, + 0.8130677223960504, + 0.6422200351976349, + 0.6427230832969295, + 1.0345398705514788, + 0.8415727462776835, + 0.6089870555080975, + 0.8527565801245708, + 1.0992667100332167, + 0.513267522372738, + 0.18734356688581966, + 0.20684500840965173, + 0.36745355899142085, + 0.15217388494831066, + 0.2127577055370788 + ], + "peak_layer": 2, + "peak_normalized": 2.249059736772125, + "population_cv": 0.5008866561011148, + "rest_mean": 0.04108421097475069, + "spike_contrast": 0.7671510653512432, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03151779621839523, + "top_five_layers": [ + 2, + 6, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03193531557917595, + 0.08903905749320984, + 0.0724225789308548, + 0.06534598022699356, + 0.05870886147022247, + 0.07393107563257217, + 0.0585218146443367, + 0.04627806320786476, + 0.04808219149708748, + 0.055465612560510635, + 0.0458499900996685, + 0.03962216153740883, + 0.044698018580675125, + 0.048193685710430145, + 0.03943536430597305, + 0.04295239597558975, + 0.04301685839891434, + 0.0397208146750927, + 0.032188910990953445, + 0.025425143539905548, + 0.025445058941841125, + 0.04095687344670296, + 0.033317409455776215, + 0.02410946786403656, + 0.03376017138361931, + 0.04351937398314476, + 0.02031998336315155, + 0.007416830398142338, + 0.008188881911337376, + 0.014547287486493587, + 0.006024481728672981, + 0.00842296238988638 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5216306564980187e-05, + "first_to_last_ratio": 7.030788105499711, + "imbalance_abs_log_ratio": 1.9502988055846384, + "last_quartile_mean": 2.1642391061504895e-06, + "mean": 1.0416225535436752e-05, + "normalized": [ + 1.2614020219005322, + 1.2435865799883328, + 1.4443414880126273, + 2.0469556474720734, + 1.6464946614743052, + 1.773065362472431, + 1.0834782895079982, + 1.1872943029799679, + 1.2246868580585597, + 1.5602141574184534, + 1.327224315217328, + 1.3409942697906458, + 1.3449072126725554, + 2.065589232705634, + 1.2862289608083182, + 1.79783161732383, + 1.738192374488952, + 1.4057167409862177, + 0.9202432421079245, + 0.7366699550385442, + 0.583019146679299, + 0.5088209255652885, + 0.40525183944794096, + 0.40558477226377354, + 0.26054664683684153, + 0.30292479006597633, + 0.2336502506334751, + 0.2297086901983557, + 0.15260828683083258, + 0.17440390432050698, + 0.1566888857901092, + 0.15167457094237005 + ], + "peak_layer": 14, + "peak_normalized": 2.065589232705634, + "population_cv": 0.6121354643087803, + "rest_mean": 1.1510614630996887e-05, + "spike_contrast": 0.3915103201593089, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.506524419412017e-06, + "top_five_layers": [ + 14, + 4, + 16, + 6, + 17 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.3139047950971872e-05, + 1.295347829000093e-05, + 1.5044586689327843e-05, + 2.1321551685105078e-05, + 1.7150259736808948e-05, + 1.8468648704583757e-05, + 1.1285754226264544e-05, + 1.236712523677852e-05, + 1.2756614523823373e-05, + 1.625154254725203e-05, + 1.3824667803419288e-05, + 1.3968098755867686e-05, + 1.4008856851432938e-05, + 2.151564331143163e-05, + 1.339765094598988e-05, + 1.8726619600784034e-05, + 1.8105403796653263e-05, + 1.4642262613051571e-05, + 9.585461157257669e-06, + 7.673320396861527e-06, + 6.072858923289459e-06, + 5.29999351783772e-06, + 4.221194558340358e-06, + 4.224662461638218e-06, + 2.7139126359543297e-06, + 3.1553329336020397e-06, + 2.4337537070096005e-06, + 2.3926975245558424e-06, + 1.5896023342065746e-06, + 1.816630401663133e-06, + 1.6321067732860683e-06, + 1.579876538926328e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.190560773371544e-05, + "first_to_last_ratio": 5.742880674184589, + "imbalance_abs_log_ratio": 1.7479609440560961, + "last_quartile_mean": 9.038252869686403e-06, + "mean": 3.1702642814934734e-05, + "normalized": [ + 0.7426774192575819, + 2.3203889953928316, + 1.8256902534826824, + 1.543739301356747, + 1.5734163075504266, + 2.109348732800137, + 1.64875428919227, + 1.334099715975053, + 1.3912665329141354, + 1.6086824817138736, + 1.3339909299684964, + 1.1314499757741263, + 1.1960452245153466, + 1.3418021783506682, + 1.0883087504651683, + 1.0855161172841996, + 1.0978647060664168, + 1.0745881727648192, + 0.87642608484069, + 0.6880064264214529, + 0.6659679650868783, + 0.8134892489335985, + 0.743952040109719, + 0.4837711990268687, + 0.5137493175424936, + 0.6236389627256068, + 0.32417337747920116, + 0.18870731593182713, + 0.15601040790107645, + 0.21053976663531476, + 0.12020463563707022, + 0.14373316690322271 + ], + "peak_layer": 2, + "peak_normalized": 2.3203889953928316, + "population_cv": 0.5767311578617529, + "rest_mean": 3.3791577186127176e-05, + "spike_contrast": 0.6043635400030921, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0422397210495546e-05, + "top_five_layers": [ + 2, + 6, + 3, + 7, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.354483694944065e-05, + 7.356246351264417e-05, + 5.787920599686913e-05, + 4.8940615670289844e-05, + 4.9881455197464675e-05, + 6.687192944809794e-05, + 5.226986831985414e-05, + 4.229448677506298e-05, + 4.410682595334947e-05, + 5.0999486120417714e-05, + 4.2291037971153855e-05, + 3.586995444493368e-05, + 3.791779454331845e-05, + 4.253867518855259e-05, + 3.450226358836517e-05, + 3.4413729736115783e-05, + 3.480521263554692e-05, + 3.406728501431644e-05, + 2.778502312139608e-05, + 2.1811621991219e-05, + 2.111294452333823e-05, + 2.57897590927314e-05, + 2.3585245799040422e-05, + 1.533682552690152e-05, + 1.628721111046616e-05, + 1.977100328076631e-05, + 1.0277152796334121e-05, + 5.982520633551758e-06, + 4.945942237100098e-06, + 6.674667019979097e-06, + 3.810804628301412e-06, + 4.556721250992268e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.938292866223492e-05, + "first_to_last_ratio": 9.18982862399783, + "imbalance_abs_log_ratio": 2.2180972880977254, + "last_quartile_mean": 4.285491087330229e-06, + "mean": 2.3341486304673253e-05, + "normalized": [ + 0.6673051270990256, + 2.308149633585561, + 1.872866846541638, + 1.4969405906506341, + 1.5875800328734282, + 2.3234736824333693, + 1.784836015034525, + 1.4568500921073086, + 1.548779422738, + 1.7929545502301463, + 1.477805011955662, + 1.2083348392465652, + 1.2158304842751781, + 1.3732002989348482, + 1.1418738689499803, + 1.0221659591685974, + 1.0518199886832604, + 1.1236379858609276, + 0.9217957770992289, + 0.7176364137990805, + 0.6755225547783117, + 0.7056040273764278, + 0.6474467916351409, + 0.4087919922057317, + 0.30494607832868875, + 0.3212400761898423, + 0.2149387803961774, + 0.18637979615916483, + 0.12482940461335829, + 0.12661742786534272, + 0.09393484526198889, + 0.09591160392285902 + ], + "peak_layer": 6, + "peak_normalized": 2.3234736824333693, + "population_cv": 0.6451544009698722, + "rest_mean": 2.5293256915877766e-05, + "spike_contrast": 0.5061398398295045, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.280192500416888e-05, + "top_five_layers": [ + 6, + 2, + 3, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.557589348522015e-05, + 5.3875643061473966e-05, + 4.371549584902823e-05, + 3.4940818295581266e-05, + 3.705647759488784e-05, + 5.423332913778722e-05, + 4.166072540101595e-05, + 3.4005046472884715e-05, + 3.615081368479878e-05, + 4.185022407909855e-05, + 3.449416544754058e-05, + 2.820433110173326e-05, + 2.8379290597513318e-05, + 3.205253597116098e-05, + 2.665303327376023e-05, + 2.3858872737037018e-05, + 2.45510418608319e-05, + 2.622738065838348e-05, + 2.151608350686729e-05, + 1.6750700524426065e-05, + 1.576770046085585e-05, + 1.646984674152918e-05, + 1.5112370419956278e-05, + 9.541812687530182e-06, + 7.117894710972905e-06, + 7.498220838897396e-06, + 5.016990598960547e-06, + 4.350381459516939e-06, + 2.9137038382032188e-06, + 2.9554389584518503e-06, + 2.192578904214315e-06, + 2.2387193894246593e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012773637081409106, + "first_to_last_ratio": 4.324151431064901, + "imbalance_abs_log_ratio": 1.4642159202240765, + "last_quartile_mean": 2.9540216814893938e-05, + "mean": 8.050168943896097e-05, + "normalized": [ + 0.8871717025827831, + 2.337156289954447, + 1.7098779659370984, + 1.4615981277611254, + 1.6595169230948639, + 1.8749716736387922, + 1.5249505044111165, + 1.238788125932271, + 1.2542052402145494, + 1.4655117877457478, + 1.219570330002609, + 1.060650662474342, + 1.227816664029005, + 1.4059417450611726, + 1.0036779465715522, + 1.2092205201021657, + 1.1438081416422972, + 1.014111450824222, + 0.8576088870341769, + 0.6691271722099967, + 0.6720599542707117, + 0.8395852269377942, + 0.8228857633365203, + 0.5045750826570258, + 0.6467370801446735, + 0.9037936178533763, + 0.4164763377402587, + 0.19712216248220002, + 0.18271060059268188, + 0.2652336386100389, + 0.14113561188950058, + 0.1824030622608841 + ], + "peak_layer": 2, + "peak_normalized": 2.337156289954447, + "population_cv": 0.5285114363123314, + "rest_mean": 8.501621491800675e-05, + "spike_contrast": 0.6601476189717621, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.6123251852113754e-05, + "top_five_layers": [ + 2, + 6, + 3, + 5, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.141882088035345e-05, + 0.0001881450298242271, + 0.00013764806499239057, + 0.00011766111856559291, + 0.0001335939159616828, + 0.00015093838737811893, + 0.00012276109191589057, + 9.972453699447215e-05, + 0.0001009656407404691, + 0.00011797617480624467, + 9.817747195484117e-05, + 8.538417023373768e-05, + 9.884131577564403e-05, + 0.00011318068573018536, + 8.079777035163715e-05, + 9.734429477248341e-05, + 9.207848779624328e-05, + 8.163768507074565e-05, + 6.903896428411826e-05, + 5.386586781241931e-05, + 5.410196172306314e-05, + 6.758802919648588e-05, + 6.624369416385889e-05, + 4.0619146602693945e-05, + 5.2063427574466914e-05, + 7.275691314134747e-05, + 3.352704879944213e-05, + 1.586866710567847e-05, + 1.4708512026118115e-05, + 2.1351756004150957e-05, + 1.1361655197106302e-05, + 1.4683754670841154e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ad0dc0ee191d8145bd7a67f8b576436d75a4c4558d60d541ecf9bca80ffb9f93", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.017109708860516548, + 0.02606838569045067, + 0.030872320756316185, + 0.034764163196086884, + 0.025188947096467018, + 0.026965361088514328, + 0.028485212475061417, + 0.030349206179380417, + 0.022602196782827377, + 0.023387571796774864, + 0.02450813725590706, + 0.025169003754854202, + 0.020798001438379288, + 0.02118327096104622, + 0.021654823794960976, + 0.022377775982022285, + 0.018887808546423912, + 0.01950881816446781, + 0.020426612347364426, + 0.021234560757875443, + 0.01860968954861164, + 0.019469313323497772, + 0.020844420418143272, + 0.02213398739695549, + 0.0209572184830904, + 0.022487536072731018, + 0.023487187922000885, + 0.024889562278985977, + 0.023824406787753105, + 0.025868752971291542, + 0.0271493811160326, + 0.028410611674189568 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.406384535584948e-05, + "first_to_last_ratio": 1.6917204830931702, + "imbalance_abs_log_ratio": 0.5257460484196427, + "last_quartile_mean": 8.313338696552819e-06, + "mean": 1.2088094919704417e-05, + "normalized": [ + 2.384784726621517, + 1.1311337715172713, + 0.9796665485569107, + 0.8224622106532729, + 1.211988479625169, + 1.0625055316005352, + 0.8419214674481715, + 0.8731050481608821, + 1.162924153782958, + 1.064627644161191, + 0.9632127839920408, + 0.9176822777828343, + 1.137313817736726, + 1.047865701151065, + 1.0364330031252973, + 0.9266530092349528, + 1.3076030994322017, + 1.245640723171692, + 1.1356311002249702, + 1.055064557020718, + 1.1753193075391253, + 1.1259884855902011, + 0.9820417647039537, + 0.9065953024025623, + 0.9143977245534269, + 0.8160033291510744, + 0.782411126229384, + 0.7182286763887548, + 0.6000700810180117, + 0.5741865902243426, + 0.5489069666673982, + 0.5476309905313882 + ], + "peak_layer": 1, + "peak_normalized": 2.384784726621517, + "population_cv": 0.31980213589111106, + "rest_mean": 1.2041379991219233e-05, + "spike_contrast": 1.0248290098413306, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2340355533524417e-05, + "top_five_layers": [ + 1, + 17, + 18, + 5, + 21 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.8827504138462245e-05, + 1.3673252396984026e-05, + 1.1842302228615154e-05, + 9.942001270246692e-06, + 1.4650631783297285e-05, + 1.284366771869827e-05, + 1.017722661345033e-05, + 1.055417669704184e-05, + 1.4057537555345334e-05, + 1.2869320016761776e-05, + 1.1643407560768537e-05, + 1.1093030479969457e-05, + 1.3747957382292952e-05, + 1.2666700058616698e-05, + 1.2528500519692898e-05, + 1.1201469533261843e-05, + 1.5806430383236147e-05, + 1.5057423297548667e-05, + 1.3727616533287801e-05, + 1.2753720511682332e-05, + 1.4207371350494213e-05, + 1.3611055692308582e-05, + 1.1871014066855423e-05, + 1.0959010069200303e-05, + 1.105332648876356e-05, + 9.863925697572995e-06, + 9.457859960093629e-06, + 8.682016414240934e-06, + 7.253704097820446e-06, + 6.9408220042532776e-06, + 6.635239515162539e-06, + 6.619815394515172e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.4028067552280845e-05, + "first_to_last_ratio": 1.6345037059364802, + "imbalance_abs_log_ratio": 0.4913392145007449, + "last_quartile_mean": 3.305472318970715e-05, + "mean": 4.737582446523447e-05, + "normalized": [ + 2.3514155771111755, + 1.133401461379931, + 0.9385744423540482, + 0.7603362412028586, + 1.162677024813552, + 1.060905926051319, + 0.8429727875353996, + 0.8730309155458181, + 1.162986103649938, + 1.0649320906402773, + 0.9658838036496159, + 0.9259372261987153, + 1.1426484858464174, + 1.062663413586317, + 1.038525930654914, + 0.9333315438116647, + 1.325499141503536, + 1.241150951130487, + 1.1365919996589382, + 1.0669045895397178, + 1.1830135675602507, + 1.1385949073084929, + 0.990415755644998, + 0.9159030297690505, + 0.924477682983016, + 0.8252864902247735, + 0.7951806757653125, + 0.7255609942258914, + 0.6087012425945892, + 0.5836005087504837, + 0.560707980014089, + 0.5581875092944122 + ], + "peak_layer": 1, + "peak_normalized": 2.3514155771111755, + "population_cv": 0.3142051905018903, + "rest_mean": 4.71084055822791e-05, + "spike_contrast": 1.0363306893909856, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.881988643319346e-05, + "top_five_layers": [ + 1, + 17, + 18, + 21, + 9 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00011140025162603706, + 5.3695828682975844e-05, + 4.446573802852072e-05, + 3.602155629778281e-05, + 5.508278263732791e-05, + 5.026129292673431e-05, + 3.9936530811246485e-05, + 4.136055940762162e-05, + 5.5097425502026454e-05, + 5.045203579356894e-05, + 4.57595415355172e-05, + 4.386703949421644e-05, + 5.413391409092583e-05, + 5.034455534769222e-05, + 4.920102219330147e-05, + 4.4217351387487724e-05, + 6.279661465669051e-05, + 5.880054959561676e-05, + 5.38469830644317e-05, + 5.0545484555186704e-05, + 5.6046243116725236e-05, + 5.394187246565707e-05, + 4.692176298703998e-05, + 4.339166116551496e-05, + 4.379789243103005e-05, + 3.9098627894418314e-05, + 3.767234011320397e-05, + 3.4374050301266834e-05, + 2.8837723220931366e-05, + 2.7648555260384455e-05, + 2.6564002837403677e-05, + 2.6444593459018506e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04791263025254011, + "first_to_last_ratio": 1.6345036692801245, + "imbalance_abs_log_ratio": 0.4913391920741478, + "last_quartile_mean": 0.029313259525224566, + "mean": 0.04201335500692949, + "normalized": [ + 2.3514156784172187, + 1.133401461661499, + 0.9385744066074354, + 0.7603362049299565, + 1.1626770145536474, + 1.0609058838539445, + 0.8429727897119484, + 0.8730308482354255, + 1.1629861153616714, + 1.0649320859164637, + 0.965883808806204, + 0.9259371845017617, + 1.142648505828565, + 1.062663395935368, + 1.0385258681211755, + 0.9333315745429377, + 1.325499101605811, + 1.2411509353845518, + 1.1365920452458722, + 1.0669045321225719, + 1.183013648725627, + 1.138594993654493, + 0.9904157371924246, + 0.9159030239053129, + 0.9244776895791931, + 0.8252865663984437, + 0.7951807151797501, + 0.7255609453201705, + 0.6087012883299899, + 0.5836005027338216, + 0.5607079790634799, + 0.5581874685732653 + ], + "peak_layer": 1, + "peak_normalized": 2.3514156784172187, + "population_cv": 0.31420520653960876, + "rest_mean": 0.04177620503361578, + "spike_contrast": 1.036330725301315, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.043293964862823484, + "top_five_layers": [ + 1, + 17, + 18, + 21, + 9 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.09879086166620255, + 0.04761799797415733, + 0.03943265974521637, + 0.03194427490234375, + 0.04884796217083931, + 0.044572215527296066, + 0.03541611507534981, + 0.03667895495891571, + 0.04886094853281975, + 0.044741369783878326, + 0.04058001935482025, + 0.03890172764658928, + 0.04800649732351303, + 0.04464605450630188, + 0.04363195598125458, + 0.039212390780448914, + 0.05568866431713104, + 0.052144914865493774, + 0.04775204509496689, + 0.04482423886656761, + 0.049702372401952744, + 0.047836195677518845, + 0.04161068797111511, + 0.038480158895254135, + 0.038840409368276596, + 0.0346730574965477, + 0.033408209681510925, + 0.030483249574899673, + 0.02557358331978321, + 0.024519015103578568, + 0.02355722337961197, + 0.02345132827758789 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.917115407148231e-06, + "first_to_last_ratio": 1.7761794884414766, + "imbalance_abs_log_ratio": 0.5744647027750681, + "last_quartile_mean": 2.2053601185234584e-06, + "mean": 3.1665725686025326e-06, + "normalized": [ + 2.7105980142014188, + 1.2584335069420332, + 1.1184553125758785, + 0.9137724260402323, + 1.3069438057161062, + 0.9543737632121961, + 0.8185107477756304, + 0.8150769920988516, + 1.106441368261451, + 1.0103937914191876, + 0.8971971105841021, + 0.8698038240167904, + 1.023913831911454, + 0.9814473768267671, + 0.9475786306403914, + 0.9043188811303046, + 1.2933347278335885, + 1.1779841320022393, + 1.0971261185238004, + 1.0522252088592114, + 1.1621449547113154, + 1.1289087330174585, + 0.9715840445086426, + 0.907831047153982, + 0.875626869339223, + 0.7871309556685123, + 0.7978996689976676, + 0.7370601330215918, + 0.6234295385177253, + 0.6039908093128533, + 0.5786627593732889, + 0.5678009158061047 + ], + "peak_layer": 1, + "peak_normalized": 2.7105980142014188, + "population_cv": 0.364237936254411, + "rest_mean": 3.161166449461706e-06, + "spike_contrast": 1.0109450619113027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.1957656119629975e-06, + "top_five_layers": [ + 1, + 5, + 17, + 2, + 18 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.583305316278711e-06, + 3.9849210224929266e-06, + 3.5416699120105477e-06, + 2.8935266982443864e-06, + 4.13853240388562e-06, + 3.0220937787817093e-06, + 2.5918736810126575e-06, + 2.5810004444792867e-06, + 3.5036268855037633e-06, + 3.1994852633943083e-06, + 2.8410397590050707e-06, + 2.754296929197153e-06, + 3.2422974527435144e-06, + 3.1078243409865536e-06, + 3.0005764983798144e-06, + 2.8635913622565567e-06, + 4.095438271178864e-06, + 3.7301722386473557e-06, + 3.4741294712148374e-06, + 3.331947482365649e-06, + 3.680016334328684e-06, + 3.5747714264289243e-06, + 3.0765913834329695e-06, + 2.8747128908435116e-06, + 2.7727360247808974e-06, + 2.492507292117807e-06, + 2.526607204345055e-06, + 2.333954398636706e-06, + 1.974134875126765e-06, + 1.9125807284581242e-06, + 1.8323776203033049e-06, + 1.7979828044190072e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.53617046787258e-05, + "first_to_last_ratio": 1.6235922245104368, + "imbalance_abs_log_ratio": 0.48464111692853945, + "last_quartile_mean": 2.7939099481955054e-05, + "mean": 3.8807642113170004e-05, + "normalized": [ + 2.437808381833339, + 1.182834547067526, + 0.9777525684870058, + 0.7857025355052735, + 1.173309606758207, + 1.0576717542347258, + 0.859768212992415, + 0.8762393870580075, + 1.137990945103205, + 1.0501479644557203, + 0.949971945680228, + 0.913039951911735, + 1.1178137006265285, + 1.0305352456029508, + 0.9972430453249006, + 0.8949050115271279, + 1.2575810394572957, + 1.1563573391187367, + 1.0783657149583266, + 1.0249789523026176, + 1.184595619530283, + 1.1506932403605004, + 1.002430737703081, + 0.9427578856689722, + 0.9703835498479987, + 0.8658087871584248, + 0.8379408888026836, + 0.7685722951163864, + 0.6138918991902961, + 0.5823916131103506, + 0.561129142648094, + 0.5593864908570572 + ], + "peak_layer": 1, + "peak_normalized": 2.437808381833339, + "population_cv": 0.3181748559585258, + "rest_mean": 3.8447074513425364e-05, + "spike_contrast": 1.0600210202615004, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.075470715179108e-05, + "top_five_layers": [ + 1, + 17, + 21, + 2, + 5 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 9.460559522267431e-05, + 4.590301978169009e-05, + 3.7944271753076464e-05, + 3.0491262805298902e-05, + 4.553337930701673e-05, + 4.104574691154994e-05, + 3.336557711008936e-05, + 3.4004784538410604e-05, + 4.416274532559328e-05, + 4.075376637047157e-05, + 3.686617128551006e-05, + 3.543292768881656e-05, + 4.337971404311247e-05, + 3.999264299636707e-05, + 3.8700651202816516e-05, + 3.4729153412627056e-05, + 4.8803754907567054e-05, + 4.487550177145749e-05, + 4.1848830733215436e-05, + 3.9777016354491934e-05, + 4.5971362851560116e-05, + 4.4655691453954205e-05, + 3.8901973312022164e-05, + 3.658621062641032e-05, + 3.76582975150086e-05, + 3.359999755048193e-05, + 3.251851012464613e-05, + 2.98264785669744e-05, + 2.3823697119951248e-05, + 2.2601245291298255e-05, + 2.1776098947157152e-05, + 2.170847074012272e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.840028375634574e-05, + "first_to_last_ratio": 1.5392922378986411, + "imbalance_abs_log_ratio": 0.43132272500352103, + "last_quartile_mean": 2.4946714347606758e-05, + "mean": 3.270594595505827e-05, + "normalized": [ + 2.461740434904075, + 1.197496439702706, + 0.9903120585800815, + 0.7910638146450573, + 1.1699175648793894, + 1.0483897843618244, + 0.8646493044292616, + 0.8692874959222262, + 1.129883380630568, + 1.0428338098095176, + 0.9376272398479191, + 0.8900945088315015, + 1.0865591482264656, + 1.0023234373075094, + 0.9670931816451467, + 0.8520871001401004, + 1.1852808372815498, + 1.07429226741598, + 1.0193151600779609, + 0.9854449496787876, + 1.1764973274911055, + 1.1637500315753844, + 1.0219017710659555, + 0.9700968049576442, + 1.0255967071543362, + 0.9382176643393733, + 0.9120563422137004, + 0.8495166179296606, + 0.6405458356516568, + 0.590640394607506, + 0.5733315467004426, + 0.5721570379956076 + ], + "peak_layer": 1, + "peak_normalized": 2.461740434904075, + "population_cv": 0.3136325717265237, + "rest_mean": 3.227247995070044e-05, + "spike_contrast": 1.0859612410365724, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.504666237859055e-05, + "top_five_layers": [ + 1, + 2, + 17, + 21, + 5 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.051354961935431e-05, + 3.916525383829139e-05, + 3.238909266656265e-05, + 2.5872490368783474e-05, + 3.8263260648818687e-05, + 3.428857962717302e-05, + 2.8279173420742154e-05, + 2.8430869861040264e-05, + 3.695390478242189e-05, + 3.41068662237376e-05, + 3.06659858324565e-05, + 2.9111382900737226e-05, + 3.553694477886893e-05, + 3.278193617006764e-05, + 3.1629697332391515e-05, + 2.7868314646184444e-05, + 3.876573100569658e-05, + 3.513574483804405e-05, + 3.3337666536681354e-05, + 3.222990926587954e-05, + 3.847845800919458e-05, + 3.806154563790187e-05, + 3.3422264095861465e-05, + 3.172793367411941e-05, + 3.354311047587544e-05, + 3.068529622396454e-05, + 2.9829665436409414e-05, + 2.7784244593931362e-05, + 2.0949657482560724e-05, + 1.9317452824907377e-05, + 1.8751350580714643e-05, + 1.871293716249056e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.757476163940737e-05, + "first_to_last_ratio": 1.686806792110223, + "imbalance_abs_log_ratio": 0.5228372695066706, + "last_quartile_mean": 5.7845843457471346e-05, + "mean": 8.550922041195008e-05, + "normalized": [ + 2.337533863407778, + 1.1216672060979693, + 0.9305333491453122, + 0.7843126666651307, + 1.1828292881294888, + 1.0607795384885752, + 0.84110598600667, + 0.8700558991883418, + 1.1556301815058487, + 1.0558057877237697, + 0.9593441405966971, + 0.94860955894161, + 1.1637725041314626, + 1.0815711321728936, + 1.0583492052405916, + 0.9627433056618342, + 1.3670140359267535, + 1.2705709383637194, + 1.1622428458162537, + 1.0675745501188607, + 1.1957367214693495, + 1.1112711965314517, + 0.9959768067004482, + 0.903076335509746, + 0.8931558925185026, + 0.7912932590255418, + 0.7799621982699588, + 0.6868562778940714, + 0.5876192160650839, + 0.5636586250712405, + 0.5509064805128926, + 0.5584410071021524 + ], + "peak_layer": 1, + "peak_normalized": 2.337533863407778, + "population_cv": 0.31926473671477396, + "rest_mean": 8.519499951270754e-05, + "spike_contrast": 1.023604833225597, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.720601326785981e-05, + "top_five_layers": [ + 1, + 17, + 18, + 21, + 5 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.00019988069834653288, + 9.591288835508749e-05, + 7.95691812527366e-05, + 6.706596468575299e-05, + 0.00010114281030837446, + 9.070643136510625e-05, + 7.192231714725494e-05, + 7.439780165441334e-05, + 9.88170359050855e-05, + 9.028112981468439e-05, + 8.203276956919581e-05, + 8.111486386042088e-05, + 9.951327956514433e-05, + 9.248430433217436e-05, + 9.049861546372995e-05, + 8.23234295239672e-05, + 0.00011689230450429022, + 0.00010864553041756153, + 9.938247967511415e-05, + 9.12874675123021e-05, + 0.00010224651487078518, + 9.50239336816594e-05, + 8.516520028933883e-05, + 7.722135342191905e-05, + 7.637306407559663e-05, + 6.766286969650537e-05, + 6.669395952485502e-05, + 5.873254485777579e-05, + 5.024686106480658e-05, + 4.819800960831344e-05, + 4.7107583668548614e-05, + 4.775185516336933e-05 + ] + } + } + } + } + } + }, + "step": 500 + }, + { + "modes": { + "learned": { + "bits_per_byte": 2.0533380169036586, + "forward": { + "activation_sha256": { + "attention_branch_output": "9fd6aaea3d231a1caa717e4fc44c63b6ac0b7fcac0845e349ac60b57e28fd322", + "mlp_branch_output": "884b74f81575660d3a5ef373d83980c2b6a40a5e790421e3f7efc3bf3cdbb781", + "post_attention_state": "2d82179ff03320f5589762540fc93ada0edba22cabc07ab3982cf783bff4a116", + "post_mlp_state": "b054da1b5190b8b19869952768070eae883b70fbd71c9a2eaf9205894fe44b7f", + "pre_attention_input": "4d1314009c174d2647fbc47c0cd287abca38783c7de30eced47b74a084c87fbc", + "pre_mlp_input": "fbd2d69d0c904dc853dab2fd491c5cf23fa8369610dbace5a1307601df767adc" + }, + "logits_sha256": "6387ffea973321ec6422be9fe114fef42d7afbe7d0316fcff56991b4114eed58", + "loss_nats": 1.4232654571533203, + "mixer_summary_sha256": "483552b4d1a3270c8696995d30bd808a9918e953c1f6ea772d07f8deb85d4145" + }, + "loss_nats": 1.4232654571533203, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.39672940969467163, + "group": 1, + "latest_source_mass": 0.1456991732120514, + "layer": 1, + "max_source_mass": 0.854300856590271, + "mixer_index": 1, + "normalized_entropy": 0.5723595519413086, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.854300856590271, + "median": 0.8727816939353943, + "p05": 0.7123822569847107, + "p95": 0.9339146018028259 + }, + { + "label": "current_group_partial", + "mean": 0.1456991732120514, + "median": 0.12721827626228333, + "p05": 0.06608537584543228, + "p95": 0.2876177430152893 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6258258819580078, + "group": 1, + "latest_source_mass": 0.40511131286621094, + "layer": 2, + "max_source_mass": 0.5948886871337891, + "mixer_index": 2, + "normalized_entropy": 0.9028758963607797, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5948886871337891, + "median": 0.5920770168304443, + "p05": 0.33635830879211426, + "p95": 0.8293883800506592 + }, + { + "label": "current_group_partial", + "mean": 0.40511131286621094, + "median": 0.4079229235649109, + "p05": 0.17061161994934082, + "p95": 0.6636416912078857 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5053832530975342, + "group": 1, + "latest_source_mass": 0.221052348613739, + "layer": 2, + "max_source_mass": 0.778947651386261, + "mixer_index": 3, + "normalized_entropy": 0.7291139129921445, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.778947651386261, + "median": 0.8033280372619629, + "p05": 0.6048110723495483, + "p95": 0.8962967395782471 + }, + { + "label": "current_group_partial", + "mean": 0.221052348613739, + "median": 0.19667191803455353, + "p05": 0.10370326042175293, + "p95": 0.39518892765045166 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6737473011016846, + "group": 1, + "latest_source_mass": 0.45933952927589417, + "layer": 3, + "max_source_mass": 0.5406605005264282, + "mixer_index": 4, + "normalized_entropy": 0.9720118901117236, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5406605005264282, + "median": 0.5336101055145264, + "p05": 0.410135954618454, + "p95": 0.6916014552116394 + }, + { + "label": "current_group_partial", + "mean": 0.45933952927589417, + "median": 0.46638989448547363, + "p05": 0.3083984851837158, + "p95": 0.5898640155792236 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6529731750488281, + "group": 1, + "latest_source_mass": 0.43803298473358154, + "layer": 3, + "max_source_mass": 0.5619670152664185, + "mixer_index": 5, + "normalized_entropy": 0.9420411614764654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5619670152664185, + "median": 0.5604932308197021, + "p05": 0.3686542510986328, + "p95": 0.7799927592277527 + }, + { + "label": "current_group_partial", + "mean": 0.43803298473358154, + "median": 0.43950676918029785, + "p05": 0.22000724077224731, + "p95": 0.6313457489013672 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6891955733299255, + "group": 1, + "latest_source_mass": 0.49001744389533997, + "layer": 4, + "max_source_mass": 0.5099825859069824, + "mixer_index": 6, + "normalized_entropy": 0.9942990358457096, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5099825859069824, + "median": 0.5090322494506836, + "p05": 0.44019854068756104, + "p95": 0.5823723077774048 + }, + { + "label": "current_group_partial", + "mean": 0.49001744389533997, + "median": 0.4909677803516388, + "p05": 0.4176277220249176, + "p95": 0.559801459312439 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.48565131425857544, + "group": 1, + "latest_source_mass": 0.20070326328277588, + "layer": 4, + "max_source_mass": 0.7992967367172241, + "mixer_index": 7, + "normalized_entropy": 0.7006467426820543, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7992967367172241, + "median": 0.8062431216239929, + "p05": 0.6658452153205872, + "p95": 0.9043658375740051 + }, + { + "label": "current_group_partial", + "mean": 0.20070326328277588, + "median": 0.1937568634748459, + "p05": 0.09563417732715607, + "p95": 0.33415475487709045 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6875144243240356, + "group": 2, + "latest_source_mass": 0.4808617830276489, + "layer": 5, + "max_source_mass": 0.5191382169723511, + "mixer_index": 8, + "normalized_entropy": 0.9918736505119168, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5191382169723511, + "median": 0.5118080973625183, + "p05": 0.45191603899002075, + "p95": 0.6115704774856567 + }, + { + "label": "completed_group_1", + "mean": 0.4808617830276489, + "median": 0.4881919026374817, + "p05": 0.38842952251434326, + "p95": 0.5480839610099792 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.048142671585083, + "group": 2, + "latest_source_mass": 0.4353799819946289, + "layer": 5, + "max_source_mass": 0.4353799819946289, + "mixer_index": 9, + "normalized_entropy": 0.9540605747781931, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3059995770454407, + "median": 0.28220513463020325, + "p05": 0.191843643784523, + "p95": 0.5426149368286133 + }, + { + "label": "completed_group_1", + "mean": 0.25862041115760803, + "median": 0.2592811584472656, + "p05": 0.1660565435886383, + "p95": 0.3212765157222748 + }, + { + "label": "current_group_partial", + "mean": 0.4353799819946289, + "median": 0.44094622135162354, + "p05": 0.28141430020332336, + "p95": 0.5360540151596069 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.078475832939148, + "group": 2, + "latest_source_mass": 0.41806069016456604, + "layer": 6, + "max_source_mass": 0.41806069016456604, + "mixer_index": 10, + "normalized_entropy": 0.9816710081102642, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2919672429561615, + "median": 0.28502336144447327, + "p05": 0.241220623254776, + "p95": 0.36694014072418213 + }, + { + "label": "completed_group_1", + "mean": 0.28997206687927246, + "median": 0.2892274856567383, + "p05": 0.2705146372318268, + "p95": 0.3101809024810791 + }, + { + "label": "current_group_partial", + "mean": 0.41806069016456604, + "median": 0.42803525924682617, + "p05": 0.32804495096206665, + "p95": 0.465151309967041 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0040717124938965, + "group": 2, + "latest_source_mass": 0.5026139616966248, + "layer": 6, + "max_source_mass": 0.5026139616966248, + "mixer_index": 11, + "normalized_entropy": 0.9139454590583285, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27878478169441223, + "median": 0.24224042892456055, + "p05": 0.1698543280363083, + "p95": 0.5158803462982178 + }, + { + "label": "completed_group_1", + "mean": 0.21860124170780182, + "median": 0.21914872527122498, + "p05": 0.14686164259910583, + "p95": 0.26903262734413147 + }, + { + "label": "current_group_partial", + "mean": 0.5026139616966248, + "median": 0.531975269317627, + "p05": 0.31593379378318787, + "p95": 0.5875058770179749 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0727062225341797, + "group": 2, + "latest_source_mass": 0.42498481273651123, + "layer": 7, + "max_source_mass": 0.42498481273651123, + "mixer_index": 12, + "normalized_entropy": 0.9764192823973078, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3306647837162018, + "median": 0.3299825191497803, + "p05": 0.30597272515296936, + "p95": 0.35778582096099854 + }, + { + "label": "completed_group_1", + "mean": 0.2443503737449646, + "median": 0.24191564321517944, + "p05": 0.2250334620475769, + "p95": 0.27701011300086975 + }, + { + "label": "current_group_partial", + "mean": 0.42498481273651123, + "median": 0.4262530505657196, + "p05": 0.3939187526702881, + "p95": 0.44790196418762207 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.036304235458374, + "group": 2, + "latest_source_mass": 0.455292671918869, + "layer": 7, + "max_source_mass": 0.455292671918869, + "mixer_index": 13, + "normalized_entropy": 0.9432847658337463, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3365095853805542, + "median": 0.320411741733551, + "p05": 0.24313315749168396, + "p95": 0.4848058521747589 + }, + { + "label": "completed_group_1", + "mean": 0.20819777250289917, + "median": 0.20705200731754303, + "p05": 0.16223153471946716, + "p95": 0.2605344355106354 + }, + { + "label": "current_group_partial", + "mean": 0.455292671918869, + "median": 0.4683176279067993, + "p05": 0.3228694796562195, + "p95": 0.5329623222351074 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.071068525314331, + "group": 2, + "latest_source_mass": 0.42717796564102173, + "layer": 8, + "max_source_mass": 0.42717796564102173, + "mixer_index": 14, + "normalized_entropy": 0.9749285861464638, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32915356755256653, + "median": 0.3258785605430603, + "p05": 0.2938454747200012, + "p95": 0.37063664197921753 + }, + { + "label": "completed_group_1", + "mean": 0.24366845190525055, + "median": 0.24299904704093933, + "p05": 0.21890395879745483, + "p95": 0.2733282744884491 + }, + { + "label": "current_group_partial", + "mean": 0.42717796564102173, + "median": 0.42829641699790955, + "p05": 0.3896912634372711, + "p95": 0.45910394191741943 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0272725820541382, + "group": 2, + "latest_source_mass": 0.48878318071365356, + "layer": 8, + "max_source_mass": 0.48878318071365356, + "mixer_index": 15, + "normalized_entropy": 0.935063800623913, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29237720370292664, + "median": 0.2755309045314789, + "p05": 0.21469420194625854, + "p95": 0.42577075958251953 + }, + { + "label": "completed_group_1", + "mean": 0.2188395857810974, + "median": 0.21017195284366608, + "p05": 0.15934023261070251, + "p95": 0.29730042815208435 + }, + { + "label": "current_group_partial", + "mean": 0.48878318071365356, + "median": 0.4994751214981079, + "p05": 0.38404613733291626, + "p95": 0.5597336292266846 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0856674909591675, + "group": 3, + "latest_source_mass": 0.35679009556770325, + "layer": 9, + "max_source_mass": 0.3748946189880371, + "mixer_index": 16, + "normalized_entropy": 0.9882171373445715, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3748946189880371, + "median": 0.37628722190856934, + "p05": 0.32393988966941833, + "p95": 0.42887312173843384 + }, + { + "label": "completed_group_1", + "mean": 0.26831525564193726, + "median": 0.2661568522453308, + "p05": 0.2455824464559555, + "p95": 0.29672208428382874 + }, + { + "label": "completed_group_2", + "mean": 0.35679009556770325, + "median": 0.3568301498889923, + "p05": 0.3128414750099182, + "p95": 0.39937278628349304 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3326265811920166, + "group": 3, + "latest_source_mass": 0.3317619562149048, + "layer": 9, + "max_source_mass": 0.3317619562149048, + "mixer_index": 17, + "normalized_entropy": 0.9612868800212679, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.26391589641571045, + "median": 0.2413073182106018, + "p05": 0.17984914779663086, + "p95": 0.4526640474796295 + }, + { + "label": "completed_group_1", + "mean": 0.14978256821632385, + "median": 0.14740070700645447, + "p05": 0.10416575521230698, + "p95": 0.1980961263179779 + }, + { + "label": "completed_group_2", + "mean": 0.2545396089553833, + "median": 0.2590503394603729, + "p05": 0.16330178081989288, + "p95": 0.31726187467575073 + }, + { + "label": "current_group_partial", + "mean": 0.3317619562149048, + "median": 0.33179306983947754, + "p05": 0.2672363519668579, + "p95": 0.39366772770881653 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3771977424621582, + "group": 3, + "latest_source_mass": 0.2781716585159302, + "layer": 10, + "max_source_mass": 0.2781716585159302, + "mixer_index": 18, + "normalized_entropy": 0.9934381766868158, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27525436878204346, + "median": 0.27489086985588074, + "p05": 0.24440184235572815, + "p95": 0.3085845708847046 + }, + { + "label": "completed_group_1", + "mean": 0.20677447319030762, + "median": 0.20647433400154114, + "p05": 0.1837204098701477, + "p95": 0.23251454532146454 + }, + { + "label": "completed_group_2", + "mean": 0.23979949951171875, + "median": 0.24061384797096252, + "p05": 0.20753827691078186, + "p95": 0.26440829038619995 + }, + { + "label": "current_group_partial", + "mean": 0.2781716585159302, + "median": 0.27804386615753174, + "p05": 0.25431159138679504, + "p95": 0.3010777235031128 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3576545715332031, + "group": 3, + "latest_source_mass": 0.3106929659843445, + "layer": 10, + "max_source_mass": 0.3106929659843445, + "mixer_index": 19, + "normalized_entropy": 0.9793407587955913, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2848266065120697, + "median": 0.28166642785072327, + "p05": 0.22695904970169067, + "p95": 0.35531073808670044 + }, + { + "label": "completed_group_1", + "mean": 0.17341844737529755, + "median": 0.17138710618019104, + "p05": 0.14481288194656372, + "p95": 0.20774812996387482 + }, + { + "label": "completed_group_2", + "mean": 0.23106196522712708, + "median": 0.23154035210609436, + "p05": 0.18570835888385773, + "p95": 0.2726413607597351 + }, + { + "label": "current_group_partial", + "mean": 0.3106929659843445, + "median": 0.3093051314353943, + "p05": 0.2755521535873413, + "p95": 0.35260096192359924 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3772028684616089, + "group": 3, + "latest_source_mass": 0.2857057750225067, + "layer": 11, + "max_source_mass": 0.2857057750225067, + "mixer_index": 20, + "normalized_entropy": 0.9934418743138093, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2702856659889221, + "median": 0.2702763080596924, + "p05": 0.24336200952529907, + "p95": 0.2981995642185211 + }, + { + "label": "completed_group_1", + "mean": 0.21318203210830688, + "median": 0.21185705065727234, + "p05": 0.18305814266204834, + "p95": 0.24470563232898712 + }, + { + "label": "completed_group_2", + "mean": 0.23082655668258667, + "median": 0.23429268598556519, + "p05": 0.19758956134319305, + "p95": 0.24867244064807892 + }, + { + "label": "current_group_partial", + "mean": 0.2857057750225067, + "median": 0.2861127257347107, + "p05": 0.25794482231140137, + "p95": 0.31186288595199585 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3277652263641357, + "group": 3, + "latest_source_mass": 0.3302627205848694, + "layer": 11, + "max_source_mass": 0.3302627205848694, + "mixer_index": 21, + "normalized_entropy": 0.9577801537701753, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31198394298553467, + "median": 0.3185040354728699, + "p05": 0.1839737594127655, + "p95": 0.41595304012298584 + }, + { + "label": "completed_group_1", + "mean": 0.16646918654441833, + "median": 0.16503849625587463, + "p05": 0.13032540678977966, + "p95": 0.20871388912200928 + }, + { + "label": "completed_group_2", + "mean": 0.1912841945886612, + "median": 0.1874908059835434, + "p05": 0.13776838779449463, + "p95": 0.26046234369277954 + }, + { + "label": "current_group_partial", + "mean": 0.3302627205848694, + "median": 0.32645535469055176, + "p05": 0.2624559700489044, + "p95": 0.407662570476532 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3750126361846924, + "group": 3, + "latest_source_mass": 0.2897627055644989, + "layer": 12, + "max_source_mass": 0.2897627055644989, + "mixer_index": 22, + "normalized_entropy": 0.9918619556916581, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2621780037879944, + "median": 0.2640007734298706, + "p05": 0.212703138589859, + "p95": 0.3065672814846039 + }, + { + "label": "completed_group_1", + "mean": 0.21051618456840515, + "median": 0.20835110545158386, + "p05": 0.18514882028102875, + "p95": 0.24316135048866272 + }, + { + "label": "completed_group_2", + "mean": 0.23754313588142395, + "median": 0.23885108530521393, + "p05": 0.2025136798620224, + "p95": 0.26575183868408203 + }, + { + "label": "current_group_partial", + "mean": 0.2897627055644989, + "median": 0.2845831513404846, + "p05": 0.25373271107673645, + "p95": 0.3410085439682007 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3568847179412842, + "group": 3, + "latest_source_mass": 0.3233448266983032, + "layer": 12, + "max_source_mass": 0.3233448266983032, + "mixer_index": 23, + "normalized_entropy": 0.9787854268159553, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2628105878829956, + "median": 0.26141154766082764, + "p05": 0.18301242589950562, + "p95": 0.35439273715019226 + }, + { + "label": "completed_group_1", + "mean": 0.1954234391450882, + "median": 0.1943855881690979, + "p05": 0.16116495430469513, + "p95": 0.2287053018808365 + }, + { + "label": "completed_group_2", + "mean": 0.21842113137245178, + "median": 0.21915890276432037, + "p05": 0.18287011981010437, + "p95": 0.2523847222328186 + }, + { + "label": "current_group_partial", + "mean": 0.3233448266983032, + "median": 0.3182007968425751, + "p05": 0.26236361265182495, + "p95": 0.4023315906524658 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3733241558074951, + "group": 4, + "latest_source_mass": 0.2984467148780823, + "layer": 13, + "max_source_mass": 0.2984467148780823, + "mixer_index": 24, + "normalized_entropy": 0.9906439745582477, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25842779874801636, + "median": 0.2585429549217224, + "p05": 0.2317616194486618, + "p95": 0.28543001413345337 + }, + { + "label": "completed_group_1", + "mean": 0.22964224219322205, + "median": 0.22467529773712158, + "p05": 0.20248907804489136, + "p95": 0.27881020307540894 + }, + { + "label": "completed_group_2", + "mean": 0.21348324418067932, + "median": 0.2143072485923767, + "p05": 0.18921855092048645, + "p95": 0.23720777034759521 + }, + { + "label": "completed_group_3", + "mean": 0.2984467148780823, + "median": 0.30472591519355774, + "p05": 0.22567860782146454, + "p95": 0.35220274329185486 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5855276584625244, + "group": 4, + "latest_source_mass": 0.2245527058839798, + "layer": 13, + "max_source_mass": 0.2499181628227234, + "mixer_index": 25, + "normalized_entropy": 0.9851437239132672, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18060272932052612, + "median": 0.17444205284118652, + "p05": 0.12549439072608948, + "p95": 0.27444836497306824 + }, + { + "label": "completed_group_1", + "mean": 0.1563272476196289, + "median": 0.15402349829673767, + "p05": 0.1285230815410614, + "p95": 0.19161051511764526 + }, + { + "label": "completed_group_2", + "mean": 0.1885991394519806, + "median": 0.19113421440124512, + "p05": 0.1411750763654709, + "p95": 0.22396130859851837 + }, + { + "label": "completed_group_3", + "mean": 0.2499181628227234, + "median": 0.25040146708488464, + "p05": 0.20360784232616425, + "p95": 0.2959936261177063 + }, + { + "label": "current_group_partial", + "mean": 0.2245527058839798, + "median": 0.22453144192695618, + "p05": 0.1881018728017807, + "p95": 0.2609967589378357 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5906484127044678, + "group": 4, + "latest_source_mass": 0.2668720781803131, + "layer": 14, + "max_source_mass": 0.2668720781803131, + "mixer_index": 26, + "normalized_entropy": 0.9883254274150809, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19773373007774353, + "median": 0.1965319812297821, + "p05": 0.18224859237670898, + "p95": 0.2179831713438034 + }, + { + "label": "completed_group_1", + "mean": 0.18790484964847565, + "median": 0.18465439975261688, + "p05": 0.17162445187568665, + "p95": 0.21807827055454254 + }, + { + "label": "completed_group_2", + "mean": 0.15935488045215607, + "median": 0.15496674180030823, + "p05": 0.1365157663822174, + "p95": 0.1937219798564911 + }, + { + "label": "completed_group_3", + "mean": 0.18813443183898926, + "median": 0.18904224038124084, + "p05": 0.15670713782310486, + "p95": 0.21698647737503052 + }, + { + "label": "current_group_partial", + "mean": 0.2668720781803131, + "median": 0.2718026638031006, + "p05": 0.22497205436229706, + "p95": 0.2880238890647888 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5916568040847778, + "group": 4, + "latest_source_mass": 0.22534610331058502, + "layer": 14, + "max_source_mass": 0.227544903755188, + "mixer_index": 27, + "normalized_entropy": 0.9889519762073764, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2148856818675995, + "median": 0.21614974737167358, + "p05": 0.17341811954975128, + "p95": 0.2519548237323761 + }, + { + "label": "completed_group_1", + "mean": 0.14808836579322815, + "median": 0.1478005349636078, + "p05": 0.1207289919257164, + "p95": 0.17668096721172333 + }, + { + "label": "completed_group_2", + "mean": 0.18413493037223816, + "median": 0.18671013414859772, + "p05": 0.1423187255859375, + "p95": 0.21475675702095032 + }, + { + "label": "completed_group_3", + "mean": 0.227544903755188, + "median": 0.22678817808628082, + "p05": 0.19582602381706238, + "p95": 0.25986844301223755 + }, + { + "label": "current_group_partial", + "mean": 0.22534610331058502, + "median": 0.22619590163230896, + "p05": 0.18310406804084778, + "p95": 0.2634280323982239 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5474352836608887, + "group": 4, + "latest_source_mass": 0.31912651658058167, + "layer": 15, + "max_source_mass": 0.31912651658058167, + "mixer_index": 28, + "normalized_entropy": 0.9614756007086727, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19825294613838196, + "median": 0.19680598378181458, + "p05": 0.17132307589054108, + "p95": 0.2297736555337906 + }, + { + "label": "completed_group_1", + "mean": 0.17294847965240479, + "median": 0.164862722158432, + "p05": 0.14083626866340637, + "p95": 0.2345556914806366 + }, + { + "label": "completed_group_2", + "mean": 0.11353601515293121, + "median": 0.10715661942958832, + "p05": 0.0958808958530426, + "p95": 0.15557381510734558 + }, + { + "label": "completed_group_3", + "mean": 0.19613605737686157, + "median": 0.19778506457805634, + "p05": 0.15166249871253967, + "p95": 0.23660960793495178 + }, + { + "label": "current_group_partial", + "mean": 0.31912651658058167, + "median": 0.327135294675827, + "p05": 0.24559810757637024, + "p95": 0.3559069037437439 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5919160842895508, + "group": 4, + "latest_source_mass": 0.2247331142425537, + "layer": 15, + "max_source_mass": 0.2257765829563141, + "mixer_index": 29, + "normalized_entropy": 0.9891130760564416, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21136541664600372, + "median": 0.2003786563873291, + "p05": 0.1631811559200287, + "p95": 0.28519296646118164 + }, + { + "label": "completed_group_1", + "mean": 0.1572093367576599, + "median": 0.15547874569892883, + "p05": 0.12932342290878296, + "p95": 0.1883375495672226 + }, + { + "label": "completed_group_2", + "mean": 0.18091553449630737, + "median": 0.18437424302101135, + "p05": 0.14374977350234985, + "p95": 0.20782355964183807 + }, + { + "label": "completed_group_3", + "mean": 0.2257765829563141, + "median": 0.22509169578552246, + "p05": 0.18708458542823792, + "p95": 0.2700234353542328 + }, + { + "label": "current_group_partial", + "mean": 0.2247331142425537, + "median": 0.22534477710723877, + "p05": 0.18621239066123962, + "p95": 0.261106938123703 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.542572021484375, + "group": 4, + "latest_source_mass": 0.32280486822128296, + "layer": 16, + "max_source_mass": 0.32280486822128296, + "mixer_index": 30, + "normalized_entropy": 0.9584538860224823, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2053474634885788, + "median": 0.20845530927181244, + "p05": 0.1648181527853012, + "p95": 0.24006053805351257 + }, + { + "label": "completed_group_1", + "mean": 0.19789552688598633, + "median": 0.19339492917060852, + "p05": 0.17084503173828125, + "p95": 0.24170872569084167 + }, + { + "label": "completed_group_2", + "mean": 0.10896679013967514, + "median": 0.10333000868558884, + "p05": 0.09237737953662872, + "p95": 0.14624080061912537 + }, + { + "label": "completed_group_3", + "mean": 0.16498532891273499, + "median": 0.16568085551261902, + "p05": 0.12339095771312714, + "p95": 0.20368334650993347 + }, + { + "label": "current_group_partial", + "mean": 0.32280486822128296, + "median": 0.3227658271789551, + "p05": 0.27662473917007446, + "p95": 0.37092989683151245 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5774884223937988, + "group": 4, + "latest_source_mass": 0.24561989307403564, + "layer": 16, + "max_source_mass": 0.24561989307403564, + "mixer_index": 31, + "normalized_entropy": 0.9801486656965963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1874420940876007, + "median": 0.1747647374868393, + "p05": 0.12934981286525726, + "p95": 0.3250933289527893 + }, + { + "label": "completed_group_1", + "mean": 0.14777487516403198, + "median": 0.1479286551475525, + "p05": 0.11702761054039001, + "p95": 0.18031248450279236 + }, + { + "label": "completed_group_2", + "mean": 0.18392466008663177, + "median": 0.1873953938484192, + "p05": 0.13232499361038208, + "p95": 0.21972279250621796 + }, + { + "label": "completed_group_3", + "mean": 0.23523849248886108, + "median": 0.23474374413490295, + "p05": 0.18369318544864655, + "p95": 0.28937381505966187 + }, + { + "label": "current_group_partial", + "mean": 0.24561989307403564, + "median": 0.24738794565200806, + "p05": 0.18173503875732422, + "p95": 0.3010544776916504 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.585869312286377, + "group": 5, + "latest_source_mass": 0.26391667127609253, + "layer": 17, + "max_source_mass": 0.26391667127609253, + "mixer_index": 32, + "normalized_entropy": 0.9853560053695527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2123720794916153, + "median": 0.21364213526248932, + "p05": 0.18289057910442352, + "p95": 0.24095472693443298 + }, + { + "label": "completed_group_1", + "mean": 0.19153766334056854, + "median": 0.19043566286563873, + "p05": 0.16703367233276367, + "p95": 0.22240948677062988 + }, + { + "label": "completed_group_2", + "mean": 0.14032581448554993, + "median": 0.1387615203857422, + "p05": 0.1285465657711029, + "p95": 0.15656951069831848 + }, + { + "label": "completed_group_3", + "mean": 0.1918477714061737, + "median": 0.19085454940795898, + "p05": 0.15591931343078613, + "p95": 0.22882042825222015 + }, + { + "label": "completed_group_4", + "mean": 0.26391667127609253, + "median": 0.26347947120666504, + "p05": 0.23446886241436005, + "p95": 0.29670727252960205 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.747511625289917, + "group": 5, + "latest_source_mass": 0.2640940546989441, + "layer": 17, + "max_source_mass": 0.2640940546989441, + "mixer_index": 33, + "normalized_entropy": 0.9753048080961441, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13209694623947144, + "median": 0.12275595963001251, + "p05": 0.09756084531545639, + "p95": 0.2136262208223343 + }, + { + "label": "completed_group_1", + "mean": 0.11971477419137955, + "median": 0.11888641864061356, + "p05": 0.10256937146186829, + "p95": 0.1397632211446762 + }, + { + "label": "completed_group_2", + "mean": 0.14703533053398132, + "median": 0.1483881175518036, + "p05": 0.12076307833194733, + "p95": 0.1677870750427246 + }, + { + "label": "completed_group_3", + "mean": 0.17972922325134277, + "median": 0.17958298325538635, + "p05": 0.1569097489118576, + "p95": 0.20113393664360046 + }, + { + "label": "completed_group_4", + "mean": 0.15732964873313904, + "median": 0.15857598185539246, + "p05": 0.12811288237571716, + "p95": 0.18277260661125183 + }, + { + "label": "current_group_partial", + "mean": 0.2640940546989441, + "median": 0.2643871307373047, + "p05": 0.23343844711780548, + "p95": 0.29374754428863525 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6791677474975586, + "group": 5, + "latest_source_mass": 0.3440830111503601, + "layer": 18, + "max_source_mass": 0.3440830111503601, + "mixer_index": 34, + "normalized_entropy": 0.937161363640509, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1429731547832489, + "median": 0.1472356617450714, + "p05": 0.10260166227817535, + "p95": 0.18026825785636902 + }, + { + "label": "completed_group_1", + "mean": 0.128341943025589, + "median": 0.1243518516421318, + "p05": 0.10659048706293106, + "p95": 0.16665902733802795 + }, + { + "label": "completed_group_2", + "mean": 0.08682706952095032, + "median": 0.08546946942806244, + "p05": 0.07587063312530518, + "p95": 0.10337291657924652 + }, + { + "label": "completed_group_3", + "mean": 0.12974600493907928, + "median": 0.12982705235481262, + "p05": 0.10628668963909149, + "p95": 0.15255016088485718 + }, + { + "label": "completed_group_4", + "mean": 0.16802877187728882, + "median": 0.16956090927124023, + "p05": 0.13419006764888763, + "p95": 0.19487163424491882 + }, + { + "label": "current_group_partial", + "mean": 0.3440830111503601, + "median": 0.3435801863670349, + "p05": 0.3040219247341156, + "p95": 0.3866944909095764 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7079570293426514, + "group": 5, + "latest_source_mass": 0.3138047456741333, + "layer": 18, + "max_source_mass": 0.3138047456741333, + "mixer_index": 35, + "normalized_entropy": 0.9532289677690342, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12207645177841187, + "median": 0.11115410923957825, + "p05": 0.08012475818395615, + "p95": 0.22810664772987366 + }, + { + "label": "completed_group_1", + "mean": 0.11454500257968903, + "median": 0.11326663196086884, + "p05": 0.09493672847747803, + "p95": 0.13798359036445618 + }, + { + "label": "completed_group_2", + "mean": 0.14774823188781738, + "median": 0.14947353303432465, + "p05": 0.11176952719688416, + "p95": 0.17675937712192535 + }, + { + "label": "completed_group_3", + "mean": 0.1675407588481903, + "median": 0.16761797666549683, + "p05": 0.13766297698020935, + "p95": 0.19680744409561157 + }, + { + "label": "completed_group_4", + "mean": 0.1342848241329193, + "median": 0.13416096568107605, + "p05": 0.10613290220499039, + "p95": 0.1641635000705719 + }, + { + "label": "current_group_partial", + "mean": 0.3138047456741333, + "median": 0.31486788392066956, + "p05": 0.2550574839115143, + "p95": 0.36806803941726685 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.689455270767212, + "group": 5, + "latest_source_mass": 0.33726364374160767, + "layer": 19, + "max_source_mass": 0.33726364374160767, + "mixer_index": 36, + "normalized_entropy": 0.9429029396981957, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14791089296340942, + "median": 0.15329936146736145, + "p05": 0.09563843905925751, + "p95": 0.19142045080661774 + }, + { + "label": "completed_group_1", + "mean": 0.15263086557388306, + "median": 0.15167269110679626, + "p05": 0.13290324807167053, + "p95": 0.17540162801742554 + }, + { + "label": "completed_group_2", + "mean": 0.09421356767416, + "median": 0.0932738184928894, + "p05": 0.0817713513970375, + "p95": 0.10977791249752045 + }, + { + "label": "completed_group_3", + "mean": 0.12123478949069977, + "median": 0.12004932761192322, + "p05": 0.10300692170858383, + "p95": 0.1423986405134201 + }, + { + "label": "completed_group_4", + "mean": 0.14674627780914307, + "median": 0.14631736278533936, + "p05": 0.12582728266716003, + "p95": 0.1688128113746643 + }, + { + "label": "current_group_partial", + "mean": 0.33726364374160767, + "median": 0.33654576539993286, + "p05": 0.28587767481803894, + "p95": 0.3911621868610382 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6673545837402344, + "group": 5, + "latest_source_mass": 0.3504054546356201, + "layer": 19, + "max_source_mass": 0.3504054546356201, + "mixer_index": 37, + "normalized_entropy": 0.9305683114143563, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.119766965508461, + "median": 0.10288102924823761, + "p05": 0.06773775815963745, + "p95": 0.26150354743003845 + }, + { + "label": "completed_group_1", + "mean": 0.10584045946598053, + "median": 0.10460227727890015, + "p05": 0.08871804922819138, + "p95": 0.12676651775836945 + }, + { + "label": "completed_group_2", + "mean": 0.1391725242137909, + "median": 0.13989759981632233, + "p05": 0.10571101307868958, + "p95": 0.16792798042297363 + }, + { + "label": "completed_group_3", + "mean": 0.15867863595485687, + "median": 0.15944495797157288, + "p05": 0.11882050335407257, + "p95": 0.19538073241710663 + }, + { + "label": "completed_group_4", + "mean": 0.1261359602212906, + "median": 0.1259205937385559, + "p05": 0.09519180655479431, + "p95": 0.15761253237724304 + }, + { + "label": "current_group_partial", + "mean": 0.3504054546356201, + "median": 0.350231796503067, + "p05": 0.2736193835735321, + "p95": 0.4254133105278015 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7276721000671387, + "group": 5, + "latest_source_mass": 0.2906533479690552, + "layer": 20, + "max_source_mass": 0.2906533479690552, + "mixer_index": 38, + "normalized_entropy": 0.9642321582435799, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16197484731674194, + "median": 0.16698119044303894, + "p05": 0.10772223770618439, + "p95": 0.20956003665924072 + }, + { + "label": "completed_group_1", + "mean": 0.16130127012729645, + "median": 0.15952633321285248, + "p05": 0.1394604593515396, + "p95": 0.18821176886558533 + }, + { + "label": "completed_group_2", + "mean": 0.1033584251999855, + "median": 0.10190790891647339, + "p05": 0.08996254205703735, + "p95": 0.12321754544973373 + }, + { + "label": "completed_group_3", + "mean": 0.12991489470005035, + "median": 0.1302758753299713, + "p05": 0.10520754754543304, + "p95": 0.15511953830718994 + }, + { + "label": "completed_group_4", + "mean": 0.15279719233512878, + "median": 0.15249209105968475, + "p05": 0.1319577395915985, + "p95": 0.17580842971801758 + }, + { + "label": "current_group_partial", + "mean": 0.2906533479690552, + "median": 0.28811272978782654, + "p05": 0.24316680431365967, + "p95": 0.34837183356285095 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.655259609222412, + "group": 5, + "latest_source_mass": 0.34562793374061584, + "layer": 20, + "max_source_mass": 0.34562793374061584, + "mixer_index": 39, + "normalized_entropy": 0.9238179776080931, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12291314452886581, + "median": 0.10247613489627838, + "p05": 0.04299236834049225, + "p95": 0.3265020251274109 + }, + { + "label": "completed_group_1", + "mean": 0.10535868257284164, + "median": 0.10351927578449249, + "p05": 0.08489269018173218, + "p95": 0.13092145323753357 + }, + { + "label": "completed_group_2", + "mean": 0.14431552588939667, + "median": 0.1464514434337616, + "p05": 0.1031925305724144, + "p95": 0.17885252833366394 + }, + { + "label": "completed_group_3", + "mean": 0.1644165813922882, + "median": 0.16633014380931854, + "p05": 0.11522950232028961, + "p95": 0.20475128293037415 + }, + { + "label": "completed_group_4", + "mean": 0.11736814677715302, + "median": 0.11602836102247238, + "p05": 0.08080777525901794, + "p95": 0.15889662504196167 + }, + { + "label": "current_group_partial", + "mean": 0.34562793374061584, + "median": 0.3455731272697449, + "p05": 0.2439023107290268, + "p95": 0.4484919309616089 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7456457614898682, + "group": 6, + "latest_source_mass": 0.2435421347618103, + "layer": 21, + "max_source_mass": 0.2435421347618103, + "mixer_index": 40, + "normalized_entropy": 0.9742634496816395, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17144063115119934, + "median": 0.17784935235977173, + "p05": 0.11719495803117752, + "p95": 0.2109185755252838 + }, + { + "label": "completed_group_1", + "mean": 0.17272350192070007, + "median": 0.17066533863544464, + "p05": 0.15510167181491852, + "p95": 0.19745397567749023 + }, + { + "label": "completed_group_2", + "mean": 0.09372986108064651, + "median": 0.09023232758045197, + "p05": 0.0791064128279686, + "p95": 0.11955329030752182 + }, + { + "label": "completed_group_3", + "mean": 0.13482090830802917, + "median": 0.1349901258945465, + "p05": 0.10589948296546936, + "p95": 0.16432839632034302 + }, + { + "label": "completed_group_4", + "mean": 0.183742955327034, + "median": 0.18362680077552795, + "p05": 0.16126227378845215, + "p95": 0.2063293308019638 + }, + { + "label": "completed_group_5", + "mean": 0.2435421347618103, + "median": 0.2420862764120102, + "p05": 0.20024174451828003, + "p95": 0.29210829734802246 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.8196223974227905, + "group": 6, + "latest_source_mass": 0.27700191736221313, + "layer": 21, + "max_source_mass": 0.27700191736221313, + "mixer_index": 41, + "normalized_entropy": 0.9351009337744438, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0976112112402916, + "median": 0.083127960562706, + "p05": 0.057163339108228683, + "p95": 0.21396708488464355 + }, + { + "label": "completed_group_1", + "mean": 0.08662202209234238, + "median": 0.0854487419128418, + "p05": 0.06999586522579193, + "p95": 0.10710393637418747 + }, + { + "label": "completed_group_2", + "mean": 0.10460365563631058, + "median": 0.10444506257772446, + "p05": 0.08147601783275604, + "p95": 0.12855757772922516 + }, + { + "label": "completed_group_3", + "mean": 0.12231925129890442, + "median": 0.12159790843725204, + "p05": 0.09556058049201965, + "p95": 0.15222840011119843 + }, + { + "label": "completed_group_4", + "mean": 0.09176328033208847, + "median": 0.09087449312210083, + "p05": 0.06908228248357773, + "p95": 0.11806865781545639 + }, + { + "label": "completed_group_5", + "mean": 0.22007866203784943, + "median": 0.21696066856384277, + "p05": 0.16129010915756226, + "p95": 0.29291966557502747 + }, + { + "label": "current_group_partial", + "mean": 0.27700191736221313, + "median": 0.2762490510940552, + "p05": 0.20922920107841492, + "p95": 0.3463406562805176 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8201665878295898, + "group": 6, + "latest_source_mass": 0.31871896982192993, + "layer": 22, + "max_source_mass": 0.31871896982192993, + "mixer_index": 42, + "normalized_entropy": 0.9353805923224315, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12032122910022736, + "median": 0.12179268896579742, + "p05": 0.07555080205202103, + "p95": 0.17048288881778717 + }, + { + "label": "completed_group_1", + "mean": 0.12977322936058044, + "median": 0.12879429757595062, + "p05": 0.10893770307302475, + "p95": 0.15374676883220673 + }, + { + "label": "completed_group_2", + "mean": 0.07435770332813263, + "median": 0.07368695735931396, + "p05": 0.06492579728364944, + "p95": 0.08556335419416428 + }, + { + "label": "completed_group_3", + "mean": 0.09230788052082062, + "median": 0.09124474227428436, + "p05": 0.07656185328960419, + "p95": 0.11202958226203918 + }, + { + "label": "completed_group_4", + "mean": 0.1133352667093277, + "median": 0.1122380942106247, + "p05": 0.09279464185237885, + "p95": 0.138209730386734 + }, + { + "label": "completed_group_5", + "mean": 0.15118570625782013, + "median": 0.14947570860385895, + "p05": 0.11790992319583893, + "p95": 0.190475732088089 + }, + { + "label": "current_group_partial", + "mean": 0.31871896982192993, + "median": 0.3194882273674011, + "p05": 0.25439688563346863, + "p95": 0.3806491792201996 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7483060359954834, + "group": 6, + "latest_source_mass": 0.35795146226882935, + "layer": 22, + "max_source_mass": 0.35795146226882935, + "mixer_index": 43, + "normalized_entropy": 0.8984515738531086, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09602338075637817, + "median": 0.08368823677301407, + "p05": 0.048330605030059814, + "p95": 0.22025157511234283 + }, + { + "label": "completed_group_1", + "mean": 0.07951871305704117, + "median": 0.0780613124370575, + "p05": 0.06370338052511215, + "p95": 0.09924023598432541 + }, + { + "label": "completed_group_2", + "mean": 0.09892410784959793, + "median": 0.09849533438682556, + "p05": 0.07656967639923096, + "p95": 0.12212418019771576 + }, + { + "label": "completed_group_3", + "mean": 0.11023539304733276, + "median": 0.10842189192771912, + "p05": 0.08411939442157745, + "p95": 0.14120185375213623 + }, + { + "label": "completed_group_4", + "mean": 0.08284324407577515, + "median": 0.08092047274112701, + "p05": 0.05862290784716606, + "p95": 0.11340560019016266 + }, + { + "label": "completed_group_5", + "mean": 0.17450372874736786, + "median": 0.17053596675395966, + "p05": 0.10771246254444122, + "p95": 0.2589290738105774 + }, + { + "label": "current_group_partial", + "mean": 0.35795146226882935, + "median": 0.35653698444366455, + "p05": 0.2500389516353607, + "p95": 0.47316011786460876 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.851926565170288, + "group": 6, + "latest_source_mass": 0.29032042622566223, + "layer": 23, + "max_source_mass": 0.29032042622566223, + "mixer_index": 44, + "normalized_entropy": 0.9517019920315172, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14006078243255615, + "median": 0.14251285791397095, + "p05": 0.09944885969161987, + "p95": 0.17503000795841217 + }, + { + "label": "completed_group_1", + "mean": 0.14811867475509644, + "median": 0.1468043476343155, + "p05": 0.13084301352500916, + "p95": 0.16874222457408905 + }, + { + "label": "completed_group_2", + "mean": 0.08932579308748245, + "median": 0.08844955265522003, + "p05": 0.07788823544979095, + "p95": 0.10412273555994034 + }, + { + "label": "completed_group_3", + "mean": 0.09198777377605438, + "median": 0.09086033701896667, + "p05": 0.07522772252559662, + "p95": 0.11275116354227066 + }, + { + "label": "completed_group_4", + "mean": 0.11175549030303955, + "median": 0.11076019704341888, + "p05": 0.09502518177032471, + "p95": 0.1322603076696396 + }, + { + "label": "completed_group_5", + "mean": 0.12843109667301178, + "median": 0.12729130685329437, + "p05": 0.09835763275623322, + "p95": 0.16321906447410583 + }, + { + "label": "current_group_partial", + "mean": 0.29032042622566223, + "median": 0.29004043340682983, + "p05": 0.2191990315914154, + "p95": 0.3636181354522705 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7523608207702637, + "group": 6, + "latest_source_mass": 0.3702468276023865, + "layer": 23, + "max_source_mass": 0.3702468276023865, + "mixer_index": 45, + "normalized_entropy": 0.9005353210275343, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10785886645317078, + "median": 0.10064198076725006, + "p05": 0.06490226089954376, + "p95": 0.1922605037689209 + }, + { + "label": "completed_group_1", + "mean": 0.08607445657253265, + "median": 0.08407071977853775, + "p05": 0.06460381299257278, + "p95": 0.11544128507375717 + }, + { + "label": "completed_group_2", + "mean": 0.09595336019992828, + "median": 0.09583206474781036, + "p05": 0.07447511702775955, + "p95": 0.11774584650993347 + }, + { + "label": "completed_group_3", + "mean": 0.10649852454662323, + "median": 0.10573073476552963, + "p05": 0.08023984730243683, + "p95": 0.1353880763053894 + }, + { + "label": "completed_group_4", + "mean": 0.0838143527507782, + "median": 0.08303111791610718, + "p05": 0.0620865523815155, + "p95": 0.10783524066209793 + }, + { + "label": "completed_group_5", + "mean": 0.14955362677574158, + "median": 0.14380723237991333, + "p05": 0.09627015888690948, + "p95": 0.222757488489151 + }, + { + "label": "current_group_partial", + "mean": 0.3702468276023865, + "median": 0.36859041452407837, + "p05": 0.2604178488254547, + "p95": 0.483437716960907 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8693451881408691, + "group": 6, + "latest_source_mass": 0.2815432548522949, + "layer": 24, + "max_source_mass": 0.2815432548522949, + "mixer_index": 46, + "normalized_entropy": 0.9606533935024625, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1378144919872284, + "median": 0.13807497918605804, + "p05": 0.10932769626379013, + "p95": 0.16912776231765747 + }, + { + "label": "completed_group_1", + "mean": 0.1244172602891922, + "median": 0.12220869958400726, + "p05": 0.10719747841358185, + "p95": 0.14835487306118011 + }, + { + "label": "completed_group_2", + "mean": 0.09968579560518265, + "median": 0.09896545112133026, + "p05": 0.08706579357385635, + "p95": 0.11497638374567032 + }, + { + "label": "completed_group_3", + "mean": 0.10974415391683578, + "median": 0.10904944688081741, + "p05": 0.09279274940490723, + "p95": 0.1297232210636139 + }, + { + "label": "completed_group_4", + "mean": 0.11655184626579285, + "median": 0.11616086959838867, + "p05": 0.09858430176973343, + "p95": 0.13470277190208435 + }, + { + "label": "completed_group_5", + "mean": 0.130243182182312, + "median": 0.1280980110168457, + "p05": 0.09857633709907532, + "p95": 0.16894926130771637 + }, + { + "label": "current_group_partial", + "mean": 0.2815432548522949, + "median": 0.2809160351753235, + "p05": 0.21616990864276886, + "p95": 0.35214728116989136 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7898914813995361, + "group": 6, + "latest_source_mass": 0.33616429567337036, + "layer": 24, + "max_source_mass": 0.33616429567337036, + "mixer_index": 47, + "normalized_entropy": 0.9198222653129591, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10115307569503784, + "median": 0.08613650500774384, + "p05": 0.050611600279808044, + "p95": 0.23426659405231476 + }, + { + "label": "completed_group_1", + "mean": 0.09615936875343323, + "median": 0.09461446106433868, + "p05": 0.07746638357639313, + "p95": 0.11935343593358994 + }, + { + "label": "completed_group_2", + "mean": 0.11218561232089996, + "median": 0.1115623190999031, + "p05": 0.08799588680267334, + "p95": 0.13799631595611572 + }, + { + "label": "completed_group_3", + "mean": 0.1163020133972168, + "median": 0.11555752158164978, + "p05": 0.08836641162633896, + "p95": 0.14650842547416687 + }, + { + "label": "completed_group_4", + "mean": 0.09238721430301666, + "median": 0.0899561196565628, + "p05": 0.06484603136777878, + "p95": 0.1281205415725708 + }, + { + "label": "completed_group_5", + "mean": 0.14564841985702515, + "median": 0.1386992335319519, + "p05": 0.08829846978187561, + "p95": 0.22654561698436737 + }, + { + "label": "current_group_partial", + "mean": 0.33616429567337036, + "median": 0.33723515272140503, + "p05": 0.22659176588058472, + "p95": 0.4460947513580322 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9094455242156982, + "group": 7, + "latest_source_mass": 0.22462961077690125, + "layer": 25, + "max_source_mass": 0.22462961077690125, + "mixer_index": 48, + "normalized_entropy": 0.981260889739787, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14588291943073273, + "median": 0.14661915600299835, + "p05": 0.11508320271968842, + "p95": 0.1800285279750824 + }, + { + "label": "completed_group_1", + "mean": 0.14179158210754395, + "median": 0.14040151238441467, + "p05": 0.12424254417419434, + "p95": 0.16426950693130493 + }, + { + "label": "completed_group_2", + "mean": 0.1023377925157547, + "median": 0.10112032294273376, + "p05": 0.08753453195095062, + "p95": 0.12110165506601334 + }, + { + "label": "completed_group_3", + "mean": 0.11511939018964767, + "median": 0.11405108869075775, + "p05": 0.10003029555082321, + "p95": 0.13439135253429413 + }, + { + "label": "completed_group_4", + "mean": 0.13270339369773865, + "median": 0.13297244906425476, + "p05": 0.11011375486850739, + "p95": 0.15357504785060883 + }, + { + "label": "completed_group_5", + "mean": 0.13753530383110046, + "median": 0.13631805777549744, + "p05": 0.11099010705947876, + "p95": 0.16822901368141174 + }, + { + "label": "completed_group_6", + "mean": 0.22462961077690125, + "median": 0.22316956520080566, + "p05": 0.17812120914459229, + "p95": 0.27728700637817383 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9082129001617432, + "group": 7, + "latest_source_mass": 0.2262895107269287, + "layer": 25, + "max_source_mass": 0.2519546449184418, + "mixer_index": 49, + "normalized_entropy": 0.9176564293412313, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09145340323448181, + "median": 0.09255078434944153, + "p05": 0.03969334065914154, + "p95": 0.15699249505996704 + }, + { + "label": "completed_group_1", + "mean": 0.07698439061641693, + "median": 0.07560532540082932, + "p05": 0.05995766818523407, + "p95": 0.09742964059114456 + }, + { + "label": "completed_group_2", + "mean": 0.08525429666042328, + "median": 0.08423174172639847, + "p05": 0.06901301443576813, + "p95": 0.10490919649600983 + }, + { + "label": "completed_group_3", + "mean": 0.08438773453235626, + "median": 0.08191201090812683, + "p05": 0.06093179062008858, + "p95": 0.11479861289262772 + }, + { + "label": "completed_group_4", + "mean": 0.06885559856891632, + "median": 0.06811907887458801, + "p05": 0.04831434786319733, + "p95": 0.09210654348134995 + }, + { + "label": "completed_group_5", + "mean": 0.11482042074203491, + "median": 0.10688380897045135, + "p05": 0.061463840305805206, + "p95": 0.19296644628047943 + }, + { + "label": "completed_group_6", + "mean": 0.2519546449184418, + "median": 0.250648558139801, + "p05": 0.1514165848493576, + "p95": 0.35778117179870605 + }, + { + "label": "current_group_partial", + "mean": 0.2262895107269287, + "median": 0.22269269824028015, + "p05": 0.0977732241153717, + "p95": 0.37383198738098145 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.025789499282837, + "group": 7, + "latest_source_mass": 0.20079906284809113, + "layer": 26, + "max_source_mass": 0.20079906284809113, + "mixer_index": 50, + "normalized_entropy": 0.9741988215000951, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11047068238258362, + "median": 0.11173020303249359, + "p05": 0.0543295294046402, + "p95": 0.16616837680339813 + }, + { + "label": "completed_group_1", + "mean": 0.12069687247276306, + "median": 0.11997807025909424, + "p05": 0.10084380954504013, + "p95": 0.14284764230251312 + }, + { + "label": "completed_group_2", + "mean": 0.09073689579963684, + "median": 0.08891503512859344, + "p05": 0.07266610860824585, + "p95": 0.11606847494840622 + }, + { + "label": "completed_group_3", + "mean": 0.09690164029598236, + "median": 0.09599806368350983, + "p05": 0.07936231791973114, + "p95": 0.11817526817321777 + }, + { + "label": "completed_group_4", + "mean": 0.10987946391105652, + "median": 0.10931764543056488, + "p05": 0.08768688142299652, + "p95": 0.13469792902469635 + }, + { + "label": "completed_group_5", + "mean": 0.11353722959756851, + "median": 0.1125042587518692, + "p05": 0.08439873158931732, + "p95": 0.14467501640319824 + }, + { + "label": "completed_group_6", + "mean": 0.15697816014289856, + "median": 0.15547990798950195, + "p05": 0.11792093515396118, + "p95": 0.19920390844345093 + }, + { + "label": "current_group_partial", + "mean": 0.20079906284809113, + "median": 0.19964557886123657, + "p05": 0.11376827955245972, + "p95": 0.2935184836387634 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.9302079677581787, + "group": 7, + "latest_source_mass": 0.24795080721378326, + "layer": 26, + "max_source_mass": 0.24795080721378326, + "mixer_index": 51, + "normalized_entropy": 0.9282338209896963, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0971018448472023, + "median": 0.08827555179595947, + "p05": 0.06328719854354858, + "p95": 0.159974604845047 + }, + { + "label": "completed_group_1", + "mean": 0.08240298926830292, + "median": 0.08067642897367477, + "p05": 0.06337254494428635, + "p95": 0.10580241680145264 + }, + { + "label": "completed_group_2", + "mean": 0.0830029845237732, + "median": 0.08191139996051788, + "p05": 0.06423131376504898, + "p95": 0.10520751774311066 + }, + { + "label": "completed_group_3", + "mean": 0.08926018327474594, + "median": 0.08775374293327332, + "p05": 0.06515306234359741, + "p95": 0.11704635620117188 + }, + { + "label": "completed_group_4", + "mean": 0.07480385899543762, + "median": 0.07413309812545776, + "p05": 0.055974941700696945, + "p95": 0.09636931121349335 + }, + { + "label": "completed_group_5", + "mean": 0.10335960239171982, + "median": 0.09859646111726761, + "p05": 0.0651523768901825, + "p95": 0.1582750678062439 + }, + { + "label": "completed_group_6", + "mean": 0.22211773693561554, + "median": 0.2181653380393982, + "p05": 0.12620511651039124, + "p95": 0.3312388062477112 + }, + { + "label": "current_group_partial", + "mean": 0.24795080721378326, + "median": 0.2450115978717804, + "p05": 0.12909922003746033, + "p95": 0.37674421072006226 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.010532855987549, + "group": 7, + "latest_source_mass": 0.20431259274482727, + "layer": 27, + "max_source_mass": 0.20431259274482727, + "mixer_index": 52, + "normalized_entropy": 0.9668619269591872, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10700424760580063, + "median": 0.10718298703432083, + "p05": 0.05378019064664841, + "p95": 0.16453370451927185 + }, + { + "label": "completed_group_1", + "mean": 0.09611140936613083, + "median": 0.09571994841098785, + "p05": 0.0705467239022255, + "p95": 0.1255740523338318 + }, + { + "label": "completed_group_2", + "mean": 0.09769980609416962, + "median": 0.09758654981851578, + "p05": 0.08190305531024933, + "p95": 0.11550790816545486 + }, + { + "label": "completed_group_3", + "mean": 0.10596007108688354, + "median": 0.10523851960897446, + "p05": 0.08640100061893463, + "p95": 0.12765458226203918 + }, + { + "label": "completed_group_4", + "mean": 0.09634587913751602, + "median": 0.09565047174692154, + "p05": 0.07201918959617615, + "p95": 0.12302076816558838 + }, + { + "label": "completed_group_5", + "mean": 0.11705268919467926, + "median": 0.11422611027956009, + "p05": 0.08651825040578842, + "p95": 0.15585871040821075 + }, + { + "label": "completed_group_6", + "mean": 0.17551329731941223, + "median": 0.17338381707668304, + "p05": 0.12190233916044235, + "p95": 0.23716403543949127 + }, + { + "label": "current_group_partial", + "mean": 0.20431259274482727, + "median": 0.1999019980430603, + "p05": 0.11230528354644775, + "p95": 0.3120884299278259 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.9408204555511475, + "group": 7, + "latest_source_mass": 0.19202858209609985, + "layer": 27, + "max_source_mass": 0.22555868327617645, + "mixer_index": 53, + "normalized_entropy": 0.9333373488264999, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08590779453516006, + "median": 0.062378786504268646, + "p05": 0.03650445491075516, + "p95": 0.22098970413208008 + }, + { + "label": "completed_group_1", + "mean": 0.09530991315841675, + "median": 0.09346900880336761, + "p05": 0.06249644607305527, + "p95": 0.13247521221637726 + }, + { + "label": "completed_group_2", + "mean": 0.09476494789123535, + "median": 0.09293568134307861, + "p05": 0.06641139835119247, + "p95": 0.13004404306411743 + }, + { + "label": "completed_group_3", + "mean": 0.1030701994895935, + "median": 0.099480040371418, + "p05": 0.06904599815607071, + "p95": 0.14615577459335327 + }, + { + "label": "completed_group_4", + "mean": 0.08048167824745178, + "median": 0.07773597538471222, + "p05": 0.052633628249168396, + "p95": 0.12012942135334015 + }, + { + "label": "completed_group_5", + "mean": 0.12287819385528564, + "median": 0.11439820379018784, + "p05": 0.07120100408792496, + "p95": 0.2025740146636963 + }, + { + "label": "completed_group_6", + "mean": 0.22555868327617645, + "median": 0.22237449884414673, + "p05": 0.12643620371818542, + "p95": 0.3325231373310089 + }, + { + "label": "current_group_partial", + "mean": 0.19202858209609985, + "median": 0.17407171428203583, + "p05": 0.06696207821369171, + "p95": 0.3700798749923706 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0024678707122803, + "group": 7, + "latest_source_mass": 0.19086343050003052, + "layer": 28, + "max_source_mass": 0.19086343050003052, + "mixer_index": 54, + "normalized_entropy": 0.9629834888720297, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10720443725585938, + "median": 0.0965266302227974, + "p05": 0.044251296669244766, + "p95": 0.18923941254615784 + }, + { + "label": "completed_group_1", + "mean": 0.09982497245073318, + "median": 0.09860500693321228, + "p05": 0.05300002545118332, + "p95": 0.15633785724639893 + }, + { + "label": "completed_group_2", + "mean": 0.10896851122379303, + "median": 0.1086052879691124, + "p05": 0.07999510318040848, + "p95": 0.13937662541866302 + }, + { + "label": "completed_group_3", + "mean": 0.1345594972372055, + "median": 0.1320592314004898, + "p05": 0.10167843103408813, + "p95": 0.17577221989631653 + }, + { + "label": "completed_group_4", + "mean": 0.09260416775941849, + "median": 0.09119044244289398, + "p05": 0.06200484186410904, + "p95": 0.1281491219997406 + }, + { + "label": "completed_group_5", + "mean": 0.1016254797577858, + "median": 0.09927940368652344, + "p05": 0.06724471598863602, + "p95": 0.14429867267608643 + }, + { + "label": "completed_group_6", + "mean": 0.1643495112657547, + "median": 0.16063156723976135, + "p05": 0.11097142100334167, + "p95": 0.2319793701171875 + }, + { + "label": "current_group_partial", + "mean": 0.19086343050003052, + "median": 0.17437744140625, + "p05": 0.08801081776618958, + "p95": 0.33255329728126526 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.9389225244522095, + "group": 7, + "latest_source_mass": 0.1535017192363739, + "layer": 28, + "max_source_mass": 0.2661469578742981, + "mixer_index": 55, + "normalized_entropy": 0.932424636898371, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08632777631282806, + "median": 0.06584042310714722, + "p05": 0.04890177398920059, + "p95": 0.24412202835083008 + }, + { + "label": "completed_group_1", + "mean": 0.09041796624660492, + "median": 0.08850662410259247, + "p05": 0.06795615702867508, + "p95": 0.1180017739534378 + }, + { + "label": "completed_group_2", + "mean": 0.09941554069519043, + "median": 0.0980331301689148, + "p05": 0.06698913872241974, + "p95": 0.13627989590168 + }, + { + "label": "completed_group_3", + "mean": 0.10805265605449677, + "median": 0.10531660169363022, + "p05": 0.07596777379512787, + "p95": 0.14889071881771088 + }, + { + "label": "completed_group_4", + "mean": 0.07865813374519348, + "median": 0.07749675214290619, + "p05": 0.05567489564418793, + "p95": 0.10581911355257034 + }, + { + "label": "completed_group_5", + "mean": 0.11747924983501434, + "median": 0.11117055267095566, + "p05": 0.0703330934047699, + "p95": 0.18614599108695984 + }, + { + "label": "completed_group_6", + "mean": 0.2661469578742981, + "median": 0.2658160924911499, + "p05": 0.15578627586364746, + "p95": 0.3771526515483856 + }, + { + "label": "current_group_partial", + "mean": 0.1535017192363739, + "median": 0.13699908554553986, + "p05": 0.0626269057393074, + "p95": 0.29617446660995483 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.008256673812866, + "group": 8, + "latest_source_mass": 0.11735987663269043, + "layer": 29, + "max_source_mass": 0.20522482693195343, + "mixer_index": 56, + "normalized_entropy": 0.9657673147139957, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1016545444726944, + "median": 0.07978972047567368, + "p05": 0.04056411236524582, + "p95": 0.2331526130437851 + }, + { + "label": "completed_group_1", + "mean": 0.10481879115104675, + "median": 0.1020151898264885, + "p05": 0.07684041559696198, + "p95": 0.1442326009273529 + }, + { + "label": "completed_group_2", + "mean": 0.10823093354701996, + "median": 0.10697583854198456, + "p05": 0.08568178862333298, + "p95": 0.13496363162994385 + }, + { + "label": "completed_group_3", + "mean": 0.1270347535610199, + "median": 0.12582050263881683, + "p05": 0.1004788875579834, + "p95": 0.15752868354320526 + }, + { + "label": "completed_group_4", + "mean": 0.0919865146279335, + "median": 0.09061764925718307, + "p05": 0.06585606187582016, + "p95": 0.1224639043211937 + }, + { + "label": "completed_group_5", + "mean": 0.14368976652622223, + "median": 0.14165253937244415, + "p05": 0.09602011740207672, + "p95": 0.20366834104061127 + }, + { + "label": "completed_group_6", + "mean": 0.20522482693195343, + "median": 0.2038867175579071, + "p05": 0.14235222339630127, + "p95": 0.27111250162124634 + }, + { + "label": "completed_group_7", + "mean": 0.11735987663269043, + "median": 0.10386677086353302, + "p05": 0.057839710265398026, + "p95": 0.21596315503120422 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.094825267791748, + "group": 8, + "latest_source_mass": 0.16477900743484497, + "layer": 29, + "max_source_mass": 0.19253811240196228, + "mixer_index": 57, + "normalized_entropy": 0.9533960658365591, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08519881963729858, + "median": 0.07928745448589325, + "p05": 0.0396789014339447, + "p95": 0.16862331330776215 + }, + { + "label": "completed_group_1", + "mean": 0.09903685003519058, + "median": 0.09788446873426437, + "p05": 0.0575668141245842, + "p95": 0.14432772994041443 + }, + { + "label": "completed_group_2", + "mean": 0.08900875598192215, + "median": 0.08856695145368576, + "p05": 0.0673285573720932, + "p95": 0.11194870620965958 + }, + { + "label": "completed_group_3", + "mean": 0.09697237610816956, + "median": 0.09451373666524887, + "p05": 0.0689435601234436, + "p95": 0.13275021314620972 + }, + { + "label": "completed_group_4", + "mean": 0.071251779794693, + "median": 0.0703696608543396, + "p05": 0.049036141484975815, + "p95": 0.09625756740570068 + }, + { + "label": "completed_group_5", + "mean": 0.09201780706644058, + "median": 0.08780405670404434, + "p05": 0.04908040910959244, + "p95": 0.14967313408851624 + }, + { + "label": "completed_group_6", + "mean": 0.19253811240196228, + "median": 0.18946462869644165, + "p05": 0.12099648267030716, + "p95": 0.27126166224479675 + }, + { + "label": "completed_group_7", + "mean": 0.1091964989900589, + "median": 0.09459474682807922, + "p05": 0.04855653643608093, + "p95": 0.2098814696073532 + }, + { + "label": "current_group_partial", + "mean": 0.16477900743484497, + "median": 0.16038388013839722, + "p05": 0.08161723613739014, + "p95": 0.2644755244255066 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.140371322631836, + "group": 8, + "latest_source_mass": 0.10623672604560852, + "layer": 30, + "max_source_mass": 0.14476820826530457, + "mixer_index": 58, + "normalized_entropy": 0.9741249687033317, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10001853853464127, + "median": 0.09240126609802246, + "p05": 0.025385506451129913, + "p95": 0.22696438431739807 + }, + { + "label": "completed_group_1", + "mean": 0.10924500226974487, + "median": 0.10823708027601242, + "p05": 0.08299213647842407, + "p95": 0.13906806707382202 + }, + { + "label": "completed_group_2", + "mean": 0.10232415795326233, + "median": 0.09996328502893448, + "p05": 0.07388727366924286, + "p95": 0.1385582983493805 + }, + { + "label": "completed_group_3", + "mean": 0.11669868975877762, + "median": 0.11613668501377106, + "p05": 0.08647200465202332, + "p95": 0.1492128223180771 + }, + { + "label": "completed_group_4", + "mean": 0.10123330354690552, + "median": 0.10036572813987732, + "p05": 0.07442998141050339, + "p95": 0.13040892779827118 + }, + { + "label": "completed_group_5", + "mean": 0.1287832111120224, + "median": 0.1245323196053505, + "p05": 0.08930385112762451, + "p95": 0.18039579689502716 + }, + { + "label": "completed_group_6", + "mean": 0.14476820826530457, + "median": 0.14273962378501892, + "p05": 0.09932437539100647, + "p95": 0.19615304470062256 + }, + { + "label": "completed_group_7", + "mean": 0.09069214761257172, + "median": 0.08483204245567322, + "p05": 0.04427404701709747, + "p95": 0.15117761492729187 + }, + { + "label": "current_group_partial", + "mean": 0.10623672604560852, + "median": 0.08798803389072418, + "p05": 0.0472906231880188, + "p95": 0.21512800455093384 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.117004632949829, + "group": 8, + "latest_source_mass": 0.13358455896377563, + "layer": 30, + "max_source_mass": 0.17978665232658386, + "mixer_index": 59, + "normalized_entropy": 0.963490329930842, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09184888750314713, + "median": 0.08941377699375153, + "p05": 0.05124857276678085, + "p95": 0.14178073406219482 + }, + { + "label": "completed_group_1", + "mean": 0.09620077908039093, + "median": 0.0951167494058609, + "p05": 0.06255991011857986, + "p95": 0.1391918957233429 + }, + { + "label": "completed_group_2", + "mean": 0.09620209783315659, + "median": 0.09759913384914398, + "p05": 0.06909230351448059, + "p95": 0.11898315697908401 + }, + { + "label": "completed_group_3", + "mean": 0.10222947597503662, + "median": 0.10126610100269318, + "p05": 0.07587265968322754, + "p95": 0.1305830329656601 + }, + { + "label": "completed_group_4", + "mean": 0.06924222409725189, + "median": 0.06880369037389755, + "p05": 0.05338519439101219, + "p95": 0.08691392093896866 + }, + { + "label": "completed_group_5", + "mean": 0.0821654349565506, + "median": 0.07825584709644318, + "p05": 0.053967058658599854, + "p95": 0.12216144800186157 + }, + { + "label": "completed_group_6", + "mean": 0.17978665232658386, + "median": 0.17625530064105988, + "p05": 0.11167110502719879, + "p95": 0.2575375437736511 + }, + { + "label": "completed_group_7", + "mean": 0.14873988926410675, + "median": 0.1422305703163147, + "p05": 0.08190920948982239, + "p95": 0.23667879402637482 + }, + { + "label": "current_group_partial", + "mean": 0.13358455896377563, + "median": 0.12330813705921173, + "p05": 0.06055498123168945, + "p95": 0.23775091767311096 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1423683166503906, + "group": 8, + "latest_source_mass": 0.11159604042768478, + "layer": 31, + "max_source_mass": 0.1649385243654251, + "mixer_index": 60, + "normalized_entropy": 0.9750338398488454, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09596475213766098, + "median": 0.0976155698299408, + "p05": 0.04209773987531662, + "p95": 0.15498679876327515 + }, + { + "label": "completed_group_1", + "mean": 0.07681103050708771, + "median": 0.07558202743530273, + "p05": 0.04380175098776817, + "p95": 0.11200148612260818 + }, + { + "label": "completed_group_2", + "mean": 0.11225531995296478, + "median": 0.11153945326805115, + "p05": 0.08037804812192917, + "p95": 0.1473693549633026 + }, + { + "label": "completed_group_3", + "mean": 0.11592596024274826, + "median": 0.115887850522995, + "p05": 0.08687994629144669, + "p95": 0.14547690749168396 + }, + { + "label": "completed_group_4", + "mean": 0.0810609757900238, + "median": 0.08056386560201645, + "p05": 0.06259842216968536, + "p95": 0.10083132982254028 + }, + { + "label": "completed_group_5", + "mean": 0.1177809089422226, + "median": 0.11515317857265472, + "p05": 0.08443494141101837, + "p95": 0.15806551277637482 + }, + { + "label": "completed_group_6", + "mean": 0.1649385243654251, + "median": 0.16307872533798218, + "p05": 0.12056200951337814, + "p95": 0.21692173182964325 + }, + { + "label": "completed_group_7", + "mean": 0.12366649508476257, + "median": 0.12186841666698456, + "p05": 0.07334212958812714, + "p95": 0.18257394433021545 + }, + { + "label": "current_group_partial", + "mean": 0.11159604042768478, + "median": 0.10134477913379669, + "p05": 0.05939295515418053, + "p95": 0.20170332491397858 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.093705415725708, + "group": 8, + "latest_source_mass": 0.09642469882965088, + "layer": 31, + "max_source_mass": 0.21980023384094238, + "mixer_index": 61, + "normalized_entropy": 0.9528863991972947, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08838656544685364, + "median": 0.07248079776763916, + "p05": 0.05256194248795509, + "p95": 0.199167400598526 + }, + { + "label": "completed_group_1", + "mean": 0.0955723449587822, + "median": 0.09313549846410751, + "p05": 0.07088664174079895, + "p95": 0.12816612422466278 + }, + { + "label": "completed_group_2", + "mean": 0.09992480278015137, + "median": 0.10067024827003479, + "p05": 0.06702706217765808, + "p95": 0.13047263026237488 + }, + { + "label": "completed_group_3", + "mean": 0.10721968114376068, + "median": 0.10629363358020782, + "p05": 0.07515021413564682, + "p95": 0.14141342043876648 + }, + { + "label": "completed_group_4", + "mean": 0.07486072182655334, + "median": 0.07471001148223877, + "p05": 0.055637944489717484, + "p95": 0.09505966305732727 + }, + { + "label": "completed_group_5", + "mean": 0.09290309250354767, + "median": 0.09035329520702362, + "p05": 0.057027287781238556, + "p95": 0.1383095532655716 + }, + { + "label": "completed_group_6", + "mean": 0.21980023384094238, + "median": 0.21675530076026917, + "p05": 0.13081300258636475, + "p95": 0.3184159994125366 + }, + { + "label": "completed_group_7", + "mean": 0.12490786612033844, + "median": 0.11867299675941467, + "p05": 0.06415548175573349, + "p95": 0.21230830252170563 + }, + { + "label": "current_group_partial", + "mean": 0.09642469882965088, + "median": 0.08079533278942108, + "p05": 0.036796458065509796, + "p95": 0.20610423386096954 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1627297401428223, + "group": 8, + "latest_source_mass": 0.09715649485588074, + "layer": 32, + "max_source_mass": 0.13960200548171997, + "mixer_index": 62, + "normalized_entropy": 0.9843007230352316, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0994885265827179, + "median": 0.10293936729431152, + "p05": 0.037643272429704666, + "p95": 0.1762332022190094 + }, + { + "label": "completed_group_1", + "mean": 0.09216944873332977, + "median": 0.09255991876125336, + "p05": 0.07000453770160675, + "p95": 0.11256910115480423 + }, + { + "label": "completed_group_2", + "mean": 0.11588040739297867, + "median": 0.11537714302539825, + "p05": 0.09686640650033951, + "p95": 0.1361614316701889 + }, + { + "label": "completed_group_3", + "mean": 0.12231601774692535, + "median": 0.12174015492200851, + "p05": 0.10175298154354095, + "p95": 0.14450547099113464 + }, + { + "label": "completed_group_4", + "mean": 0.09873614460229874, + "median": 0.09799051284790039, + "p05": 0.07405688613653183, + "p95": 0.12650127708911896 + }, + { + "label": "completed_group_5", + "mean": 0.13436099886894226, + "median": 0.13138467073440552, + "p05": 0.1039634495973587, + "p95": 0.17342084646224976 + }, + { + "label": "completed_group_6", + "mean": 0.13960200548171997, + "median": 0.1386084109544754, + "p05": 0.10816053301095963, + "p95": 0.17382028698921204 + }, + { + "label": "completed_group_7", + "mean": 0.1002899557352066, + "median": 0.10035285353660583, + "p05": 0.06299241632223129, + "p95": 0.14122843742370605 + }, + { + "label": "current_group_partial", + "mean": 0.09715649485588074, + "median": 0.08847378194332123, + "p05": 0.05721396207809448, + "p95": 0.1772533655166626 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1300058364868164, + "group": 8, + "latest_source_mass": 0.07372864335775375, + "layer": 32, + "max_source_mass": 0.1931532621383667, + "mixer_index": 63, + "normalized_entropy": 0.9694074326572047, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0899420753121376, + "median": 0.07823143899440765, + "p05": 0.051866769790649414, + "p95": 0.19706517457962036 + }, + { + "label": "completed_group_1", + "mean": 0.1065598577260971, + "median": 0.10517844557762146, + "p05": 0.08068235963582993, + "p95": 0.13615867495536804 + }, + { + "label": "completed_group_2", + "mean": 0.11013364791870117, + "median": 0.11150452494621277, + "p05": 0.07762482017278671, + "p95": 0.1385573297739029 + }, + { + "label": "completed_group_3", + "mean": 0.12377548217773438, + "median": 0.12257742881774902, + "p05": 0.0912313312292099, + "p95": 0.15917643904685974 + }, + { + "label": "completed_group_4", + "mean": 0.08726659417152405, + "median": 0.08603408932685852, + "p05": 0.06540867686271667, + "p95": 0.11203364282846451 + }, + { + "label": "completed_group_5", + "mean": 0.11411209404468536, + "median": 0.11123606562614441, + "p05": 0.07641662657260895, + "p95": 0.16032427549362183 + }, + { + "label": "completed_group_6", + "mean": 0.1931532621383667, + "median": 0.19350019097328186, + "p05": 0.12057775259017944, + "p95": 0.26759904623031616 + }, + { + "label": "completed_group_7", + "mean": 0.10132836550474167, + "median": 0.09569840133190155, + "p05": 0.06355049461126328, + "p95": 0.15627986192703247 + }, + { + "label": "current_group_partial", + "mean": 0.07372864335775375, + "median": 0.0687510147690773, + "p05": 0.04834525287151337, + "p95": 0.1152075007557869 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.9258099794387817, + "group": null, + "latest_source_mass": 0.1844884157180786, + "layer": null, + "max_source_mass": 0.2706507444381714, + "mixer_index": null, + "normalized_entropy": 0.8764738931573011, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11242504417896271, + "median": 0.11513695120811462, + "p05": 0.04474085196852684, + "p95": 0.16795574128627777 + }, + { + "label": "completed_group_1", + "mean": 0.0684729665517807, + "median": 0.06829436123371124, + "p05": 0.046843402087688446, + "p95": 0.09099384397268295 + }, + { + "label": "completed_group_2", + "mean": 0.07014584541320801, + "median": 0.06858030706644058, + "p05": 0.05111496150493622, + "p95": 0.09375781565904617 + }, + { + "label": "completed_group_3", + "mean": 0.059657223522663116, + "median": 0.057276882231235504, + "p05": 0.037933796644210815, + "p95": 0.08897658437490463 + }, + { + "label": "completed_group_4", + "mean": 0.05741159990429878, + "median": 0.05489874631166458, + "p05": 0.03844546526670456, + "p95": 0.08331313729286194 + }, + { + "label": "completed_group_5", + "mean": 0.05127572640776634, + "median": 0.04766879230737686, + "p05": 0.026877565309405327, + "p95": 0.08776431530714035 + }, + { + "label": "completed_group_6", + "mean": 0.12547241151332855, + "median": 0.10524329543113708, + "p05": 0.04063908010721207, + "p95": 0.28077831864356995 + }, + { + "label": "completed_group_7", + "mean": 0.2706507444381714, + "median": 0.2597617506980896, + "p05": 0.09951092302799225, + "p95": 0.4855192005634308 + }, + { + "label": "completed_group_8", + "mean": 0.1844884157180786, + "median": 0.16994649171829224, + "p05": 0.03097841702401638, + "p95": 0.39343374967575073 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "e4c617a86986c3d60218f9eb527fd075b82f9473140115b430b5480c040907aa", + "optimizer_state_before": "e4c617a86986c3d60218f9eb527fd075b82f9473140115b430b5480c040907aa", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "9fd6aaea3d231a1caa717e4fc44c63b6ac0b7fcac0845e349ac60b57e28fd322", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.05045261234045029, + 0.09517527371644974, + 0.08140470087528229, + 0.035610511898994446, + 0.1042611226439476, + 0.050615254789590836, + 0.04264022409915924, + 0.03225839138031006, + 0.040291935205459595, + 0.036216676235198975, + 0.02871432714164257, + 0.03194425627589226, + 0.05663999170064926, + 0.03894488140940666, + 0.03595346212387085, + 0.03414257988333702, + 0.03680795058608055, + 0.022694969549775124, + 0.022243596613407135, + 0.02950950525701046, + 0.035728659480810165, + 0.024367405101656914, + 0.0301151555031538, + 0.022455131635069847, + 0.025910304859280586, + 0.025968369096517563, + 0.01597527042031288, + 0.015805471688508987, + 0.013562481850385666, + 0.02184087596833706, + 0.012465676292777061, + 0.01876053772866726 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.710775745777937e-05, + "first_to_last_ratio": 1.9385534866869027, + "imbalance_abs_log_ratio": 0.6619420695392257, + "last_quartile_mean": 8.82501183241402e-06, + "mean": 1.510921526914899e-05, + "normalized": [ + 1.0736286012354828, + 1.090890635760861, + 1.0608178444904952, + 1.0075413201072734, + 1.2489868422885204, + 1.2985786082191968, + 1.1828591103209587, + 1.094881544464814, + 1.179146782695164, + 1.0539743088863864, + 0.9822323375070559, + 0.9179899035878774, + 0.7825253397308062, + 0.7478217700804238, + 0.7367490149494519, + 0.7112256790174882, + 1.3539887939888489, + 1.3049151840479707, + 1.2111415907817773, + 1.1081141456773322, + 1.8022267118607482, + 1.6529820157870687, + 1.4576498485204101, + 1.2664807468037573, + 0.9749572058937866, + 0.8591646316280283, + 0.7163279624937049, + 0.6428460632699132, + 0.42935673617045356, + 0.3797395981731299, + 0.34123235543755254, + 0.32902676612326176 + ], + "peak_layer": 21, + "peak_normalized": 1.8022267118607482, + "population_cv": 0.35198707913936206, + "rest_mean": 1.3903669713289236e-05, + "spike_contrast": 1.5549248303940866, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1619161270791664e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.6221685655182227e-05, + 1.648250145080965e-05, + 1.602812517376151e-05, + 1.5223158698063344e-05, + 1.8871211068471894e-05, + 1.962050373549573e-05, + 1.787207293091342e-05, + 1.6542800949537195e-05, + 1.781598257366568e-05, + 1.5924724721116945e-05, + 1.4840759831713513e-05, + 1.3870107068214566e-05, + 1.1823343811556697e-05, + 1.1299000107101165e-05, + 1.1131699466204736e-05, + 1.0746061889221892e-05, + 2.045770816039294e-05, + 1.9716244423761964e-05, + 1.829939901654143e-05, + 1.6742735169827938e-05, + 2.7230231353314593e-05, + 2.4975261112558655e-05, + 2.2023945348337293e-05, + 1.9135530237690546e-05, + 1.4730838302057236e-05, + 1.2981303370906971e-05, + 1.082315338862827e-05, + 9.712899554870091e-06, + 6.487243354058592e-06, + 5.737567335017957e-06, + 5.155753115104744e-06, + 4.9713362386683e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.80973648741201e-05, + "first_to_last_ratio": 1.9447119155409294, + "imbalance_abs_log_ratio": 0.6651138506971189, + "last_quartile_mean": 3.501668516037171e-05, + "mean": 6.006126523061539e-05, + "normalized": [ + 1.0680384089723292, + 1.089396516867671, + 1.0622658589778824, + 1.0104708179682353, + 1.2588472863894493, + 1.3055677385553706, + 1.1820759209316258, + 1.0937244453707788, + 1.1769346432718848, + 1.0548790883233023, + 0.9819146790647293, + 0.9183563002786295, + 0.7871743717726897, + 0.7495710863521058, + 0.7365673123693164, + 0.7105643678097476, + 1.3531016610201878, + 1.3024588646491044, + 1.2137050756314243, + 1.110290286467306, + 1.806416737608712, + 1.6488489338485182, + 1.4524952538921891, + 1.2622054710738435, + 0.9668831047344751, + 0.8526160887571205, + 0.7131274958192858, + 0.6402858642446936, + 0.4318749856664095, + 0.3827318699453911, + 0.3444766877197535, + 0.332132775645838 + ], + "peak_layer": 21, + "peak_normalized": 1.806416737608712, + "population_cv": 0.35152664329216815, + "rest_mean": 5.530786209554343e-05, + "spike_contrast": 1.5500444043905992, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.572964216000401e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 6 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.414773815777153e-05, + 6.543053314089775e-05, + 6.380103150149807e-05, + 6.069015580578707e-05, + 7.560796075267717e-05, + 7.841405022190884e-05, + 7.099697540979832e-05, + 6.569047400262207e-05, + 7.068818376865238e-05, + 6.335737271001562e-05, + 5.89750379731413e-05, + 5.5157641327241436e-05, + 4.727868872578256e-05, + 4.502018782659434e-05, + 4.4239164708415046e-05, + 4.26773949584458e-05, + 8.126899774651974e-05, + 7.822732732165605e-05, + 7.289666245924309e-05, + 6.668543937848881e-05, + 0.00010849567479453981, + 9.903195314109325e-05, + 8.723870269022882e-05, + 7.580965757369995e-05, + 5.807222260045819e-05, + 5.120920104673132e-05, + 4.2831339669646695e-05, + 3.845637911581434e-05, + 2.5938958060578443e-05, + 2.298736035299953e-05, + 2.0689705706899986e-05, + 1.994831472984515e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06038942374289036, + "first_to_last_ratio": 1.9447119017990018, + "imbalance_abs_log_ratio": 0.6651138436308138, + "last_quartile_mean": 0.03105314658023417, + "mean": 0.05326293042162433, + "normalized": [ + 1.0680384098761833, + 1.0893965216947878, + 1.062265786546042, + 1.0104708738830177, + 1.2588473385979138, + 1.3055677142391287, + 1.1820758895935386, + 1.09372443299358, + 1.1769346283619155, + 1.0548791227653083, + 0.9819147305623657, + 0.9183562837560109, + 0.7871743890171391, + 0.7495711002960661, + 0.73656729318353, + 0.7105643650402791, + 1.3531016496377335, + 1.3024589535705866, + 1.2137049835246356, + 1.1102903617054871, + 1.8064166988298391, + 1.6488489461272462, + 1.4524952635466242, + 1.26220537084274, + 0.9668831089685697, + 0.8526160811834912, + 0.7131275119134806, + 0.6402858670776477, + 0.43187496591238844, + 0.38273188596957125, + 0.34447667234318236, + 0.3321327984399701 + ], + "peak_layer": 21, + "peak_normalized": 1.8064166988298391, + "population_cv": 0.35152663902064574, + "rest_mean": 0.049047565225649764, + "spike_contrast": 1.5500443728474567, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.07602590247988701, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 6 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05688685551285744, + 0.05802445113658905, + 0.05657938867807388, + 0.053820639848709106, + 0.06704989820718765, + 0.06953836232423782, + 0.06296082586050034, + 0.058254968374967575, + 0.062686987221241, + 0.05618595331907272, + 0.052299655973911285, + 0.0489143468439579, + 0.041927214711904526, + 0.03992435336112976, + 0.03923173248767853, + 0.03784674033522606, + 0.0720701590180397, + 0.06937278062105179, + 0.06464548408985138, + 0.05913731828331947, + 0.09621504694223404, + 0.08782252669334412, + 0.07736415416002274, + 0.0672287568449974, + 0.051499027758836746, + 0.045412831008434296, + 0.03798326104879379, + 0.034103501588106155, + 0.023002926260232925, + 0.020385421812534332, + 0.018347837030887604, + 0.017690366134047508 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.7076921753632632e-06, + "first_to_last_ratio": 1.1762812234778344, + "imbalance_abs_log_ratio": 0.16235795649847604, + "last_quartile_mean": 2.301908864410507e-06, + "mean": 3.0470990566300316e-06, + "normalized": [ + 1.1138437702053292, + 0.9595105662781346, + 0.9024337193377198, + 0.9101147747351473, + 0.8469008290036252, + 0.8140327082666439, + 0.7830671665056109, + 0.7790013618751644, + 0.8724389801211856, + 0.848368375017665, + 0.8340257934438313, + 0.8332842972958924, + 0.6804006560304527, + 0.6823355453447383, + 0.6859143070691168, + 0.7155834804515504, + 1.343985609139468, + 1.324647908954466, + 1.3004430629764219, + 1.269459612482823, + 2.009601561760871, + 1.969745677435912, + 1.8244349675950344, + 1.6528833381228312, + 1.244173825255001, + 1.0943835444416905, + 0.9361801141676105, + 0.8608013633619623, + 0.5518818827305038, + 0.48726512359119484, + 0.4398138473062473, + 0.429042229696154 + ], + "peak_layer": 21, + "peak_normalized": 2.009601561760871, + "population_cv": 0.4073949027659483, + "rest_mean": 2.629438902793616e-06, + "spike_contrast": 2.0165761911079723, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.302463887346676e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.3939923014258966e-06, + 2.923723741332651e-06, + 2.7498049348650966e-06, + 2.773209871520521e-06, + 2.580590717116138e-06, + 2.4804382974252803e-06, + 2.3860832243371988e-06, + 2.3736943148833234e-06, + 2.6584079932945315e-06, + 2.58506247519108e-06, + 2.541359208407812e-06, + 2.5390997961949324e-06, + 2.073248197120847e-06, + 2.07914399652509e-06, + 2.0900488379993476e-06, + 2.180453748223954e-06, + 4.095257281733211e-06, + 4.0363333937420975e-06, + 3.962578830396524e-06, + 3.868169187626336e-06, + 6.123455023043789e-06, + 6.00201019551605e-06, + 5.559234068641672e-06, + 5.036499260313576e-06, + 3.7911208892182913e-06, + 3.334695065859705e-06, + 2.852633542715921e-06, + 2.6229470222460805e-06, + 1.6816387642393238e-06, + 1.4847450984234456e-06, + 1.3401563592196908e-06, + 1.3073341733615962e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.342586018741713e-05, + "first_to_last_ratio": 2.2394319733492734, + "imbalance_abs_log_ratio": 0.806222250383043, + "last_quartile_mean": 2.385688014783227e-05, + "mean": 4.457031414517587e-05, + "normalized": [ + 1.1494422787070995, + 1.1718836421223593, + 1.1428090747145467, + 1.086444308176937, + 1.3229548508092461, + 1.3578460621138582, + 1.2296177221512083, + 1.1284987855754436, + 1.227111884992109, + 1.0975534107511307, + 1.0166764145258342, + 0.9402072429997458, + 0.8277192334474507, + 0.78847643593883, + 0.7746430720935581, + 0.7477481748973609, + 1.3079701078445523, + 1.262663592527712, + 1.1708787769345523, + 1.070669614331925, + 1.7921884489124142, + 1.5923472008639945, + 1.3491773347638585, + 1.1623611344243405, + 0.9084315137318004, + 0.7836848181998147, + 0.6646284136029903, + 0.6021622600764145, + 0.37448601778033414, + 0.3346427580221723, + 0.3104097006323165, + 0.30366571333408976 + ], + "peak_layer": 21, + "peak_normalized": 1.7921884489124142, + "population_cv": 0.35707967857039985, + "rest_mean": 4.1591522184952975e-05, + "spike_contrast": 1.458369098891111, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.0655790730379525e-05, + "top_five_layers": [ + 21, + 22, + 6, + 23, + 5 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.1231003453722224e-05, + 5.2231222070986405e-05, + 5.093535946798511e-05, + 4.842316411668435e-05, + 5.896451330045238e-05, + 6.051962554920465e-05, + 5.480444815475494e-05, + 5.029754538554698e-05, + 5.469276220537722e-05, + 4.8918300308287144e-05, + 4.531358717940748e-05, + 4.190533218206838e-05, + 3.689170625875704e-05, + 3.514264244586229e-05, + 3.4526085073594004e-05, + 3.3327371056657284e-05, + 5.8296638599131256e-05, + 5.6277312978636473e-05, + 5.21864349138923e-05, + 4.772008105646819e-05, + 7.987840217538178e-05, + 7.09714149706997e-05, + 6.013325764797628e-05, + 5.180680091143586e-05, + 4.0489077946403995e-05, + 3.492907853797078e-05, + 2.962269718409516e-05, + 2.683856109797489e-05, + 1.669095945544541e-05, + 1.4915132851456292e-05, + 1.3835057870892342e-05, + 1.3534476238419302e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.062916423208662e-05, + "first_to_last_ratio": 2.3115003864920087, + "imbalance_abs_log_ratio": 0.837896831650113, + "last_quartile_mean": 1.7576966229171376e-05, + "mean": 3.283533709463882e-05, + "normalized": [ + 1.212911398116158, + 1.2408980091883677, + 1.2100379395586904, + 1.1493036508582075, + 1.3478891639463513, + 1.376547190436269, + 1.2400156408077365, + 1.1212846877559697, + 1.2545199865442007, + 1.1255636869056884, + 1.0416509414039974, + 0.9496446486640949, + 0.863811056776793, + 0.8232971746250972, + 0.814426624693093, + 0.7852693022328948, + 1.2280636708101726, + 1.1957979497936206, + 1.0979825841550646, + 1.0020961529096264, + 1.784118618704249, + 1.5269728290156659, + 1.2477064495899464, + 1.0777394093111723, + 0.8973943474546292, + 0.7708740338031522, + 0.6829522402516178, + 0.6279739538251877, + 0.35258776830231575, + 0.32638760973755176, + 0.31414801689768274, + 0.3101332629247351 + ], + "peak_layer": 21, + "peak_normalized": 1.784118618704249, + "population_cv": 0.3491548585302889, + "rest_mean": 3.096988661531618e-05, + "spike_contrast": 1.385499734498883, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2908769682981077e-05, + "top_five_layers": [ + 21, + 22, + 6, + 5, + 9 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 3.982635462307371e-05, + 4.074530443176627e-05, + 3.973200364271179e-05, + 3.7737772800028324e-05, + 4.425839506438933e-05, + 4.519939102465287e-05, + 4.071633156854659e-05, + 3.6817760701524094e-05, + 4.1192586650140584e-05, + 3.695826308103278e-05, + 3.420295979594812e-05, + 3.1181902159005404e-05, + 2.836352723534219e-05, + 2.7033240257878788e-05, + 2.6741972760646604e-05, + 2.578458224888891e-05, + 4.0323884604731575e-05, + 3.926442877855152e-05, + 3.605262827477418e-05, + 3.2904164982028306e-05, + 5.858213626197539e-05, + 5.013866757508367e-05, + 4.0968861867440864e-05, + 3.538793680490926e-05, + 2.946624590549618e-05, + 2.53119087574305e-05, + 2.2424967028200626e-05, + 2.061973646050319e-05, + 1.1577338227652945e-05, + 1.0717047189245932e-05, + 1.0315156032447703e-05, + 1.0183330232393928e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00013562339336203877, + "first_to_last_ratio": 1.8937266056606614, + "imbalance_abs_log_ratio": 0.6385466367527224, + "last_quartile_mean": 7.161719804571476e-05, + "mean": 0.00012335994665590988, + "normalized": [ + 1.019000781731899, + 1.0322486217644518, + 1.0117188048559849, + 0.9776595701209428, + 1.2173099941502679, + 1.2992497220260693, + 1.1640291527018616, + 1.0740785542805755, + 1.1554579576814683, + 1.0322293937877338, + 0.9611972370481242, + 0.9118663882402063, + 0.7737203353417383, + 0.7258140672751332, + 0.7097483849490263, + 0.6892843914209803, + 1.412411909274259, + 1.3466569474208636, + 1.2405204033466979, + 1.1273987953867963, + 1.8525995137972373, + 1.7348580010509511, + 1.5314488922065341, + 1.355054500835559, + 0.9639698405130199, + 0.8534217733756123, + 0.7099903271591395, + 0.6468017103823497, + 0.4317554879354095, + 0.37917174911531354, + 0.33471259721834695, + 0.32461419360544697 + ], + "peak_layer": 21, + "peak_normalized": 1.8525995137972373, + "population_cv": 0.370445431994308, + "rest_mean": 0.00011222131676428641, + "spike_contrast": 1.6352378795922027, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00018350854807067663, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012570388207677752, + 0.00012733813491649926, + 0.0001248055777978152, + 0.0001206040324177593, + 0.000150167295942083, + 0.00016027537640184164, + 0.00014359457418322563, + 0.0001324982731603086, + 0.0001425372320227325, + 0.00012733576295431703, + 0.00011857323988806456, + 0.00011248778901062906, + 9.544609929434955e-05, + 8.953638462116942e-05, + 8.755452290643007e-05, + 8.503008575644344e-05, + 0.00017423505778424442, + 0.00016612352919764817, + 0.00015303053078241646, + 0.00013907585525885224, + 0.00022853657719679177, + 0.0002140119904652238, + 0.00018891945364885032, + 0.00016715945093892515, + 0.0001189152681035921, + 0.00010527806443860754, + 8.758436888456345e-05, + 7.978942448971793e-05, + 5.326133396010846e-05, + 4.6774606744293123e-05, + 4.1290128137916327e-05, + 4.004438960691914e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "884b74f81575660d3a5ef373d83980c2b6a40a5e790421e3f7efc3bf3cdbb781", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0860176682472229, + 0.031032679602503777, + 0.011454286053776741, + 0.03601690009236336, + 0.01171922218054533, + 0.008982792496681213, + 0.01206296682357788, + 0.026524480432271957, + 0.024220434948801994, + 0.026388581842184067, + 0.03031591884791851, + 0.032942064106464386, + 0.029961559921503067, + 0.02937687374651432, + 0.031781863421201706, + 0.035546254366636276, + 0.030760737136006355, + 0.03042135201394558, + 0.03508196026086807, + 0.04756028205156326, + 0.0295625701546669, + 0.03350908309221268, + 0.038334235548973083, + 0.043223608285188675, + 0.04338310658931732, + 0.046161677688360214, + 0.050398919731378555, + 0.062427736818790436, + 0.0627857968211174, + 0.08928166329860687, + 0.09919292479753494, + 0.13455425202846527 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6889636526684626e-05, + "first_to_last_ratio": 2.126000037178537, + "imbalance_abs_log_ratio": 0.7542422974073087, + "last_quartile_mean": 7.944325602693425e-06, + "mean": 1.4253426712684814e-05, + "normalized": [ + 1.115336759287406, + 1.1464967880052945, + 1.1249731711222886, + 1.0622402704566627, + 1.317199232206508, + 1.3595432924960245, + 1.2293188827089512, + 1.1245132369645223, + 1.155858821804884, + 1.0614255591183779, + 0.9912125389310478, + 0.9372185786895139, + 0.8026693967866093, + 0.7700318104442434, + 0.7622598930724469, + 0.7357685661471873, + 1.3571059222187842, + 1.2941689608655147, + 1.1908997045010374, + 1.0892684211984158, + 1.7975523481237365, + 1.5821682703899984, + 1.3498998616881461, + 1.1839696810356735, + 0.918100044807967, + 0.760612540110315, + 0.6815993230072928, + 0.6358967479480955, + 0.40490244236710804, + 0.3615503937465385, + 0.34857773642902556, + 0.3476608033203826 + ], + "peak_layer": 21, + "peak_normalized": 1.7975523481237365, + "population_cv": 0.3491891089141571, + "rest_mean": 1.3286468856676947e-05, + "spike_contrast": 1.4657769001837064, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.947499913512729e-05, + "top_five_layers": [ + 21, + 22, + 6, + 17, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.5897370758466423e-05, + 1.6341507944162004e-05, + 1.603472264832817e-05, + 1.5140563846216537e-05, + 1.877460272226017e-05, + 1.93781506823143e-05, + 1.7522006601211615e-05, + 1.602816701051779e-05, + 1.6474949006806128e-05, + 1.51289514178643e-05, + 1.4128175280347932e-05, + 1.335857632511761e-05, + 1.1440789421612862e-05, + 1.0975591976603027e-05, + 1.0864815521927085e-05, + 1.0487223335076123e-05, + 1.9343409803695977e-05, + 1.8446342437528074e-05, + 1.697440166026354e-05, + 1.5525807611993514e-05, + 2.5621280656196177e-05, + 2.2551319489139132e-05, + 1.9240698748035356e-05, + 1.6875625078682788e-05, + 1.3086071703583002e-05, + 1.0841335097211413e-05, + 9.715125997900032e-06, + 9.063707693712786e-06, + 5.771247288066661e-06, + 5.153332040208625e-06, + 4.9684272198646795e-06, + 4.9553577810002025e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.725307594024343e-05, + "first_to_last_ratio": 2.1308782233368584, + "imbalance_abs_log_ratio": 0.7565342061700902, + "last_quartile_mean": 3.156120101266424e-05, + "mean": 5.6670199910513475e-05, + "normalized": [ + 1.110680833704548, + 1.1455034927935899, + 1.1258904459559615, + 1.0664054974724477, + 1.3263498819954045, + 1.3678166597071255, + 1.2291648981942673, + 1.1221482262899733, + 1.1546676748054345, + 1.0627574527858743, + 0.9896202136771384, + 0.9362005960764237, + 0.8068774857623636, + 0.771625357953291, + 0.7613046275916601, + 0.7347843884669755, + 1.3577587462273788, + 1.2928466997481425, + 1.1909272887269375, + 1.0894198852162194, + 1.8057809001369873, + 1.5765275443860074, + 1.343499485983638, + 1.1760205514263615, + 0.9111321409731876, + 0.7565798297150907, + 0.67877158866213, + 0.6338403657465395, + 0.40731991397385253, + 0.3648985473433193, + 0.351811563377079, + 0.35106721512464933 + ], + "peak_layer": 21, + "peak_normalized": 1.8057809001369873, + "population_cv": 0.34914418193489394, + "rest_mean": 5.286498357977339e-05, + "spike_contrast": 1.4606713720053888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.721836809650994e-05, + "top_five_layers": [ + 21, + 22, + 6, + 17, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.29425048828125e-05, + 6.491591193480417e-05, + 6.380443664966151e-05, + 6.043341272743419e-05, + 7.516451296396554e-05, + 7.751444354653358e-05, + 6.965702050365508e-05, + 6.359236431308091e-05, + 6.543524796143174e-05, + 6.022667730576359e-05, + 5.608197534456849e-05, + 5.305467493599281e-05, + 4.572590842144564e-05, + 4.372816329123452e-05, + 4.314328543841839e-05, + 4.164037818554789e-05, + 7.694445957895368e-05, + 7.326588092837483e-05, + 6.749008753104135e-05, + 6.17376426816918e-05, + 0.00010233396460535005, + 8.934213110478595e-05, + 7.613638445036486e-05, + 6.66453197482042e-05, + 5.1634040573844686e-05, + 4.287553019821644e-05, + 3.8466121623059735e-05, + 3.591986023820937e-05, + 2.3082900952431373e-05, + 2.067887362500187e-05, + 1.9937231627409346e-05, + 1.9895049263141118e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.059640700463205576, + "first_to_last_ratio": 2.1308782476695067, + "imbalance_abs_log_ratio": 0.7565342175891604, + "last_quartile_mean": 0.02798878843896091, + "mean": 0.05025569983990863, + "normalized": [ + 1.1106807805227414, + 1.1455035131670817, + 1.1258903978802657, + 1.0664054823966183, + 1.3263498841814558, + 1.3678166690587785, + 1.229165007333235, + 1.1221482585659923, + 1.1546676515729808, + 1.0627574840194747, + 0.9896202794839417, + 0.9362006347867332, + 0.8068774504100944, + 0.7716253757910643, + 0.7613046381394664, + 0.7347843948827595, + 1.3577587106372004, + 1.292846681318099, + 1.1909273307261892, + 1.0894199023292641, + 1.8057809158157074, + 1.576527571436041, + 1.3434993978576328, + 1.1760204469019233, + 0.9111320892566023, + 0.7565798749963789, + 0.6787715750645948, + 0.6338403295741707, + 0.40731992625007324, + 0.36489853744490924, + 0.35181157583949707, + 0.3510672323590328 + ], + "peak_layer": 21, + "peak_normalized": 1.8057809158157074, + "population_cv": 0.34914418003675807, + "rest_mean": 0.04688119633054292, + "spike_contrast": 1.46067131708135, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.06847801879048347, + "top_five_layers": [ + 21, + 22, + 6, + 17, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.055818039923906326, + 0.057568080723285675, + 0.056582409888505936, + 0.05359295383095741, + 0.06665664166212082, + 0.06874058395624161, + 0.06177254766225815, + 0.05639434605836868, + 0.058028630912303925, + 0.05340962111949921, + 0.04973405972123146, + 0.04704941809177399, + 0.04055019095540047, + 0.03877857327461243, + 0.038259897381067276, + 0.036927103996276855, + 0.0682351142168045, + 0.06497291475534439, + 0.05985088646411896, + 0.05474955961108208, + 0.0907507836818695, + 0.07922949641942978, + 0.06751850247383118, + 0.05910173058509827, + 0.045789580792188644, + 0.03802245110273361, + 0.03411214053630829, + 0.031854089349508286, + 0.0204701479524374, + 0.01833823136985302, + 0.017680536955595016, + 0.01764312945306301 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.6838719406896416e-06, + "first_to_last_ratio": 1.2889488096889063, + "imbalance_abs_log_ratio": 0.25382700997133584, + "last_quartile_mean": 2.082217633869732e-06, + "mean": 2.8460147731834695e-06, + "normalized": [ + 1.1585408375114765, + 1.0223791463270808, + 0.9675822950955996, + 0.9815034677189116, + 0.9061206974504489, + 0.8633297690787293, + 0.8261227289030565, + 0.818645960689999, + 0.8917059523481307, + 0.8654323654695489, + 0.843182297073924, + 0.8552717070234496, + 0.7079172332660071, + 0.7114323992588514, + 0.7214302379069232, + 0.7529703821484969, + 1.351183821841786, + 1.2964760501367036, + 1.2560631856060487, + 1.2250133382734523, + 2.006580964722333, + 1.883398280372172, + 1.6561147848218503, + 1.578596263002518, + 1.1809705030150337, + 0.9928005092892233, + 0.9060768367688408, + 0.8769753144678925, + 0.5241234809909122, + 0.4617266282105258, + 0.45178683106552914, + 0.4585457301445449 + ], + "peak_layer": 21, + "peak_normalized": 2.006580964722333, + "population_cv": 0.37720922393237755, + "rest_mean": 2.4975718302071242e-06, + "spike_contrast": 1.8928811608448006, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.727606665255735e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.2972243388940115e-06, + 2.909706154241576e-06, + 2.7537535061128438e-06, + 2.793373369058827e-06, + 2.5788328912312863e-06, + 2.457049276927137e-06, + 2.351157490920741e-06, + 2.329878498130711e-06, + 2.537808313718415e-06, + 2.463033297317452e-06, + 2.3997092739591608e-06, + 2.4341159132745815e-06, + 2.0147429040662246e-06, + 2.0247471184120513e-06, + 2.0532011149043683e-06, + 2.1429648313642247e-06, + 3.845489118248224e-06, + 3.6897899917676114e-06, + 3.574774382286705e-06, + 3.486406058073044e-06, + 5.710759069188498e-06, + 5.3601793297275435e-06, + 4.713327143690549e-06, + 4.492708285397384e-06, + 3.361059498274699e-06, + 2.8255249162612017e-06, + 2.578708063083468e-06, + 2.495884700692841e-06, + 1.4916631698724814e-06, + 1.3140808050593478e-06, + 1.2857919955422403e-06, + 1.3050279221715755e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.28848186149844e-05, + "first_to_last_ratio": 2.4318675102825495, + "imbalance_abs_log_ratio": 0.8886594849400852, + "last_quartile_mean": 2.1746587094639835e-05, + "mean": 4.2398125145837184e-05, + "normalized": [ + 1.1902652753716751, + 1.2242846271388579, + 1.2015411916375924, + 1.138461083714705, + 1.3837339926605239, + 1.4128864745548413, + 1.2711291935221773, + 1.1564070657346273, + 1.2059250641459591, + 1.1015552456548003, + 1.0185066417879192, + 0.9526019131078938, + 0.8449353737531005, + 0.8080602539480353, + 0.7967414350915603, + 0.7697874535477814, + 1.3107593460824956, + 1.2543499045498807, + 1.1507767871653785, + 1.055520508570848, + 1.7920151228199508, + 1.5166494788051519, + 1.2523182952766116, + 1.0874771895891975, + 0.8458404468035057, + 0.7039937567721195, + 0.6353010461472927, + 0.5963690903457104, + 0.3543163425906758, + 0.3268551237187333, + 0.3200346200309376, + 0.3206006553594606 + ], + "peak_layer": 21, + "peak_normalized": 1.7920151228199508, + "population_cv": 0.3590956605869151, + "rest_mean": 4.0051623626722074e-05, + "spike_contrast": 1.3749563279206762, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5069233349058776e-05, + "top_five_layers": [ + 21, + 22, + 6, + 5, + 17 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.046501610195264e-05, + 5.190737283555791e-05, + 5.0943093810928985e-05, + 4.826861550100148e-05, + 5.866772698936984e-05, + 5.990373756503686e-05, + 5.3893494623480365e-05, + 4.9029491492547095e-05, + 5.112896178616211e-05, + 4.6703877160325646e-05, + 4.318277206039056e-05, + 4.03885351261124e-05, + 3.582367571652867e-05, + 3.426023977226578e-05, + 3.378034307388589e-05, + 3.263754479121417e-05, + 5.557373879128136e-05, + 5.318208422977477e-05, + 4.879077823716216e-05, + 4.475209061638452e-05, + 7.597808144055307e-05, + 6.430309440474957e-05, + 5.309594780555926e-05, + 4.610699397744611e-05, + 3.586204911698587e-05, + 2.9848015401512384e-05, + 2.69355732598342e-05, + 2.5284931325586513e-05, + 1.5022348634374794e-05, + 1.3858044439984951e-05, + 1.3568867871072143e-05, + 1.3592866707767826e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.0377713048656005e-05, + "first_to_last_ratio": 2.4735894758562855, + "imbalance_abs_log_ratio": 0.905670324816002, + "last_quartile_mean": 1.632353041713941e-05, + "mean": 3.153969646518817e-05, + "normalized": [ + 1.2510101789123635, + 1.284673110645709, + 1.2593728815079348, + 1.1934472035118944, + 1.4006716541713753, + 1.4203687191598957, + 1.2779886928015973, + 1.154218133705255, + 1.2357024917885882, + 1.1249430803382512, + 1.0345099464756928, + 0.9503377951773421, + 0.8790382032687162, + 0.8374457509593257, + 0.832874530076665, + 0.8029016908949563, + 1.238017371285292, + 1.1902047112336624, + 1.0825047379103157, + 0.9955341765531912, + 1.7778096901796956, + 1.4620144036626055, + 1.1595163229484706, + 1.0144538092925819, + 0.8300456109926844, + 0.7167594293866448, + 0.6575434300155114, + 0.6234101171676332, + 0.3406962742049681, + 0.32720961254604936, + 0.32247976203421724, + 0.32229647719091353 + ], + "peak_layer": 21, + "peak_normalized": 1.7778096901796956, + "population_cv": 0.35299231892411403, + "rest_mean": 3.008672124619975e-05, + "spike_contrast": 1.3090746022283983, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.938576264772564e-05, + "top_five_layers": [ + 21, + 22, + 6, + 5, + 2 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 3.94564813177567e-05, + 4.0518199966754764e-05, + 3.9720238419249654e-05, + 3.7640962545992807e-05, + 4.4176758819958195e-05, + 4.479799827095121e-05, + 4.030737545690499e-05, + 3.640368959167972e-05, + 3.897368151228875e-05, + 3.548036329448223e-05, + 3.2628129702061415e-05, + 2.9973365599289536e-05, + 2.772459811239969e-05, + 2.64127847913187e-05, + 2.626860987220425e-05, + 2.532327562221326e-05, + 3.904669210896827e-05, + 3.753869532374665e-05, + 3.4141870855819434e-05, + 3.1398845749208704e-05, + 5.607157800113782e-05, + 4.6111490519251674e-05, + 3.6570792872225866e-05, + 3.199556522304192e-05, + 2.6179386622970924e-05, + 2.2606374841416255e-05, + 2.0738720195367932e-05, + 1.9662165868794546e-05, + 1.0745457075245213e-05, + 1.0320091860194225e-05, + 1.0170913810725324e-05, + 1.0165133062400855e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001337056455668062, + "first_to_last_ratio": 2.0677765287412297, + "imbalance_abs_log_ratio": 0.7264738892699912, + "last_quartile_mean": 6.466155491580139e-05, + "mean": 0.00011615552659804962, + "normalized": [ + 1.0561568199318774, + 1.0932504740214608, + 1.0705240070918929, + 1.0352536719647083, + 1.2895031862888213, + 1.3493114057481659, + 1.2159685847053738, + 1.0987642787605771, + 1.1299239404100763, + 1.0527850453988277, + 0.9794864078984968, + 0.9382838968850933, + 0.7953293135025563, + 0.743714816851873, + 0.7363961718879526, + 0.7207598399740149, + 1.4085271780882864, + 1.3361487817201736, + 1.2152286835241228, + 1.0939565497169452, + 1.865453114985999, + 1.651978845678226, + 1.434528077319642, + 1.2353202703920463, + 0.9104842334884138, + 0.7537977556803896, + 0.688402255010069, + 0.6517209586478964, + 0.40424092626230357, + 0.3552178063401815, + 0.34378476083116255, + 0.3457979409923738 + ], + "peak_layer": 21, + "peak_normalized": 1.865453114985999, + "population_cv": 0.3635258211024969, + "rest_mean": 0.00010713082492993972, + "spike_contrast": 1.5391360583069846, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016488891560584307, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 6 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00012267845158930868, + 0.00012698708451353014, + 0.00012434727977961302, + 0.00012025043542962521, + 0.00014978292165324092, + 0.0001567299768794328, + 0.0001412414712831378, + 0.00012762754340656102, + 0.00013124691031407565, + 0.00012228680134285241, + 0.00011377275950508192, + 0.0001089868601411581, + 9.238189522875473e-05, + 8.638658619020134e-05, + 8.553648513043299e-05, + 8.372023876290768e-05, + 0.00016360821609850973, + 0.0001552010653540492, + 0.00014115552767179906, + 0.0001270690991077572, + 0.00021668268891517073, + 0.0001918864727485925, + 0.00016662836424075067, + 0.00014348927652463317, + 0.00010575777560006827, + 8.75577752594836e-05, + 7.996172644197941e-05, + 7.570099114673212e-05, + 4.695481766248122e-05, + 4.126051135244779e-05, + 3.993249993072823e-05, + 4.016634193249047e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2d82179ff03320f5589762540fc93ada0edba22cabc07ab3982cf783bff4a116", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05045261234045029, + 0.16565440595149994, + 0.23719695210456848, + 0.24811406433582306, + 0.1042611226439476, + 0.12873972952365875, + 0.1382850706577301, + 0.14404986798763275, + 0.040291935205459595, + 0.06678888201713562, + 0.09121427685022354, + 0.12122578918933868, + 0.05663999170064926, + 0.09547740966081619, + 0.12204843014478683, + 0.14941293001174927, + 0.03680795058608055, + 0.055156365036964417, + 0.07373107224702835, + 0.09760652482509613, + 0.035728659480810165, + 0.05559300258755684, + 0.08069392293691635, + 0.10087756812572479, + 0.025910304859280586, + 0.060516636818647385, + 0.09090615063905716, + 0.1273101419210434, + 0.013562481850385666, + 0.07113923132419586, + 0.14901472628116608, + 0.23766301572322845 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.7107619441958377e-05, + "first_to_last_ratio": 1.9385227140552528, + "imbalance_abs_log_ratio": 0.6619261953958985, + "last_quartile_mean": 8.82508072663768e-06, + "mean": 1.5109197732954271e-05, + "normalized": [ + 1.0736154005764045, + 1.090864091897401, + 1.0608090833759376, + 1.007555611950532, + 1.2489908200810547, + 1.2985740959119219, + 1.182842063583229, + 1.0948707762633996, + 1.1791532076095934, + 1.0539858856617061, + 0.9822340794635126, + 0.9179957846165998, + 0.7825363606751058, + 0.7478278147754593, + 0.7367534215342862, + 0.7112451648667536, + 1.353988198457684, + 1.3048851565120956, + 1.2111308371257685, + 1.1081161541255893, + 1.8022461396740475, + 1.6529667185837162, + 1.4576382974635043, + 1.266481614772143, + 0.974968751154223, + 0.8591782697026162, + 0.7163417357615641, + 0.6428331451635377, + 0.4293628627060633, + 0.3797467205300767, + 0.34123497868874664, + 0.32902675673572673 + ], + "peak_layer": 21, + "peak_normalized": 1.8022461396740475, + "population_cv": 0.3519835486656121, + "rest_mean": 1.3903650782325443e-05, + "spike_contrast": 1.5549262279970795, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1619151266349945e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.6221467376453802e-05, + 1.648208126425743e-05, + 1.6027974197641015e-05, + 1.522335696790833e-05, + 1.8871249267249368e-05, + 1.9620412786025554e-05, + 1.7871794625534676e-05, + 1.654261905059684e-05, + 1.7816058971220627e-05, + 1.592488115420565e-05, + 1.484076892666053e-05, + 1.3870179827790707e-05, + 1.1823496606666595e-05, + 1.1299078323645517e-05, + 1.1131753126392141e-05, + 1.0746343832579441e-05, + 2.0457675418583676e-05, + 1.9715767848538235e-05, + 1.829921529861167e-05, + 1.674274608376436e-05, + 2.7230493287788704e-05, + 2.497500099707395e-05, + 2.2023745259502903e-05, + 1.9135521142743528e-05, + 1.4730995644640643e-05, + 1.2981494364794344e-05, + 1.0823348929989152e-05, + 9.712693099572789e-06, + 6.487328391813207e-06, + 5.737668288929854e-06, + 5.15578676640871e-06, + 4.971330326952739e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.809728029111284e-05, + "first_to_last_ratio": 1.944702112932975, + "imbalance_abs_log_ratio": 0.6651088100365553, + "last_quartile_mean": 3.501681817397184e-05, + "mean": 6.006133520486401e-05, + "normalized": [ + 1.0680225064611175, + 1.089383496884205, + 1.062265590525735, + 1.010490961736264, + 1.2588526037326673, + 1.3055648849448955, + 1.1820615815511215, + 1.0937235345573042, + 1.1769375120625731, + 1.054885733578057, + 0.9819171693521337, + 0.9183525046530807, + 0.7871855083191073, + 0.7495806312894487, + 0.7365719056296469, + 0.7105757147525049, + 1.3531016594414176, + 1.302448867275397, + 1.2136850057217365, + 1.1102934751860138, + 1.8064271106878023, + 1.6488367157824373, + 1.4524898062618805, + 1.2622043639721385, + 0.9668821599838774, + 0.8526186691123343, + 0.713142716323793, + 0.640285602850375, + 0.431877541350555, + 0.3827323628964117, + 0.34447643781604553, + 0.3321256653079218 + ], + "peak_layer": 21, + "peak_normalized": 1.8064271106878023, + "population_cv": 0.35152458934703573, + "rest_mean": 5.530794731856548e-05, + "spike_contrast": 1.5500417923139733, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.572962979087606e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 6 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.414685776690021e-05, + 6.542982737300918e-05, + 6.380108970915899e-05, + 6.069143637432717e-05, + 7.560836820630357e-05, + 7.841397018637508e-05, + 7.09961968823336e-05, + 6.569049583049491e-05, + 7.068843842716888e-05, + 6.335784564726055e-05, + 5.897525625186972e-05, + 5.5157477618195117e-05, + 4.727941268356517e-05, + 4.502081355894916e-05, + 4.4239492126507685e-05, + 4.267812619218603e-05, + 8.126909233396873e-05, + 7.822681800462306e-05, + 7.28955419617705e-05, + 6.668570858892053e-05, + 0.00010849642421817407, + 9.903133468469605e-05, + 8.723847713554278e-05, + 7.58096794015728e-05, + 5.807223351439461e-05, + 5.120941568748094e-05, + 4.2832303734030575e-05, + 3.84564082196448e-05, + 2.59391417785082e-05, + 2.298741674167104e-05, + 2.0689714801847003e-05, + 1.9947910914197564e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06038934923708439, + "first_to_last_ratio": 1.9447021390890675, + "imbalance_abs_log_ratio": 0.6651088234864776, + "last_quartile_mean": 0.031053264159709215, + "mean": 0.053262992412783206, + "normalized": [ + 1.068022549058515, + 1.089383503618041, + 1.062265669270574, + 1.0104909600242913, + 1.2588525878422883, + 1.3055647959002497, + 1.1820616445871126, + 1.0937235097485394, + 1.176937455050195, + 1.0548857284650295, + 0.9819172246968001, + 0.91835248719328, + 0.7871855727179047, + 0.7495806491676233, + 0.7365718913476978, + 0.7105757078465339, + 1.3531016135159784, + 1.3024487649403136, + 1.213684966508138, + 1.1102934757832938, + 1.8064271858558933, + 1.6488366757497936, + 1.4524897961943863, + 1.262204461332232, + 0.9668821235259304, + 0.8526186558632091, + 0.713142698515675, + 0.6402856114586859, + 0.4318775756604022, + 0.3827323497583975, + 0.3444764462709729, + 0.3321256625320224 + ], + "peak_layer": 21, + "peak_normalized": 1.8064271858558933, + "population_cv": 0.3515245852652579, + "rest_mean": 0.04904764035233745, + "spike_contrast": 1.5500418163453433, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.07602589353919029, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 6 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05688607692718506, + 0.05802382528781891, + 0.05657944828271866, + 0.05382177233695984, + 0.0670502558350563, + 0.06953828781843185, + 0.06296014040708542, + 0.05825498700141907, + 0.0626872107386589, + 0.05618637055158615, + 0.052299849689006805, + 0.04891420155763626, + 0.04192785918712616, + 0.03992490842938423, + 0.03923202306032181, + 0.03784738853573799, + 0.07207024097442627, + 0.06937231868505478, + 0.06464449316263199, + 0.059137552976608276, + 0.09621571749448776, + 0.08782197535037994, + 0.07736395299434662, + 0.0672287866473198, + 0.05149903520941734, + 0.04541302099823952, + 0.03798411414027214, + 0.034103527665138245, + 0.023003092035651207, + 0.020385470241308212, + 0.01834784634411335, + 0.017690006643533707 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.7077437607658794e-06, + "first_to_last_ratio": 1.1762941928935384, + "imbalance_abs_log_ratio": 0.16236898221575077, + "last_quartile_mean": 2.3019273385216366e-06, + "mean": 3.047136040379428e-06, + "normalized": [ + 1.1138713662086555, + 0.9595541384126708, + 0.9023959781504268, + 0.9101529768920249, + 0.8469222629807022, + 0.8140191718536866, + 0.7830494541805194, + 0.7789886983581177, + 0.8724379423486327, + 0.84833009613214, + 0.8339719440677056, + 0.8333161939360028, + 0.6804733592729626, + 0.6824055388257974, + 0.6859436644795092, + 0.7155569613636353, + 1.3439258687459783, + 1.3246194446931803, + 1.3004895113079367, + 1.2694288333012982, + 2.009558665326211, + 1.96970386169125, + 1.8244250615184896, + 1.6529419249494095, + 1.24414521844288, + 1.0943599642755901, + 0.9362285212165059, + 0.8607523377030725, + 0.5518971596612197, + 0.4872731259576552, + 0.4398348869409435, + 0.42902586680518967 + ], + "peak_layer": 21, + "peak_normalized": 2.009558665326211, + "population_cv": 0.4073851731752779, + "rest_mean": 2.629478112120759e-06, + "spike_contrast": 2.016555615555062, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.30248885297624e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.394117584321066e-06, + 2.923891997852479e-06, + 2.7497233077156125e-06, + 2.7733599381463137e-06, + 2.5806873509282013e-06, + 2.4804271561151836e-06, + 2.3860582132329e-06, + 2.3736845378152793e-06, + 2.6584370971249882e-06, + 2.584977210062789e-06, + 2.541225967434002e-06, + 2.539227807574207e-06, + 2.073494897558703e-06, + 2.07938251151063e-06, + 2.0901636617054464e-06, + 2.180399405915523e-06, + 4.095124950254103e-06, + 4.036295649711974e-06, + 3.962768460041843e-06, + 3.8681223486491945e-06, + 6.123398634372279e-06, + 6.0019556258339435e-06, + 5.559271357924445e-06, + 5.036738912167493e-06, + 3.791079734583036e-06, + 3.334663688292494e-06, + 2.852815669029951e-06, + 2.6228294700558763e-06, + 1.6817057257867418e-06, + 1.4847875036139158e-06, + 1.34023673581396e-06, + 1.3073001809971174e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.3425848363986006e-05, + "first_to_last_ratio": 2.239424829290692, + "imbalance_abs_log_ratio": 0.8062190602571265, + "last_quartile_mean": 2.3856950974732172e-05, + "mean": 4.4570399836629804e-05, + "normalized": [ + 1.1494336205459152, + 1.171873145104734, + 1.142800837422437, + 1.0864554423282164, + 1.3229603063582054, + 1.3578480224061202, + 1.229609154713601, + 1.1284956364350298, + 1.2271150761135223, + 1.0975498313673546, + 1.0166755209562053, + 0.9402071494360641, + 0.8277227843287724, + 0.7884832455736381, + 0.7746476228748241, + 0.7477519611549666, + 1.307967756375797, + 1.2626631238761457, + 1.1708682002256867, + 1.0706732694770698, + 1.7921913698445469, + 1.5923449556348939, + 1.3491832296364525, + 1.1623630624416765, + 0.9084319709996431, + 0.7836885353653049, + 0.6646335840154506, + 0.6021591842085282, + 0.3744874199934144, + 0.3346428492438081, + 0.3104099404736517, + 0.3036621910683233 + ], + "peak_layer": 21, + "peak_normalized": 1.7921913698445469, + "population_cv": 0.35707915921975414, + "rest_mean": 4.15915873654066e-05, + "spike_contrast": 1.4583715367325734, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.065598718123511e-05, + "top_five_layers": [ + 21, + 22, + 6, + 23, + 5 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.1230716053396463e-05, + 5.223085463512689e-05, + 5.0935090257553384e-05, + 4.8423753469251096e-05, + 5.8964869822375476e-05, + 6.0519829276017845e-05, + 5.48041716683656e-05, + 5.02975017298013e-05, + 5.46930095879361e-05, + 4.8918234824668616e-05, + 4.531363447313197e-05, + 4.190540857962333e-05, + 3.689193545142189e-05, + 3.5143013519700617e-05, + 3.452635428402573e-05, + 3.332760388730094e-05, + 5.829664587508887e-05, + 5.6277400290127844e-05, + 5.2186063840053976e-05, + 4.7720335714984685e-05, + 7.987868593772873e-05, + 7.097145135048777e-05, + 6.013363599777222e-05, + 5.180698644835502e-05, + 4.048917617183179e-05, + 3.4929311368614435e-05, + 2.962298458442092e-05, + 2.6838475605472922e-05, + 1.6691054042894393e-05, + 1.4915165593265556e-05, + 1.3835095160175115e-05, + 1.3534345271182247e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.0632114178151824e-05, + "first_to_last_ratio": 2.3114815443448227, + "imbalance_abs_log_ratio": 0.837888680137748, + "last_quartile_mean": 1.7578385723027168e-05, + "mean": 3.283649590457571e-05, + "normalized": [ + 1.2128463251702764, + 1.2409607981861552, + 1.2097488384048114, + 1.1492961072150816, + 1.3482524084927348, + 1.376595331959661, + 1.2403426967960511, + 1.121214539041085, + 1.2543031022204718, + 1.1257078780047793, + 1.041573188661204, + 0.9496272000687872, + 0.8637967480657868, + 0.8233044041758093, + 0.8143347880182855, + 0.7852282395591935, + 1.2279923020188777, + 1.1955877909984085, + 1.0982969260190423, + 1.0020729756097966, + 1.7841330993237274, + 1.5266975818342838, + 1.2476748262094024, + 1.077765966545238, + 0.8974242777715684, + 0.7709389519020208, + 0.6829234854491848, + 0.6280779276603982, + 0.35252181345805783, + 0.3264428428242703, + 0.31417631661714684, + 0.31014032171840134 + ], + "peak_layer": 21, + "peak_normalized": 1.7841330993237274, + "population_cv": 0.34914670400639497, + "rest_mean": 3.097126648720371e-05, + "spike_contrast": 1.385436878279194, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.290873475838452e-05, + "top_five_layers": [ + 21, + 22, + 6, + 5, + 9 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 3.9825623389333487e-05, + 4.0748804167378694e-05, + 3.9723912777844816e-05, + 3.773885691771284e-05, + 4.427188468980603e-05, + 4.520256698015146e-05, + 4.072850788361393e-05, + 3.6816756619373336e-05, + 4.1186918679159135e-05, + 3.6964302125852555e-05, + 3.4201613743789494e-05, + 3.118242966593243e-05, + 2.8364058380248025e-05, + 2.703443169593811e-05, + 2.6739900931715965e-05, + 2.5784143872442655e-05, + 4.032296419609338e-05, + 3.925891360267997e-05, + 3.606422251323238e-05, + 3.2904565159697086e-05, + 5.858467920916155e-05, + 5.0131398893427104e-05, + 4.096926932106726e-05, + 3.5390057746553794e-05, + 2.9468268621712923e-05, + 2.5314933736808598e-05, + 2.242481423309073e-05, + 2.0623878299375065e-05, + 1.157558108388912e-05, + 1.0719239071477205e-05, + 1.0316449333913624e-05, + 1.018392140395008e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00013562132971856045, + "first_to_last_ratio": 1.8937793918607297, + "imbalance_abs_log_ratio": 0.638574510609612, + "last_quartile_mean": 7.161411213019164e-05, + "mean": 0.00012335703559074318, + "normalized": [ + 1.0190898279943903, + 1.0323955479842792, + 1.0116562111461564, + 0.9775328839891945, + 1.2172178061017016, + 1.2992066539806704, + 1.164116312982372, + 1.0741536827144558, + 1.1553960428140764, + 1.0323218193398271, + 0.9612127831166123, + 0.9118644908803012, + 0.7735676616433252, + 0.7259226190596877, + 0.7097387097391373, + 0.6893988641881136, + 1.4121216600785822, + 1.346602611711891, + 1.2404286450676882, + 1.1276439912131677, + 1.8527673327840795, + 1.7347177459206955, + 1.5317035051224266, + 1.3548754374827359, + 0.9638063798796889, + 0.8533115017901868, + 0.7100586330731438, + 0.6469535195087284, + 0.43178667471980076, + 0.37906287870027017, + 0.3347439711995271, + 0.324623594073087 + ], + "peak_layer": 21, + "peak_normalized": 1.8527673327840795, + "population_cv": 0.37043820400707683, + "rest_mean": 0.00011221894426314436, + "spike_contrast": 1.6352205945680411, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001835027287597768, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012571190018206835, + 0.00012735325435642153, + 0.0001247949112439528, + 0.00012058555876137689, + 0.00015015238022897393, + 0.0001602662814548239, + 0.00014360193745233119, + 0.00013250441406853497, + 0.00014252623077481985, + 0.0001273441594094038, + 0.00011857235949719325, + 0.00011248490045545623, + 9.542501356918365e-05, + 8.95476623554714e-05, + 8.755126327741891e-05, + 8.504220022587106e-05, + 0.000174195141880773, + 0.00016611290629953146, + 0.00015301560051739216, + 0.0001391028199577704, + 0.000228551885811612, + 0.00021398963872343302, + 0.00018894640379585326, + 0.0001671334175625816, + 0.00011889229790540412, + 0.00010526197729632258, + 8.759072807151824e-05, + 7.980626833159477e-05, + 5.326392420101911e-05, + 4.676007301895879e-05, + 4.129302396904677e-05, + 4.004460424766876e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b054da1b5190b8b19869952768070eae883b70fbd71c9a2eaf9205894fe44b7f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.10241243988275528, + 0.1819411665201187, + 0.24157075583934784, + 0.26444777846336365, + 0.10636615753173828, + 0.13034582138061523, + 0.14029134809970856, + 0.15042904019355774, + 0.04812335968017578, + 0.07954440265893936, + 0.10673297941684723, + 0.14089107513427734, + 0.06885477155447006, + 0.10946234315633774, + 0.13876035809516907, + 0.1687854677438736, + 0.05059739947319031, + 0.07094386219978333, + 0.09166282415390015, + 0.12053482979536057, + 0.048727843910455704, + 0.07109228521585464, + 0.09690473228693008, + 0.11853098124265671, + 0.05259029567241669, + 0.08889466524124146, + 0.12455403059720993, + 0.1728687882423401, + 0.06489895284175873, + 0.14679722487926483, + 0.23576539754867554, + 0.35208994150161743 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6889579683265765e-05, + "first_to_last_ratio": 2.125980149627833, + "imbalance_abs_log_ratio": 0.754232942918421, + "last_quartile_mean": 7.944373180635012e-06, + "mean": 1.4253392819796318e-05, + "normalized": [ + 1.1153433675846105, + 1.1464834343722714, + 1.124956448240789, + 1.0622490496205712, + 1.3172283984218986, + 1.3595312111704494, + 1.229301131769315, + 1.124519228992338, + 1.1558693549986068, + 1.061428019254281, + 0.9912129816477457, + 0.9372316548143664, + 0.8026823443963541, + 0.770025154892527, + 0.7622460724323464, + 0.7357708899991084, + 1.3570894960856237, + 1.294169996358889, + 1.1908969211284495, + 1.0892496991453142, + 1.7975621100651502, + 1.5821389795391794, + 1.3498915859714575, + 1.1839851305574904, + 0.9181060564815129, + 0.760619453477076, + 0.6816165769744638, + 0.6358942400688073, + 0.4049116684425325, + 0.36155600724105696, + 0.3485738753445159, + 0.3476594605109013 + ], + "peak_layer": 21, + "peak_normalized": 1.7975621100651502, + "population_cv": 0.34918660027907056, + "rest_mean": 1.3286440611813709e-05, + "spike_contrast": 1.4657751697310237, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9474934742902405e-05, + "top_five_layers": [ + 21, + 22, + 6, + 17, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.5897427147137932e-05, + 1.6341278751497157e-05, + 1.603444616193883e-05, + 1.514065297669731e-05, + 1.8774973796098493e-05, + 1.9377932403585874e-05, + 1.752171192492824e-05, + 1.6028214304242283e-05, + 1.6475059965159744e-05, + 1.5128950508369599e-05, + 1.4128147995506879e-05, + 1.3358730939216912e-05, + 1.144094676419627e-05, + 1.0975471013807692e-05, + 1.086459269572515e-05, + 1.048723152052844e-05, + 1.934312967932783e-05, + 1.8446313333697617e-05, + 1.6974321624729782e-05, + 1.5525503840763122e-05, + 2.562135887274053e-05, + 2.2550848370883614e-05, + 1.9240535038989037e-05, + 1.687580515863374e-05, + 1.3086126273265108e-05, + 1.0841407856787555e-05, + 9.715348824101966e-06, + 9.063650395546574e-06, + 5.771365067630541e-06, + 5.153399797563907e-06, + 4.968360372004099e-06, + 4.955326858180342e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.725337607349502e-05, + "first_to_last_ratio": 2.130880302861834, + "imbalance_abs_log_ratio": 0.7565351820700446, + "last_quartile_mean": 3.1561311061523156e-05, + "mean": 5.6670258345548064e-05, + "normalized": [ + 1.1106881622493918, + 1.145498588275503, + 1.1258880010928651, + 1.0664271230850946, + 1.3263713679598177, + 1.367809856868302, + 1.2291495077292092, + 1.122159908308202, + 1.154668923611267, + 1.062750001572791, + 0.9896186154790125, + 0.9362122130506743, + 0.8068839078557516, + 0.7716166020495587, + 0.7612932503122151, + 0.7347861986228512, + 1.357749257546284, + 1.2928515294142118, + 1.190927729797305, + 1.089408233799192, + 1.8057756999530474, + 1.5765198843812338, + 1.3434883429213678, + 1.176024987992221, + 0.9111282484717285, + 0.7565791137700626, + 0.678779490957478, + 0.6338427293588494, + 0.4073256246452393, + 0.3649030178459797, + 0.3518060328673549, + 0.35106784815593767 + ], + "peak_layer": 21, + "peak_normalized": 1.8057756999530474, + "population_cv": 0.34914284904214865, + "rest_mean": 5.286508732953937e-05, + "spike_contrast": 1.4606649819879878, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.721818183199502e-05, + "top_five_layers": [ + 21, + 22, + 6, + 17, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.294298509601504e-05, + 6.491570093203336e-05, + 6.380436389008537e-05, + 6.04347005719319e-05, + 7.516580808442086e-05, + 7.751413795631379e-05, + 6.965622014831752e-05, + 6.359309190884233e-05, + 6.543538620462641e-05, + 6.0226317145861685e-05, + 5.608194260275923e-05, + 5.3055387979839e-05, + 4.572631951305084e-05, + 4.3727712181862444e-05, + 4.314268517191522e-05, + 4.164052370470017e-05, + 7.694400119362399e-05, + 7.326623017434031e-05, + 6.749018211849034e-05, + 6.173704605316743e-05, + 0.00010233377543045208, + 8.934178913477808e-05, + 7.613583147758618e-05, + 6.664563989033923e-05, + 5.163387322681956e-05, + 4.2875533836195245e-05, + 3.8466609112219885e-05, + 3.59200312232133e-05, + 2.308324837940745e-05, + 2.0679148292401806e-05, + 1.9936938770115376e-05, + 1.9895105651812628e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05964096589013934, + "first_to_last_ratio": 2.1308802328272587, + "imbalance_abs_log_ratio": 0.756535149203545, + "last_quartile_mean": 0.027988886926323175, + "mean": 0.05025575193576515, + "normalized": [ + 1.1106881537388107, + 1.1454985452622695, + 1.1258879706126397, + 1.0664271338250162, + 1.3263713402742836, + 1.3678099140413058, + 1.2291494267200453, + 1.122159919240132, + 1.154668900806693, + 1.0627500074578355, + 0.9896186606157457, + 0.9362122658369968, + 0.8068838784002234, + 0.7716165702350821, + 0.7612932488495279, + 0.7347862276269627, + 1.3577492974867253, + 1.2928515677722938, + 1.1909276528528374, + 1.0894083211634755, + 1.805775782344081, + 1.5765197105460131, + 1.3434883687023758, + 1.1760249355734054, + 0.9111282538244218, + 0.7565791648411243, + 0.6787795442574317, + 0.6338427117183585, + 0.40732561946444507, + 0.3649030144809506, + 0.35180602228123725, + 0.3510678691472537 + ], + "peak_layer": 21, + "peak_normalized": 1.805775782344081, + "population_cv": 0.3491428438845931, + "rest_mean": 0.04688128842799752, + "spike_contrast": 1.4606649512818284, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.06847785487771034, + "top_five_layers": [ + 21, + 22, + 6, + 17, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05581846833229065, + 0.057567890733480453, + 0.05658234655857086, + 0.05359409749507904, + 0.06665778905153275, + 0.06874031573534012, + 0.061771828681230545, + 0.05639499053359032, + 0.058028753846883774, + 0.05340930074453354, + 0.04973402991890907, + 0.047050051391124725, + 0.040550556033849716, + 0.03877817094326019, + 0.038259364664554596, + 0.0369272343814373, + 0.06823471188545227, + 0.06497322767972946, + 0.059850964695215225, + 0.054749034345149994, + 0.09075061976909637, + 0.07922918349504471, + 0.06751801818609238, + 0.05910201743245125, + 0.045789435505867004, + 0.03802245482802391, + 0.03411257639527321, + 0.03185424208641052, + 0.020470455288887024, + 0.01833847537636757, + 0.017680276185274124, + 0.01764317974448204 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.6838953033347934e-06, + "first_to_last_ratio": 1.2889440986362102, + "imbalance_abs_log_ratio": 0.25382335500748454, + "last_quartile_mean": 2.082243369727621e-06, + "mean": 2.846027708613974e-06, + "normalized": [ + 1.1585324560759196, + 1.0223734609389419, + 0.9675124661430837, + 0.9815131475159752, + 0.9061526900553396, + 0.8633991057607137, + 0.8261330350267956, + 0.8186399230205594, + 0.8916923923720145, + 0.8654164482697974, + 0.843209462677348, + 0.8552471278215477, + 0.7079228836901937, + 0.7114272483418291, + 0.7214437361806822, + 0.7529832577201679, + 1.3512097970186538, + 1.2964355644882453, + 1.2560996594542384, + 1.2250466776906157, + 2.006533496683837, + 1.883321173188558, + 1.656149120836194, + 1.5785540956364958, + 1.1809573060310585, + 0.9928361025066657, + 0.906104994776819, + 0.8769702899521535, + 0.5241597663354778, + 0.46173906989410896, + 0.45174563077862806, + 0.45853841311734106 + ], + "peak_layer": 21, + "peak_normalized": 2.006533496683837, + "population_cv": 0.3771990652330804, + "rest_mean": 2.4975985297714942e-06, + "spike_contrast": 1.8928363457980932, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.727545274363365e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.297215471320669e-06, + 2.909703198383795e-06, + 2.7535672870726557e-06, + 2.7934136141993804e-06, + 2.5789356641325867e-06, + 2.4572577785875183e-06, + 2.351197508687619e-06, + 2.329871904294123e-06, + 2.537781256251037e-06, + 2.4629991912661353e-06, + 2.3997974949452328e-06, + 2.4340570234926417e-06, + 2.0147681425441988e-06, + 2.0247416614438407e-06, + 2.0532488633762114e-06, + 2.143011215594015e-06, + 3.8455805224657524e-06, + 3.6896915389661444e-06, + 3.574894435587339e-06, + 3.4865167890529847e-06, + 5.710649929824285e-06, + 5.359984243114013e-06, + 4.713446287496481e-06, + 4.49260869572754e-06, + 3.3610372156545054e-06, + 2.8256390578462742e-06, + 2.578799922048347e-06, + 2.49588174483506e-06, + 1.491773218731396e-06, + 1.3141221870682784e-06, + 1.2856805824412731e-06, + 1.305013029195834e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.2885012337355874e-05, + "first_to_last_ratio": 2.4318717272327546, + "imbalance_abs_log_ratio": 0.888661218976389, + "last_quartile_mean": 2.1746629045082955e-05, + "mean": 4.23981676931362e-05, + "normalized": [ + 1.190270602106981, + 1.2242785076575162, + 1.2015505398967856, + 1.1384753861662993, + 1.3837446167726861, + 1.412880509028644, + 1.2711237134722582, + 1.1564115683962857, + 1.205924025589364, + 1.101554226030039, + 1.0185042468172618, + 0.9526077357578042, + 0.8449377006342662, + 0.8080550669856646, + 0.796734028555351, + 0.7697863378322973, + 1.310756743636259, + 1.254353365068151, + 1.1507733156032836, + 1.0555102681924855, + 1.792017786366534, + 1.51664778521082, + 1.252309144484671, + 1.0874779001939108, + 0.8458376244707143, + 0.7039938225479899, + 0.635304784671484, + 0.596370594103779, + 0.35431838957072104, + 0.3268559755339857, + 0.3200335480760464, + 0.32060014056966063 + ], + "peak_layer": 21, + "peak_normalized": 1.792017786366534, + "population_cv": 0.35909580795741874, + "rest_mean": 4.0051679981708595e-05, + "spike_contrast": 1.374953593956486, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.506920133484528e-05, + "top_five_layers": [ + 21, + 22, + 6, + 5, + 17 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.046529258834198e-05, + 5.1907165470765904e-05, + 5.094354128232226e-05, + 4.8269270337186754e-05, + 5.866823630640283e-05, + 5.990354475216009e-05, + 5.389331636251882e-05, + 4.902973159914836e-05, + 5.112896906211972e-05, + 4.6703880798304453e-05, + 4.318271385272965e-05, + 4.038882252643816e-05, + 3.582381032174453e-05, + 3.4260054235346615e-05, + 3.378006294951774e-05, + 3.263753023929894e-05, + 5.557368422159925e-05, + 5.318228431860916e-05, + 4.8790680011734366e-05, + 4.475170135265216e-05, + 7.597827061545104e-05, + 6.430308712879196e-05, + 5.309561311150901e-05, + 4.610707037500106e-05, + 3.586196544347331e-05, + 2.9848048143321648e-05, + 2.6935758796753362e-05, + 2.5285020456067286e-05, + 1.5022450497781392e-05, + 1.3858094462193549e-05, + 1.3568836038757581e-05, + 1.359285852231551e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.03796584578231e-05, + "first_to_last_ratio": 2.4739007999741456, + "imbalance_abs_log_ratio": 0.9057961761470462, + "last_quartile_mean": 1.6322262581525138e-05, + "mean": 3.154074499889248e-05, + "normalized": [ + 1.2508863516144793, + 1.2841867971630014, + 1.2593428954345096, + 1.1937167612399988, + 1.4009066408475082, + 1.420515275619532, + 1.2780975365837242, + 1.1542512752891054, + 1.235598435522957, + 1.124789879408544, + 1.0344513335895396, + 0.9505323884005586, + 0.8790314147700448, + 0.8375394240979268, + 0.8329084925799662, + 0.802864330251641, + 1.2379667568434787, + 1.1905528094480917, + 1.0824881288459707, + 0.9955427197632982, + 1.7778790802657947, + 1.461709510411412, + 1.1598120379240673, + 1.0144482285055212, + 0.8299461589475229, + 0.7167094189126102, + 0.6574294700400573, + 0.6234038104311064, + 0.3406256334471781, + 0.3271495125781152, + 0.3224606792775168, + 0.3222568119452214 + ], + "peak_layer": 21, + "peak_normalized": 1.7778790802657947, + "population_cv": 0.35301095948018335, + "rest_mean": 3.0087773834734604e-05, + "spike_contrast": 1.30906292707755, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.9386789285345e-05, + "top_five_layers": [ + 21, + 22, + 6, + 5, + 2 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 3.945388743886724e-05, + 4.050420830026269e-05, + 3.9720613131066784e-05, + 3.765071596717462e-05, + 4.4185639126226306e-05, + 4.4804110075347126e-05, + 4.03121484850999e-05, + 3.640594513854012e-05, + 3.897169517586008e-05, + 3.547671076375991e-05, + 3.2627365726511925e-05, + 2.9980499675730243e-05, + 2.772530569927767e-05, + 2.641661740199197e-05, + 2.6270554371876642e-05, + 2.5322939109173603e-05, + 3.904639379470609e-05, + 3.755092257051729e-05, + 3.4142482036259025e-05, + 3.1400159059558064e-05, + 5.607563070952892e-05, + 4.610340693034232e-05, + 3.658133573480882e-05, + 3.199645288987085e-05, + 2.6177120162174106e-05, + 2.2605549020227045e-05, + 2.073581526929047e-05, + 1.9662620616145432e-05, + 1.0743586244643666e-05, + 1.0318539352738298e-05, + 1.017065005726181e-05, + 1.0164219929720275e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00013370490523811895, + "first_to_last_ratio": 2.067929402982526, + "imbalance_abs_log_ratio": 0.7265478182407109, + "last_quartile_mean": 6.465641672548372e-05, + "mean": 0.00011615701851042104, + "normalized": [ + 1.0560644548880596, + 1.0932324234962196, + 1.0704146702692696, + 1.0356495330841202, + 1.2890613053020747, + 1.349240331041549, + 1.2161603019292984, + 1.0987401440739695, + 1.1302068376440797, + 1.052621189942207, + 0.9794612370099226, + 0.9384889523526194, + 0.795246875601738, + 0.7437305707818854, + 0.7363409872103527, + 0.7207683094190416, + 1.4084141263671424, + 1.3357660592114178, + 1.2156581878342292, + 1.0939775768488822, + 1.8651920040475745, + 1.6523968523362491, + 1.4344972498256239, + 1.2356342609168542, + 0.9104609510825358, + 0.7537285042826567, + 0.6882282342205499, + 0.6516164997895624, + 0.4042698411477325, + 0.355151701146589, + 0.3437385024432238, + 0.34584132445276966 + ], + "peak_layer": 21, + "peak_normalized": 1.8651920040475745, + "population_cv": 0.36353364558210677, + "rest_mean": 0.00010713040790977646, + "spike_contrast": 1.539252197123888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016490071575390176, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 6 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00012266929843463004, + 0.00012698661885224283, + 0.00012433617666829377, + 0.00012029796198476106, + 0.00014973351790104061, + 0.00015672373410779983, + 0.00014126555470284075, + 0.00012762637925334275, + 0.0001312814565608278, + 0.00012226933904457837, + 0.00011377129703760147, + 0.00010901207861024886, + 9.237350604962558e-05, + 8.638952567707747e-05, + 8.553117368137464e-05, + 8.37222978589125e-05, + 0.00016359718574676663, + 0.00015515860286541283, + 0.00014120723062660545, + 0.00012707317364402115, + 0.00021665514213964343, + 0.00019193749176338315, + 0.00016662692360114306, + 0.00014352759171742946, + 0.00010575642954790965, + 8.755085582379252e-05, + 7.99425397417508e-05, + 7.568982982775196e-05, + 4.695877942140214e-05, + 4.125336272409186e-05, + 3.992763959104195e-05, + 4.01718971261289e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4d1314009c174d2647fbc47c0cd287abca38783c7de30eced47b74a084c87fbc", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05571148172020912, + 0.061161335557699203, + 0.09595561027526855, + 0.12880638241767883, + 0.13726800680160522, + 0.10176635533571243, + 0.09925490617752075, + 0.10055620223283768, + 0.0990053340792656, + 0.07309409230947495, + 0.07328634709119797, + 0.07434380799531937, + 0.07861478626728058, + 0.06236656382679939, + 0.06600815057754517, + 0.07462523877620697, + 0.07498960196971893, + 0.05233807489275932, + 0.05626988410949707, + 0.061378348618745804, + 0.06856362521648407, + 0.04801172763109207, + 0.0518198125064373, + 0.05105668306350708, + 0.054886870086193085, + 0.04603559896349907, + 0.04718422889709473, + 0.05210597813129425, + 0.05379685014486313, + 0.05137190222740173, + 0.05757243186235428, + 0.06228902190923691 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 9.289336674100923e-06, + "first_to_last_ratio": 2.2865411488778773, + "imbalance_abs_log_ratio": 0.8270402604018311, + "last_quartile_mean": 4.0626151331935034e-06, + "mean": 9.477520439560294e-06, + "normalized": [ + 1.252991954527522, + 0.7394811736091712, + 0.5152207040408479, + 0.7320401747081184, + 0.45740983320689066, + 1.2545020339537147, + 1.465347353652639, + 1.4241603595420207, + 1.5020047812281965, + 1.3115942831695324, + 0.9033867836813385, + 0.9397941307418568, + 0.6815998176876099, + 0.7241591358217578, + 0.9921275569566015, + 0.8513079482595329, + 0.987677640009984, + 1.6262023844874374, + 1.2990996618268078, + 1.8103875474840807, + 1.9614463506839017, + 2.117788128754767, + 1.9620662739867234, + 1.0589398388998983, + 0.9090157064636322, + 0.8549381461887039, + 0.37264179907278366, + 0.3857075463499887, + 0.23244225762357623, + 0.33699601723381234, + 0.16357771526793677, + 0.17394496087861533 + ], + "peak_layer": 22, + "peak_normalized": 2.117788128754767, + "population_cv": 0.5364323230626372, + "rest_mean": 8.421213473647421e-06, + "spike_contrast": 1.8027779610382346, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.5181578055489808e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1875256859639194e-05, + 7.0084479375509545e-06, + 4.88301475343178e-06, + 6.937925718375482e-06, + 4.335111043474171e-06, + 1.1889568668266293e-05, + 1.3887859495298471e-05, + 1.349750891677104e-05, + 1.423528101440752e-05, + 1.2430661627149675e-05, + 8.56186670716852e-06, + 8.906918083084747e-06, + 6.459876203734893e-06, + 6.863233011245029e-06, + 9.40290919970721e-06, + 8.06828847998986e-06, + 9.360735020891298e-06, + 1.5412366337841377e-05, + 1.2312243597989436e-05, + 1.7157984984805807e-05, + 1.8589647879707627e-05, + 2.007138027693145e-05, + 1.859552321548108e-05, + 1.003612396743847e-05, + 8.615214937890414e-06, + 8.102693755063228e-06, + 3.5317202673468273e-06, + 3.6555511542246677e-06, + 2.2029762476449832e-06, + 3.1938866413838696e-06, + 1.5503111399084446e-06, + 1.6485669220855925e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.385339800843212e-05, + "first_to_last_ratio": 2.08046689071163, + "imbalance_abs_log_ratio": 0.7325923352126326, + "last_quartile_mean": 1.627201959308877e-05, + "mean": 3.714502105367501e-05, + "normalized": [ + 0.6435910429764604, + 0.6409303103278117, + 0.5081801030667545, + 0.7165876903215328, + 0.47047927357980707, + 1.3039514496969595, + 1.5507865803288083, + 1.4565698873921415, + 1.551285486064157, + 1.3082956708978086, + 0.9732102467191055, + 0.9041604784923716, + 0.6922959073858851, + 0.7633837554908558, + 1.007898690427689, + 0.8632027073532546, + 1.0068526925373231, + 1.6741376444679952, + 1.3331202957363155, + 1.7806470652336404, + 2.0225428919697093, + 2.301977702545363, + 2.002199408280622, + 1.019174508509882, + 0.9428974629120086, + 0.8578527415516407, + 0.38583353479489596, + 0.3927921630823954, + 0.2389950189630043, + 0.3440899675346293, + 0.16786811733193024, + 0.1742095040272426 + ], + "peak_layer": 22, + "peak_normalized": 2.301977702545363, + "population_cv": 0.562210067540178, + "rest_mean": 3.2620493382435394e-05, + "spike_contrast": 1.8876927996290056, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.157747047836892e-05, + "top_five_layers": [ + 22, + 21, + 23, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.390620284131728e-05, + 2.380736987106502e-05, + 1.8876360627473332e-05, + 2.6617664843797684e-05, + 1.747596252243966e-05, + 4.843530405196361e-05, + 5.760400017607026e-05, + 5.4104319133330137e-05, + 5.762253204011358e-05, + 4.859667023993097e-05, + 3.6149915104033425e-05, + 3.358506000950001e-05, + 2.5715346055221744e-05, + 2.8355905669741333e-05, + 3.7438418075907975e-05, + 3.206368273822591e-05, + 3.7399564462248236e-05, + 6.218587805051357e-05, + 4.9518781452206895e-05, + 6.614217272726819e-05, + 7.512739830417559e-05, + 8.550701022613794e-05, + 7.437173917423934e-05, + 3.7857258575968444e-05, + 3.502394611132331e-05, + 3.186495814588852e-05, + 1.4331794773170259e-05, + 1.4590273167414125e-05, + 8.877475011104252e-06, + 1.2781229088432156e-05, + 6.235464752535336e-06, + 6.471015694842208e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03002153232228011, + "first_to_last_ratio": 2.0804669230521093, + "imbalance_abs_log_ratio": 0.7325923507574504, + "last_quartile_mean": 0.014430189679842442, + "mean": 0.03294057691527996, + "normalized": [ + 0.6435910397676826, + 0.6409302858979954, + 0.5081800703155225, + 0.716587695511193, + 0.4704792716879319, + 1.3039513599328192, + 1.5507866493520404, + 1.456569894837028, + 1.5512854947592953, + 1.3082956463699744, + 0.973210295631563, + 0.9041604672039659, + 0.6922959258926152, + 0.7633837713422348, + 1.0078987680340359, + 0.8632027260248254, + 1.0068526741738888, + 1.6741375136914436, + 1.3331203018554725, + 1.78064694542985, + 2.0225429754607553, + 2.301977829207771, + 2.0021994467984428, + 1.0191745289341387, + 0.9428974359364356, + 0.8578527192959636, + 0.3858335126274743, + 0.39279215725797884, + 0.2389950138470689, + 0.3440899570497142, + 0.16786812365093953, + 0.17420950222193982 + ], + "peak_layer": 22, + "peak_normalized": 2.301977829207771, + "population_cv": 0.5622100766374781, + "rest_mean": 0.028928180138959928, + "spike_contrast": 1.887692874045117, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.054607519507408143, + "top_five_layers": [ + 22, + 21, + 23, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.021200260147452354, + 0.021112613379955292, + 0.016739744693040848, + 0.02360481210052967, + 0.015497858636081219, + 0.04295291006565094, + 0.05108380690217018, + 0.04798025265336037, + 0.0511002391576767, + 0.043096013367176056, + 0.03205810859799385, + 0.029783567413687706, + 0.02280462719500065, + 0.025146301835775375, + 0.033200766891241074, + 0.028434395790100098, + 0.0331663079559803, + 0.05514705553650856, + 0.04391375184059143, + 0.05865553766489029, + 0.0666237324476242, + 0.07582847774028778, + 0.06595360487699509, + 0.03357219696044922, + 0.031059585511684418, + 0.02825816348195076, + 0.01270957849919796, + 0.012938800267875195, + 0.007872633635997772, + 0.011334521695971489, + 0.005529672838747501, + 0.005738561507314444 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.777704892764632e-06, + "first_to_last_ratio": 1.0317973970310554, + "imbalance_abs_log_ratio": 0.03130232707959694, + "last_quartile_mean": 2.6921030240600885e-06, + "mean": 3.2474394480175306e-06, + "normalized": [ + 3.264922808244637, + 1.0119989909820577, + 0.406734804751791, + 0.3594970166401698, + 0.3085255046215715, + 0.44136542394078937, + 0.4976088478313049, + 0.5521652291156375, + 0.5596032700860724, + 0.7626434513115219, + 0.6804286383958748, + 0.8574740801621124, + 0.5510556809031377, + 0.657880030915651, + 0.5700052058099263, + 0.6746699382392725, + 0.8730218984285152, + 1.4178722452350283, + 1.4067082871744077, + 1.5468786674281858, + 1.5912638166777024, + 2.4472781271242594, + 2.129464926083038, + 1.7989929191022997, + 1.363241191573045, + 1.2762479066181045, + 1.0041585663694674, + 1.0201870464948222, + 0.6062702503600963, + 0.5376439897532335, + 0.43319266698828435, + 0.39099857263798254 + ], + "peak_layer": 1, + "peak_normalized": 3.264922808244637, + "population_cv": 0.670834276414478, + "rest_mean": 2.726617396917761e-06, + "spike_contrast": 2.2224894958883965, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.059878523956286e-06, + "top_five_layers": [ + 1, + 22, + 23, + 24, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.060263912222581e-05, + 3.2864054446690716e-06, + 1.3208466498326743e-06, + 1.167444793281902e-06, + 1.0019178944276064e-06, + 1.4333074886963004e-06, + 1.615954602129932e-06, + 1.793123146853759e-06, + 1.8172777345171198e-06, + 2.476638428561273e-06, + 2.2096508018876193e-06, + 2.78459515357099e-06, + 1.78951995621901e-06, + 2.1364255644584773e-06, + 1.8510573909225059e-06, + 2.1909497718297644e-06, + 2.835085751939914e-06, + 4.604454261425417e-06, + 4.5681999836233445e-06, + 5.023394805903081e-06, + 5.167532890482107e-06, + 7.947387530293781e-06, + 6.9153084041317925e-06, + 5.842120572197018e-06, + 4.42704322267673e-06, + 4.144537797401426e-06, + 3.2609441404929385e-06, + 3.3129956591437804e-06, + 1.968825927178841e-06, + 1.7459663013141835e-06, + 1.406766955369676e-06, + 1.2697441889031325e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.4231409383901337e-05, + "first_to_last_ratio": 2.412573321332518, + "imbalance_abs_log_ratio": 0.8806939460313521, + "last_quartile_mean": 1.004380226277135e-05, + "mean": 2.5713559296036692e-05, + "normalized": [ + 0.6136241747208916, + 0.6528682314803166, + 0.5678169227836256, + 0.7498796676883942, + 0.5334908499704412, + 1.3465695630143542, + 1.6172632261416102, + 1.457361031568912, + 1.4228430752167622, + 1.3681707330388233, + 0.9853905554776219, + 0.9155538605941198, + 0.6950712915766559, + 0.7940335378691747, + 1.0577609619611685, + 0.881295451045459, + 1.0180543332344463, + 1.8064117968549576, + 1.3937187971362992, + 1.682480285033005, + 1.9772967331635165, + 2.3224003214146545, + 1.9930145593470021, + 1.0228032828152094, + 0.9037764911003984, + 0.7535194773585814, + 0.3571273114497246, + 0.3679098930110221, + 0.19844991746454535, + 0.2567840165451007, + 0.14387571001082, + 0.14338393991238602 + ], + "peak_layer": 22, + "peak_normalized": 2.3224003214146545, + "population_cv": 0.5708513691298637, + "rest_mean": 2.2647654111130612e-05, + "spike_contrast": 1.8663940683267242, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2269447294529525e-05, + "top_five_layers": [ + 22, + 23, + 21, + 18, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.5778461602167226e-05, + 1.678756598266773e-05, + 1.4600594113289844e-05, + 1.9282075299997814e-05, + 1.3717948604607955e-05, + 3.462509630480781e-05, + 4.1585593862691894e-05, + 3.747393930098042e-05, + 3.658635978354141e-05, + 3.518053927109577e-05, + 2.5337898478028364e-05, + 2.354214848310221e-05, + 1.7872756870929152e-05, + 2.041742845904082e-05, + 2.719879921642132e-05, + 2.2661242837784812e-05, + 2.6177800464211032e-05, + 4.644927685149014e-05, + 3.583747093216516e-05, + 4.326255657360889e-05, + 5.0843336794059724e-05, + 5.971717837383039e-05, + 5.124749804963358e-05, + 2.6299912860849872e-05, + 2.323931039427407e-05, + 1.937566776177846e-05, + 9.183014299196657e-06, + 9.460272849537432e-06, + 5.1028537200181745e-06, + 6.602831035706913e-06, + 3.6995566006226e-06, + 3.6869114410365e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7024314388436323e-05, + "first_to_last_ratio": 2.80437609780141, + "imbalance_abs_log_ratio": 1.0311810892083095, + "last_quartile_mean": 6.070624550602588e-06, + "mean": 1.751632886026755e-05, + "normalized": [ + 0.584240566149076, + 0.6817235247296635, + 0.6361546768283869, + 0.7650115500840945, + 0.6079996047256336, + 1.3836573287680065, + 1.6554473132638503, + 1.4610542070855017, + 1.2966582857771938, + 1.3930304105863356, + 0.9987773633166749, + 0.9034199114794684, + 0.7206853676335052, + 0.821798500240951, + 1.1153051447773232, + 0.8912570241764348, + 1.0577423140442908, + 1.92155965677437, + 1.4320706567338384, + 1.5861347958986807, + 1.9171730212528781, + 2.3573454464224612, + 1.9827077484178592, + 1.0564899390528322, + 0.8322707332281934, + 0.6678826522442516, + 0.33914335537769175, + 0.32515959168264075, + 0.17199332542575016, + 0.18497778526153036, + 0.13366747042309535, + 0.11746072813753591 + ], + "peak_layer": 22, + "peak_normalized": 2.3573454464224612, + "population_cv": 0.5792420506711731, + "rest_mean": 1.547536067779152e-05, + "spike_contrast": 1.8440640990417727, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.8537557045638096e-05, + "top_five_layers": [ + 22, + 23, + 18, + 21, + 7 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.0233749890176114e-05, + 1.1941293450945523e-05, + 1.114309452532325e-05, + 1.3400193893176038e-05, + 1.0649921023286879e-05, + 2.4236596800619736e-05, + 2.8997359549975954e-05, + 2.5592305973987095e-05, + 2.2712692953064106e-05, + 2.4400778784183785e-05, + 1.74949127540458e-05, + 1.5824600268388167e-05, + 1.2623761904251296e-05, + 1.439489278709516e-05, + 1.9536051695467904e-05, + 1.561155113449786e-05, + 1.852776222222019e-05, + 3.3658670872682706e-05, + 2.5084620574489236e-05, + 2.778325870167464e-05, + 3.358183312229812e-05, + 4.129203807679005e-05, + 3.472976095508784e-05, + 1.8505825210013427e-05, + 1.457832786400104e-05, + 1.1698852176778018e-05, + 5.940546543570235e-06, + 5.695602339983452e-06, + 3.012691649928456e-06, + 3.2401317184849177e-06, + 2.341363369851024e-06, + 2.057480742223561e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.760827909602085e-05, + "first_to_last_ratio": 2.1482842886109816, + "imbalance_abs_log_ratio": 0.7646695182666318, + "last_quartile_mean": 3.1470825092583254e-05, + "mean": 7.373131964527602e-05, + "normalized": [ + 0.6107383587992412, + 0.6200732857988205, + 0.49856563619679256, + 0.7443981105032205, + 0.46530796044932227, + 1.3039254699053608, + 1.601092740237933, + 1.4915359307012412, + 1.5070070061189442, + 1.3672490492442442, + 0.9793783719831628, + 0.9514935883546256, + 0.6639415020375616, + 0.7664323809011713, + 0.9882247241637759, + 0.8679526231420076, + 0.9882466315784014, + 1.659918490517302, + 1.324670508689015, + 1.8045145321525453, + 2.0232513831671906, + 2.378151302734961, + 2.00069385121045, + 0.978587238005451, + 0.9661377082112428, + 0.817626258962968, + 0.36492362259091654, + 0.41187609689456905, + 0.22193259512312896, + 0.3124952321103628, + 0.15168754329397788, + 0.16797026622009212 + ], + "peak_layer": 22, + "peak_normalized": 2.378151302734961, + "population_cv": 0.5750991325799932, + "rest_mean": 6.459185429034901e-05, + "spike_contrast": 1.9055720557053664, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012308443256188185, + "top_five_layers": [ + 22, + 21, + 23, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.503054515225813e-05, + 4.571882163872942e-05, + 3.675990228657611e-05, + 5.488545502885245e-05, + 3.430776996538043e-05, + 9.61401456152089e-05, + 0.00011805068061221391, + 0.00010997291246894747, + 0.0001111136152758263, + 0.0001008090766845271, + 7.221085979836062e-05, + 7.015487790340558e-05, + 4.895328311249614e-05, + 5.65100708627142e-05, + 7.286311301868409e-05, + 6.399529229383916e-05, + 7.286472828127444e-05, + 0.00012238798080943525, + 9.766970470082015e-05, + 0.00013304923777468503, + 0.00014917699445504695, + 0.000175344233866781, + 0.000147513797855936, + 7.215252844616771e-05, + 7.123460818547755e-05, + 6.028466304996982e-05, + 2.6906300263362937e-05, + 3.0368168154382147e-05, + 1.6363383110729046e-05, + 2.304068584635388e-05, + 1.1184122740814928e-05, + 1.238466938957572e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "fbd2d69d0c904dc853dab2fd491c5cf23fa8369610dbace5a1307601df767adc", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04887383058667183, + 0.06501953303813934, + 0.11691275984048843, + 0.07614059001207352, + 0.09499973803758621, + 0.09968593716621399, + 0.09547581523656845, + 0.09975490719079971, + 0.06202486902475357, + 0.06455004960298538, + 0.062386661767959595, + 0.07036563754081726, + 0.05747222527861595, + 0.057312749326229095, + 0.06149744614958763, + 0.06584152579307556, + 0.04952141270041466, + 0.04880477488040924, + 0.0513896569609642, + 0.057161398231983185, + 0.04507989436388016, + 0.04335150495171547, + 0.047289349138736725, + 0.05114771053195, + 0.042588189244270325, + 0.044108107686042786, + 0.05026337504386902, + 0.05349528789520264, + 0.04645879194140434, + 0.05323651432991028, + 0.05820684880018234, + 0.058245208114385605 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.5842610063573375e-06, + "first_to_last_ratio": 0.31018923275847604, + "imbalance_abs_log_ratio": 1.1705727394853376, + "last_quartile_mean": 1.4778917261537572e-05, + "mean": 1.149474866934952e-05, + "normalized": [ + 0.964044714824398, + 0.4099629685278065, + 0.15847344197468682, + 0.44879991396551794, + 0.2564005888362708, + 0.21248149821699294, + 0.26315959961471164, + 0.47718531434582095, + 0.8325875353069343, + 0.6265632148302367, + 0.6147981550559392, + 0.5305901990282497, + 0.5691137397437396, + 0.5683891347333625, + 0.5227011984642989, + 0.5253372083161463, + 1.2593919921808647, + 1.2898533329037387, + 1.2925988880315054, + 1.3253277249748845, + 2.110587665830833, + 2.2010040974041387, + 2.209732114295895, + 2.045233009603335, + 1.9419589930337084, + 1.7718838003851192, + 1.545352082047033, + 1.491806952509735, + 1.0189371444399558, + 0.9016019294848561, + 0.8000663161346, + 0.8140755309546854 + ], + "peak_layer": 23, + "peak_normalized": 2.209732114295895, + "population_cv": 0.6223339325251772, + "rest_mean": 9.14960031439949e-06, + "spike_contrast": 2.6403940014799714, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4158549786079674e-05, + "top_five_layers": [ + 23, + 22, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1081451702921186e-05, + 4.712421286967583e-06, + 1.8216123862657696e-06, + 5.158842213859316e-06, + 2.9472603273461573e-06, + 2.442421418891172e-06, + 3.024953457497759e-06, + 5.4851252571097575e-06, + 9.570384463586379e-06, + 7.20218667993322e-06, + 7.066950274747796e-06, + 6.099000984249869e-06, + 6.54181940262788e-06, + 6.533490250149043e-06, + 6.008318905514898e-06, + 6.038619176251814e-06, + 1.4476394426310435e-05, + 1.4826539882051293e-05, + 1.4858099348202813e-05, + 1.523430910310708e-05, + 2.4260674763354473e-05, + 2.5299988919869065e-05, + 2.5400315280421637e-05, + 2.3509439415647648e-05, + 2.232233055110555e-05, + 2.0367358956718817e-05, + 1.7763433788786642e-05, + 1.7147945982287638e-05, + 1.171242638520198e-05, + 1.0363687579229008e-05, + 9.196561222779565e-06, + 9.357593626191374e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.844370848402832e-05, + "first_to_last_ratio": 0.3140073886393369, + "imbalance_abs_log_ratio": 1.158338762652201, + "last_quartile_mean": 5.8736542996484786e-05, + "mean": 4.581337286424514e-05, + "normalized": [ + 0.983221977853255, + 0.4150243087528777, + 0.15746378077315915, + 0.45454211061157807, + 0.2523150793847112, + 0.21218614843533373, + 0.2635465417903775, + 0.4823681378821299, + 0.8343480408888806, + 0.6208313208522753, + 0.6128114426671225, + 0.5334391065031241, + 0.5690611192260892, + 0.5657304418592991, + 0.525481722937821, + 0.5279767829162021, + 1.2652030229954638, + 1.2822561927565206, + 1.2949405359606927, + 1.3366411232201898, + 2.107649239013807, + 2.207816441558945, + 2.189332645277925, + 2.049149351497267, + 1.941094009743254, + 1.7640868530853437, + 1.5394721367578272, + 1.480393360068856, + 1.0182100711985076, + 0.90044042915216, + 0.8028775926136404, + 0.8100889317653636 + ], + "peak_layer": 22, + "peak_normalized": 2.207816441558945, + "population_cv": 0.6207014414064956, + "rest_mean": 3.648943235627181e-05, + "spike_contrast": 2.635356194867543, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.616265160730108e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.504471507971175e-05, + 1.9013663404621184e-05, + 7.213946901174495e-06, + 2.0824107195949182e-05, + 1.1559404811123386e-05, + 9.72096313489601e-06, + 1.2073955986124929e-05, + 2.2098911358625628e-05, + 3.8224297895794734e-05, + 2.8442376788007095e-05, + 2.8074959118384868e-05, + 2.44386446865974e-05, + 2.6070609237649478e-05, + 2.5918019673554227e-05, + 2.4074090106296353e-05, + 2.4188397219404578e-05, + 5.79632178414613e-05, + 5.874448106624186e-05, + 5.9325593610992655e-05, + 6.123603816376999e-05, + 9.655852045398206e-05, + 0.00010114751785295084, + 0.00010030071280198172, + 9.387844329467043e-05, + 8.892806363292038e-05, + 8.081876876531169e-05, + 7.052841101540253e-05, + 6.78218129905872e-05, + 4.6647637645946816e-05, + 4.125221312278882e-05, + 3.678253051475622e-05, + 3.711290628416464e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.016356064879801124, + "first_to_last_ratio": 0.3140073792809234, + "imbalance_abs_log_ratio": 1.1583387924553648, + "last_quartile_mean": 0.05208815447986126, + "mean": 0.040627757538459264, + "normalized": [ + 0.9832219159051098, + 0.4150243255028214, + 0.15746378595866897, + 0.4545420901919184, + 0.25231507934570285, + 0.212186133434977, + 0.26354653718783205, + 0.48236814287311225, + 0.8343480548464586, + 0.6208312921874214, + 0.6128114355572287, + 0.5334391169468627, + 0.5690611113916649, + 0.5657304465211455, + 0.5254817034060001, + 0.5279767678755438, + 1.2652029871996548, + 1.282256277419591, + 1.2949405686374758, + 1.3366410906562367, + 2.10764920469435, + 2.2078163557568278, + 2.1893326511557634, + 2.049149474394762, + 1.94109395479328, + 1.7640869604276401, + 1.5394721077288682, + 1.480393334684612, + 1.0182101267471966, + 0.9004404430197213, + 0.8028775630631926, + 0.8100889604883591 + ], + "peak_layer": 22, + "peak_normalized": 2.2078163557568278, + "population_cv": 0.6207014436535331, + "rest_mean": 0.03235919385527571, + "spike_contrast": 2.6353561775689003, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08527800142765045, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03994610160589218, + 0.016861507669091225, + 0.006397400517016649, + 0.018467025831341743, + 0.010250995866954327, + 0.00862064678221941, + 0.010707304812967777, + 0.019597535952925682, + 0.03389769047498703, + 0.025222983211278915, + 0.024897154420614243, + 0.021672435104846954, + 0.023119676858186722, + 0.02298435941338539, + 0.021349143236875534, + 0.02145051211118698, + 0.05140236020088196, + 0.0520951971411705, + 0.05261053144931793, + 0.05430473014712334, + 0.08562906086444855, + 0.08969862759113312, + 0.08894767612218857, + 0.08325234800577164, + 0.07886229455471039, + 0.07167089730501175, + 0.0625452995300293, + 0.0601450614631176, + 0.04136759415268898, + 0.03658287599682808, + 0.032619114965200424, + 0.03291209787130356 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.079009859239477e-06, + "first_to_last_ratio": 0.27720812628615055, + "imbalance_abs_log_ratio": 1.2829866964894792, + "last_quartile_mean": 3.892417851147911e-06, + "mean": 2.952973377823298e-06, + "normalized": [ + 0.9923364246284452, + 0.41483623106591405, + 0.15635970486874604, + 0.4924804983623727, + 0.23205015211521984, + 0.1632731916136273, + 0.19761849511604435, + 0.2742273866813069, + 0.647135722646919, + 0.5558013269361449, + 0.5842350348266575, + 0.48137427600621085, + 0.5660568730264243, + 0.5692692385054805, + 0.5090179437241957, + 0.5218754519537611, + 1.2935636777127382, + 1.3103201829927758, + 1.3424099585692733, + 1.3728171691781692, + 2.133225965099078, + 2.191548108133119, + 2.305749086881416, + 2.147337119746363, + 1.9823368774471204, + 1.787241267304806, + 1.6332376019052974, + 1.5389626829092304, + 1.0508228818507006, + 0.9064559278972918, + 0.8245776443221877, + 0.821445895972963 + ], + "peak_layer": 23, + "peak_normalized": 2.305749086881416, + "population_cv": 0.6589200861025274, + "rest_mean": 2.3229841609143612e-06, + "spike_contrast": 2.735668738537661, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.354915149131557e-06, + "top_five_layers": [ + 23, + 22, + 24, + 21, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.9303430437721545e-06, + 1.2250003464941983e-06, + 4.6172604584171495e-07, + 1.4542818007612368e-06, + 6.852379215160909e-07, + 4.821413881472836e-07, + 5.835621550431824e-07, + 8.097861723399546e-07, + 1.9109745608147932e-06, + 1.641266521801299e-06, + 1.725230504234787e-06, + 1.421485421815305e-06, + 1.671550876380934e-06, + 1.6810369061204256e-06, + 1.5031164366519079e-06, + 1.5410843161589582e-06, + 3.819859102804912e-06, + 3.869340616802219e-06, + 3.964100869779941e-06, + 4.053892553201877e-06, + 6.299359483818989e-06, + 6.4715832195361145e-06, + 6.8088156695012e-06, + 6.341029347822769e-06, + 5.853788024978712e-06, + 5.277675882098265e-06, + 4.822907158086309e-06, + 4.544515832094476e-06, + 3.1030519949126756e-06, + 2.6767402232508175e-06, + 2.434955831631669e-06, + 2.4257078621303663e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.3062004313724174e-05, + "first_to_last_ratio": 0.3462796036926242, + "imbalance_abs_log_ratio": 1.0605087271060973, + "last_quartile_mean": 3.7720975115007604e-05, + "mean": 3.0910587128119005e-05, + "normalized": [ + 1.044258273718076, + 0.4193917891961965, + 0.1513346310446105, + 0.48311870475518875, + 0.26867569695647586, + 0.22205928542141973, + 0.2887419879047208, + 0.503010044141552, + 0.8687652035040012, + 0.6835300441309964, + 0.6794437217438256, + 0.5876684752752086, + 0.6255961849481598, + 0.6155440908016775, + 0.5767443891193211, + 0.585169369008815, + 1.2725148048203212, + 1.2730810288123915, + 1.2985548706938432, + 1.36266128932131, + 2.142396449976532, + 2.2433771027099114, + 2.089704311420349, + 1.9520549176365862, + 1.910920056896515, + 1.6930503405879038, + 1.4702439655082709, + 1.4111274031667742, + 0.9417698480554898, + 0.8189675040741922, + 0.7471564493409493, + 0.7693677653084159 + ], + "peak_layer": 22, + "peak_normalized": 2.2433771027099114, + "population_cv": 0.5973036170926191, + "rest_mean": 2.4798931143002982e-05, + "spike_contrast": 2.577269523399549, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.391352944774553e-05, + "top_five_layers": [ + 22, + 21, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.227863635402173e-05, + 1.2963646440766752e-05, + 4.677842298406176e-06, + 1.4933482816559263e-05, + 8.304923539981246e-06, + 6.863982889626641e-06, + 8.925184374675155e-06, + 1.554833579575643e-05, + 2.6854042516788468e-05, + 2.112831498379819e-05, + 2.1002004359615967e-05, + 1.8165177607443184e-05, + 1.9337545381858945e-05, + 1.902682924992405e-05, + 1.7827507690526545e-05, + 1.8087928765453398e-05, + 3.933417974621989e-05, + 3.9351682062260807e-05, + 4.013909347122535e-05, + 4.212066050968133e-05, + 6.622273212997243e-05, + 6.934410339454189e-05, + 6.459398719016463e-05, + 6.033916361047886e-05, + 5.906766091356985e-05, + 5.233318006503396e-05, + 4.54461041954346e-05, + 4.361877654446289e-05, + 2.9110658942954615e-05, + 2.5314766389783472e-05, + 2.3095044525689445e-05, + 2.3781609343132004e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.172314918259872e-06, + "first_to_last_ratio": 0.3673647725037251, + "imbalance_abs_log_ratio": 1.0013999940215956, + "last_quartile_mean": 2.4967867375380592e-05, + "mean": 2.0697781202727583e-05, + "normalized": [ + 1.111303541294376, + 0.4345514774860271, + 0.14873123246072173, + 0.515966262987765, + 0.28568711280472064, + 0.23243024785129948, + 0.309127325753849, + 0.5074388141476497, + 0.8970449242709069, + 0.738367224211955, + 0.7462688121005764, + 0.642179562850806, + 0.6755482383061984, + 0.6507702437283572, + 0.6234304508304926, + 0.6319968759160601, + 1.225562031333336, + 1.197045832524259, + 1.2456604157243978, + 1.337071883147354, + 2.1461057208048535, + 2.2767618292912672, + 1.9401568217351242, + 1.8303413600277973, + 1.8918927171524897, + 1.644370796755471, + 1.4630724614723636, + 1.4103465215309783, + 0.9354546739436189, + 0.7941395510132133, + 0.7495417621189062, + 0.7616332744228086 + ], + "peak_layer": 22, + "peak_normalized": 2.2767618292912672, + "population_cv": 0.5708172615963157, + "rest_mean": 1.6799500953210452e-05, + "spike_contrast": 2.485103257912062, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.174849455012009e-05, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.3001517547527328e-05, + 8.994251402327791e-06, + 3.0784065074840328e-06, + 1.067935681930976e-05, + 5.9130893532710616e-06, + 4.810790414921939e-06, + 6.3982497522374615e-06, + 1.05028575489996e-05, + 1.8566839571576566e-05, + 1.5282563254004344e-05, + 1.5446108591277152e-05, + 1.329169208474923e-05, + 1.3982349628349766e-05, + 1.346950011793524e-05, + 1.2903627066407353e-05, + 1.3080933058517985e-05, + 2.5366414774907753e-05, + 2.4776192731224e-05, + 2.578240673756227e-05, + 2.7674421289702877e-05, + 4.441962664714083e-05, + 4.7123918193392456e-05, + 4.015694139525294e-05, + 3.788400499615818e-05, + 3.9157981518656015e-05, + 3.403482696739957e-05, + 3.0282353691291064e-05, + 2.9191043722676113e-05, + 1.9361836166353896e-05, + 1.643692667130381e-05, + 1.5513851394644007e-05, + 1.5764118870720267e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.607548808304273e-05, + "first_to_last_ratio": 0.29287710552991697, + "imbalance_abs_log_ratio": 1.228002193004033, + "last_quartile_mean": 0.00012317619712121086, + "mean": 9.582550958953107e-05, + "normalized": [ + 0.9271980053828993, + 0.37240108217726614, + 0.14097411813076574, + 0.43019423957314057, + 0.23484470025293644, + 0.20099018929480036, + 0.24920840443437586, + 0.45595414361828196, + 0.7897161290201836, + 0.5997815779888516, + 0.573550343218901, + 0.5091114036209083, + 0.5450578599083878, + 0.5437232517489079, + 0.5034351618175156, + 0.5173076228323285, + 1.3237975068923464, + 1.3288440675518258, + 1.3248895211467493, + 1.3241782160754627, + 2.1143087668589984, + 2.2312871682409448, + 2.2963419698621235, + 2.179530309079723, + 1.935901488532883, + 1.8140845725947157, + 1.6046775137322655, + 1.5564323816789656, + 0.9821558132941463, + 0.8522601786353591, + 0.771975946680249, + 0.7658863461227908 + ], + "peak_layer": 23, + "peak_normalized": 2.2963419698621235, + "population_cv": 0.6534359520074113, + "rest_mean": 7.539206938047385e-05, + "spike_contrast": 2.734585857804243, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020616608671844005, + "top_five_layers": [ + 23, + 22, + 24, + 21, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.884922135621309e-05, + 3.568552347132936e-05, + 1.3508916708815377e-05, + 4.1223582229577005e-05, + 2.250411307613831e-05, + 1.9259987311670557e-05, + 2.388052234891802e-05, + 4.36920381616801e-05, + 7.567495049443096e-05, + 5.747437535319477e-05, + 5.496075391420163e-05, + 4.878585968981497e-05, + 5.2230447181500494e-05, + 5.210255767451599e-05, + 4.824193092645146e-05, + 4.957126657245681e-05, + 0.00012685357069130987, + 0.00012733715993817896, + 0.00012695821351371706, + 0.00012689005234278738, + 0.00020260471501387656, + 0.00021381422993727028, + 0.0002200481394538656, + 0.0002088546025333926, + 0.00018550874665379524, + 0.0001738355786073953, + 0.00015376904048025608, + 0.0001491459261160344, + 9.41155813052319e-05, + 8.166826592059806e-05, + 7.397498848149553e-05, + 7.33914494048804e-05 + ] + } + } + } + } + } + }, + "step": 2000 + }, + { + "modes": { + "learned": { + "bits_per_byte": 1.9147359946947653, + "forward": { + "activation_sha256": { + "attention_branch_output": "729d18a894ede5429f33682ebed5494f0e192bf108efbc8a123308e96fadab4d", + "mlp_branch_output": "0c4b69b5a62d474637d9a6156b8452d836a142d489dd60b2de0cdb09db585af9", + "post_attention_state": "4cec366fa91114ffbf2b3855220297ceecccb362041740eb1c3b94ecdc658134", + "post_mlp_state": "4fbc436930b501f120196fb3bf6ab490fc20b93e2a3a7db48af6d653a1fa07c9", + "pre_attention_input": "e4d8ebf5b6b3ad968bf338e0327e2eabdbae498ab18a1157cbd942424ccea56f", + "pre_mlp_input": "6920ec3aaf9b121de9e3648519271c2cd5d5a0dcd243425164f6c664720841a8" + }, + "logits_sha256": "2ffb0289fec25aa61039516a8b1b69c57c89a6b65a12db0380cd5800a2c4f358", + "loss_nats": 1.3271938562393188, + "mixer_summary_sha256": "8d41a981bb22af8def07f1f5f21c359a1017aaf93614b906571e789e41229366" + }, + "loss_nats": 1.3271938562393188, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22879935801029205, + "group": 1, + "latest_source_mass": 0.06730473041534424, + "layer": 1, + "max_source_mass": 0.9326952695846558, + "mixer_index": 1, + "normalized_entropy": 0.3300876991600269, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9326952695846558, + "median": 0.9494354128837585, + "p05": 0.83271324634552, + "p95": 0.980820894241333 + }, + { + "label": "current_group_partial", + "mean": 0.06730473041534424, + "median": 0.05056454986333847, + "p05": 0.019179098308086395, + "p95": 0.1672867089509964 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5610548257827759, + "group": 1, + "latest_source_mass": 0.3240153193473816, + "layer": 2, + "max_source_mass": 0.6759846806526184, + "mixer_index": 2, + "normalized_entropy": 0.8094310148236321, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6759846806526184, + "median": 0.6873152256011963, + "p05": 0.3489363193511963, + "p95": 0.9112191796302795 + }, + { + "label": "current_group_partial", + "mean": 0.3240153193473816, + "median": 0.3126847445964813, + "p05": 0.08878080546855927, + "p95": 0.6510636806488037 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.43457120656967163, + "group": 1, + "latest_source_mass": 0.1704302430152893, + "layer": 2, + "max_source_mass": 0.8295696973800659, + "mixer_index": 3, + "normalized_entropy": 0.6269537246311986, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8295696973800659, + "median": 0.8586000800132751, + "p05": 0.6682394742965698, + "p95": 0.9325915575027466 + }, + { + "label": "current_group_partial", + "mean": 0.1704302430152893, + "median": 0.14139993488788605, + "p05": 0.06740844249725342, + "p95": 0.3317604660987854 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6696063280105591, + "group": 1, + "latest_source_mass": 0.45400765538215637, + "layer": 3, + "max_source_mass": 0.545992374420166, + "mixer_index": 4, + "normalized_entropy": 0.9660377287687022, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.545992374420166, + "median": 0.5355842709541321, + "p05": 0.40380191802978516, + "p95": 0.7150261998176575 + }, + { + "label": "current_group_partial", + "mean": 0.45400765538215637, + "median": 0.4644157290458679, + "p05": 0.2849738299846649, + "p95": 0.5961980819702148 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6690812110900879, + "group": 1, + "latest_source_mass": 0.479726105928421, + "layer": 3, + "max_source_mass": 0.5202739238739014, + "mixer_index": 5, + "normalized_entropy": 0.9652801451916515, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5202739238739014, + "median": 0.5058591365814209, + "p05": 0.3796083331108093, + "p95": 0.7244806289672852 + }, + { + "label": "current_group_partial", + "mean": 0.479726105928421, + "median": 0.4941408932209015, + "p05": 0.2755194306373596, + "p95": 0.6203916072845459 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6865538358688354, + "group": 1, + "latest_source_mass": 0.4638097584247589, + "layer": 4, + "max_source_mass": 0.5361902713775635, + "mixer_index": 6, + "normalized_entropy": 0.9904878143112643, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5361902713775635, + "median": 0.5348554849624634, + "p05": 0.4683650732040405, + "p95": 0.6130049228668213 + }, + { + "label": "current_group_partial", + "mean": 0.4638097584247589, + "median": 0.4651445150375366, + "p05": 0.3869951069355011, + "p95": 0.5316349267959595 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4152516722679138, + "group": 1, + "latest_source_mass": 0.15472738444805145, + "layer": 4, + "max_source_mass": 0.8452726602554321, + "mixer_index": 7, + "normalized_entropy": 0.5990815283017684, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8452726602554321, + "median": 0.8536937236785889, + "p05": 0.7318258285522461, + "p95": 0.9432691931724548 + }, + { + "label": "current_group_partial", + "mean": 0.15472738444805145, + "median": 0.14630627632141113, + "p05": 0.05673077702522278, + "p95": 0.26817411184310913 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6853918433189392, + "group": 2, + "latest_source_mass": 0.4779263138771057, + "layer": 5, + "max_source_mass": 0.5220736861228943, + "mixer_index": 8, + "normalized_entropy": 0.9888114134219791, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5220736861228943, + "median": 0.5176622867584229, + "p05": 0.4356726408004761, + "p95": 0.6261813640594482 + }, + { + "label": "completed_group_1", + "mean": 0.4779263138771057, + "median": 0.4823376536369324, + "p05": 0.37381869554519653, + "p95": 0.5643272995948792 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0265861749649048, + "group": 2, + "latest_source_mass": 0.4458175599575043, + "layer": 5, + "max_source_mass": 0.4458175599575043, + "mixer_index": 9, + "normalized_entropy": 0.934439005965858, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3228098154067993, + "median": 0.3112887740135193, + "p05": 0.17013706266880035, + "p95": 0.5363445281982422 + }, + { + "label": "completed_group_1", + "mean": 0.23137259483337402, + "median": 0.23920756578445435, + "p05": 0.15646742284297943, + "p95": 0.27411726117134094 + }, + { + "label": "current_group_partial", + "mean": 0.4458175599575043, + "median": 0.46458810567855835, + "p05": 0.26785901188850403, + "p95": 0.584333062171936 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.066486120223999, + "group": 2, + "latest_source_mass": 0.44416186213493347, + "layer": 6, + "max_source_mass": 0.44416186213493347, + "mixer_index": 10, + "normalized_entropy": 0.9707575012809491, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2909931242465973, + "median": 0.28486818075180054, + "p05": 0.24587924778461456, + "p95": 0.3592495620250702 + }, + { + "label": "completed_group_1", + "mean": 0.26484501361846924, + "median": 0.2616615891456604, + "p05": 0.23822297155857086, + "p95": 0.30263856053352356 + }, + { + "label": "current_group_partial", + "mean": 0.44416186213493347, + "median": 0.4583350419998169, + "p05": 0.3442475199699402, + "p95": 0.4912717938423157 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.8915231227874756, + "group": 2, + "latest_source_mass": 0.6131048202514648, + "layer": 6, + "max_source_mass": 0.6131048202514648, + "mixer_index": 11, + "normalized_entropy": 0.8114993178060147, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21873247623443604, + "median": 0.1719968020915985, + "p05": 0.11714419722557068, + "p95": 0.4722311198711395 + }, + { + "label": "completed_group_1", + "mean": 0.16816270351409912, + "median": 0.16899234056472778, + "p05": 0.11237625032663345, + "p95": 0.20729371905326843 + }, + { + "label": "current_group_partial", + "mean": 0.6131048202514648, + "median": 0.6546929478645325, + "p05": 0.37714827060699463, + "p95": 0.7158147096633911 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0651111602783203, + "group": 2, + "latest_source_mass": 0.41314971446990967, + "layer": 7, + "max_source_mass": 0.41314971446990967, + "mixer_index": 12, + "normalized_entropy": 0.9695059588033517, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.363992840051651, + "median": 0.3703998625278473, + "p05": 0.312745600938797, + "p95": 0.4009946286678314 + }, + { + "label": "completed_group_1", + "mean": 0.22285741567611694, + "median": 0.219387024641037, + "p05": 0.19873447716236115, + "p95": 0.2561606168746948 + }, + { + "label": "current_group_partial", + "mean": 0.41314971446990967, + "median": 0.412320613861084, + "p05": 0.37329763174057007, + "p95": 0.450311541557312 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0005524158477783, + "group": 2, + "latest_source_mass": 0.49950775504112244, + "layer": 7, + "max_source_mass": 0.49950775504112244, + "mixer_index": 13, + "normalized_entropy": 0.9107420572008955, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31661832332611084, + "median": 0.29119664430618286, + "p05": 0.20734748244285583, + "p95": 0.5084977149963379 + }, + { + "label": "completed_group_1", + "mean": 0.18387392163276672, + "median": 0.18630169332027435, + "p05": 0.13286109268665314, + "p95": 0.22609204053878784 + }, + { + "label": "current_group_partial", + "mean": 0.49950775504112244, + "median": 0.5250610113143921, + "p05": 0.3313663601875305, + "p95": 0.5866644382476807 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0648009777069092, + "group": 2, + "latest_source_mass": 0.41697072982788086, + "layer": 8, + "max_source_mass": 0.41697072982788086, + "mixer_index": 14, + "normalized_entropy": 0.9692236184594373, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35599708557128906, + "median": 0.3621290922164917, + "p05": 0.286081463098526, + "p95": 0.4091617166996002 + }, + { + "label": "completed_group_1", + "mean": 0.2270321547985077, + "median": 0.2260679304599762, + "p05": 0.19767868518829346, + "p95": 0.2600743770599365 + }, + { + "label": "current_group_partial", + "mean": 0.41697072982788086, + "median": 0.41424113512039185, + "p05": 0.3673546314239502, + "p95": 0.4745914936065674 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0006463527679443, + "group": 2, + "latest_source_mass": 0.5162267684936523, + "layer": 8, + "max_source_mass": 0.5162267684936523, + "mixer_index": 15, + "normalized_entropy": 0.9108275622704591, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28650256991386414, + "median": 0.265026330947876, + "p05": 0.18542741239070892, + "p95": 0.454831063747406 + }, + { + "label": "completed_group_1", + "mean": 0.19727063179016113, + "median": 0.1941412389278412, + "p05": 0.14316384494304657, + "p95": 0.2586134374141693 + }, + { + "label": "current_group_partial", + "mean": 0.5162267684936523, + "median": 0.5309736728668213, + "p05": 0.40128540992736816, + "p95": 0.5870281457901001 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.072684407234192, + "group": 3, + "latest_source_mass": 0.34105706214904785, + "layer": 9, + "max_source_mass": 0.41332393884658813, + "mixer_index": 16, + "normalized_entropy": 0.9763994252555183, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.41332393884658813, + "median": 0.4128713607788086, + "p05": 0.3400208353996277, + "p95": 0.48814913630485535 + }, + { + "label": "completed_group_1", + "mean": 0.2456190288066864, + "median": 0.24588967859745026, + "p05": 0.21115674078464508, + "p95": 0.2809695601463318 + }, + { + "label": "completed_group_2", + "mean": 0.34105706214904785, + "median": 0.3410349190235138, + "p05": 0.29047802090644836, + "p95": 0.392678439617157 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.304518699645996, + "group": 3, + "latest_source_mass": 0.35304954648017883, + "layer": 9, + "max_source_mass": 0.35304954648017883, + "mixer_index": 17, + "normalized_entropy": 0.9410113293630988, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2721603810787201, + "median": 0.23975250124931335, + "p05": 0.16686967015266418, + "p95": 0.4960964620113373 + }, + { + "label": "completed_group_1", + "mean": 0.12663133442401886, + "median": 0.12792861461639404, + "p05": 0.07451300323009491, + "p95": 0.16275638341903687 + }, + { + "label": "completed_group_2", + "mean": 0.2481587529182434, + "median": 0.2613898515701294, + "p05": 0.14526164531707764, + "p95": 0.30340635776519775 + }, + { + "label": "current_group_partial", + "mean": 0.35304954648017883, + "median": 0.35470759868621826, + "p05": 0.26913243532180786, + "p95": 0.42660659551620483 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3646502494812012, + "group": 3, + "latest_source_mass": 0.2896400988101959, + "layer": 10, + "max_source_mass": 0.29890745878219604, + "mixer_index": 18, + "normalized_entropy": 0.9843870737372079, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29890745878219604, + "median": 0.29588401317596436, + "p05": 0.25596654415130615, + "p95": 0.35064268112182617 + }, + { + "label": "completed_group_1", + "mean": 0.18950368463993073, + "median": 0.18751488626003265, + "p05": 0.1636066436767578, + "p95": 0.2226080447435379 + }, + { + "label": "completed_group_2", + "mean": 0.2219487726688385, + "median": 0.22622263431549072, + "p05": 0.17594511806964874, + "p95": 0.2489231377840042 + }, + { + "label": "current_group_partial", + "mean": 0.2896400988101959, + "median": 0.2882089614868164, + "p05": 0.2503519654273987, + "p95": 0.33445414900779724 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.331923484802246, + "group": 3, + "latest_source_mass": 0.3192984461784363, + "layer": 10, + "max_source_mass": 0.3192984461784363, + "mixer_index": 19, + "normalized_entropy": 0.9607797031838735, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31035515666007996, + "median": 0.31498879194259644, + "p05": 0.23021729290485382, + "p95": 0.40513044595718384 + }, + { + "label": "completed_group_1", + "mean": 0.14580611884593964, + "median": 0.14570140838623047, + "p05": 0.11249937862157822, + "p95": 0.1775612086057663 + }, + { + "label": "completed_group_2", + "mean": 0.22454024851322174, + "median": 0.22962218523025513, + "p05": 0.15898653864860535, + "p95": 0.2764281928539276 + }, + { + "label": "current_group_partial", + "mean": 0.3192984461784363, + "median": 0.31587594747543335, + "p05": 0.2565876841545105, + "p95": 0.3967522978782654 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3639233112335205, + "group": 3, + "latest_source_mass": 0.29643675684928894, + "layer": 11, + "max_source_mass": 0.29643675684928894, + "mixer_index": 20, + "normalized_entropy": 0.9838626986347272, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2952677607536316, + "median": 0.2923831343650818, + "p05": 0.24788890779018402, + "p95": 0.35271933674812317 + }, + { + "label": "completed_group_1", + "mean": 0.19548878073692322, + "median": 0.19292637705802917, + "p05": 0.1706344485282898, + "p95": 0.22774341702461243 + }, + { + "label": "completed_group_2", + "mean": 0.21280676126480103, + "median": 0.21625755727291107, + "p05": 0.1678445041179657, + "p95": 0.24309992790222168 + }, + { + "label": "current_group_partial", + "mean": 0.29643675684928894, + "median": 0.2966676950454712, + "p05": 0.25015711784362793, + "p95": 0.34584304690361023 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2764989137649536, + "group": 3, + "latest_source_mass": 0.3178994059562683, + "layer": 11, + "max_source_mass": 0.3656603693962097, + "mixer_index": 21, + "normalized_entropy": 0.9207993262944236, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3656603693962097, + "median": 0.3675394058227539, + "p05": 0.19079747796058655, + "p95": 0.5280343294143677 + }, + { + "label": "completed_group_1", + "mean": 0.14041098952293396, + "median": 0.1384333223104477, + "p05": 0.09619437158107758, + "p95": 0.18906019628047943 + }, + { + "label": "completed_group_2", + "mean": 0.17602922022342682, + "median": 0.17342498898506165, + "p05": 0.10058781504631042, + "p95": 0.2688048481941223 + }, + { + "label": "current_group_partial", + "mean": 0.3178994059562683, + "median": 0.3060161769390106, + "p05": 0.2126653641462326, + "p95": 0.4575258493423462 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.364103078842163, + "group": 3, + "latest_source_mass": 0.30165016651153564, + "layer": 12, + "max_source_mass": 0.30165016651153564, + "mixer_index": 22, + "normalized_entropy": 0.9839923735534777, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27470695972442627, + "median": 0.28049176931381226, + "p05": 0.20271319150924683, + "p95": 0.3434835374355316 + }, + { + "label": "completed_group_1", + "mean": 0.18989428877830505, + "median": 0.18842613697052002, + "p05": 0.1683974415063858, + "p95": 0.21698811650276184 + }, + { + "label": "completed_group_2", + "mean": 0.23374855518341064, + "median": 0.23892146348953247, + "p05": 0.17832231521606445, + "p95": 0.2664119601249695 + }, + { + "label": "current_group_partial", + "mean": 0.30165016651153564, + "median": 0.29482388496398926, + "p05": 0.25988030433654785, + "p95": 0.36617302894592285 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3429455757141113, + "group": 3, + "latest_source_mass": 0.3075411319732666, + "layer": 12, + "max_source_mass": 0.3075411319732666, + "mixer_index": 23, + "normalized_entropy": 0.9687304611332612, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2830207943916321, + "median": 0.28728431463241577, + "p05": 0.15201547741889954, + "p95": 0.4170074760913849 + }, + { + "label": "completed_group_1", + "mean": 0.17677830159664154, + "median": 0.17705857753753662, + "p05": 0.1337193250656128, + "p95": 0.21641075611114502 + }, + { + "label": "completed_group_2", + "mean": 0.23265975713729858, + "median": 0.2354048639535904, + "p05": 0.1727992296218872, + "p95": 0.2813771963119507 + }, + { + "label": "current_group_partial", + "mean": 0.3075411319732666, + "median": 0.29946592450141907, + "p05": 0.2323879599571228, + "p95": 0.41285809874534607 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3732160329818726, + "group": 4, + "latest_source_mass": 0.2892792224884033, + "layer": 13, + "max_source_mass": 0.2892792224884033, + "mixer_index": 24, + "normalized_entropy": 0.9905659804260815, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2642192244529724, + "median": 0.26541125774383545, + "p05": 0.22824814915657043, + "p95": 0.2958231270313263 + }, + { + "label": "completed_group_1", + "mean": 0.22012627124786377, + "median": 0.21323975920677185, + "p05": 0.19119325280189514, + "p95": 0.27720844745635986 + }, + { + "label": "completed_group_2", + "mean": 0.22637531161308289, + "median": 0.2285899519920349, + "p05": 0.193231463432312, + "p95": 0.2564765214920044 + }, + { + "label": "completed_group_3", + "mean": 0.2892792224884033, + "median": 0.2939226031303406, + "p05": 0.21020042896270752, + "p95": 0.35712480545043945 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5599325895309448, + "group": 4, + "latest_source_mass": 0.2374764084815979, + "layer": 13, + "max_source_mass": 0.26679670810699463, + "mixer_index": 25, + "normalized_entropy": 0.9692406134336154, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18314200639724731, + "median": 0.17654985189437866, + "p05": 0.09326011687517166, + "p95": 0.32041969895362854 + }, + { + "label": "completed_group_1", + "mean": 0.13039523363113403, + "median": 0.1256764680147171, + "p05": 0.1007242202758789, + "p95": 0.17066726088523865 + }, + { + "label": "completed_group_2", + "mean": 0.18218961358070374, + "median": 0.18480567634105682, + "p05": 0.1226210743188858, + "p95": 0.23115605115890503 + }, + { + "label": "completed_group_3", + "mean": 0.26679670810699463, + "median": 0.2663910388946533, + "p05": 0.20957797765731812, + "p95": 0.3251228928565979 + }, + { + "label": "current_group_partial", + "mean": 0.2374764084815979, + "median": 0.23747703433036804, + "p05": 0.1834154576063156, + "p95": 0.2942982316017151 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5772080421447754, + "group": 4, + "latest_source_mass": 0.2955484390258789, + "layer": 14, + "max_source_mass": 0.2955484390258789, + "mixer_index": 26, + "normalized_entropy": 0.9799744556529175, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19838765263557434, + "median": 0.1983858346939087, + "p05": 0.1743457317352295, + "p95": 0.22720122337341309 + }, + { + "label": "completed_group_1", + "mean": 0.18937475979328156, + "median": 0.18892334401607513, + "p05": 0.17057949304580688, + "p95": 0.21014319360256195 + }, + { + "label": "completed_group_2", + "mean": 0.15325719118118286, + "median": 0.15288054943084717, + "p05": 0.13430020213127136, + "p95": 0.17301824688911438 + }, + { + "label": "completed_group_3", + "mean": 0.16343194246292114, + "median": 0.1631144881248474, + "p05": 0.14038164913654327, + "p95": 0.18891242146492004 + }, + { + "label": "current_group_partial", + "mean": 0.2955484390258789, + "median": 0.2958310842514038, + "p05": 0.2728894054889679, + "p95": 0.31609299778938293 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5662866830825806, + "group": 4, + "latest_source_mass": 0.2633046805858612, + "layer": 14, + "max_source_mass": 0.2633046805858612, + "mixer_index": 27, + "normalized_entropy": 0.9731886337347067, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22973918914794922, + "median": 0.22824904322624207, + "p05": 0.15654821693897247, + "p95": 0.29926687479019165 + }, + { + "label": "completed_group_1", + "mean": 0.12454323470592499, + "median": 0.12315000593662262, + "p05": 0.0935351550579071, + "p95": 0.15979833900928497 + }, + { + "label": "completed_group_2", + "mean": 0.16623356938362122, + "median": 0.1699516773223877, + "p05": 0.1137569397687912, + "p95": 0.20656129717826843 + }, + { + "label": "completed_group_3", + "mean": 0.21617934107780457, + "median": 0.21567070484161377, + "p05": 0.1733308732509613, + "p95": 0.2610541582107544 + }, + { + "label": "current_group_partial", + "mean": 0.2633046805858612, + "median": 0.2623734474182129, + "p05": 0.21259599924087524, + "p95": 0.3165385127067566 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5421115159988403, + "group": 4, + "latest_source_mass": 0.3367220163345337, + "layer": 15, + "max_source_mass": 0.3367220163345337, + "mixer_index": 28, + "normalized_entropy": 0.9581677578767632, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2021060585975647, + "median": 0.2054135799407959, + "p05": 0.16289709508419037, + "p95": 0.24026834964752197 + }, + { + "label": "completed_group_1", + "mean": 0.1677371859550476, + "median": 0.1653897762298584, + "p05": 0.14229144155979156, + "p95": 0.20148998498916626 + }, + { + "label": "completed_group_2", + "mean": 0.11891373246908188, + "median": 0.11754471063613892, + "p05": 0.10472346842288971, + "p95": 0.13705110549926758 + }, + { + "label": "completed_group_3", + "mean": 0.17452099919319153, + "median": 0.17379853129386902, + "p05": 0.1422983705997467, + "p95": 0.21011367440223694 + }, + { + "label": "current_group_partial", + "mean": 0.3367220163345337, + "median": 0.33760347962379456, + "p05": 0.29649966955184937, + "p95": 0.3747695982456207 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.564535140991211, + "group": 4, + "latest_source_mass": 0.25831347703933716, + "layer": 15, + "max_source_mass": 0.25831347703933716, + "mixer_index": 29, + "normalized_entropy": 0.9721003394439871, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22419840097427368, + "median": 0.20440472662448883, + "p05": 0.14446967840194702, + "p95": 0.3542698621749878 + }, + { + "label": "completed_group_1", + "mean": 0.12987476587295532, + "median": 0.12998200953006744, + "p05": 0.10328994691371918, + "p95": 0.15754634141921997 + }, + { + "label": "completed_group_2", + "mean": 0.1717071831226349, + "median": 0.17557166516780853, + "p05": 0.12064125388860703, + "p95": 0.21374036371707916 + }, + { + "label": "completed_group_3", + "mean": 0.21590615808963776, + "median": 0.2161775827407837, + "p05": 0.1625441014766693, + "p95": 0.2702201008796692 + }, + { + "label": "current_group_partial", + "mean": 0.25831347703933716, + "median": 0.2593906819820404, + "p05": 0.1950702667236328, + "p95": 0.3162909150123596 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5267750024795532, + "group": 4, + "latest_source_mass": 0.33957332372665405, + "layer": 16, + "max_source_mass": 0.33957332372665405, + "mixer_index": 30, + "normalized_entropy": 0.9486386462528844, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.215010404586792, + "median": 0.22210124135017395, + "p05": 0.1559298038482666, + "p95": 0.2592989504337311 + }, + { + "label": "completed_group_1", + "mean": 0.19819970428943634, + "median": 0.19619059562683105, + "p05": 0.1762683242559433, + "p95": 0.22793664038181305 + }, + { + "label": "completed_group_2", + "mean": 0.10917141288518906, + "median": 0.10697916150093079, + "p05": 0.09485334903001785, + "p95": 0.13162456452846527 + }, + { + "label": "completed_group_3", + "mean": 0.13804511725902557, + "median": 0.13598395884037018, + "p05": 0.1083223894238472, + "p95": 0.17449748516082764 + }, + { + "label": "current_group_partial", + "mean": 0.33957332372665405, + "median": 0.3368798792362213, + "p05": 0.2983384430408478, + "p95": 0.3911387622356415 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.556459903717041, + "group": 4, + "latest_source_mass": 0.2654845416545868, + "layer": 16, + "max_source_mass": 0.2654845416545868, + "mixer_index": 31, + "normalized_entropy": 0.9670829124206874, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2037975788116455, + "median": 0.1888633668422699, + "p05": 0.12331099063158035, + "p95": 0.3977951407432556 + }, + { + "label": "completed_group_1", + "mean": 0.12750470638275146, + "median": 0.12862002849578857, + "p05": 0.09542393684387207, + "p95": 0.1604449599981308 + }, + { + "label": "completed_group_2", + "mean": 0.1812792420387268, + "median": 0.18877843022346497, + "p05": 0.11129779368638992, + "p95": 0.22781090438365936 + }, + { + "label": "completed_group_3", + "mean": 0.22193393111228943, + "median": 0.22035372257232666, + "p05": 0.1620493084192276, + "p95": 0.2887580990791321 + }, + { + "label": "current_group_partial", + "mean": 0.2654845416545868, + "median": 0.2675127387046814, + "p05": 0.18758812546730042, + "p95": 0.33534300327301025 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5851528644561768, + "group": 5, + "latest_source_mass": 0.26602527499198914, + "layer": 17, + "max_source_mass": 0.26602527499198914, + "mixer_index": 32, + "normalized_entropy": 0.9849108513038598, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21840544044971466, + "median": 0.2208200842142105, + "p05": 0.17802955210208893, + "p95": 0.2532227635383606 + }, + { + "label": "completed_group_1", + "mean": 0.1923592984676361, + "median": 0.19034898281097412, + "p05": 0.17059199512004852, + "p95": 0.22078171372413635 + }, + { + "label": "completed_group_2", + "mean": 0.1442098617553711, + "median": 0.14471027255058289, + "p05": 0.13257130980491638, + "p95": 0.15438252687454224 + }, + { + "label": "completed_group_3", + "mean": 0.1790001392364502, + "median": 0.17785562574863434, + "p05": 0.15098975598812103, + "p95": 0.2121024876832962 + }, + { + "label": "completed_group_4", + "mean": 0.26602527499198914, + "median": 0.26503798365592957, + "p05": 0.23478449881076813, + "p95": 0.2991819977760315 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7158764600753784, + "group": 5, + "latest_source_mass": 0.3043471872806549, + "layer": 17, + "max_source_mass": 0.3043471872806549, + "mixer_index": 33, + "normalized_entropy": 0.9576488862172057, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13390132784843445, + "median": 0.12297401577234268, + "p05": 0.08219312131404877, + "p95": 0.2245325893163681 + }, + { + "label": "completed_group_1", + "mean": 0.10310791432857513, + "median": 0.10256334394216537, + "p05": 0.08573498576879501, + "p95": 0.122348852455616 + }, + { + "label": "completed_group_2", + "mean": 0.14190615713596344, + "median": 0.1444874107837677, + "p05": 0.10590264946222305, + "p95": 0.16855967044830322 + }, + { + "label": "completed_group_3", + "mean": 0.16341811418533325, + "median": 0.16260069608688354, + "p05": 0.13960278034210205, + "p95": 0.18985378742218018 + }, + { + "label": "completed_group_4", + "mean": 0.15331928431987762, + "median": 0.15326738357543945, + "p05": 0.12543924152851105, + "p95": 0.18207281827926636 + }, + { + "label": "current_group_partial", + "mean": 0.3043471872806549, + "median": 0.3056420385837555, + "p05": 0.2622474730014801, + "p95": 0.34185561537742615 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6410574913024902, + "group": 5, + "latest_source_mass": 0.3788188695907593, + "layer": 18, + "max_source_mass": 0.3788188695907593, + "mixer_index": 34, + "normalized_entropy": 0.9158916246774509, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14665581285953522, + "median": 0.15054050087928772, + "p05": 0.10432849824428558, + "p95": 0.18623962998390198 + }, + { + "label": "completed_group_1", + "mean": 0.12231245636940002, + "median": 0.12052746117115021, + "p05": 0.10417830944061279, + "p95": 0.14690443873405457 + }, + { + "label": "completed_group_2", + "mean": 0.08100371807813644, + "median": 0.08060519397258759, + "p05": 0.07169733196496964, + "p95": 0.09173959493637085 + }, + { + "label": "completed_group_3", + "mean": 0.11105544120073318, + "median": 0.109858438372612, + "p05": 0.09216424077749252, + "p95": 0.13396628201007843 + }, + { + "label": "completed_group_4", + "mean": 0.16015371680259705, + "median": 0.16027043759822845, + "p05": 0.13385453820228577, + "p95": 0.18605849146842957 + }, + { + "label": "current_group_partial", + "mean": 0.3788188695907593, + "median": 0.3796834945678711, + "p05": 0.325242280960083, + "p95": 0.43094635009765625 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6537573337554932, + "group": 5, + "latest_source_mass": 0.363794207572937, + "layer": 18, + "max_source_mass": 0.363794207572937, + "mixer_index": 35, + "normalized_entropy": 0.9229795417059984, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12217442691326141, + "median": 0.11129313707351685, + "p05": 0.06773082911968231, + "p95": 0.23439142107963562 + }, + { + "label": "completed_group_1", + "mean": 0.09171080589294434, + "median": 0.09032920002937317, + "p05": 0.07566524296998978, + "p95": 0.11195167899131775 + }, + { + "label": "completed_group_2", + "mean": 0.1348741054534912, + "median": 0.13740724325180054, + "p05": 0.09137580543756485, + "p95": 0.16799552738666534 + }, + { + "label": "completed_group_3", + "mean": 0.16064059734344482, + "median": 0.15993739664554596, + "p05": 0.12922504544258118, + "p95": 0.1927689015865326 + }, + { + "label": "completed_group_4", + "mean": 0.1268058568239212, + "median": 0.1260996162891388, + "p05": 0.09770616143941879, + "p95": 0.1587117612361908 + }, + { + "label": "current_group_partial", + "mean": 0.363794207572937, + "median": 0.36618149280548096, + "p05": 0.2940343916416168, + "p95": 0.4259193539619446 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6571812629699707, + "group": 5, + "latest_source_mass": 0.3666725754737854, + "layer": 19, + "max_source_mass": 0.3666725754737854, + "mixer_index": 36, + "normalized_entropy": 0.9248904729851576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15173450112342834, + "median": 0.15636742115020752, + "p05": 0.0919112116098404, + "p95": 0.20387139916419983 + }, + { + "label": "completed_group_1", + "mean": 0.14253422617912292, + "median": 0.1411309391260147, + "p05": 0.12321583926677704, + "p95": 0.16562411189079285 + }, + { + "label": "completed_group_2", + "mean": 0.09320533275604248, + "median": 0.09343434870243073, + "p05": 0.08043995499610901, + "p95": 0.10511597990989685 + }, + { + "label": "completed_group_3", + "mean": 0.10436245799064636, + "median": 0.10301217436790466, + "p05": 0.08662116527557373, + "p95": 0.1267353594303131 + }, + { + "label": "completed_group_4", + "mean": 0.1414909064769745, + "median": 0.1412898600101471, + "p05": 0.11758942902088165, + "p95": 0.1667475551366806 + }, + { + "label": "current_group_partial", + "mean": 0.3666725754737854, + "median": 0.3659632205963135, + "p05": 0.30865201354026794, + "p95": 0.429829478263855 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6046829223632812, + "group": 5, + "latest_source_mass": 0.40004032850265503, + "layer": 19, + "max_source_mass": 0.40004032850265503, + "mixer_index": 37, + "normalized_entropy": 0.8955905912162574, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12253663688898087, + "median": 0.10175791382789612, + "p05": 0.06238773465156555, + "p95": 0.2659108340740204 + }, + { + "label": "completed_group_1", + "mean": 0.08343374729156494, + "median": 0.08214844763278961, + "p05": 0.06734234094619751, + "p95": 0.10468925535678864 + }, + { + "label": "completed_group_2", + "mean": 0.1302124410867691, + "median": 0.13176652789115906, + "p05": 0.08769658207893372, + "p95": 0.167977973818779 + }, + { + "label": "completed_group_3", + "mean": 0.14439499378204346, + "median": 0.1426759511232376, + "p05": 0.10537415742874146, + "p95": 0.188823863863945 + }, + { + "label": "completed_group_4", + "mean": 0.11938183009624481, + "median": 0.11938531696796417, + "p05": 0.08638042211532593, + "p95": 0.15211962163448334 + }, + { + "label": "current_group_partial", + "mean": 0.40004032850265503, + "median": 0.40058958530426025, + "p05": 0.30273354053497314, + "p95": 0.49229907989501953 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7041404247283936, + "group": 5, + "latest_source_mass": 0.3168083429336548, + "layer": 20, + "max_source_mass": 0.3168083429336548, + "mixer_index": 38, + "normalized_entropy": 0.9510988801764724, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1659325659275055, + "median": 0.17128247022628784, + "p05": 0.10529995709657669, + "p95": 0.22297045588493347 + }, + { + "label": "completed_group_1", + "mean": 0.15505464375019073, + "median": 0.15309038758277893, + "p05": 0.1348525583744049, + "p95": 0.18119457364082336 + }, + { + "label": "completed_group_2", + "mean": 0.10219550132751465, + "median": 0.10201787948608398, + "p05": 0.08870653063058853, + "p95": 0.11639079451560974 + }, + { + "label": "completed_group_3", + "mean": 0.11347829550504684, + "median": 0.11260586977005005, + "p05": 0.09271398186683655, + "p95": 0.13755615055561066 + }, + { + "label": "completed_group_4", + "mean": 0.1465306580066681, + "median": 0.14615076780319214, + "p05": 0.1223587766289711, + "p95": 0.17159976065158844 + }, + { + "label": "current_group_partial", + "mean": 0.3168083429336548, + "median": 0.31445345282554626, + "p05": 0.2601228952407837, + "p95": 0.3804415166378021 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5808616876602173, + "group": 5, + "latest_source_mass": 0.40230536460876465, + "layer": 20, + "max_source_mass": 0.40230536460876465, + "mixer_index": 39, + "normalized_entropy": 0.882295706990906, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12713032960891724, + "median": 0.10352589935064316, + "p05": 0.03667577728629112, + "p95": 0.34593111276626587 + }, + { + "label": "completed_group_1", + "mean": 0.08526259660720825, + "median": 0.08378827571868896, + "p05": 0.0644085630774498, + "p95": 0.11194434762001038 + }, + { + "label": "completed_group_2", + "mean": 0.1294247955083847, + "median": 0.13107044994831085, + "p05": 0.08035385608673096, + "p95": 0.1728026270866394 + }, + { + "label": "completed_group_3", + "mean": 0.1489948183298111, + "median": 0.14636963605880737, + "p05": 0.10222268849611282, + "p95": 0.2024523913860321 + }, + { + "label": "completed_group_4", + "mean": 0.10688205808401108, + "median": 0.10608744621276855, + "p05": 0.0690753310918808, + "p95": 0.14802516996860504 + }, + { + "label": "current_group_partial", + "mean": 0.40230536460876465, + "median": 0.4073683023452759, + "p05": 0.26591286063194275, + "p95": 0.5286550521850586 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7305450439453125, + "group": 6, + "latest_source_mass": 0.25957077741622925, + "layer": 21, + "max_source_mass": 0.25957077741622925, + "mixer_index": 40, + "normalized_entropy": 0.9658355787514741, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1789149045944214, + "median": 0.18693166971206665, + "p05": 0.11472857743501663, + "p95": 0.22855065762996674 + }, + { + "label": "completed_group_1", + "mean": 0.17673785984516144, + "median": 0.17547538876533508, + "p05": 0.15558482706546783, + "p95": 0.20077811181545258 + }, + { + "label": "completed_group_2", + "mean": 0.09075746685266495, + "median": 0.090105801820755, + "p05": 0.07735756039619446, + "p95": 0.10616939514875412 + }, + { + "label": "completed_group_3", + "mean": 0.11627361923456192, + "median": 0.11497560888528824, + "p05": 0.09235620498657227, + "p95": 0.14503517746925354 + }, + { + "label": "completed_group_4", + "mean": 0.17774535715579987, + "median": 0.17699119448661804, + "p05": 0.15380802750587463, + "p95": 0.2042580246925354 + }, + { + "label": "completed_group_5", + "mean": 0.25957077741622925, + "median": 0.2581925392150879, + "p05": 0.20536908507347107, + "p95": 0.31587323546409607 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7513642311096191, + "group": 6, + "latest_source_mass": 0.3233836889266968, + "layer": 21, + "max_source_mass": 0.3233836889266968, + "mixer_index": 41, + "normalized_entropy": 0.9000231752529063, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09819091856479645, + "median": 0.08258114755153656, + "p05": 0.05219355598092079, + "p95": 0.2192530781030655 + }, + { + "label": "completed_group_1", + "mean": 0.06520213931798935, + "median": 0.06374529749155045, + "p05": 0.051295794546604156, + "p95": 0.08378917723894119 + }, + { + "label": "completed_group_2", + "mean": 0.091708242893219, + "median": 0.09206989407539368, + "p05": 0.06343865394592285, + "p95": 0.11800471693277359 + }, + { + "label": "completed_group_3", + "mean": 0.11103156208992004, + "median": 0.10943961143493652, + "p05": 0.0842345654964447, + "p95": 0.14296333491802216 + }, + { + "label": "completed_group_4", + "mean": 0.08388154208660126, + "median": 0.08271945267915726, + "p05": 0.06058824062347412, + "p95": 0.11240207403898239 + }, + { + "label": "completed_group_5", + "mean": 0.22660189867019653, + "median": 0.22274313867092133, + "p05": 0.14589408040046692, + "p95": 0.32426023483276367 + }, + { + "label": "current_group_partial", + "mean": 0.3233836889266968, + "median": 0.32124656438827515, + "p05": 0.23837368190288544, + "p95": 0.41599756479263306 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.766061782836914, + "group": 6, + "latest_source_mass": 0.36142250895500183, + "layer": 22, + "max_source_mass": 0.36142250895500183, + "mixer_index": 42, + "normalized_entropy": 0.9075762227224567, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12103402614593506, + "median": 0.12304593622684479, + "p05": 0.07208628952503204, + "p95": 0.17526710033416748 + }, + { + "label": "completed_group_1", + "mean": 0.1159343421459198, + "median": 0.11432909965515137, + "p05": 0.09892642498016357, + "p95": 0.13806137442588806 + }, + { + "label": "completed_group_2", + "mean": 0.06956179440021515, + "median": 0.06937768310308456, + "p05": 0.059766486287117004, + "p95": 0.08032344281673431 + }, + { + "label": "completed_group_3", + "mean": 0.07680170238018036, + "median": 0.07588358968496323, + "p05": 0.06175529211759567, + "p95": 0.09538999199867249 + }, + { + "label": "completed_group_4", + "mean": 0.10650800168514252, + "median": 0.10561707615852356, + "p05": 0.08491266518831253, + "p95": 0.13118049502372742 + }, + { + "label": "completed_group_5", + "mean": 0.1487376093864441, + "median": 0.1469218134880066, + "p05": 0.11073103547096252, + "p95": 0.19332705438137054 + }, + { + "label": "current_group_partial", + "mean": 0.36142250895500183, + "median": 0.36172276735305786, + "p05": 0.2831430435180664, + "p95": 0.43728554248809814 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.6586570739746094, + "group": 6, + "latest_source_mass": 0.4119114875793457, + "layer": 22, + "max_source_mass": 0.4119114875793457, + "mixer_index": 43, + "normalized_entropy": 0.8523811208754127, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09605097770690918, + "median": 0.08114366233348846, + "p05": 0.047440964728593826, + "p95": 0.2204016000032425 + }, + { + "label": "completed_group_1", + "mean": 0.06012085825204849, + "median": 0.05865291506052017, + "p05": 0.04616369307041168, + "p95": 0.07940929383039474 + }, + { + "label": "completed_group_2", + "mean": 0.08562932163476944, + "median": 0.08538836240768433, + "p05": 0.05786459147930145, + "p95": 0.11294843256473541 + }, + { + "label": "completed_group_3", + "mean": 0.10102558135986328, + "median": 0.09871675074100494, + "p05": 0.07363428175449371, + "p95": 0.1344296932220459 + }, + { + "label": "completed_group_4", + "mean": 0.07161714881658554, + "median": 0.07036904245615005, + "p05": 0.049177274107933044, + "p95": 0.09921920299530029 + }, + { + "label": "completed_group_5", + "mean": 0.17364460229873657, + "median": 0.16451683640480042, + "p05": 0.09478501230478287, + "p95": 0.2832472622394562 + }, + { + "label": "current_group_partial", + "mean": 0.4119114875793457, + "median": 0.4100137948989868, + "p05": 0.275837242603302, + "p95": 0.5469895601272583 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7821578979492188, + "group": 6, + "latest_source_mass": 0.3514069616794586, + "layer": 23, + "max_source_mass": 0.3514069616794586, + "mixer_index": 44, + "normalized_entropy": 0.9158479895972629, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13399384915828705, + "median": 0.1371074914932251, + "p05": 0.08264479786157608, + "p95": 0.1757817566394806 + }, + { + "label": "completed_group_1", + "mean": 0.12986589968204498, + "median": 0.12847267091274261, + "p05": 0.1112535148859024, + "p95": 0.1538200080394745 + }, + { + "label": "completed_group_2", + "mean": 0.0802837461233139, + "median": 0.07979932427406311, + "p05": 0.06686097383499146, + "p95": 0.09519629925489426 + }, + { + "label": "completed_group_3", + "mean": 0.07629825174808502, + "median": 0.074957937002182, + "p05": 0.060364846140146255, + "p95": 0.09640902280807495 + }, + { + "label": "completed_group_4", + "mean": 0.10134232044219971, + "median": 0.10061150044202805, + "p05": 0.0801728367805481, + "p95": 0.12531554698944092 + }, + { + "label": "completed_group_5", + "mean": 0.12680892646312714, + "median": 0.1246437132358551, + "p05": 0.09191874414682388, + "p95": 0.16937871277332306 + }, + { + "label": "current_group_partial", + "mean": 0.3514069616794586, + "median": 0.3516124486923218, + "p05": 0.2630184292793274, + "p95": 0.4369184076786041 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.655133843421936, + "group": 6, + "latest_source_mass": 0.43040627241134644, + "layer": 23, + "max_source_mass": 0.43040627241134644, + "mixer_index": 45, + "normalized_entropy": 0.8505705385346074, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10968074202537537, + "median": 0.10342554748058319, + "p05": 0.058600373566150665, + "p95": 0.19649700820446014 + }, + { + "label": "completed_group_1", + "mean": 0.0657510757446289, + "median": 0.06393904238939285, + "p05": 0.04856989532709122, + "p95": 0.08747231960296631 + }, + { + "label": "completed_group_2", + "mean": 0.08727139234542847, + "median": 0.08742742240428925, + "p05": 0.06016327068209648, + "p95": 0.11422628164291382 + }, + { + "label": "completed_group_3", + "mean": 0.09466101229190826, + "median": 0.09291957318782806, + "p05": 0.06727220118045807, + "p95": 0.12599577009677887 + }, + { + "label": "completed_group_4", + "mean": 0.0703636035323143, + "median": 0.06896525621414185, + "p05": 0.05082611367106438, + "p95": 0.0946161150932312 + }, + { + "label": "completed_group_5", + "mean": 0.14186589419841766, + "median": 0.13478878140449524, + "p05": 0.07877384126186371, + "p95": 0.2290385663509369 + }, + { + "label": "current_group_partial", + "mean": 0.43040627241134644, + "median": 0.42919087409973145, + "p05": 0.2915465831756592, + "p95": 0.5688671469688416 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8160101175308228, + "group": 6, + "latest_source_mass": 0.3277372717857361, + "layer": 24, + "max_source_mass": 0.3277372717857361, + "mixer_index": 46, + "normalized_entropy": 0.933244589125786, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13769462704658508, + "median": 0.13593362271785736, + "p05": 0.09772415459156036, + "p95": 0.18957549333572388 + }, + { + "label": "completed_group_1", + "mean": 0.11569900810718536, + "median": 0.11375441402196884, + "p05": 0.0982208400964737, + "p95": 0.1386219710111618 + }, + { + "label": "completed_group_2", + "mean": 0.08787183463573456, + "median": 0.08727885782718658, + "p05": 0.07391972839832306, + "p95": 0.10365156829357147 + }, + { + "label": "completed_group_3", + "mean": 0.0929793268442154, + "median": 0.09170648455619812, + "p05": 0.07486406713724136, + "p95": 0.11526438593864441 + }, + { + "label": "completed_group_4", + "mean": 0.10649851709604263, + "median": 0.10617373883724213, + "p05": 0.08636443316936493, + "p95": 0.12787018716335297 + }, + { + "label": "completed_group_5", + "mean": 0.1315194070339203, + "median": 0.12966084480285645, + "p05": 0.09266573190689087, + "p95": 0.17860902845859528 + }, + { + "label": "current_group_partial", + "mean": 0.3277372717857361, + "median": 0.32655179500579834, + "p05": 0.24662655591964722, + "p95": 0.4135902523994446 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.6999847888946533, + "group": 6, + "latest_source_mass": 0.3966374695301056, + "layer": 24, + "max_source_mass": 0.3966374695301056, + "mixer_index": 47, + "normalized_entropy": 0.873619365066753, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10220035910606384, + "median": 0.0844031423330307, + "p05": 0.04488028585910797, + "p95": 0.24769487977027893 + }, + { + "label": "completed_group_1", + "mean": 0.07546016573905945, + "median": 0.07393044233322144, + "p05": 0.05928082764148712, + "p95": 0.09592372179031372 + }, + { + "label": "completed_group_2", + "mean": 0.1028895378112793, + "median": 0.10241077095270157, + "p05": 0.07516536116600037, + "p95": 0.13291233777999878 + }, + { + "label": "completed_group_3", + "mean": 0.10639216750860214, + "median": 0.10462376475334167, + "p05": 0.07692230492830276, + "p95": 0.14082345366477966 + }, + { + "label": "completed_group_4", + "mean": 0.07665040343999863, + "median": 0.07448552548885345, + "p05": 0.05249025300145149, + "p95": 0.10721437633037567 + }, + { + "label": "completed_group_5", + "mean": 0.13976991176605225, + "median": 0.12892484664916992, + "p05": 0.0740278959274292, + "p95": 0.24029651284217834 + }, + { + "label": "current_group_partial", + "mean": 0.3966374695301056, + "median": 0.39967864751815796, + "p05": 0.25664323568344116, + "p95": 0.525518000125885 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8706753253936768, + "group": 7, + "latest_source_mass": 0.27128130197525024, + "layer": 25, + "max_source_mass": 0.27128130197525024, + "mixer_index": 48, + "normalized_entropy": 0.9613369488318045, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1469482183456421, + "median": 0.14645828306674957, + "p05": 0.10641424357891083, + "p95": 0.1947372704744339 + }, + { + "label": "completed_group_1", + "mean": 0.12422746419906616, + "median": 0.12275843322277069, + "p05": 0.10553248971700668, + "p95": 0.14753170311450958 + }, + { + "label": "completed_group_2", + "mean": 0.09348422288894653, + "median": 0.09236403554677963, + "p05": 0.07779940217733383, + "p95": 0.11272361129522324 + }, + { + "label": "completed_group_3", + "mean": 0.1027352511882782, + "median": 0.10125023126602173, + "p05": 0.08420244604349136, + "p95": 0.12524712085723877 + }, + { + "label": "completed_group_4", + "mean": 0.12104703485965729, + "median": 0.12076126039028168, + "p05": 0.09965822100639343, + "p95": 0.14355391263961792 + }, + { + "label": "completed_group_5", + "mean": 0.14027650654315948, + "median": 0.139203742146492, + "p05": 0.10420101135969162, + "p95": 0.18390552699565887 + }, + { + "label": "completed_group_6", + "mean": 0.27128130197525024, + "median": 0.2698158919811249, + "p05": 0.20765253901481628, + "p95": 0.3398803472518921 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7911043167114258, + "group": 7, + "latest_source_mass": 0.27496105432510376, + "layer": 25, + "max_source_mass": 0.27496105432510376, + "mixer_index": 49, + "normalized_entropy": 0.8613391051447965, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09078208357095718, + "median": 0.0916827917098999, + "p05": 0.030423305928707123, + "p95": 0.16150596737861633 + }, + { + "label": "completed_group_1", + "mean": 0.05932023376226425, + "median": 0.05782822519540787, + "p05": 0.04296479746699333, + "p95": 0.08013182133436203 + }, + { + "label": "completed_group_2", + "mean": 0.06744871288537979, + "median": 0.06634571403265, + "p05": 0.04800695925951004, + "p95": 0.09060627222061157 + }, + { + "label": "completed_group_3", + "mean": 0.0717800185084343, + "median": 0.0682687982916832, + "p05": 0.04594217985868454, + "p95": 0.10765853524208069 + }, + { + "label": "completed_group_4", + "mean": 0.05460817739367485, + "median": 0.052827149629592896, + "p05": 0.03489609435200691, + "p95": 0.08040964603424072 + }, + { + "label": "completed_group_5", + "mean": 0.10924962162971497, + "median": 0.09424155950546265, + "p05": 0.047475166618824005, + "p95": 0.2151142656803131 + }, + { + "label": "completed_group_6", + "mean": 0.2718501091003418, + "median": 0.2685811221599579, + "p05": 0.14285480976104736, + "p95": 0.40964099764823914 + }, + { + "label": "current_group_partial", + "mean": 0.27496105432510376, + "median": 0.2828528583049774, + "p05": 0.06592769920825958, + "p95": 0.48006877303123474 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9753689765930176, + "group": 7, + "latest_source_mass": 0.23982003331184387, + "layer": 26, + "max_source_mass": 0.23982003331184387, + "mixer_index": 50, + "normalized_entropy": 0.9499516754855512, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10981251299381256, + "median": 0.10871805250644684, + "p05": 0.050737276673316956, + "p95": 0.1706625372171402 + }, + { + "label": "completed_group_1", + "mean": 0.10515634715557098, + "median": 0.10322865843772888, + "p05": 0.08713022619485855, + "p95": 0.12963688373565674 + }, + { + "label": "completed_group_2", + "mean": 0.07847881317138672, + "median": 0.07656338810920715, + "p05": 0.05796374753117561, + "p95": 0.10510093718767166 + }, + { + "label": "completed_group_3", + "mean": 0.080687016248703, + "median": 0.07948547601699829, + "p05": 0.06504379212856293, + "p95": 0.09981580078601837 + }, + { + "label": "completed_group_4", + "mean": 0.10027816891670227, + "median": 0.09996622800827026, + "p05": 0.07846689224243164, + "p95": 0.1242886409163475 + }, + { + "label": "completed_group_5", + "mean": 0.1099884957075119, + "median": 0.10788669437170029, + "p05": 0.07500496506690979, + "p95": 0.15274949371814728 + }, + { + "label": "completed_group_6", + "mean": 0.1757785975933075, + "median": 0.1735863983631134, + "p05": 0.12249398976564407, + "p95": 0.23682411015033722 + }, + { + "label": "current_group_partial", + "mean": 0.23982003331184387, + "median": 0.23704850673675537, + "p05": 0.11852282285690308, + "p95": 0.3622197210788727 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.81705904006958, + "group": 7, + "latest_source_mass": 0.3062949776649475, + "layer": 26, + "max_source_mass": 0.3062949776649475, + "mixer_index": 51, + "normalized_entropy": 0.8738206887036145, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0980612188577652, + "median": 0.09061099588871002, + "p05": 0.06147613376379013, + "p95": 0.1617073118686676 + }, + { + "label": "completed_group_1", + "mean": 0.060419049113988876, + "median": 0.05925498157739639, + "p05": 0.04340171068906784, + "p95": 0.08076231181621552 + }, + { + "label": "completed_group_2", + "mean": 0.06808654963970184, + "median": 0.06729011237621307, + "p05": 0.04935084655880928, + "p95": 0.08873726427555084 + }, + { + "label": "completed_group_3", + "mean": 0.07581339031457901, + "median": 0.07308594882488251, + "p05": 0.049850281327962875, + "p95": 0.10994531959295273 + }, + { + "label": "completed_group_4", + "mean": 0.06074263155460358, + "median": 0.05964510142803192, + "p05": 0.04034031927585602, + "p95": 0.08569400757551193 + }, + { + "label": "completed_group_5", + "mean": 0.09064903855323792, + "median": 0.08298017829656601, + "p05": 0.047057293355464935, + "p95": 0.15634870529174805 + }, + { + "label": "completed_group_6", + "mean": 0.23993313312530518, + "median": 0.23315298557281494, + "p05": 0.11595168709754944, + "p95": 0.3906308114528656 + }, + { + "label": "current_group_partial", + "mean": 0.3062949776649475, + "median": 0.3047541677951813, + "p05": 0.12540288269519806, + "p95": 0.4860513210296631 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9814136028289795, + "group": 7, + "latest_source_mass": 0.23084799945354462, + "layer": 27, + "max_source_mass": 0.23084799945354462, + "mixer_index": 52, + "normalized_entropy": 0.9528585262504343, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11045937240123749, + "median": 0.10626375675201416, + "p05": 0.05494675412774086, + "p95": 0.17410020530223846 + }, + { + "label": "completed_group_1", + "mean": 0.08757395297288895, + "median": 0.08668297529220581, + "p05": 0.07028551399707794, + "p95": 0.10834703594446182 + }, + { + "label": "completed_group_2", + "mean": 0.08714326471090317, + "median": 0.0863339900970459, + "p05": 0.06873977184295654, + "p95": 0.10879559814929962 + }, + { + "label": "completed_group_3", + "mean": 0.09700852632522583, + "median": 0.0958750918507576, + "p05": 0.07455667108297348, + "p95": 0.12356264889240265 + }, + { + "label": "completed_group_4", + "mean": 0.09435248374938965, + "median": 0.09339165687561035, + "p05": 0.0721961036324501, + "p95": 0.11932304501533508 + }, + { + "label": "completed_group_5", + "mean": 0.1148696318268776, + "median": 0.11229501664638519, + "p05": 0.07511283457279205, + "p95": 0.16472017765045166 + }, + { + "label": "completed_group_6", + "mean": 0.1777447611093521, + "median": 0.17455129325389862, + "p05": 0.1177421361207962, + "p95": 0.24785079061985016 + }, + { + "label": "current_group_partial", + "mean": 0.23084799945354462, + "median": 0.2267751693725586, + "p05": 0.11723043769598007, + "p95": 0.3665117919445038 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8373757600784302, + "group": 7, + "latest_source_mass": 0.25260502099990845, + "layer": 27, + "max_source_mass": 0.25260502099990845, + "mixer_index": 53, + "normalized_entropy": 0.8835909657715805, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08649781346321106, + "median": 0.05945904552936554, + "p05": 0.026423925533890724, + "p95": 0.23923249542713165 + }, + { + "label": "completed_group_1", + "mean": 0.07379771023988724, + "median": 0.07217878103256226, + "p05": 0.04322340711951256, + "p95": 0.11003518849611282 + }, + { + "label": "completed_group_2", + "mean": 0.0769633874297142, + "median": 0.0755230188369751, + "p05": 0.04856198653578758, + "p95": 0.10847672075033188 + }, + { + "label": "completed_group_3", + "mean": 0.0880836620926857, + "median": 0.08413928747177124, + "p05": 0.04961243271827698, + "p95": 0.13771715760231018 + }, + { + "label": "completed_group_4", + "mean": 0.066677525639534, + "median": 0.0648220106959343, + "p05": 0.0391211211681366, + "p95": 0.10059694200754166 + }, + { + "label": "completed_group_5", + "mean": 0.11875502020120621, + "median": 0.10550784319639206, + "p05": 0.05534175783395767, + "p95": 0.22876456379890442 + }, + { + "label": "completed_group_6", + "mean": 0.23661985993385315, + "median": 0.22938117384910583, + "p05": 0.11185489594936371, + "p95": 0.386798620223999 + }, + { + "label": "current_group_partial", + "mean": 0.25260502099990845, + "median": 0.23003548383712769, + "p05": 0.06494839489459991, + "p95": 0.5125646591186523 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.968585729598999, + "group": 7, + "latest_source_mass": 0.22389832139015198, + "layer": 28, + "max_source_mass": 0.22389832139015198, + "mixer_index": 54, + "normalized_entropy": 0.946689623219086, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10324695706367493, + "median": 0.0941208153963089, + "p05": 0.039645131677389145, + "p95": 0.1829081028699875 + }, + { + "label": "completed_group_1", + "mean": 0.0860632136464119, + "median": 0.08438493311405182, + "p05": 0.04368945211172104, + "p95": 0.14022991061210632 + }, + { + "label": "completed_group_2", + "mean": 0.09797552973031998, + "median": 0.09780187904834747, + "p05": 0.06858807802200317, + "p95": 0.1295054405927658 + }, + { + "label": "completed_group_3", + "mean": 0.1271316111087799, + "median": 0.12579670548439026, + "p05": 0.09182935208082199, + "p95": 0.16805431246757507 + }, + { + "label": "completed_group_4", + "mean": 0.08284065127372742, + "median": 0.08147171139717102, + "p05": 0.0564071387052536, + "p95": 0.11312974244356155 + }, + { + "label": "completed_group_5", + "mean": 0.10640716552734375, + "median": 0.10340218245983124, + "p05": 0.06448834389448166, + "p95": 0.15891367197036743 + }, + { + "label": "completed_group_6", + "mean": 0.17243653535842896, + "median": 0.16770902276039124, + "p05": 0.1087951734662056, + "p95": 0.24907639622688293 + }, + { + "label": "current_group_partial", + "mean": 0.22389832139015198, + "median": 0.21250957250595093, + "p05": 0.09934276342391968, + "p95": 0.3857041895389557 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8529754877090454, + "group": 7, + "latest_source_mass": 0.20538833737373352, + "layer": 28, + "max_source_mass": 0.2897399663925171, + "mixer_index": 55, + "normalized_entropy": 0.8910928490022162, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08385773003101349, + "median": 0.06364437937736511, + "p05": 0.04014988988637924, + "p95": 0.2496711015701294 + }, + { + "label": "completed_group_1", + "mean": 0.07188580930233002, + "median": 0.07024423032999039, + "p05": 0.05124984309077263, + "p95": 0.09714871644973755 + }, + { + "label": "completed_group_2", + "mean": 0.08174896240234375, + "median": 0.07993435859680176, + "p05": 0.049406472593545914, + "p95": 0.11908631026744843 + }, + { + "label": "completed_group_3", + "mean": 0.09558321535587311, + "median": 0.0924227386713028, + "p05": 0.05770375207066536, + "p95": 0.14457376301288605 + }, + { + "label": "completed_group_4", + "mean": 0.06307300925254822, + "median": 0.06175791472196579, + "p05": 0.040213022381067276, + "p95": 0.0907377302646637 + }, + { + "label": "completed_group_5", + "mean": 0.108722984790802, + "median": 0.1007031798362732, + "p05": 0.05666481703519821, + "p95": 0.1869736760854721 + }, + { + "label": "completed_group_6", + "mean": 0.2897399663925171, + "median": 0.28985595703125, + "p05": 0.14349502325057983, + "p95": 0.4364045262336731 + }, + { + "label": "current_group_partial", + "mean": 0.20538833737373352, + "median": 0.18734587728977203, + "p05": 0.07006806135177612, + "p95": 0.39856192469596863 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9637342691421509, + "group": 8, + "latest_source_mass": 0.13441291451454163, + "layer": 29, + "max_source_mass": 0.2349606156349182, + "mixer_index": 56, + "normalized_entropy": 0.9443565639050314, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10303820669651031, + "median": 0.0819694995880127, + "p05": 0.030241798609495163, + "p95": 0.2497645914554596 + }, + { + "label": "completed_group_1", + "mean": 0.08717728406190872, + "median": 0.08418120443820953, + "p05": 0.06677442789077759, + "p95": 0.11682014167308807 + }, + { + "label": "completed_group_2", + "mean": 0.09220535308122635, + "median": 0.09127489477396011, + "p05": 0.06960588693618774, + "p95": 0.11840611696243286 + }, + { + "label": "completed_group_3", + "mean": 0.11336663365364075, + "median": 0.11189766228199005, + "p05": 0.08372759819030762, + "p95": 0.1468089371919632 + }, + { + "label": "completed_group_4", + "mean": 0.08865514397621155, + "median": 0.08804303407669067, + "p05": 0.06345562636852264, + "p95": 0.11571189761161804 + }, + { + "label": "completed_group_5", + "mean": 0.14618384838104248, + "median": 0.1397639960050583, + "p05": 0.08802400529384613, + "p95": 0.22461283206939697 + }, + { + "label": "completed_group_6", + "mean": 0.2349606156349182, + "median": 0.23364676535129547, + "p05": 0.1564730703830719, + "p95": 0.317343533039093 + }, + { + "label": "completed_group_7", + "mean": 0.13441291451454163, + "median": 0.11298182606697083, + "p05": 0.049395374953746796, + "p95": 0.2780357003211975 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.962258219718933, + "group": 8, + "latest_source_mass": 0.24145527184009552, + "layer": 29, + "max_source_mass": 0.24145527184009552, + "mixer_index": 57, + "normalized_entropy": 0.8930622021795581, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07841142266988754, + "median": 0.06509651243686676, + "p05": 0.027810024097561836, + "p95": 0.17502810060977936 + }, + { + "label": "completed_group_1", + "mean": 0.07437640428543091, + "median": 0.07133704423904419, + "p05": 0.039178602397441864, + "p95": 0.12137842923402786 + }, + { + "label": "completed_group_2", + "mean": 0.07400830835103989, + "median": 0.07284313440322876, + "p05": 0.047712553292512894, + "p95": 0.10446867346763611 + }, + { + "label": "completed_group_3", + "mean": 0.07925957441329956, + "median": 0.07685956358909607, + "p05": 0.04267336428165436, + "p95": 0.12466679513454437 + }, + { + "label": "completed_group_4", + "mean": 0.05560165271162987, + "median": 0.05358406901359558, + "p05": 0.031528957188129425, + "p95": 0.08545175939798355 + }, + { + "label": "completed_group_5", + "mean": 0.08937598019838333, + "median": 0.08107791095972061, + "p05": 0.038014721125364304, + "p95": 0.1699092984199524 + }, + { + "label": "completed_group_6", + "mean": 0.19564677774906158, + "median": 0.1907363384962082, + "p05": 0.10008887201547623, + "p95": 0.3112020492553711 + }, + { + "label": "completed_group_7", + "mean": 0.1118645966053009, + "median": 0.08986739069223404, + "p05": 0.03501077741384506, + "p95": 0.26519542932510376 + }, + { + "label": "current_group_partial", + "mean": 0.24145527184009552, + "median": 0.22476747632026672, + "p05": 0.05287282541394234, + "p95": 0.49067121744155884 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0993216037750244, + "group": 8, + "latest_source_mass": 0.12042839825153351, + "layer": 30, + "max_source_mass": 0.1680138111114502, + "mixer_index": 58, + "normalized_entropy": 0.955442436530595, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1043272390961647, + "median": 0.0907609686255455, + "p05": 0.016956502571702003, + "p95": 0.284129798412323 + }, + { + "label": "completed_group_1", + "mean": 0.09323035180568695, + "median": 0.0917273461818695, + "p05": 0.068887859582901, + "p95": 0.1232273131608963 + }, + { + "label": "completed_group_2", + "mean": 0.08778750896453857, + "median": 0.08604135364294052, + "p05": 0.061383333057165146, + "p95": 0.12025082856416702 + }, + { + "label": "completed_group_3", + "mean": 0.10311444103717804, + "median": 0.10148647427558899, + "p05": 0.07428233325481415, + "p95": 0.13808897137641907 + }, + { + "label": "completed_group_4", + "mean": 0.09694567322731018, + "median": 0.0953250527381897, + "p05": 0.06249001622200012, + "p95": 0.13581177592277527 + }, + { + "label": "completed_group_5", + "mean": 0.13869863748550415, + "median": 0.1327243149280548, + "p05": 0.08499303460121155, + "p95": 0.21259371936321259 + }, + { + "label": "completed_group_6", + "mean": 0.1680138111114502, + "median": 0.16382917761802673, + "p05": 0.10980678349733353, + "p95": 0.23901790380477905 + }, + { + "label": "completed_group_7", + "mean": 0.0874539315700531, + "median": 0.07735057175159454, + "p05": 0.03232480213046074, + "p95": 0.17377637326717377 + }, + { + "label": "current_group_partial", + "mean": 0.12042839825153351, + "median": 0.10099855065345764, + "p05": 0.041284509003162384, + "p95": 0.2534906268119812 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0489909648895264, + "group": 8, + "latest_source_mass": 0.16280405223369598, + "layer": 30, + "max_source_mass": 0.18801958858966827, + "mixer_index": 59, + "normalized_entropy": 0.9325359756232098, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0919051468372345, + "median": 0.08924420177936554, + "p05": 0.04205578938126564, + "p95": 0.14709380269050598 + }, + { + "label": "completed_group_1", + "mean": 0.07878251373767853, + "median": 0.07600399851799011, + "p05": 0.04804171249270439, + "p95": 0.11802247911691666 + }, + { + "label": "completed_group_2", + "mean": 0.07574650645256042, + "median": 0.07504475116729736, + "p05": 0.051915232092142105, + "p95": 0.10225261747837067 + }, + { + "label": "completed_group_3", + "mean": 0.09311886876821518, + "median": 0.0916299819946289, + "p05": 0.05949338898062706, + "p95": 0.13063858449459076 + }, + { + "label": "completed_group_4", + "mean": 0.05773407965898514, + "median": 0.05621109902858734, + "p05": 0.03913509100675583, + "p95": 0.0801405981183052 + }, + { + "label": "completed_group_5", + "mean": 0.07325181365013123, + "median": 0.0683174878358841, + "p05": 0.039874374866485596, + "p95": 0.12237872183322906 + }, + { + "label": "completed_group_6", + "mean": 0.17863741517066956, + "median": 0.17224830389022827, + "p05": 0.09081655740737915, + "p95": 0.29137635231018066 + }, + { + "label": "completed_group_7", + "mean": 0.18801958858966827, + "median": 0.1791573464870453, + "p05": 0.08688034117221832, + "p95": 0.3170011341571808 + }, + { + "label": "current_group_partial", + "mean": 0.16280405223369598, + "median": 0.14800652861595154, + "p05": 0.058800291270017624, + "p95": 0.3165009021759033 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.126072406768799, + "group": 8, + "latest_source_mass": 0.11243770271539688, + "layer": 31, + "max_source_mass": 0.17183929681777954, + "mixer_index": 60, + "normalized_entropy": 0.967617251644945, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09415344893932343, + "median": 0.09509380161762238, + "p05": 0.032531045377254486, + "p95": 0.17404073476791382 + }, + { + "label": "completed_group_1", + "mean": 0.07414552569389343, + "median": 0.07293231785297394, + "p05": 0.043177857995033264, + "p95": 0.10848568379878998 + }, + { + "label": "completed_group_2", + "mean": 0.10936354845762253, + "median": 0.10600043833255768, + "p05": 0.07479241490364075, + "p95": 0.15432706475257874 + }, + { + "label": "completed_group_3", + "mean": 0.11765885353088379, + "median": 0.11672843247652054, + "p05": 0.0860721692442894, + "p95": 0.1508050411939621 + }, + { + "label": "completed_group_4", + "mean": 0.07923046499490738, + "median": 0.07799427211284637, + "p05": 0.05631105974316597, + "p95": 0.1065656989812851 + }, + { + "label": "completed_group_5", + "mean": 0.12059831619262695, + "median": 0.11797575652599335, + "p05": 0.07954882085323334, + "p95": 0.17233142256736755 + }, + { + "label": "completed_group_6", + "mean": 0.17183929681777954, + "median": 0.16947603225708008, + "p05": 0.11957718431949615, + "p95": 0.23197905719280243 + }, + { + "label": "completed_group_7", + "mean": 0.12057285010814667, + "median": 0.11689606308937073, + "p05": 0.058023348450660706, + "p95": 0.2011682540178299 + }, + { + "label": "current_group_partial", + "mean": 0.11243770271539688, + "median": 0.10209996998310089, + "p05": 0.057224418967962265, + "p95": 0.2040310949087143 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.028747081756592, + "group": 8, + "latest_source_mass": 0.11574049293994904, + "layer": 31, + "max_source_mass": 0.24226921796798706, + "mixer_index": 61, + "normalized_entropy": 0.9233225873597866, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08796507120132446, + "median": 0.07151719927787781, + "p05": 0.048716649413108826, + "p95": 0.21155157685279846 + }, + { + "label": "completed_group_1", + "mean": 0.0757753923535347, + "median": 0.07367455959320068, + "p05": 0.05229219049215317, + "p95": 0.10657010972499847 + }, + { + "label": "completed_group_2", + "mean": 0.0847562700510025, + "median": 0.08441580832004547, + "p05": 0.0502907857298851, + "p95": 0.12154337763786316 + }, + { + "label": "completed_group_3", + "mean": 0.09340266138315201, + "median": 0.09229434281587601, + "p05": 0.05622609704732895, + "p95": 0.13404318690299988 + }, + { + "label": "completed_group_4", + "mean": 0.06450396031141281, + "median": 0.06322714686393738, + "p05": 0.043605200946331024, + "p95": 0.08994095027446747 + }, + { + "label": "completed_group_5", + "mean": 0.08635804057121277, + "median": 0.0823119580745697, + "p05": 0.043849315494298935, + "p95": 0.14317797124385834 + }, + { + "label": "completed_group_6", + "mean": 0.24226921796798706, + "median": 0.23795896768569946, + "p05": 0.11723512411117554, + "p95": 0.3776393532752991 + }, + { + "label": "completed_group_7", + "mean": 0.14922890067100525, + "median": 0.14130288362503052, + "p05": 0.06516221165657043, + "p95": 0.2617015242576599 + }, + { + "label": "current_group_partial", + "mean": 0.11574049293994904, + "median": 0.08995743095874786, + "p05": 0.03192901611328125, + "p95": 0.2981606125831604 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1297550201416016, + "group": 8, + "latest_source_mass": 0.10419845581054688, + "layer": 32, + "max_source_mass": 0.17309343814849854, + "mixer_index": 62, + "normalized_entropy": 0.9692932812191579, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10085826367139816, + "median": 0.10132397711277008, + "p05": 0.025741146877408028, + "p95": 0.20014458894729614 + }, + { + "label": "completed_group_1", + "mean": 0.08009681105613708, + "median": 0.0792180746793747, + "p05": 0.058846138417720795, + "p95": 0.10435555130243301 + }, + { + "label": "completed_group_2", + "mean": 0.0982624888420105, + "median": 0.09733013808727264, + "p05": 0.0730227455496788, + "p95": 0.12785884737968445 + }, + { + "label": "completed_group_3", + "mean": 0.11403533071279526, + "median": 0.11354124546051025, + "p05": 0.08656539022922516, + "p95": 0.14382149279117584 + }, + { + "label": "completed_group_4", + "mean": 0.09301646053791046, + "median": 0.09111075103282928, + "p05": 0.06575828790664673, + "p95": 0.12499162554740906 + }, + { + "label": "completed_group_5", + "mean": 0.12771157920360565, + "median": 0.12228857725858688, + "p05": 0.08901788294315338, + "p95": 0.18423326313495636 + }, + { + "label": "completed_group_6", + "mean": 0.17309343814849854, + "median": 0.17134559154510498, + "p05": 0.12187358736991882, + "p95": 0.2286318838596344 + }, + { + "label": "completed_group_7", + "mean": 0.10872717201709747, + "median": 0.10637345165014267, + "p05": 0.053436052054166794, + "p95": 0.1774827241897583 + }, + { + "label": "current_group_partial", + "mean": 0.10419845581054688, + "median": 0.09291182458400726, + "p05": 0.04793589562177658, + "p95": 0.21689072251319885 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1051812171936035, + "group": 8, + "latest_source_mass": 0.08249962329864502, + "layer": 32, + "max_source_mass": 0.21416787803173065, + "mixer_index": 63, + "normalized_entropy": 0.9581092615238248, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09203393757343292, + "median": 0.08020778000354767, + "p05": 0.047239016741514206, + "p95": 0.20447812974452972 + }, + { + "label": "completed_group_1", + "mean": 0.08722756803035736, + "median": 0.08618196845054626, + "p05": 0.06572069227695465, + "p95": 0.11272746324539185 + }, + { + "label": "completed_group_2", + "mean": 0.10295312851667404, + "median": 0.1034228652715683, + "p05": 0.06958446651697159, + "p95": 0.13481324911117554 + }, + { + "label": "completed_group_3", + "mean": 0.11847956478595734, + "median": 0.11627265810966492, + "p05": 0.08390682935714722, + "p95": 0.1599627137184143 + }, + { + "label": "completed_group_4", + "mean": 0.07401066273450851, + "median": 0.07330383360385895, + "p05": 0.05390108376741409, + "p95": 0.09712604433298111 + }, + { + "label": "completed_group_5", + "mean": 0.11382973194122314, + "median": 0.10948765277862549, + "p05": 0.07062914222478867, + "p95": 0.1708521842956543 + }, + { + "label": "completed_group_6", + "mean": 0.21416787803173065, + "median": 0.21409451961517334, + "p05": 0.1212707906961441, + "p95": 0.3072466552257538 + }, + { + "label": "completed_group_7", + "mean": 0.11479790508747101, + "median": 0.10708929598331451, + "p05": 0.06293575465679169, + "p95": 0.18719080090522766 + }, + { + "label": "current_group_partial", + "mean": 0.08249962329864502, + "median": 0.07665085047483444, + "p05": 0.049256522208452225, + "p95": 0.13453297317028046 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.8321013450622559, + "group": null, + "latest_source_mass": 0.219556525349617, + "layer": null, + "max_source_mass": 0.30178403854370117, + "mixer_index": null, + "normalized_entropy": 0.8338252557157281, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09867185354232788, + "median": 0.10158779472112656, + "p05": 0.035266630351543427, + "p95": 0.1513454020023346 + }, + { + "label": "completed_group_1", + "mean": 0.05898427963256836, + "median": 0.05800648033618927, + "p05": 0.04004587605595589, + "p95": 0.0809728130698204 + }, + { + "label": "completed_group_2", + "mean": 0.057430200278759, + "median": 0.05592039227485657, + "p05": 0.03935804218053818, + "p95": 0.08066077530384064 + }, + { + "label": "completed_group_3", + "mean": 0.05067210644483566, + "median": 0.0480031743645668, + "p05": 0.03069971315562725, + "p95": 0.0778159350156784 + }, + { + "label": "completed_group_4", + "mean": 0.04944504797458649, + "median": 0.04722342640161514, + "p05": 0.030949361622333527, + "p95": 0.07596699148416519 + }, + { + "label": "completed_group_5", + "mean": 0.047245003283023834, + "median": 0.04242067039012909, + "p05": 0.021957436576485634, + "p95": 0.09008318185806274 + }, + { + "label": "completed_group_6", + "mean": 0.1162109300494194, + "median": 0.09733854234218597, + "p05": 0.03573312610387802, + "p95": 0.2731606960296631 + }, + { + "label": "completed_group_7", + "mean": 0.30178403854370117, + "median": 0.29523494839668274, + "p05": 0.09429201483726501, + "p95": 0.5453053712844849 + }, + { + "label": "completed_group_8", + "mean": 0.219556525349617, + "median": 0.19983074069023132, + "p05": 0.03722763806581497, + "p95": 0.4634954333305359 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "942e0d186e83d3b52788e6a64c0684235f3d3e5e1f34cb38f76047fded397c74", + "optimizer_state_before": "942e0d186e83d3b52788e6a64c0684235f3d3e5e1f34cb38f76047fded397c74", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "729d18a894ede5429f33682ebed5494f0e192bf108efbc8a123308e96fadab4d", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.08130212873220444, + 0.1432749330997467, + 0.12958019971847534, + 0.04409155994653702, + 0.14100632071495056, + 0.0481393076479435, + 0.04208913818001747, + 0.04236427694559097, + 0.05848926678299904, + 0.04504085332155228, + 0.04397343844175339, + 0.05665678158402443, + 0.06478945910930634, + 0.04678606986999512, + 0.04817425087094307, + 0.049439266324043274, + 0.04376237466931343, + 0.02850259095430374, + 0.02922668866813183, + 0.04021619260311127, + 0.04409762844443321, + 0.030135832726955414, + 0.03803896903991699, + 0.030100898817181587, + 0.03295852243900299, + 0.0306381918489933, + 0.02206242084503174, + 0.02074088715016842, + 0.021799447014927864, + 0.026608876883983612, + 0.01902451552450657, + 0.02519184537231922 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3376500305639638e-05, + "first_to_last_ratio": 1.4098205594119886, + "imbalance_abs_log_ratio": 0.3434624334630992, + "last_quartile_mean": 9.48808713019389e-06, + "mean": 1.3753251778325648e-05, + "normalized": [ + 0.872126231956151, + 0.8994990546880632, + 0.8826478210575978, + 0.8463839625815878, + 1.12968786363437, + 1.1469805773400135, + 1.040016753229815, + 0.9635087058816061, + 1.057144936913433, + 0.941033957776458, + 0.8736259813008125, + 0.8166546176089677, + 0.726185589501133, + 0.6844792789299848, + 0.6647323036859834, + 0.638869348101534, + 1.45211763876736, + 1.3695497549648363, + 1.2684543887383168, + 1.1495085731929047, + 2.0935002358588783, + 1.9169888836924016, + 1.6426233009262203, + 1.404643849952257, + 1.1986055749986184, + 1.025360289254095, + 0.8286550591551723, + 0.7176699044021386, + 0.5714448392683698, + 0.43393140073490627, + 0.38174836988061706, + 0.36162095202539646 + ], + "peak_layer": 21, + "peak_normalized": 2.0935002358588783, + "population_cv": 0.3980646966565043, + "rest_mean": 1.2094527173084552e-05, + "spike_contrast": 1.8777389410615202, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.2710364646627568e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.199457165057538e-05, + 1.2371036973490845e-05, + 1.2139277714595664e-05, + 1.1640531738521531e-05, + 1.55368816194823e-05, + 1.577471266500652e-05, + 1.4303612260846421e-05, + 1.3251377822598442e-05, + 1.4539180483552627e-05, + 1.2942276953253895e-05, + 1.2015198080916889e-05, + 1.1231656571908388e-05, + 9.987413250200916e-06, + 9.413815860170871e-06, + 9.142230737779755e-06, + 8.78653099789517e-06, + 1.9971339497715235e-05, + 1.8835762602975592e-05, + 1.7445372577640228e-05, + 1.5809480828465894e-05, + 2.8792435841751285e-05, + 2.636483077367302e-05, + 2.2591411834582686e-05, + 1.9318420527270064e-05, + 1.6484724255860783e-05, + 1.4102038221608382e-05, + 1.139670166594442e-05, + 9.87029488896951e-06, + 7.85922475188272e-06, + 5.967967808828689e-06, + 5.2502814469335135e-06, + 4.973464001523098e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.305891409079777e-05, + "first_to_last_ratio": 1.392774690210974, + "imbalance_abs_log_ratio": 0.3312979374306361, + "last_quartile_mean": 3.8095834497653414e-05, + "mean": 5.492098136983259e-05, + "normalized": [ + 0.861703606856031, + 0.8875085509159336, + 0.8724772501581232, + 0.8369715559103585, + 1.1319676146266233, + 1.1404157619380209, + 1.0352725679446342, + 0.9624472568066351, + 1.064445563081042, + 0.9432467955493706, + 0.8769115703251862, + 0.8183121449066616, + 0.7308699981720386, + 0.6902004800301615, + 0.6672877834263703, + 0.6409674903247492, + 1.4550601983922828, + 1.3755688672717281, + 1.2726074279553434, + 1.1425630054516895, + 2.0822419693705934, + 1.9086329329387715, + 1.644199125843742, + 1.4089356642756514, + 1.2059871047522959, + 1.0325249891655295, + 0.8327095265990165, + 0.7226098406524658, + 0.5732866609317387, + 0.43596455609961304, + 0.38312094357080967, + 0.3629811957567887 + ], + "peak_layer": 21, + "peak_normalized": 2.0822419693705934, + "population_cv": 0.3966570033750761, + "rest_mean": 4.831012901612902e-05, + "spike_contrast": 1.8757884925038641, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.061958407983184e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.732560773845762e-05, + 4.874284059042111e-05, + 4.791730680153705e-05, + 4.5967299229232594e-05, + 6.216877227416262e-05, + 6.263275281526148e-05, + 5.6858185416786e-05, + 5.2858547860523686e-05, + 5.8460394939174876e-05, + 5.1804039685521275e-05, + 4.816084401682019e-05, + 4.494250606512651e-05, + 4.014009755337611e-05, + 3.790648770518601e-05, + 3.6648099921876565e-05, + 3.52025635947939e-05, + 7.991333404788747e-05, + 7.554759213235229e-05, + 6.989284884184599e-05, + 6.275068153627217e-05, + 0.00011435877240728587, + 0.0001048239937517792, + 9.030102955875918e-05, + 7.738012936897576e-05, + 6.623399531235918e-05, + 5.670728569384664e-05, + 4.57332243968267e-05, + 3.968644159613177e-05, + 3.148546602460556e-05, + 2.3943601263454184e-05, + 2.104137820424512e-05, + 1.993528348975815e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04705317597836256, + "first_to_last_ratio": 1.3927746759399198, + "imbalance_abs_log_ratio": 0.33129792718414436, + "last_quartile_mean": 0.03378376760520041, + "mean": 0.04870447557186708, + "normalized": [ + 0.8617036111459598, + 0.8875085515420177, + 0.8724772009144255, + 0.8369715628486869, + 1.1319676319756757, + 1.1404158445724537, + 1.0352725718667957, + 0.9624472384239273, + 1.0644455689163006, + 0.9432467761096968, + 0.8769115717297861, + 0.8183120975983121, + 0.7308699765260276, + 0.6902005050409423, + 0.6672877917020854, + 0.6409674776009149, + 1.4550601346467584, + 1.3755688270610786, + 1.2726074407181456, + 1.1425630055475084, + 2.082241939293198, + 1.9086328990500294, + 1.6441992249175394, + 1.4089356413043088, + 1.2059871669112603, + 1.032524982933128, + 0.8327095186818301, + 0.7226098470311895, + 0.5732867026926071, + 0.43596454525011596, + 0.38312095710023625, + 0.36298118834705717 + ], + "peak_layer": 21, + "peak_normalized": 2.082241939293198, + "population_cv": 0.3966570015078369, + "rest_mean": 0.04284190545203509, + "spike_contrast": 1.8757885152641456, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0803623542189598, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.04196882247924805, + 0.04322563856840134, + 0.0424935445189476, + 0.04076426103711128, + 0.05513188987970352, + 0.05554335564374924, + 0.05042240768671036, + 0.0468754880130291, + 0.05184326320886612, + 0.0459403395652771, + 0.04270951822400093, + 0.039855461567640305, + 0.035596638917922974, + 0.033615853637456894, + 0.032499901950359344, + 0.03121798485517502, + 0.07086794078350067, + 0.06699635833501816, + 0.0619816780090332, + 0.05564793199300766, + 0.1014145016670227, + 0.092958964407444, + 0.08007986098527908, + 0.06862147152423859, + 0.05873697251081467, + 0.05028858780860901, + 0.04055668041110039, + 0.035194333642721176, + 0.027921628206968307, + 0.02123342454433441, + 0.01865970529615879, + 0.01767880842089653 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0335509560709397e-06, + "first_to_last_ratio": 0.800252825630956, + "imbalance_abs_log_ratio": 0.2228275692031206, + "last_quartile_mean": 2.541135614819723e-06, + "mean": 2.913915704283454e-06, + "normalized": [ + 0.7803382065461789, + 0.7320562648837056, + 0.7098851296044338, + 0.713059869815856, + 0.6729150824257989, + 0.6708688942390978, + 0.6491951630171852, + 0.6546869735527502, + 0.7715235933317293, + 0.7335599865229504, + 0.7284058520495426, + 0.7336793728603509, + 0.6162135948862999, + 0.6241820086654949, + 0.6341697291979046, + 0.6616511765677783, + 1.3815401009435135, + 1.3825758749583201, + 1.3822762386605307, + 1.332295187522962, + 2.3406602841103434, + 2.278960644638916, + 2.0404236213013194, + 1.7983249835948945, + 1.517129115086387, + 1.3281010596567688, + 1.062368999113631, + 0.926894533430433, + 0.6999310401252395, + 0.5293580836703028, + 0.4667888356778559, + 0.44598049934152495 + ], + "peak_layer": 21, + "peak_normalized": 2.3406602841103434, + "population_cv": 0.5165207800134821, + "rest_mean": 2.3769459394989773e-06, + "spike_contrast": 2.445807596004911, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.813552434119629e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2738397547072964e-06, + 2.1331502466637176e-06, + 2.068545427391655e-06, + 2.077796352750738e-06, + 1.9608178263297305e-06, + 1.9548554064385826e-06, + 1.8916999806606327e-06, + 1.907702653625165e-06, + 2.248154714834527e-06, + 2.137531964763184e-06, + 2.1225132513791323e-06, + 2.1378798464866122e-06, + 1.7955944713321514e-06, + 1.8188137573815766e-06, + 1.8479171330909594e-06, + 1.9279957541584736e-06, + 4.0256913962366525e-06, + 4.028709554404486e-06, + 4.027836439490784e-06, + 3.882195869664429e-06, + 6.820486760261701e-06, + 6.640699211857282e-06, + 5.94562243350083e-06, + 5.240167411102448e-06, + 4.420786353875883e-06, + 3.869974534609355e-06, + 3.0956537102611037e-06, + 2.700892537177424e-06, + 2.039540049736388e-06, + 1.5425048331962898e-06, + 1.360183318865893e-06, + 1.2995495808354462e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.0862232708605006e-05, + "first_to_last_ratio": 1.6228918616520855, + "imbalance_abs_log_ratio": 0.48420965763249596, + "last_quartile_mean": 2.5178654027513403e-05, + "mean": 3.966477586914152e-05, + "normalized": [ + 0.9416433745088736, + 0.9689750084042642, + 0.950763641261749, + 0.911671542713804, + 1.193012300282416, + 1.190876368625092, + 1.0841056502696969, + 1.00046752668501, + 1.1216839411912192, + 0.9926384676731087, + 0.9187729101347912, + 0.8506497325833076, + 0.7805624114195948, + 0.7377957179764669, + 0.7154649230403652, + 0.6867945255570269, + 1.3962697506908583, + 1.3266925657293973, + 1.2247822658863927, + 1.1061834866215106, + 2.1088522482676924, + 1.872182986128609, + 1.5438875021669929, + 1.296981213404733, + 1.145045831270383, + 0.9535938791130757, + 0.7733123689206051, + 0.6739013879397645, + 0.49031224706376303, + 0.37696827424181495, + 0.33882165196875497, + 0.32633429825886623 + ], + "peak_layer": 21, + "peak_normalized": 2.1088522482676924, + "population_cv": 0.38950134534158565, + "rest_mean": 3.530613149551856e-05, + "spike_contrast": 1.7900985695566152, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.320145548670553e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.735007339855656e-05, + 3.843417653115466e-05, + 3.7711826735176146e-05, + 3.6161247408017516e-05, + 4.732056549983099e-05, + 4.723584424937144e-05, + 4.300080763641745e-05, + 3.968332021031529e-05, + 4.449134212336503e-05, + 3.937278233934194e-05, + 3.64429215551354e-05, + 3.3740830986062065e-05, + 3.096083310083486e-05, + 2.926450179074891e-05, + 2.8378755814628676e-05, + 2.724155092437286e-05, + 5.538272671401501e-05, + 5.262296326691285e-05, + 4.858071406488307e-05, + 4.387652006698772e-05, + 8.364715176867321e-05, + 7.425971853081137e-05, + 6.123795174062252e-05, + 5.1444469136185944e-05, + 4.541798625723459e-05, + 3.782408748520538e-05, + 3.067326179007068e-05, + 2.6730147510534152e-05, + 1.9448125385679305e-05, + 1.4952362107578665e-05, + 1.343928488495294e-05, + 1.2943976798851509e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.1457733257411746e-05, + "first_to_last_ratio": 1.8977474170968827, + "imbalance_abs_log_ratio": 0.6406676128793697, + "last_quartile_mean": 1.657635414176184e-05, + "mean": 2.8327688568197118e-05, + "normalized": [ + 1.0378057368348226, + 1.06983861568018, + 1.0491977904887932, + 1.0092011590321537, + 1.2852683271687153, + 1.2533015871147788, + 1.1416273294114696, + 1.0377128856771576, + 1.1745255348133223, + 1.0416840385088497, + 0.9634808894084209, + 0.8849637415797892, + 0.8443903541769905, + 0.7965684762965399, + 0.7722003128278548, + 0.7374273616451005, + 1.3424651539896326, + 1.2854458102944317, + 1.1706594334988774, + 1.0674068913781267, + 2.103772472082861, + 1.7707976898599065, + 1.3606069602640058, + 1.1183364576939212, + 1.065869260796216, + 0.8913709209455979, + 0.7395545264445227, + 0.6679526066396835, + 0.41071849146647305, + 0.32387129128319875, + 0.29724047481130667, + 0.2847374178863 + ], + "peak_layer": 21, + "peak_normalized": 2.103772472082861, + "population_cv": 0.379228075638394, + "rest_mean": 2.5789335840845188e-05, + "spike_contrast": 1.6299292682567954, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2034793295897543e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.9398637707345188e-05, + 3.0306055123219267e-05, + 2.9721348255407065e-05, + 2.8588336135726422e-05, + 3.640868089860305e-05, + 3.5503137041814625e-05, + 3.233966344851069e-05, + 2.939600744866766e-05, + 3.3271593565586954e-05, + 2.950850102934055e-05, + 2.7293186576571316e-05, + 2.506897726561874e-05, + 2.391962698311545e-05, + 2.256494371977169e-05, + 2.1874649974051863e-05, + 2.0889612642349675e-05, + 3.80289347958751e-05, + 3.641370858531445e-05, + 3.3162075851578265e-05, + 3.0237169994506985e-05, + 5.9595011407509446e-05, + 5.0162605475634336e-05, + 3.8542850234080106e-05, + 3.167988688801415e-05, + 3.0193612474249676e-05, + 2.525047784729395e-05, + 2.094987030432094e-05, + 1.892155341920443e-05, + 1.1634705515461974e-05, + 9.174525075650308e-06, + 8.420135600317735e-06, + 8.065952897595707e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010518337694520596, + "first_to_last_ratio": 1.298853979021645, + "imbalance_abs_log_ratio": 0.2614823210709866, + "last_quartile_mean": 8.098167973003001e-05, + "mean": 0.0001129772994090672, + "normalized": [ + 0.810962534989689, + 0.8287901520895661, + 0.8193344011879904, + 0.7967594592504931, + 1.090444814007616, + 1.1188741501855428, + 1.0312115132038047, + 0.9517299838647473, + 1.0314031089846865, + 0.9115982349983334, + 0.8441619298911692, + 0.7979269376759806, + 0.6929773379071392, + 0.654575232028, + 0.6380663091161348, + 0.6205945771541241, + 1.4091870327874871, + 1.3398512567689524, + 1.2663035936681761, + 1.1428583354431259, + 2.1533778907233563, + 2.0039677010770927, + 1.7844997896295434, + 1.5261754809958494, + 1.2411778195662533, + 1.0739169569244635, + 0.8779998056570737, + 0.7656704516242112, + 0.5726664578534288, + 0.445094451914986, + 0.3923455251595323, + 0.36549677367145117 + ], + "peak_layer": 21, + "peak_normalized": 2.1533778907233563, + "population_cv": 0.4230707862688655, + "rest_mean": 9.745673459191393e-05, + "spike_contrast": 2.019238077755404, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019678834942169488, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.162035712506622e-05, + 9.363447315990925e-05, + 9.256618795916438e-05, + 9.001573198474944e-05, + 0.000123195510241203, + 0.00012640737986657768, + 0.00011650349188130349, + 0.00010752388334367424, + 0.00011652513785520568, + 0.00010298990673618391, + 9.537113510305062e-05, + 9.014763054437935e-05, + 7.82907081884332e-05, + 7.3952141974587e-05, + 7.208700844785199e-05, + 7.011309935478494e-05, + 0.00015920614532660693, + 0.0001513727765996009, + 0.0001430635602446273, + 0.0001291170483455062, + 0.00024328281870111823, + 0.0002264028589706868, + 0.00020160796702839434, + 0.00017242318426724523, + 0.00014022491814102978, + 0.00012132823758292943, + 9.919404692482203e-05, + 8.650337986182421e-05, + 6.469830987043679e-05, + 5.0285569159314036e-05, + 4.432613786775619e-05, + 4.129283843212761e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "0c4b69b5a62d474637d9a6156b8452d836a142d489dd60b2de0cdb09db585af9", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13576549291610718, + 0.041234951466321945, + 0.023600688204169273, + 0.0554339736700058, + 0.01666521653532982, + 0.008341255597770214, + 0.01377889234572649, + 0.03695881366729736, + 0.033706482499837875, + 0.0386466309428215, + 0.04499480128288269, + 0.057638294994831085, + 0.04709118604660034, + 0.044345736503601074, + 0.04982399195432663, + 0.060589101165533066, + 0.040811967104673386, + 0.04198655113577843, + 0.05064959079027176, + 0.07217346876859665, + 0.039372652769088745, + 0.04702961444854736, + 0.054020222276449203, + 0.06186771020293236, + 0.054687224328517914, + 0.05866995081305504, + 0.06297128647565842, + 0.07938925921916962, + 0.07573848962783813, + 0.1105475053191185, + 0.12047550827264786, + 0.16982334852218628 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.32639113417099e-05, + "first_to_last_ratio": 1.6142215524258654, + "imbalance_abs_log_ratio": 0.4788528295882149, + "last_quartile_mean": 8.216908838676318e-06, + "mean": 1.2790947593543933e-05, + "normalized": [ + 0.9316524047936077, + 0.9631591197877182, + 0.9539238462095513, + 0.9083131861987929, + 1.2160660233571214, + 1.219529953106465, + 1.0980857820278223, + 1.0050812307438088, + 1.0448625967092706, + 0.9611285157606685, + 0.8964095006895693, + 0.8477627470675287, + 0.7494675898854604, + 0.706090181898008, + 0.6924087401774147, + 0.671952384866888, + 1.468062717452813, + 1.366424579378114, + 1.2391161308106104, + 1.1215385546051009, + 2.1279101909054225, + 1.8304940369610547, + 1.5253256372483166, + 1.316031782178504, + 1.115126772003547, + 0.8926197698458935, + 0.7767158527757819, + 0.6977024035715537, + 0.4688772635179848, + 0.4106255281575897, + 0.3886908734905123, + 0.38884410381750617 + ], + "peak_layer": 21, + "peak_normalized": 2.1279101909054225, + "population_cv": 0.387928054896289, + "rest_mean": 1.1410051852349338e-05, + "spike_contrast": 1.7745567555703712, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0247784595994746e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1916717085114215e-05, + 1.2319717825448606e-05, + 1.2201589925098233e-05, + 1.1618186363193672e-05, + 1.5554636775050312e-05, + 1.5598943718941882e-05, + 1.4045557691133581e-05, + 1.2855941349698696e-05, + 1.3364782716962509e-05, + 1.2293744475755375e-05, + 1.1465926945675164e-05, + 1.0843688869499601e-05, + 9.5864006652846e-06, + 9.031562512973323e-06, + 8.856563908921089e-06, + 8.594907740189228e-06, + 1.8777913282974623e-05, + 1.7477865185355768e-05, + 1.5849469491513446e-05, + 1.4345540876092855e-05, + 2.7217987735639326e-05, + 2.3413753297063522e-05, + 1.9510360289132223e-05, + 1.683329355728347e-05, + 1.4263528100855183e-05, + 1.1417452697060071e-05, + 9.934931767929811e-06, + 8.924274879973382e-06, + 5.997384505462833e-06, + 5.252289611235028e-06, + 4.971724592905957e-06, + 4.973684553988278e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.259526642475976e-05, + "first_to_last_ratio": 1.5935840922413673, + "imbalance_abs_log_ratio": 0.465985625520115, + "last_quartile_mean": 3.3004387205437524e-05, + "mean": 5.1062648935840116e-05, + "normalized": [ + 0.9216268503960833, + 0.950589815056072, + 0.9422144196635877, + 0.8985231786108131, + 1.2178691387217164, + 1.2132293533449192, + 1.0941896193182434, + 1.0018732508280441, + 1.0531925942812075, + 0.964189208098705, + 0.8991854758416759, + 0.850007127879547, + 0.7546609062980287, + 0.7123005293021083, + 0.6948749020960197, + 0.6737990168624725, + 1.4721384112868208, + 1.371930474910031, + 1.2436471490563514, + 1.1111982472028779, + 2.113050053165766, + 1.8242395861657554, + 1.5311286043798866, + 1.3195351834383122, + 1.1232494064757303, + 0.8974402485431373, + 0.7821413966916887, + 0.7035327854962891, + 0.4712604586854498, + 0.41227083555557753, + 0.3905192233334204, + 0.390392549013662 + ], + "peak_layer": 21, + "peak_normalized": 2.113050053165766, + "population_cv": 0.38599887162413726, + "rest_mean": 4.5556955296046067e-05, + "spike_contrast": 1.773459048474649, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.079339459072798e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.706070831161924e-05, + 4.853963400819339e-05, + 4.811196413356811e-05, + 4.5880973630119115e-05, + 6.218762428034097e-05, + 6.195070454850793e-05, + 5.5872220400488004e-05, + 5.11583020852413e-05, + 5.377880370360799e-05, + 4.923405504086986e-05, + 4.591479228110984e-05, + 4.3403615563875064e-05, + 3.8534984923899174e-05, + 3.6371951864566654e-05, + 3.548215318005532e-05, + 3.440596265136264e-05, + 7.517128688050434e-05, + 7.005440420471132e-05, + 6.350391777232289e-05, + 5.674072599504143e-05, + 0.0001078979330486618, + 9.315050556324422e-05, + 7.818348240107298e-05, + 6.737896183039993e-05, + 5.735609011026099e-05, + 4.582567635225132e-05, + 3.993821155745536e-05, + 3.592424764065072e-05, + 2.406380735919811e-05, + 2.1051640942459926e-05, + 1.9940946003771387e-05, + 1.9934477677452378e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.046642007771879435, + "first_to_last_ratio": 1.5935840941315902, + "imbalance_abs_log_ratio": 0.46598562670626065, + "last_quartile_mean": 0.029268620302900672, + "mean": 0.04528286767890677, + "normalized": [ + 0.9216268273678067, + 0.9505898655982172, + 0.9422144164682038, + 0.8985231812169583, + 1.2178690166672288, + 1.2132293989104181, + 1.0941896425839293, + 1.0018732026333346, + 1.053192572659169, + 0.9641892757112167, + 0.8991854313888171, + 0.8500071449620881, + 0.7546608899084634, + 0.7123004895677968, + 0.694874918803495, + 0.6737989921850662, + 1.4721383659439504, + 1.3719304542249628, + 1.2436471663071618, + 1.1111982836748173, + 2.1130500968616075, + 1.824239588496973, + 1.531128649637463, + 1.3195352773050415, + 1.1232494269264028, + 0.8974402993520915, + 0.7821413926920331, + 0.7035327086278959, + 0.4712604718453035, + 0.41227080515505665, + 0.3905192188271276, + 0.39039252748990244 + ], + "peak_layer": 21, + "peak_normalized": 2.1130500968616075, + "population_cv": 0.3859988823671551, + "rest_mean": 0.04040036315995234, + "spike_contrast": 1.7734591097013597, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.07164839208126068, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.04173390567302704, + 0.04304543510079384, + 0.04266617074608803, + 0.04068770632147789, + 0.05514860153198242, + 0.054938506335020065, + 0.04954804480075836, + 0.04536769166588783, + 0.04769157990813255, + 0.04366125538945198, + 0.04071769490838051, + 0.03849076107144356, + 0.034173209220170975, + 0.032255008816719055, + 0.03146592900156975, + 0.03051155060529709, + 0.06666264683008194, + 0.06212494522333145, + 0.05631591007113457, + 0.050318244844675064, + 0.0956849679350853, + 0.08260679990053177, + 0.06933389604091644, + 0.059752341359853745, + 0.05086395516991615, + 0.040638670325279236, + 0.03541760519146919, + 0.03185797855257988, + 0.02134002558887005, + 0.018668804317712784, + 0.017683830112218857, + 0.017678093165159225 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0265283637854736e-06, + "first_to_last_ratio": 0.9156476671615019, + "imbalance_abs_log_ratio": 0.08812363113882582, + "last_quartile_mean": 2.2132185080181443e-06, + "mean": 2.6874861127623717e-06, + "normalized": [ + 0.8352375485728037, + 0.7929371121311408, + 0.7730150142183774, + 0.7741669057859127, + 0.7344628997751053, + 0.7231354442993345, + 0.6957352730984985, + 0.7037974988161008, + 0.790273167233088, + 0.7708879700832757, + 0.7554230960744762, + 0.7732858335285265, + 0.6462830832935508, + 0.6542078265434891, + 0.6700938622286156, + 0.7131401727840788, + 1.4047798643220162, + 1.3792830850081157, + 1.3465103950870556, + 1.2824167454988713, + 2.3822796171807563, + 2.1737357192581888, + 1.9010524006612797, + 1.735640094726425, + 1.463796392135114, + 1.1467668979777017, + 1.0088922880136448, + 0.9362270084384399, + 0.5805339755855011, + 0.495871510556662, + 0.4753741013945969, + 0.4807571956892568 + ], + "peak_layer": 21, + "peak_normalized": 2.3822796171807563, + "population_cv": 0.48438790023185435, + "rest_mean": 2.2239938743950916e-06, + "spike_contrast": 2.3337942877011817, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.190344199945684e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.244689312647097e-06, + 2.1310074771463405e-06, + 2.0774671156686964e-06, + 2.080562808259856e-06, + 1.973858843484777e-06, + 1.943416464200709e-06, + 1.8697788846111507e-06, + 1.8914460042651626e-06, + 2.1238481622276595e-06, + 2.071750714094378e-06, + 2.0301890799601097e-06, + 2.07819493880379e-06, + 1.7368768112646649e-06, + 1.7581744486960815e-06, + 1.8008679489867063e-06, + 1.91655431081017e-06, + 3.775326376853627e-06, + 3.706804136527353e-06, + 3.6187279874866363e-06, + 3.4464771943021333e-06, + 6.4023433878901415e-06, + 5.841884558321908e-06, + 5.109051926410757e-06, + 4.664508651330834e-06, + 3.933932475774782e-06, + 3.081920112890657e-06, + 2.7113840133097256e-06, + 2.516097083571367e-06, + 1.5601769973727642e-06, + 1.332647798335529e-06, + 1.2775612958648708e-06, + 1.2920282870254596e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.059575303472229e-05, + "first_to_last_ratio": 1.8428664869924756, + "imbalance_abs_log_ratio": 0.6113222327604917, + "last_quartile_mean": 2.20285915020213e-05, + "mean": 3.723078083339715e-05, + "normalized": [ + 1.000747308098028, + 1.0297510550328743, + 1.0169292808330181, + 0.9702628967336459, + 1.2715949638568862, + 1.257179274381204, + 1.1379428423791025, + 1.0386440313235143, + 1.1094267873975205, + 1.0094867752605219, + 0.9345792064389409, + 0.8772088096510522, + 0.8023959259632547, + 0.7583457927560233, + 0.7417727186299412, + 0.7178167934970746, + 1.4137765645172868, + 1.3271740625307213, + 1.203509903524359, + 1.0853458857052019, + 2.1304053478424945, + 1.7810705561765154, + 1.4345481768124502, + 1.2166701599774927, + 1.051280441541134, + 0.8337316241032007, + 0.7246574768415295, + 0.6567125387812885, + 0.4057821793994631, + 0.362051816161588, + 0.34956720507227845, + 0.3496315987803933 + ], + "peak_layer": 21, + "peak_normalized": 2.1304053478424945, + "population_cv": 0.3839892025017025, + "rest_mean": 3.3626324592559184e-05, + "spike_contrast": 1.6860256129945155, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.669484453392215e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.725860369740985e-05, + 3.833843584288843e-05, + 3.7861071177758276e-05, + 3.6123645259067416e-05, + 4.73424734082073e-05, + 4.6805766032775864e-05, + 4.236650056554936e-05, + 3.8669528294121847e-05, + 4.130482557229698e-05, + 3.758398088393733e-05, + 3.4795113606378436e-05, + 3.265916893724352e-05, + 2.98738268611487e-05, + 2.8233806006028317e-05, + 2.761677751550451e-05, + 2.6724879717221484e-05, + 5.263600542093627e-05, + 4.941172664985061e-05, + 4.4807613448938355e-05, + 4.040827479911968e-05, + 7.931665459182113e-05, + 6.63106475258246e-05, + 5.340934876585379e-05, + 4.5297580072656274e-05, + 3.913999171345495e-05, + 3.104047937085852e-05, + 2.6979563699569553e-05, + 2.444992060190998e-05, + 1.5107587387319654e-05, + 1.3479471817845479e-05, + 1.3014659998589195e-05, + 1.3017057426623069e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.131921857857378e-05, + "first_to_last_ratio": 2.116701522335747, + "imbalance_abs_log_ratio": 0.7498589913461419, + "last_quartile_mean": 1.4796237564951298e-05, + "mean": 2.6929862087854417e-05, + "normalized": [ + 1.0947226696497885, + 1.1271282027493932, + 1.106919822700648, + 1.0632330527264946, + 1.3526853947996393, + 1.3079534266756163, + 1.1814225789176866, + 1.069872161641373, + 1.157814771927838, + 1.0519056823159265, + 0.9685157022801635, + 0.9017704048401847, + 0.8577058532160233, + 0.8140538691438461, + 0.7944030091628936, + 0.7643614989434289, + 1.351455932687456, + 1.2802515554321876, + 1.1546255463982888, + 1.0491990014149588, + 2.127009711159991, + 1.6881429365545844, + 1.2776017477923152, + 1.0617569146647619, + 0.9776304191432695, + 0.792423657567716, + 0.7111660870958268, + 0.6578041081913825, + 0.34259196138670317, + 0.31373499556133577, + 0.3002789981315749, + 0.299858325126703 + ], + "peak_layer": 21, + "peak_normalized": 2.127009711159991, + "population_cv": 0.3819549344885706, + "rest_mean": 2.4803259024068853e-05, + "spike_contrast": 1.548728680977784, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.841351863229647e-05, + "top_five_layers": [ + 21, + 22, + 5, + 17, + 6 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.9480730518116616e-05, + 3.0353407055372372e-05, + 2.9809198167640716e-05, + 2.863271947717294e-05, + 3.642763113020919e-05, + 3.522300539771095e-05, + 3.18155471177306e-05, + 2.8811509764636867e-05, + 3.117979213129729e-05, + 2.83276749541983e-05, + 2.608199429232627e-05, + 2.428455263725482e-05, + 2.3097900339053012e-05, + 2.1922358428128064e-05, + 2.1393163478933275e-05, + 2.058414975181222e-05, + 3.639452188508585e-05, + 3.4476997825549915e-05, + 3.109390672761947e-05, + 2.825478441081941e-05, + 5.728007818106562e-05, + 4.546145646600053e-05, + 3.440563887124881e-05, + 2.859296728274785e-05, + 2.6327452360419556e-05, + 2.1339859813451767e-05, + 1.915160464704968e-05, + 1.7714573914417997e-05, + 9.225954272551462e-06, + 8.44884016260039e-06, + 8.086472007562406e-06, + 8.075143341557123e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00010410473532829201, + "first_to_last_ratio": 1.4802470022359204, + "imbalance_abs_log_ratio": 0.39220896725351806, + "last_quartile_mean": 7.032929988781689e-05, + "mean": 0.00010518467865949788, + "normalized": [ + 0.8620082316979699, + 0.8793601137677285, + 0.8838341652030255, + 0.853808582522237, + 1.1744815620635005, + 1.1878682305333963, + 1.0874202108294881, + 0.9890819727977851, + 1.0219046417637305, + 0.9359499228966669, + 0.8632032673756425, + 0.8297183344841335, + 0.7158097001920478, + 0.6803843215474668, + 0.6659599171875242, + 0.6502865912508059, + 1.4528893199520632, + 1.3424784269798153, + 1.242892858354237, + 1.1348824127127712, + 2.1809149327188573, + 1.9248156603663433, + 1.6620583320953677, + 1.4289735266476418, + 1.1532148244613336, + 0.9405995358363591, + 0.8280572099898275, + 0.7404736013254825, + 0.4795450387252912, + 0.4212381094212713, + 0.392435197099403, + 0.3934512472007861 + ], + "peak_layer": 21, + "peak_normalized": 2.1809149327188573, + "population_cv": 0.40887132992350467, + "rest_mean": 9.213407557345581e-05, + "spike_contrast": 1.906546890830614, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000175657935324125, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.067005885299295e-05, + 9.249521099263802e-05, + 9.296581265516579e-05, + 8.980758138932288e-05, + 0.0001235374656971544, + 0.00012494553811848164, + 0.00011437994544394314, + 0.00010403626947663724, + 0.00010748871136456728, + 9.844759188126773e-05, + 9.079575829673558e-05, + 8.727365639060736e-05, + 7.529221329605207e-05, + 7.156600622693077e-05, + 7.004877988947555e-05, + 6.840018613729626e-05, + 0.00015282169624697417, + 0.00014120816194918007, + 0.0001307332859141752, + 0.0001193722418975085, + 0.00022939883638173342, + 0.00020246111671440303, + 0.00017482307157479227, + 0.00015030612121336162, + 0.00012130053073633462, + 9.89366599242203e-05, + 8.709893154446036e-05, + 7.788647781126201e-05, + 5.044079080107622e-05, + 4.430779517861083e-05, + 4.1278170101577416e-05, + 4.138504300499335e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4cec366fa91114ffbf2b3855220297ceecccb362041740eb1c3b94ecdc658134", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.08130212873220444, + 0.2561342120170593, + 0.3685380816459656, + 0.38441672921180725, + 0.14100632071495056, + 0.16686734557151794, + 0.18023288249969482, + 0.18965557217597961, + 0.05848926678299904, + 0.09072534739971161, + 0.12624414265155792, + 0.17554737627506256, + 0.06478945910930634, + 0.11512516438961029, + 0.15369470417499542, + 0.19735252857208252, + 0.04376237466931343, + 0.07002050429582596, + 0.09636416286230087, + 0.12901709973812103, + 0.04409762844443321, + 0.07137759029865265, + 0.10297419875860214, + 0.1298016905784607, + 0.03295852243900299, + 0.07529281079769135, + 0.11068660020828247, + 0.1495535671710968, + 0.021799447014927864, + 0.08539005368947983, + 0.17447474598884583, + 0.27695441246032715 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3376565448197653e-05, + "first_to_last_ratio": 1.4098290045963058, + "imbalance_abs_log_ratio": 0.343468423699918, + "last_quartile_mean": 9.488076500474563e-06, + "mean": 1.3753222575019208e-05, + "normalized": [ + 0.8721340354721765, + 0.8994799354622096, + 0.8826639131112579, + 0.8463853629982472, + 1.1296999173073485, + 1.1469756063027932, + 1.0400388665804514, + 0.9635277470755461, + 1.0571451316162943, + 0.9410534141549793, + 0.8736341847787974, + 0.8166490774236239, + 0.7261875943754862, + 0.6844771613422266, + 0.6647247876726992, + 0.6388650175203425, + 1.452098767146314, + 1.3695701212448788, + 1.2684285141735008, + 1.1495085011117947, + 2.0934894713524743, + 1.9169745701660839, + 1.6426207049178487, + 1.4046256710812022, + 1.198612087866968, + 1.0253598212999628, + 0.8286524541524178, + 0.7176736105632963, + 0.5714418203699311, + 0.43393410763421036, + 0.3817475933674649, + 0.36162043035717156 + ], + "peak_layer": 21, + "peak_normalized": 2.0934894713524743, + "population_cv": 0.39806173416327506, + "rest_mean": 1.2094521530848902e-05, + "spike_contrast": 1.8777268828380727, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.2710208213538863e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.199465350509854e-05, + 1.2370747754175682e-05, + 1.2139473255956545e-05, + 1.164052628155332e-05, + 1.553701440570876e-05, + 1.577461080159992e-05, + 1.4303886018751655e-05, + 1.3251611562736798e-05, + 1.4539152289216872e-05, + 1.2942517059855163e-05, + 1.201528539240826e-05, + 1.1231556527491193e-05, + 9.987419616663828e-06, + 9.413766747456975e-06, + 9.142107955995016e-06, + 8.786452781350818e-06, + 1.9971037545474246e-05, + 1.883600270957686e-05, + 1.7444979675929062e-05, + 1.5809446267667226e-05, + 2.8792226657969877e-05, + 2.6364577934145927e-05, + 2.2591328161070123e-05, + 1.9318129488965496e-05, + 1.648477882554289e-05, + 1.410200184182031e-05, + 1.1396641639294103e-05, + 9.87032490229467e-06, + 7.859166544221807e-06, + 5.967992365185637e-06, + 5.250259619060671e-06, + 4.973446266376413e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.305920649334439e-05, + "first_to_last_ratio": 1.392778608294149, + "imbalance_abs_log_ratio": 0.3313007505760008, + "last_quartile_mean": 3.8095937270554714e-05, + "mean": 5.492100513038167e-05, + "normalized": [ + 0.8617029028549882, + 0.8875029339754659, + 0.8724915117830877, + 0.8369809973610108, + 1.131954274301571, + 1.1404132813521581, + 1.0352898061888103, + 0.9624677060873116, + 1.0644389422291614, + 0.9432618214397073, + 0.8769178812067566, + 0.8183058955006484, + 0.7308681584499725, + 0.6902021023938238, + 0.6672861699325884, + 0.6409658882179515, + 1.4550351924910432, + 1.3755767509041323, + 1.2725799838609873, + 1.142576421586798, + 2.0822225212691614, + 1.908623231016431, + 1.6442116625532532, + 1.4089265759792677, + 1.2059942668685917, + 1.0325358032982117, + 0.8327031384838283, + 0.7226180730156292, + 0.5732837633016289, + 0.4359643343675707, + 0.38311796261183234, + 0.36298004511661974 + ], + "peak_layer": 21, + "peak_normalized": 2.0822225212691614, + "population_cv": 0.3966538790797015, + "rest_mean": 4.8310187628009805e-05, + "spike_contrast": 1.87578281295371, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.061941964318975e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.7325589548563585e-05, + 4.874255319009535e-05, + 4.791811079485342e-05, + 4.5967837650096044e-05, + 6.216806650627404e-05, + 6.263264367589727e-05, + 5.6859156757127494e-05, + 5.285969382384792e-05, + 5.8460056607145816e-05, + 5.180488733458333e-05, + 4.816121145267971e-05, + 4.494218228501268e-05, + 4.0140013879863545e-05, + 3.7906593206571415e-05, + 3.664802716230042e-05, + 3.520249083521776e-05, + 7.991199527168646e-05, + 7.55480577936396e-05, + 6.989137182245031e-05, + 6.275144551182166e-05, + 0.00011435775377321988, + 0.00010482350626261905, + 9.03017571545206e-05, + 7.737966370768845e-05, + 6.62344173179008e-05, + 5.670790415024385e-05, + 4.5732893340755254e-05, + 3.968691089539789e-05, + 3.1485320505453274e-05, + 2.394359944446478e-05, + 2.104122359014582e-05, + 1.9935228920076042e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04705343535169959, + "first_to_last_ratio": 1.3927786002877673, + "imbalance_abs_log_ratio": 0.33130074482750527, + "last_quartile_mean": 0.03378385864198208, + "mean": 0.048704497050493956, + "normalized": [ + 0.8617028486976422, + 0.8875029589946798, + 0.8724915017730162, + 0.836980984158547, + 1.131954282861735, + 1.1404132764850687, + 1.0352898604369885, + 0.9624676951017845, + 1.064438980488691, + 0.9432618106350297, + 0.8769178394343718, + 0.8183058471780916, + 0.7308681244611269, + 0.6902021128534255, + 0.6672861971396676, + 0.6409658946457253, + 1.4550351699087516, + 1.3755767870484994, + 1.272579955863019, + 1.1425763459349243, + 2.0822225110266683, + 1.908623337758237, + 1.6442116556955237, + 1.4089266063288464, + 1.2059942838327453, + 1.032535771268638, + 0.8327031854247592, + 0.7226180949727652, + 0.5732837728076834, + 0.4359643147464535, + 0.3831179581029778, + 0.36298003393391687 + ], + "peak_layer": 21, + "peak_normalized": 2.0822225110266683, + "population_cv": 0.39665388748012176, + "rest_mean": 0.042841957675086126, + "spike_contrast": 1.8757828549097615, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08036220967769622, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.041968803852796555, + 0.04322538524866104, + 0.042494259774684906, + 0.040764737874269485, + 0.05513126403093338, + 0.05554325506091118, + 0.0504232719540596, + 0.04687650501728058, + 0.05184296518564224, + 0.04594109207391739, + 0.0427098423242569, + 0.03985517472028732, + 0.035596564412117004, + 0.033615946769714355, + 0.03249983862042427, + 0.031217921525239944, + 0.07086675614118576, + 0.06699677556753159, + 0.061980366706848145, + 0.05564860627055168, + 0.10141360014677048, + 0.09295853972434998, + 0.08008050173521042, + 0.06862106174230576, + 0.05873734503984451, + 0.05028913542628288, + 0.04055638983845711, + 0.035194750875234604, + 0.02792149782180786, + 0.021233422681689262, + 0.018659567460417747, + 0.01767875999212265 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0335669290716396e-06, + "first_to_last_ratio": 0.8002578806956262, + "imbalance_abs_log_ratio": 0.22282125238855607, + "last_quartile_mean": 2.5411395228047695e-06, + "mean": 2.9139069361860948e-06, + "normalized": [ + 0.7803650562078773, + 0.7320955321763626, + 0.70991137711656, + 0.7130517934518413, + 0.6728847245961602, + 0.6708950243493448, + 0.649173395200872, + 0.6546893336924704, + 0.7715659445434139, + 0.7335472900136112, + 0.7283727740652475, + 0.7337376064535494, + 0.6162116646247685, + 0.6242056183615969, + 0.6341550559619227, + 0.6616468470383687, + 1.3816834645183462, + 1.3825650533409872, + 1.382166161319272, + 1.3322845267305112, + 2.3405652633576954, + 2.2789024246956493, + 2.040432570140748, + 1.7983076099361612, + 1.5171012195101121, + 1.3281261242224407, + 1.0623618177778928, + 0.9268841353462449, + 0.6999640463402925, + 0.5293512882553055, + 0.46681517102107106, + 0.44598008563330255 + ], + "peak_layer": 21, + "peak_normalized": 2.3405652633576954, + "population_cv": 0.5165053196020641, + "rest_mean": 2.3769592450696144e-06, + "spike_contrast": 2.4457400690708226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.813424468215089e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2739111500413856e-06, + 2.133258249159553e-06, + 2.0686156858573668e-06, + 2.077766566799255e-06, + 1.9607234662544215e-06, + 1.9549256649042945e-06, + 1.891630859063298e-06, + 1.9077037904935423e-06, + 2.2482713575300295e-06, + 2.1374885363911744e-06, + 2.122410478477832e-06, + 2.1380431007855805e-06, + 1.7955834437088924e-06, + 1.8188770809501875e-06, + 1.8478688161849277e-06, + 1.9279773368907627e-06, + 4.026097030873643e-06, + 4.028665898658801e-06, + 4.027503564429935e-06, + 3.8821531234134454e-06, + 6.820189355494222e-06, + 6.640509582211962e-06, + 5.945630618953146e-06, + 5.240101017989218e-06, + 4.4206917664268985e-06, + 3.870035925501725e-06, + 3.0956234695622697e-06, + 2.700854111026274e-06, + 2.0396300897118635e-06, + 1.5424803905261797e-06, + 1.360255964755197e-06, + 1.2995444649277488e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.08623318435275e-05, + "first_to_last_ratio": 1.6228911311744263, + "imbalance_abs_log_ratio": 0.48420920752374813, + "last_quartile_mean": 2.517872644602903e-05, + "mean": 3.966481972383917e-05, + "normalized": [ + 0.941643984322134, + 0.9689728364581338, + 0.9507669925320091, + 0.9116714519207078, + 1.193004560988191, + 1.1908787206748392, + 1.084110505037148, + 1.0004772432625977, + 1.1216820589953018, + 0.9926455330841178, + 0.9187755630308546, + 0.8506477831794215, + 0.7805623738669399, + 0.7377941685016672, + 0.7154642237174069, + 0.6867938120739691, + 1.3962577510757062, + 1.3266966019768376, + 1.2247798111115489, + 1.1061916188277885, + 2.108839277362418, + 1.87217926525862, + 1.5438876295558621, + 1.296977303034086, + 1.145052086148766, + 0.9535982361523758, + 0.7733129814099837, + 0.6739034402524328, + 0.4903104209064994, + 0.3769676510880434, + 0.33882058947149796, + 0.3263335247220941 + ], + "peak_layer": 21, + "peak_normalized": 2.108839277362418, + "population_cv": 0.38949977839098365, + "rest_mean": 3.530619142111391e-05, + "spike_contrast": 1.790094315320562, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.32014125585556e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.735013888217509e-05, + 3.843413287540898e-05, + 3.7712001358158886e-05, + 3.616128378780559e-05, + 4.7320310841314495e-05, + 4.723598976852372e-05, + 4.3001047743018717e-05, + 3.9683749491814524e-05, + 4.449131665751338e-05, + 3.937310611945577e-05, + 3.644306707428768e-05, + 3.374079096829519e-05, + 3.0960865842644125e-05, + 2.9264472686918452e-05, + 2.8378759452607483e-05, + 2.7241552743362263e-05, + 5.5382311984431e-05, + 5.262318154564127e-05, + 4.858067040913738e-05, + 4.3876891140826046e-05, + 8.364672976313159e-05, + 7.425965304719284e-05, + 6.123802450019866e-05, + 5.144437091075815e-05, + 4.541828457149677e-05, + 3.7824302125955e-05, + 3.0673319997731596e-05, + 2.673025846888777e-05, + 1.9448074453976005e-05, + 1.4952353922126349e-05, + 1.3439257600111887e-05, + 1.2943960427946877e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.14603882998199e-05, + "first_to_last_ratio": 1.8980819473898056, + "imbalance_abs_log_ratio": 0.6408438749089113, + "last_quartile_mean": 1.6574831420257397e-05, + "mean": 2.8326866441830134e-05, + "normalized": [ + 1.0377835864880585, + 1.0701104047897898, + 1.0491032802041116, + 1.0093792976317837, + 1.285175659609536, + 1.2536418234821836, + 1.1421213928087102, + 1.0376456542092856, + 1.1747807768794196, + 1.0416012540102209, + 0.9636221263630744, + 0.8849957187485022, + 0.8443134022244022, + 0.796513189344223, + 0.7723615555596486, + 0.7373587354640148, + 1.3422069324537358, + 1.2852755770330793, + 1.1704786768102076, + 1.0672891502889117, + 2.103806816412106, + 1.770591712609653, + 1.360589169814368, + 1.1182332944394449, + 1.0655109283784003, + 0.891301818145435, + 0.7396229310643554, + 0.6680012742401278, + 0.4107558405430897, + 0.3238447630597274, + 0.29727709900388205, + 0.2847061578865119 + ], + "peak_layer": 21, + "peak_normalized": 2.103806816412106, + "population_cv": 0.37921215100884365, + "rest_mean": 2.5789270289856682e-05, + "spike_contrast": 1.6297431137094942, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.202988566248678e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.93971570499707e-05, + 3.031287451449316e-05, + 2.9717808502027765e-05, + 2.859255255316384e-05, + 3.6404999264050275e-05, + 3.5511744499672204e-05, + 3.2352720154449344e-05, + 2.9393249860731885e-05, + 3.3277858165092766e-05, + 2.950529960799031e-05, + 2.729639527387917e-05, + 2.506915552658029e-05, + 2.3916752979857847e-05, + 2.256272273370996e-05, + 2.187858262914233e-05, + 2.0887062419205904e-05, + 3.802051651291549e-05, + 3.640782961156219e-05, + 3.315599315101281e-05, + 3.0232957215048373e-05, + 5.959425470791757e-05, + 5.0155314966104925e-05, + 3.854122769553214e-05, + 3.1676045182393864e-05, + 3.018258576048538e-05, + 2.5247787561966106e-05, + 2.095119998557493e-05, + 1.8922382878372446e-05, + 1.1635425835265778e-05, + 9.173507351079024e-06, + 8.420928679697681e-06, + 8.064833309617825e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001051802182701067, + "first_to_last_ratio": 1.2988284379869668, + "imbalance_abs_log_ratio": 0.26146265659274887, + "last_quartile_mean": 8.098084026642027e-05, + "mean": 0.00011297554704015056, + "normalized": [ + 0.8108794110949893, + 0.8285381181909731, + 0.8190502123585883, + 0.796943644872694, + 1.0905965877081263, + 1.1190586951317285, + 1.0312475376881012, + 0.951684658247413, + 1.0315797924663694, + 0.9117273340922073, + 0.8439995901115003, + 0.7977917028142015, + 0.6929493161394222, + 0.6545568546712421, + 0.6381324299605099, + 0.620599759422206, + 1.4094209052378723, + 1.3397655167973181, + 1.2663323805793527, + 1.1428262144942543, + 2.153338001318954, + 2.00370497849062, + 1.7845980547125555, + 1.5262805588928774, + 1.2411593314085447, + 1.074164756659053, + 0.8778398584235759, + 0.7656336393496486, + 0.5727781276048317, + 0.4449834340085659, + 0.3922665989654889, + 0.36557199808621427 + ], + "peak_layer": 21, + "peak_normalized": 2.153338001318954, + "population_cv": 0.42307789431213444, + "rest_mean": 9.745571568836803e-05, + "spike_contrast": 2.019200566635041, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019678263633977623, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 17 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.160954505205154e-05, + 9.360454714624211e-05, + 9.2532645794563e-05, + 9.003514423966408e-05, + 0.0001232107460964471, + 0.0001264262682525441, + 0.0001165057547041215, + 0.00010751709487522021, + 0.00011654329136945307, + 0.00010300289432052523, + 9.535131539450958e-05, + 9.013095404952765e-05, + 7.828632806194946e-05, + 7.394891872536391e-05, + 7.209336035884917e-05, + 7.011259731370956e-05, + 0.00015923009777907282, + 0.00015136074216570705, + 0.0001430645934306085, + 0.0001291114167543128, + 0.0002432745386613533, + 0.0002263696660520509, + 0.0002016159414779395, + 0.00017243238107766956, + 0.00014022065442986786, + 0.00012135435099480674, + 9.917443821905181e-05, + 8.649787923786789e-05, + 6.470992229878902e-05, + 5.0272246880922467e-05, + 4.4316533603705466e-05, + 4.130069646635093e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "4fbc436930b501f120196fb3bf6ab490fc20b93e2a3a7db48af6d653a1fa07c9", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16142085194587708, + 0.2773628532886505, + 0.3768865168094635, + 0.40868353843688965, + 0.14338424801826477, + 0.16827327013015747, + 0.18227216601371765, + 0.19825012981891632, + 0.06985136866569519, + 0.1086883395910263, + 0.14860449731349945, + 0.21192558109760284, + 0.08672910183668137, + 0.13778631389141083, + 0.18114051222801208, + 0.23228809237480164, + 0.062035780400037766, + 0.0909910649061203, + 0.12021689862012863, + 0.1606491208076477, + 0.061457011848688126, + 0.092374786734581, + 0.12395651638507843, + 0.15162576735019684, + 0.06645520776510239, + 0.10725662857294083, + 0.14660172164440155, + 0.20186874270439148, + 0.07922796159982681, + 0.17203646898269653, + 0.2743971645832062, + 0.41855114698410034 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3263900200399803e-05, + "first_to_last_ratio": 1.6142150262450288, + "imbalance_abs_log_ratio": 0.478848786652461, + "last_quartile_mean": 8.21693515717925e-06, + "mean": 1.2790936466444691e-05, + "normalized": [ + 0.9316267643378715, + 0.9631635128907836, + 0.9539325686606005, + 0.9083160383931895, + 1.2160436167127753, + 1.219552487596532, + 1.0980943454698229, + 1.005082460607795, + 1.044869976177374, + 0.9611479101725001, + 0.8963889491080845, + 0.8477546675806928, + 0.7494694506413357, + 0.7060841834105682, + 0.692386873457513, + 0.6719497697041285, + 1.4680346994465847, + 1.3664213595724557, + 1.2391345582710243, + 1.1215611460596746, + 2.127901945162036, + 1.830484252606031, + 1.5253501442672783, + 1.3160248210947718, + 1.1151359191068313, + 0.8926157823445571, + 0.7767052939266422, + 0.6977092677239847, + 0.46888414192457656, + 0.41063288917446084, + 0.3886900028421406, + 0.3888502015553834 + ], + "peak_layer": 21, + "peak_normalized": 2.127901945162036, + "population_cv": 0.3879272801672154, + "rest_mean": 1.1410037822551439e-05, + "spike_contrast": 1.774559336118009, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0247789143468255e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1916378753085155e-05, + 1.2319763300183695e-05, + 1.220169087901013e-05, + 1.1618212738540024e-05, + 1.5554336641798727e-05, + 1.5599218386341818e-05, + 1.404565500706667e-05, + 1.2855945897172205e-05, + 1.336486548098037e-05, + 1.2293981853872538e-05, + 1.1465654097264633e-05, + 1.0843576092156582e-05, + 9.58641612669453e-06, + 9.031477929966059e-06, + 8.856276508595329e-06, + 8.594866812927648e-06, + 1.8777538571157493e-05, + 1.7477808796684258e-05, + 1.584969140822068e-05, + 1.4345817362482194e-05, + 2.7217858587391675e-05, + 2.341360777791124e-05, + 1.9510656784405e-05, + 1.6833189874887466e-05, + 1.4263632692745887e-05, + 1.1417391760915052e-05, + 9.934788067766931e-06, + 8.924354915507138e-06, + 5.997467269480694e-06, + 5.252379196463153e-06, + 4.971709131496027e-06, + 4.9737582230591215e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.259535555524053e-05, + "first_to_last_ratio": 1.59358297228639, + "imbalance_abs_log_ratio": 0.46598492272985975, + "last_quartile_mean": 3.300446633147658e-05, + "mean": 5.106260448428657e-05, + "normalized": [ + 0.921606350307445, + 0.9505980521014451, + 0.9422292752458163, + 0.8985273805847644, + 1.2178505351666271, + 1.2132371065714032, + 1.0942079557361923, + 1.0018801076079868, + 1.0531977858451338, + 0.9641972432463806, + 0.8991742893725557, + 0.8499991046449304, + 0.7546616344988103, + 0.712299296999438, + 0.6948617566317875, + 0.6737987484795189, + 1.4721176067352022, + 1.3719352314903852, + 1.2436616258358075, + 1.111210257581308, + 2.1130451955668876, + 1.8242133884920444, + 1.531138344238427, + 1.319527925169485, + 1.1232592899812257, + 0.8974371112918603, + 0.7821448561426008, + 0.7035301918972675, + 0.4712647874324093, + 0.41227521981846393, + 0.3905171409466258, + 0.3903952043397641 + ], + "peak_layer": 21, + "peak_normalized": 2.1130451955668876, + "population_cv": 0.3859968358313239, + "rest_mean": 4.555695098436748e-05, + "spike_contrast": 1.7734534826874866, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.079313338384963e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.705962055595592e-05, + 4.8540012357989326e-05, + 4.8112680815393105e-05, + 4.5881148253101856e-05, + 6.218662019819021e-05, + 6.19510465185158e-05, + 5.5873108067316934e-05, + 5.11586076754611e-05, + 5.377902198233642e-05, + 4.923442247672938e-05, + 4.591418110067025e-05, + 4.340316809248179e-05, + 3.853498856187798e-05, + 3.637185727711767e-05, + 3.548145105014555e-05, + 3.440591899561696e-05, + 7.517015910707414e-05, + 7.005458610365167e-05, + 6.350460171233863e-05, + 5.674128988175653e-05, + 0.00010789759107865393, + 9.314908675150946e-05, + 7.818391168257222e-05, + 6.73785325489007e-05, + 5.735654485761188e-05, + 4.582547626341693e-05, + 3.993835343862884e-05, + 3.59240839316044e-05, + 2.40640074480325e-05, + 2.1051846488262527e-05, + 1.9940822312491946e-05, + 1.993459591176361e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04664208646863699, + "first_to_last_ratio": 1.5935829291342798, + "imbalance_abs_log_ratio": 0.46598489565118767, + "last_quartile_mean": 0.029268691083416343, + "mean": 0.04528282774845138, + "normalized": [ + 0.9216063328612899, + 0.9505981078752931, + 0.9422292327337689, + 0.8985273464900081, + 1.2178505932652952, + 1.2132370501116239, + 1.0942079658156032, + 1.0018800915901536, + 1.053197779258807, + 0.9641972831787459, + 0.8991742955513574, + 0.8499990919117887, + 0.7546616376365752, + 0.7122993077974369, + 0.6948617929269221, + 0.6737987636701324, + 1.4721176164746403, + 1.3719352014835002, + 1.243661590239114, + 1.1112102873322856, + 2.1130452142435634, + 1.8242133908087463, + 1.531138226506127, + 1.3195278850867733, + 1.123259302264044, + 0.8974371418917119, + 0.7821448794692506, + 0.7035302028513013, + 0.47126479509418523, + 0.4122752409211241, + 0.39051713630582213, + 0.39039521635300933 + ], + "peak_layer": 21, + "peak_normalized": 2.1130452142435634, + "population_cv": 0.3859968269414632, + "rest_mean": 0.04040035943466204, + "spike_contrast": 1.7734534456008406, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.07164815664291382, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.04173294082283974, + 0.0430457703769207, + 0.042666804045438766, + 0.04068785905838013, + 0.055147718638181686, + 0.05493880435824394, + 0.04954883083701134, + 0.04536796361207962, + 0.04769177362322807, + 0.04366157948970795, + 0.04071715474128723, + 0.03849036246538162, + 0.03417321294546127, + 0.03225492686033249, + 0.031465306878089905, + 0.030511513352394104, + 0.06666164845228195, + 0.062125105410814285, + 0.05631651356816292, + 0.05031874403357506, + 0.09568466246128082, + 0.08260554075241089, + 0.06933426856994629, + 0.059751953929662704, + 0.05086435750126839, + 0.04063849151134491, + 0.03541773185133934, + 0.03185783699154854, + 0.021340202540159225, + 0.018668988719582558, + 0.017683720216155052, + 0.01767819933593273 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.026510429686823e-06, + "first_to_last_ratio": 0.915629822205116, + "imbalance_abs_log_ratio": 0.0881431202184329, + "last_quartile_mean": 2.2132420554044074e-06, + "mean": 2.6874848515490157e-06, + "normalized": [ + 0.8352326104505317, + 0.7929193788577268, + 0.7729974408049657, + 0.7742255616893798, + 0.734464344312438, + 0.7231120097580448, + 0.6957460905757977, + 0.7037397057177938, + 0.7902611858247006, + 0.7709003457125179, + 0.755409152404643, + 0.7733638634799724, + 0.646281017658817, + 0.6542165517187554, + 0.6700454867303076, + 0.7131122495064298, + 1.404762079762564, + 1.37930285294127, + 1.346554090752029, + 1.2824702252229325, + 2.382204590992269, + 2.173774134622282, + 1.9010011763534234, + 1.7356112976249671, + 1.4638471650261105, + 1.1467541532177463, + 1.0089145048692982, + 0.9362415767757399, + 0.5805525649285846, + 0.49587787710091685, + 0.4753518619529011, + 0.48075285265414314 + ], + "peak_layer": 21, + "peak_normalized": 2.382204590992269, + "population_cv": 0.4843847200620537, + "rest_mean": 2.2239993861014092e-06, + "spike_contrast": 2.3337714917558996, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.190306364966091e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.244674988105544e-06, + 2.1309588191797957e-06, + 2.0774189124495024e-06, + 2.0807194687222363e-06, + 1.9738617993425578e-06, + 1.9433525721979095e-06, + 1.8698070789469057e-06, + 1.8912897985501331e-06, + 2.1238149656710448e-06, + 2.071783001156291e-06, + 2.0301506538089598e-06, + 2.078403667837847e-06, + 1.7368704448017525e-06, + 1.7581970723767881e-06, + 1.8007370954364887e-06, + 1.916478368002572e-06, + 3.7752768093923805e-06, + 3.706855522978003e-06, + 3.618843720687437e-06, + 3.4466193028492853e-06, + 6.4021387515822425e-06, + 5.841985057486454e-06, + 5.108911864226684e-06, + 4.664429070544429e-06, + 3.934067080990644e-06, + 3.0818844152236125e-06, + 2.7114424483443145e-06, + 2.516135054975166e-06, + 1.5602262237734976e-06, + 1.3326642829269986e-06, + 1.2775009281540406e-06, + 1.2920160088469856e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.0595810332888504e-05, + "first_to_last_ratio": 1.8428634767077197, + "imbalance_abs_log_ratio": 0.6113205992795879, + "last_quartile_mean": 2.2028658577255555e-05, + "mean": 3.7230811983590684e-05, + "normalized": [ + 1.000742073656224, + 1.0297534180314578, + 1.0169311659883222, + 0.9702710746421311, + 1.271580024525988, + 1.257182912808566, + 1.1379498051373607, + 1.0386461914531528, + 1.1094287905906044, + 1.0094888620711941, + 0.9345745159294909, + 0.8772057305692527, + 0.8023970134719064, + 0.7583468682633459, + 0.7417669191528679, + 0.7178190266273485, + 1.4137682485056282, + 1.3271728543994634, + 1.203515443425389, + 1.0853495701875495, + 2.130407864804216, + 1.78105851286132, + 1.4345507874114394, + 1.2166669923043094, + 1.0512864996659526, + 0.8337322945371913, + 0.7246595088199664, + 0.6567104258973786, + 0.4057836476031255, + 0.3620542980954201, + 0.34956534917004795, + 0.3496333093923887 + ], + "peak_layer": 21, + "peak_normalized": 2.130407864804216, + "population_cv": 0.38398827396059443, + "rest_mean": 3.362635827754814e-05, + "spike_contrast": 1.686024443333039, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.6694861996220425e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.7258439988363534e-05, + 3.8338555896189064e-05, + 3.7861173041164875e-05, + 3.612397995311767e-05, + 4.734195681521669e-05, + 4.6805940655758604e-05, + 4.236679524183273e-05, + 3.8669641071464866e-05, + 4.130493471166119e-05, + 3.758409002330154e-05, + 3.479496808722615e-05, + 3.265908162575215e-05, + 2.9873892344767228e-05, + 2.823386967065744e-05, + 2.7616584702627733e-05, + 2.672498521860689e-05, + 5.2635739848483354e-05, + 4.94117230118718e-05, + 4.480785719351843e-05, + 4.0408445784123614e-05, + 7.931681466288865e-05, + 6.631025462411344e-05, + 5.340949064702727e-05, + 4.529750003712252e-05, + 3.914025000995025e-05, + 3.104053030256182e-05, + 2.6979661924997345e-05, + 2.4449862394249067e-05, + 1.5107654689927585e-05, + 1.347957550024148e-05, + 1.3014601790928282e-05, + 1.3017132005188614e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.132075494249875e-05, + "first_to_last_ratio": 2.1168727266692042, + "imbalance_abs_log_ratio": 0.749939870680473, + "last_quartile_mean": 1.4795766674069455e-05, + "mean": 2.6930997876206675e-05, + "normalized": [ + 1.0946769735958595, + 1.1271800223787605, + 1.106456131482209, + 1.0633216760799757, + 1.3525082558946082, + 1.3082808262675252, + 1.181611989443304, + 1.0699654356061856, + 1.1577934996059576, + 1.051929807416546, + 0.9684829611938787, + 0.9018017398021899, + 0.8575415520179537, + 0.8139981937633387, + 0.7942835242774724, + 0.7646872384868243, + 1.3514805278136963, + 1.2804770533277787, + 1.1546995085484912, + 1.0491442158344637, + 2.1271928786890193, + 1.6882992242363928, + 1.2773347017743915, + 1.0616887663741317, + 0.9776669976077785, + 0.7923208716371506, + 0.7110474784846503, + 0.6578201335496455, + 0.3425598167485049, + 0.3136933962077473, + 0.3002437073917716, + 0.29981089446179765 + ], + "peak_layer": 21, + "peak_normalized": 2.1271928786890193, + "population_cv": 0.3819948907959122, + "rest_mean": 2.480426438624919e-05, + "spike_contrast": 1.5487400925815615, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.84153587219771e-05, + "top_five_layers": [ + 21, + 22, + 5, + 17, + 6 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.948074325104244e-05, + 3.0356082788784988e-05, + 2.9797967727063224e-05, + 2.8636313800234348e-05, + 3.642439696704969e-05, + 3.523330815369263e-05, + 3.182198997819796e-05, + 2.8815236873924732e-05, + 3.118053427897394e-05, + 2.8329519409453496e-05, + 2.6082212571054697e-05, + 2.428642073937226e-05, + 2.3094449716154486e-05, + 2.1921783627476543e-05, + 2.139084790542256e-05, + 2.0593790395651013e-05, + 3.6396719224285334e-05, + 3.448452480370179e-05, + 3.109721001237631e-05, + 2.8254500648472458e-05, + 5.7287426898255944e-05, + 4.546758282231167e-05, + 3.439989814069122e-05, + 2.8592337912414223e-05, + 2.632954783621244e-05, + 2.1337991711334325e-05, + 1.9149218132952228e-05, + 1.7715752619551495e-05, + 9.225477697327733e-06, + 8.448076187050901e-06, + 8.085862646112218e-06, + 8.074206562014297e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001040923161781393, + "first_to_last_ratio": 1.4801753983355805, + "imbalance_abs_log_ratio": 0.3921605931428911, + "last_quartile_mean": 7.032431176412501e-05, + "mean": 0.00010517841462842625, + "normalized": [ + 0.8617262734887446, + 0.8795612163230485, + 0.8836052514767463, + 0.8537133297175012, + 1.173918399219713, + 1.1881939629889764, + 1.0876047193780078, + 0.9890668591031853, + 1.0219410138030538, + 0.9359575171630558, + 0.8633693032863031, + 0.8295890644515174, + 0.71591189277355, + 0.6802765265722901, + 0.6658347297769002, + 0.6504541971660552, + 1.4527609839189186, + 1.3426235449558381, + 1.2433488773729529, + 1.1347940072566047, + 2.180691877285598, + 1.925348956107517, + 1.6618204246441632, + 1.428933143337476, + 1.1531390634220187, + 0.9405881065525119, + 0.8279860191271594, + 0.7404700379978636, + 0.4795815886023818, + 0.4212812520523834, + 0.3924877272922389, + 0.3934201333857252 + ], + "peak_layer": 21, + "peak_normalized": 2.180691877285598, + "population_cv": 0.4088800352175212, + "rest_mean": 9.212875940849901e-05, + "spike_contrast": 1.9065333555314292, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017564655281603335, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.063500328920782e-05, + 9.251085430150852e-05, + 9.293619950767606e-05, + 8.979221456684172e-05, + 0.0001234708761330694, + 0.00012497235729824752, + 0.0001143925401265733, + 0.00010402848420199007, + 0.00010748613567557186, + 9.844252781476825e-05, + 9.080781455850229e-05, + 8.725486259208992e-05, + 7.529847789555788e-05, + 7.155040657380596e-05, + 7.003144128248096e-05, + 6.841374124633148e-05, + 0.00015279909712262452, + 0.00014121501590125263, + 0.00013077346375212073, + 0.00011935583461308852, + 0.00022936171444598585, + 0.00020250515080988407, + 0.0001747876376612112, + 0.0001502929226262495, + 0.00012128533853683621, + 9.892956586554646e-05, + 8.708625682629645e-05, + 7.788146467646584e-05, + 5.044163117418066e-05, + 4.430969420354813e-05, + 4.1281236917711794e-05, + 4.1379305912414566e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e4d8ebf5b6b3ad968bf338e0327e2eabdbae498ab18a1157cbd942424ccea56f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059127259999513626, + 0.0761958509683609, + 0.13674014806747437, + 0.18700072169303894, + 0.2049511820077896, + 0.14231130480766296, + 0.13109618425369263, + 0.13412337005138397, + 0.13132649660110474, + 0.09715579450130463, + 0.09982985258102417, + 0.10234102606773376, + 0.11231652647256851, + 0.08827291429042816, + 0.08991213887929916, + 0.1065397635102272, + 0.10877490043640137, + 0.07123783230781555, + 0.07851143926382065, + 0.08676032721996307, + 0.10016339272260666, + 0.06517573446035385, + 0.07041674107313156, + 0.07159053534269333, + 0.07631570100784302, + 0.06180137023329735, + 0.06370948255062103, + 0.07006713002920151, + 0.0730312243103981, + 0.06770572811365128, + 0.07336394488811493, + 0.08089767396450043 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 6.732241587315002e-06, + "first_to_last_ratio": 1.4278944469783923, + "imbalance_abs_log_ratio": 0.356200944503622, + "last_quartile_mean": 4.7148033957000735e-06, + "mean": 8.330031846526254e-06, + "normalized": [ + 0.5885835786473461, + 0.7084430032945174, + 0.4907562240746722, + 0.6082386339015993, + 0.45167837229458263, + 1.0561301434994, + 1.3931893382898604, + 1.1684941266915543, + 1.5667867561754885, + 1.2816056832829195, + 0.9678205108695733, + 0.9346667715729674, + 0.48015470085429257, + 0.7950059323694191, + 0.968685455585295, + 0.9109115267981576, + 0.9428117950257354, + 1.5508886748757895, + 1.3922681645248334, + 1.687647767587324, + 1.8711243564793436, + 2.3658432545646693, + 2.1045605102704337, + 1.1856995606832565, + 1.1632563088618004, + 0.9930387481612112, + 0.515645548723529, + 0.5621148182174204, + 0.3607436892275036, + 0.4577143696679217, + 0.23690245631552195, + 0.23858921861206098 + ], + "peak_layer": 22, + "peak_normalized": 2.3658432545646693, + "population_cv": 0.5306581723841309, + "rest_mean": 7.191444840120514e-06, + "spike_contrast": 2.0132813367827516, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4478401681117248e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 9 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.9029199544747826e-06, + 5.901352778892033e-06, + 4.088014975422993e-06, + 5.066647190687945e-06, + 3.7624952256010147e-06, + 8.797597729426343e-06, + 1.1605311556195375e-05, + 9.73359328781953e-06, + 1.3051383575657383e-05, + 1.0675816156435758e-05, + 8.061975677264854e-06, + 7.785803973092698e-06, + 3.999703949375544e-06, + 6.622424734814558e-06, + 8.0691806942923e-06, + 7.587922027596505e-06, + 7.853652277844958e-06, + 1.2918952052132227e-05, + 1.1597638149396516e-05, + 1.4058159649721347e-05, + 1.5586525478283875e-05, + 1.9707549654413015e-05, + 1.7531056073494256e-05, + 9.876915100903716e-06, + 9.689962098491378e-06, + 8.272044397017453e-06, + 4.295343842386501e-06, + 4.682434337155428e-06, + 3.005006419698475e-06, + 3.8127752759464784e-06, + 1.9734050056285923e-06, + 1.9874557892762823e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.6136923338526685e-05, + "first_to_last_ratio": 1.4118866191026362, + "imbalance_abs_log_ratio": 0.3449268377610348, + "last_quartile_mean": 1.8512055419250828e-05, + "mean": 3.305522808716432e-05, + "normalized": [ + 0.5803678005983632, + 0.6305039863571337, + 0.4562023660826862, + 0.6002682493018332, + 0.4447469941024251, + 1.006280776549637, + 1.3680201611097922, + 1.2392470554464203, + 1.559214741299254, + 1.3270070893842953, + 0.9349629066298444, + 0.9435481692108943, + 0.505187459486473, + 0.8266440594730092, + 0.9312125841240197, + 0.8763433716202387, + 1.000973249174259, + 1.52217970123748, + 1.4468612378475991, + 1.722721252602393, + 1.878854565585447, + 2.4072912511567135, + 2.1076999580640594, + 1.2033880908130195, + 1.1649327575261703, + 1.0028277195998705, + 0.5138852559553679, + 0.5315879646037021, + 0.35380571795268584, + 0.44968368242147705, + 0.22921632783533166, + 0.2343334968481047 + ], + "peak_layer": 22, + "peak_normalized": 2.4072912511567135, + "population_cv": 0.5417197270843308, + "rest_mean": 2.8449328982349942e-05, + "spike_contrast": 2.036149368904277, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.792708325316198e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 9 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.91841900232248e-05, + 2.0841453078901395e-05, + 1.507987326476723e-05, + 1.9842003894154914e-05, + 1.4701213331136387e-05, + 3.326284058857709e-05, + 4.522021845332347e-05, + 4.0963594074128196e-05, + 5.1540198910515755e-05, + 4.3864522012881935e-05, + 3.0905412131687626e-05, + 3.118919994449243e-05, + 1.6699086700100452e-05, + 2.7324907932779752e-05, + 3.078144436585717e-05, + 2.8967730031581596e-05, + 3.3087399060605094e-05, + 5.031599721405655e-05, + 4.78263282275293e-05, + 5.694494393537752e-05, + 6.210596620803699e-05, + 7.957356137922034e-05, + 6.967050285311416e-05, + 3.9778267819201574e-05, + 3.850711800623685e-05, + 3.314869900350459e-05, + 1.6986594346235506e-05, + 1.7571761418366805e-05, + 1.169512870546896e-05, + 1.486439668951789e-05, + 7.576797997899121e-06, + 7.745947186776903e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.023178485920652747, + "first_to_last_ratio": 1.4118866250838005, + "imbalance_abs_log_ratio": 0.3449268419973269, + "last_quartile_mean": 0.016416676458902657, + "mean": 0.029313707374967635, + "normalized": [ + 0.5803678083378483, + 0.6305039884996685, + 0.456202372531724, + 0.600268267835863, + 0.4447469968655515, + 1.006280799172435, + 1.3680202139428703, + 1.2392470777204811, + 1.559214634558375, + 1.3270070448652505, + 0.9349629030577731, + 0.9435480965148331, + 0.5051874298044523, + 0.826644056489135, + 0.9312126036518835, + 0.8763433869124639, + 1.000973281360552, + 1.5221796892520219, + 1.4468612026594825, + 1.7227212328102954, + 1.8788545699778456, + 2.407291269948361, + 2.1077000369999817, + 1.203388036597476, + 1.1649328054213401, + 1.00282774821406, + 0.513885282837708, + 0.5315879274732056, + 0.35380571518519355, + 0.44968368484851295, + 0.2292163286219473, + 0.2343335070314083 + ], + "peak_layer": 22, + "peak_normalized": 2.407291269948361, + "population_cv": 0.5417197285436497, + "rest_mean": 0.0252291498084863, + "spike_contrast": 2.0361493995603235, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.05137031823396683, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 9 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.017012732103466988, + 0.018482409417629242, + 0.01337298285216093, + 0.017596088349819183, + 0.013037183322012424, + 0.029497820883989334, + 0.04010174423456192, + 0.036326926201581955, + 0.045706361532211304, + 0.0388994961977005, + 0.02740722894668579, + 0.027658892795443535, + 0.014808916486799717, + 0.02423200197517872, + 0.02729729376733303, + 0.02568887360394001, + 0.02934223785996437, + 0.044620729982852936, + 0.04241286590695381, + 0.05049934610724449, + 0.05507619306445122, + 0.07056663185358047, + 0.06178450211882591, + 0.035275764763355255, + 0.03414849936962128, + 0.029396599158644676, + 0.015063882805407047, + 0.015582812950015068, + 0.010371357202529907, + 0.013181895948946476, + 0.006719180382788181, + 0.006869183853268623 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.833606326329118e-06, + "first_to_last_ratio": 0.6149451250612414, + "imbalance_abs_log_ratio": 0.48622224269933, + "last_quartile_mean": 2.9817397546594293e-06, + "mean": 2.912170225854993e-06, + "normalized": [ + 2.082671717873138, + 1.0258662958379037, + 0.2962831198895636, + 0.2730480128818211, + 0.2012400182045682, + 0.2720270773081725, + 0.42178614984566853, + 0.4641631742428811, + 0.5401984753999569, + 0.6459927680651849, + 0.5728072013454448, + 0.7481885282952891, + 0.4630758731723221, + 0.5702400277708374, + 0.578231799947324, + 0.5768551843487628, + 0.8350551079719575, + 1.4095672389330323, + 1.4695978758937063, + 1.4976484633572726, + 1.689917115612428, + 2.7658156568776397, + 2.289043897069376, + 2.1195653126073872, + 1.6714052019812762, + 1.6382121473547555, + 1.1947076934524972, + 1.295704736654288, + 0.8014556622731791, + 0.6524420889778894, + 0.48392041690429416, + 0.4532659596501817 + ], + "peak_layer": 22, + "peak_normalized": 2.7658156568776397, + "population_cv": 0.6680125549192626, + "rest_mean": 2.315094739649929e-06, + "spike_contrast": 2.6505947019214586, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.13637785136234e-06, + "top_five_layers": [ + 22, + 23, + 24, + 1, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 6.065094567020424e-06, + 2.9874972824472934e-06, + 8.628268801658123e-07, + 7.9516229334331e-07, + 5.860451892658602e-07, + 7.921891551632143e-07, + 1.2283130672585685e-06, + 1.3517221759684617e-06, + 1.5731499161120155e-06, + 1.8812409052770818e-06, + 1.6681120769135305e-06, + 2.178852355427807e-06, + 1.3485557701642392e-06, + 1.660636030464957e-06, + 1.6839094314491376e-06, + 1.6799004924905603e-06, + 2.431822622384061e-06, + 4.104899744561408e-06, + 4.279719178157393e-06, + 4.361407263786532e-06, + 4.921326308249263e-06, + 8.054526006162632e-06, + 6.6660854827205185e-06, + 6.172534995130263e-06, + 4.867416464549024e-06, + 4.7707526391604915e-06, + 3.4791921734722564e-06, + 3.773312755583902e-06, + 2.333975317014847e-06, + 1.9000224256160436e-06, + 1.4092586297920207e-06, + 1.3199876320868498e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.8523472363085602e-05, + "first_to_last_ratio": 1.6367299197356435, + "imbalance_abs_log_ratio": 0.49270029988784414, + "last_quartile_mean": 1.1317366499952186e-05, + "mean": 2.2688649906399405e-05, + "normalized": [ + 0.5580404711156489, + 0.6466177790146251, + 0.5012363612452404, + 0.6198604481052087, + 0.4857210377349011, + 1.0577981440873514, + 1.4196693381283139, + 1.2424181153378644, + 1.3858726044228813, + 1.3021505795717108, + 0.9395686627187273, + 0.9061129797569311, + 0.5424003201469547, + 0.8400230538707855, + 0.9733318846209535, + 0.8833365784843897, + 1.0149403960317096, + 1.6494378056474266, + 1.491698228985569, + 1.6819650994199165, + 1.9383906831259385, + 2.5307914955464006, + 2.1573696327610388, + 1.2407538293982432, + 1.0963126662413099, + 0.9513736363429608, + 0.49399328204720055, + 0.41259651935941966, + 0.32015875996296644, + 0.33597611060399785, + 0.19288429859973458, + 0.1871991975636804 + ], + "peak_layer": 22, + "peak_normalized": 2.5307914955464006, + "population_cv": 0.5672379252078826, + "rest_mean": 1.935794071633373e-05, + "spike_contrast": 2.1011780193351854, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.067447953275405e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.2661184882745147e-05, + 1.4670884411316365e-05, + 1.1372376320650801e-05, + 1.4063796697882935e-05, + 1.1020354577340186e-05, + 2.4000011762836948e-05, + 3.221038059564307e-05, + 2.8188789656269364e-05, + 3.14435783366207e-05, + 2.9544038625317626e-05, + 2.1317544451449066e-05, + 2.055848017334938e-05, + 1.230633097293321e-05, + 1.905898898257874e-05, + 2.2083586372900754e-05, + 2.004171437874902e-05, + 2.3027627321425825e-05, + 3.742351691471413e-05, + 3.3844618883449584e-05, + 3.816151729552075e-05, + 4.3979467591270804e-05, + 5.742024222854525e-05, + 4.894780431641266e-05, + 2.8151029255241156e-05, + 2.4873854272300377e-05, + 2.158538336516358e-05, + 1.1208040632482152e-05, + 9.361257980344817e-06, + 7.263970019266708e-06, + 7.622844350407831e-06, + 4.376284323370783e-06, + 4.247297056281241e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.2930837215208157e-05, + "first_to_last_ratio": 1.8364715924470074, + "imbalance_abs_log_ratio": 0.6078461178470359, + "last_quartile_mean": 7.041131084406516e-06, + "mean": 1.536685377345748e-05, + "normalized": [ + 0.5313414842881189, + 0.6754785013800788, + 0.5526059414596252, + 0.6240789848356078, + 0.5290051964319934, + 1.0858225994860433, + 1.4732941115929494, + 1.2601804357195125, + 1.2171383832168858, + 1.2767137827335708, + 0.9427961388447176, + 0.8464954353259013, + 0.5939861247138315, + 0.8658328722737771, + 1.029153088938301, + 0.8803697744896689, + 1.0401785162972257, + 1.7727172708005683, + 1.5249027896433116, + 1.5954681052275985, + 1.9631529625513957, + 2.591841549742642, + 2.193273942807079, + 1.268551867535944, + 1.0451422845983234, + 0.9043570007378636, + 0.4862798750552231, + 0.35652120226339873, + 0.2993186376401588, + 0.2463406294000189, + 0.17093996322188035, + 0.15672054674678554 + ], + "peak_layer": 22, + "peak_normalized": 2.591841549742642, + "population_cv": 0.582798597235848, + "rest_mean": 1.3055017276470877e-05, + "spike_contrast": 2.133338491047477, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.7850770857185124e-05, + "top_five_layers": [ + 22, + 23, + 21, + 18, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.165046892827377e-06, + 1.0379979357821867e-05, + 8.491814696753863e-06, + 9.590130503056571e-06, + 8.129145498969592e-06, + 1.668567711021751e-05, + 2.2639895178144798e-05, + 1.936500848387368e-05, + 1.8703587556956336e-05, + 1.9619074009824544e-05, + 1.4487810403807089e-05, + 1.3007971574552357e-05, + 9.127697921940126e-06, + 1.3305127140483819e-05, + 1.581484502821695e-05, + 1.3528513591154478e-05, + 1.5984271158231422e-05, + 2.7241087082074955e-05, + 2.343295818718616e-05, + 2.4517325073247775e-05, + 3.0167484510457143e-05, + 3.98284500988666e-05, + 3.370371996425092e-05, + 1.949365105247125e-05, + 1.6060548659879714e-05, + 1.3897121789341327e-05, + 7.472591732948786e-06, + 5.478609182318905e-06, + 4.5995857362868264e-06, + 3.785480430451571e-06, + 2.6268094188708346e-06, + 2.4083017251541605e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 5.1106853561577736e-05, + "first_to_last_ratio": 1.4788913820367076, + "imbalance_abs_log_ratio": 0.3912927408943512, + "last_quartile_mean": 3.4557543699520465e-05, + "mean": 6.540310639024938e-05, + "normalized": [ + 0.5486856091946496, + 0.6274437757279189, + 0.43571932821426484, + 0.625224380313627, + 0.45136709492800875, + 1.0286093422994689, + 1.3184260053458061, + 1.2158304195841385, + 1.5307439392265534, + 1.3325715083590115, + 0.9176469695695924, + 0.9806287946397408, + 0.5097569130124642, + 0.831748301564442, + 0.897166288563406, + 0.8855516750577264, + 1.004404805032616, + 1.5575747054430396, + 1.497114703177346, + 1.776321321168683, + 2.025217116434768, + 2.4423949256211186, + 2.124634016222874, + 1.2081963668463171, + 1.151934526222184, + 1.004674358671154, + 0.4849562539246849, + 0.4229542168376202, + 0.3250708970312119, + 0.41524843702243736, + 0.21300096811608038, + 0.209182036627046 + ], + "peak_layer": 22, + "peak_normalized": 2.4423949256211186, + "population_cv": 0.5647711514503198, + "rest_mean": 5.582911638123699e-05, + "spike_contrast": 2.097519359598391, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011710265243891627, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.588574327295646e-05, + 4.1036772017832845e-05, + 2.849739757948555e-05, + 4.0891616663429886e-05, + 2.9520810130634345e-05, + 6.72742462484166e-05, + 8.622915629530326e-05, + 7.951908628456295e-05, + 0.0001001154087134637, + 8.715431613381952e-05, + 6.001696237944998e-05, + 6.413616938516498e-05, + 3.333968561491929e-05, + 5.4398922657128423e-05, + 5.867746222065762e-05, + 5.7917830417864025e-05, + 6.569119432242587e-05, + 0.00010187022417085245, + 9.791595221031457e-05, + 0.00011617693235166371, + 0.0001324554905295372, + 0.00015974021516740322, + 0.00013895766460336745, + 7.901979552116245e-05, + 7.534009637311101e-05, + 6.570882396772504e-05, + 3.171764547005296e-05, + 2.7662519642035477e-05, + 2.1260646462906152e-05, + 2.7158537704963237e-05, + 1.3930924978922121e-05, + 1.368115499644773e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "6920ec3aaf9b121de9e3648519271c2cd5d5a0dcd243425164f6c664720841a8", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.055979277938604355, + 0.07538121938705444, + 0.18601374328136444, + 0.08912894874811172, + 0.12973569333553314, + 0.13784728944301605, + 0.1306019276380539, + 0.13534265756607056, + 0.08038423955440521, + 0.0852823480963707, + 0.08248664438724518, + 0.09743349254131317, + 0.07765212655067444, + 0.07436380535364151, + 0.08191535621881485, + 0.09093168377876282, + 0.06605636328458786, + 0.06527329236268997, + 0.0699063166975975, + 0.08022409677505493, + 0.05980068817734718, + 0.05788872763514519, + 0.0643942728638649, + 0.07148924469947815, + 0.056988880038261414, + 0.05852367728948593, + 0.06811124831438065, + 0.07283915579319, + 0.057786133140325546, + 0.07090020179748535, + 0.07763758301734924, + 0.07564623653888702 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.8137599602805494e-06, + "first_to_last_ratio": 0.257680145287283, + "imbalance_abs_log_ratio": 1.3560362101310268, + "last_quartile_mean": 1.480036405610008e-05, + "mean": 1.100234549866741e-05, + "normalized": [ + 0.8846955474380374, + 0.33662903052191306, + 0.12213284218224829, + 0.434361012681639, + 0.26037541556803, + 0.1262485959911879, + 0.1939520037247958, + 0.4146578693220764, + 0.7427772495066182, + 0.5340713440899699, + 0.5540395486368574, + 0.4982682858424812, + 0.5539022855457759, + 0.5320020234280948, + 0.4968872232332075, + 0.4886950428755402, + 1.256050776052354, + 1.2616225581229157, + 1.3245147622521796, + 1.345633356324533, + 2.1682720990903794, + 2.4117590499712667, + 2.234761662567843, + 2.0620839500138963, + 2.0505140064546468, + 1.8952882564355573, + 1.5733863445633343, + 1.481268881612435, + 1.1894375476641659, + 0.9329830578726799, + 0.8176375882880192, + 0.8210907821253209 + ], + "peak_layer": 22, + "peak_normalized": 2.4117590499712667, + "population_cv": 0.672071089583683, + "rest_mean": 8.586967678807182e-06, + "spike_contrast": 2.800218496834122, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4045385725912637e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.73372607404599e-06, + 3.7037088986835442e-06, + 1.3437477264233166e-06, + 4.778989932674449e-06, + 2.8647402814385714e-06, + 1.3890306718167267e-06, + 2.1339269551390316e-06, + 4.562209142022766e-06, + 8.1722919276217e-06, + 5.876037448615534e-06, + 6.095734534028452e-06, + 5.482119831867749e-06, + 6.094224318076158e-06, + 5.8532700677460525e-06, + 5.4669249038852286e-06, + 5.376791705202777e-06, + 1.3819504601997323e-05, + 1.3880807273380924e-05, + 1.4572769032383803e-05, + 1.4805123100813944e-05, + 2.385607876931317e-05, + 2.6535006327321753e-05, + 2.4587619918747805e-05, + 2.2687760065309703e-05, + 2.2560463548870757e-05, + 2.0852616216870956e-05, + 1.731094016577117e-05, + 1.6297432011924684e-05, + 1.3086602848488837e-05, + 1.0265001947118435e-05, + 8.995931239041965e-06, + 9.033924470713828e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.4820166825302294e-05, + "first_to_last_ratio": 0.250213415325602, + "imbalance_abs_log_ratio": 1.38544106397906, + "last_quartile_mean": 5.9230104852758814e-05, + "mean": 4.3850411316270765e-05, + "normalized": [ + 0.8638548954533332, + 0.33176746760552467, + 0.11339954764263065, + 0.425856726056105, + 0.25141584739259687, + 0.12462507515024858, + 0.1898225194515901, + 0.4030258231564756, + 0.744960226144929, + 0.5352705464819547, + 0.5547974738280123, + 0.5031678209626799, + 0.552108547371514, + 0.533071767737807, + 0.49794336735911743, + 0.49026058404355205, + 1.2586675503373956, + 1.2670879196781222, + 1.333871541046564, + 1.341755883919874, + 2.1805808903209534, + 2.388311265067082, + 2.2357839008695124, + 2.0727457387753443, + 2.055304013291544, + 1.9056304238078259, + 1.572799386274169, + 1.4807257856235785, + 1.1970813365071429, + 0.9443362704093409, + 0.820902093251126, + 0.8290677649823542 + ], + "peak_layer": 22, + "peak_normalized": 2.388311265067082, + "population_cv": 0.6740429802830727, + "rest_mean": 3.421513476361589e-05, + "spike_contrast": 2.802295106040794, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.588090470060706e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.788039248320274e-05, + 1.4548139915859792e-05, + 4.9726168072083965e-06, + 1.8673992599360645e-05, + 1.1024688319594134e-05, + 5.464860805659555e-06, + 8.323795555043034e-06, + 1.767284811649006e-05, + 3.266681233071722e-05, + 2.347183362871874e-05, + 2.4328097424586304e-05, + 2.20641159103252e-05, + 2.4210186893469654e-05, + 2.337541627639439e-05, + 2.1835021470906213e-05, + 2.149812826246489e-05, + 5.519308979273774e-05, + 5.556232645176351e-05, + 5.8490815717959777e-05, + 5.883654739591293e-05, + 9.561936894897372e-05, + 0.00010472843132447451, + 9.804004366742447e-05, + 9.089075319934636e-05, + 9.012592636281624e-05, + 8.356267790077254e-05, + 6.896790000610054e-05, + 6.493043474620208e-05, + 5.2492508984869346e-05, + 4.140953387832269e-05, + 3.5996894439449534e-05, + 3.6354962503537536e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.013142672483809292, + "first_to_last_ratio": 0.25021342279299313, + "imbalance_abs_log_ratio": 1.3854410341349725, + "last_quartile_mean": 0.05252584908157587, + "mean": 0.03888698373339139, + "normalized": [ + 0.8638549177389022, + 0.3317674544941228, + 0.11339954573412385, + 0.4258567477072783, + 0.25141586355000733, + 0.12462508052643778, + 0.18982251383769766, + 0.4030258156098557, + 0.7449602264335986, + 0.5352705630942872, + 0.5547974744780005, + 0.5031678339346759, + 0.5521085240028383, + 0.5330717624323057, + 0.4979434011756376, + 0.49026060341985517, + 1.2586675506734653, + 1.2670878962474197, + 1.3338714669036535, + 1.3417559191804453, + 2.180580864665263, + 2.3883112699220628, + 2.2357840053368334, + 2.0727457982123334, + 2.0553038796184118, + 1.9056304336464245, + 1.5727993114276466, + 1.4807258688900127, + 1.1970813019784332, + 0.9443362429514629, + 0.8209021215723841, + 0.8290677406041228 + ], + "peak_layer": 22, + "peak_normalized": 2.3883112699220628, + "population_cv": 0.6740429788021378, + "rest_mean": 0.030342324015994866, + "spike_contrast": 2.802295109712436, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08502814620733261, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03359271213412285, + 0.012901435606181622, + 0.004409766290336847, + 0.016560284420847893, + 0.009776804596185684, + 0.004846293479204178, + 0.007381625007838011, + 0.015672458335757256, + 0.028969256207346916, + 0.020815057680010796, + 0.02157440036535263, + 0.019566679373383522, + 0.021469835191965103, + 0.02072955295443535, + 0.019363516941666603, + 0.019064756110310555, + 0.04894578456878662, + 0.04927322641015053, + 0.05187023803591728, + 0.05217684060335159, + 0.08479621261358261, + 0.09287422150373459, + 0.0869428962469101, + 0.08060283213853836, + 0.0799245685338974, + 0.07410421967506409, + 0.06116142123937607, + 0.057580962777137756, + 0.04655088111758232, + 0.03672238811850548, + 0.03192240744829178, + 0.032239943742752075 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 8.559198363400355e-07, + "first_to_last_ratio": 0.2186950684356118, + "imbalance_abs_log_ratio": 1.5200769010021633, + "last_quartile_mean": 3.913759201168432e-06, + "mean": 2.80401877361669e-06, + "normalized": [ + 0.9040556019078396, + 0.32702629052223714, + 0.09351520076589508, + 0.43074186706516326, + 0.22724111410332884, + 0.09525147785722864, + 0.14581699925344985, + 0.2183317787379819, + 0.6348218757245028, + 0.43653474839421297, + 0.5014505522601324, + 0.42281266866764283, + 0.5237432404167371, + 0.5038080787557457, + 0.48028933248389244, + 0.4797106038616353, + 1.2212117799762032, + 1.2800650021631181, + 1.3574152409248479, + 1.4176280739859934, + 2.2743522375387655, + 2.3285282750638068, + 2.3417080754487287, + 2.187797237735209, + 2.1078913327288835, + 1.993431185351473, + 1.6687004494389643, + 1.5599769890603215, + 1.2136943895243368, + 0.9467738344272854, + 0.8188081541617154, + 0.8568663116927213 + ], + "peak_layer": 23, + "peak_normalized": 2.3417080754487287, + "population_cv": 0.7184404459833599, + "rest_mean": 2.155950095651928e-06, + "spike_contrast": 2.9238105498542586, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.303589634626405e-06, + "top_five_layers": [ + 23, + 22, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.5349888801429188e-06, + 9.169878580905788e-07, + 2.6221837856610364e-07, + 1.2078082818334224e-06, + 6.371883500833064e-07, + 2.6708693212640355e-07, + 4.0887360341912427e-07, + 6.122064064584265e-07, + 1.780052457434067e-06, + 1.2240516298334114e-06, + 1.406076762577868e-06, + 1.1855746606670436e-06, + 1.4685858786833705e-06, + 1.4126873111308669e-06, + 1.3467403050526627e-06, + 1.3451175391310244e-06, + 3.4243007576151285e-06, + 3.589326297515072e-06, + 3.8062178191466955e-06, + 3.975055733462796e-06, + 6.377326371875824e-06, + 6.529236998176202e-06, + 6.566193405888043e-06, + 6.134624527476262e-06, + 5.910566869715694e-06, + 5.589618467638502e-06, + 4.679067387769464e-06, + 4.374204763735179e-06, + 3.4032218536594883e-06, + 2.654771606103168e-06, + 2.295953436259879e-06, + 2.4026692244660808e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.0053945828758515e-05, + "first_to_last_ratio": 0.27064297247327324, + "imbalance_abs_log_ratio": 1.3069547714438037, + "last_quartile_mean": 3.714837203006027e-05, + "mean": 2.8984760795935927e-05, + "normalized": [ + 0.8999349409837173, + 0.32728285540106666, + 0.1058957173747875, + 0.44016555637738164, + 0.2552979443105837, + 0.12918922677367176, + 0.20615311725153693, + 0.41104115254674733, + 0.764240914256914, + 0.5920784377209981, + 0.6110717682804259, + 0.557954391842857, + 0.6124804063036768, + 0.5885550227171041, + 0.5526510071226007, + 0.5526852723084109, + 1.2329478642521525, + 1.2506174039150972, + 1.3311309260294062, + 1.3670973218339744, + 2.302706839732118, + 2.480544409221905, + 2.19254363978492, + 1.9825191370367767, + 2.0485913267373626, + 1.8175804645452507, + 1.4838659208054645, + 1.396779884204161, + 1.116946905853611, + 0.8583881521618855, + 0.7577608404635063, + 0.773301231849928 + ], + "peak_layer": 22, + "peak_normalized": 2.480544409221905, + "population_cv": 0.6636797496656769, + "rest_mean": 2.2536289878661154e-05, + "spike_contrast": 2.8312780893733493, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.38065037492197e-05, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.608439899631776e-05, + 9.486215276410803e-06, + 3.069362037422252e-06, + 1.2758093362208456e-05, + 7.399749847536441e-06, + 3.7445188354467973e-06, + 5.97529879087233e-06, + 1.191392948385328e-05, + 2.215134009020403e-05, + 1.7161251889774576e-05, + 1.771176903275773e-05, + 1.6172174582607113e-05, + 1.775259806890972e-05, + 1.7059126548701897e-05, + 1.6018457245081663e-05, + 1.6019450413296e-05, + 3.573669891920872e-05, + 3.624884629971348e-05, + 3.858251147903502e-05, + 3.962498885812238e-05, + 6.674340693280101e-05, + 7.189798634499311e-05, + 6.355035293381661e-05, + 5.746284296037629e-05, + 5.9377929574111477e-05, + 5.2682134992210194e-05, + 4.300949876778759e-05, + 4.048533082823269e-05, + 3.237443888792768e-05, + 2.48801752604777e-05, + 2.1963516701362096e-05, + 2.2413951228372753e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 6.8028442683498724e-06, + "first_to_last_ratio": 0.30342053392291607, + "imbalance_abs_log_ratio": 1.1926355349810676, + "last_quartile_mean": 2.2420513801080233e-05, + "mean": 1.860519103047409e-05, + "normalized": [ + 0.979119532254684, + 0.3415768918027291, + 0.09970038372228712, + 0.4595782792628581, + 0.258771439136117, + 0.13567266851847518, + 0.22194219475035443, + 0.4287767646116532, + 0.7944047903163591, + 0.6593126061472351, + 0.6705300974303611, + 0.6146988416484243, + 0.681950359220749, + 0.661289911896569, + 0.6390706540768326, + 0.6305979949978427, + 1.2424026640613004, + 1.27008934410493, + 1.3446579537308916, + 1.4010843712596734, + 2.4186677284743303, + 2.542833867301576, + 2.061542914299684, + 1.8011865606538167, + 1.9777885269447963, + 1.6774560753368692, + 1.3713733102501011, + 1.3230990688095643, + 1.0599527142158984, + 0.7835783704069301, + 0.7234063690287212, + 0.7238867513273863 + ], + "peak_layer": 22, + "peak_normalized": 2.542833867301576, + "population_cv": 0.6397326686677515, + "rest_mean": 1.4607128697982128e-05, + "spike_contrast": 2.7517199620130204, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.019472762593068e-05, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.821670593926683e-05, + 6.355103323585354e-06, + 1.854944684964721e-06, + 8.550541679142043e-06, + 4.814492058358155e-06, + 2.5242159154004185e-06, + 4.1292769310530275e-06, + 7.97747361502843e-06, + 1.4780052879359573e-05, + 1.2266636986169033e-05, + 1.247534055437427e-05, + 1.1436589375080075e-05, + 1.2687816706602462e-05, + 1.2303425137361046e-05, + 1.1890031601069495e-05, + 1.1732396160368808e-05, + 2.3115138901630417e-05, + 2.363025487284176e-05, + 2.5017618099809624e-05, + 2.6067442377097905e-05, + 4.4999775127507746e-05, + 4.730990985990502e-05, + 3.83553997380659e-05, + 3.351142004248686e-05, + 3.6797133361687884e-05, + 3.120939072687179e-05, + 2.551466241129674e-05, + 2.4616510927444324e-05, + 1.97206227312563e-05, + 1.457862526876852e-05, + 1.3459113688440993e-05, + 1.3468051292875316e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.855429124792863e-05, + "first_to_last_ratio": 0.22282350811084511, + "imbalance_abs_log_ratio": 1.5013752643278568, + "last_quartile_mean": 0.00012814757064916193, + "mean": 9.17358819947367e-05, + "normalized": [ + 0.7746039998248817, + 0.2866381886199903, + 0.1036281000087033, + 0.4039563945354074, + 0.2464815307667851, + 0.1160751938545964, + 0.18551560419629914, + 0.3732316220987844, + 0.7092844774450237, + 0.5116442907474149, + 0.5377274775333667, + 0.4967237837368528, + 0.5197471875160191, + 0.5027122037293308, + 0.4626857797839783, + 0.4753195792444178, + 1.1837655470220732, + 1.1937355792503366, + 1.3296888336896793, + 1.292665760701516, + 2.2124766020724245, + 2.4217320353409204, + 2.3397097366792714, + 2.14490074947905, + 2.1260014458866174, + 2.0351633744735502, + 1.6931959406501869, + 1.5581194253201036, + 1.1956665626368055, + 0.9514093786344858, + 0.8264167114021962, + 0.7893769031189312 + ], + "peak_layer": 22, + "peak_normalized": 2.4217320353409204, + "population_cv": 0.708541458383297, + "rest_mean": 7.05183218896208e-05, + "spike_contrast": 2.925632672956847, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020631070656236262, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.10589811205864e-05, + 2.629500704642851e-05, + 9.50641515373718e-06, + 3.7057296140119433e-05, + 2.2611200620303862e-05, + 1.0648260285961442e-05, + 1.701843757473398e-05, + 3.423873204155825e-05, + 6.506683712359518e-05, + 4.693614027928561e-05, + 4.932890442432836e-05, + 4.556739440886304e-05, + 4.767946666106582e-05, + 4.611674739862792e-05, + 4.2444888094905764e-05, + 4.3603860831353813e-05, + 0.00010859377653105184, + 0.00010950838623102754, + 0.0001219801779370755, + 0.00011858383368235081, + 0.00020296349248383194, + 0.00022215972421690822, + 0.0002146353363059461, + 0.00019676436204463243, + 0.00019503061776049435, + 0.00018669750716071576, + 0.00015532682300545275, + 0.00014293545973487198, + 0.00010968552669510245, + 8.727837848709896e-05, + 7.581206591567025e-05, + 7.241418643388897e-05 + ] + } + } + } + } + } + }, + "step": 4000 + }, + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 1.8565786854442996, + "forward": { + "activation_sha256": { + "attention_branch_output": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", + "mlp_branch_output": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", + "post_attention_state": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", + "post_mlp_state": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", + "pre_attention_input": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", + "pre_mlp_input": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f" + }, + "logits_sha256": "a911233a29684ae3ced35122e9aee12a584fd2d6b7e0272c227ab94eb4e607ec", + "loss_nats": 1.2868822813034058, + "mixer_summary_sha256": "998a9ce6e3ee4cab0180aeb81cc12c18ccf6b1832b43c8f455133227a7e72a26" + }, + "loss_nats": 1.2868822813034058, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.21598131954669952, + "group": 1, + "latest_source_mass": 0.061274416744709015, + "layer": 1, + "max_source_mass": 0.9387255907058716, + "mixer_index": 1, + "normalized_entropy": 0.31159517863467795, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9387255907058716, + "median": 0.9528161883354187, + "p05": 0.8532108068466187, + "p95": 0.9815421104431152 + }, + { + "label": "current_group_partial", + "mean": 0.061274416744709015, + "median": 0.047183819115161896, + "p05": 0.018457919359207153, + "p95": 0.14678917825222015 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5377621054649353, + "group": 1, + "latest_source_mass": 0.2917928099632263, + "layer": 2, + "max_source_mass": 0.7082071900367737, + "mixer_index": 2, + "normalized_entropy": 0.7758267227322699, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7082071900367737, + "median": 0.7268136143684387, + "p05": 0.38468095660209656, + "p95": 0.923936128616333 + }, + { + "label": "current_group_partial", + "mean": 0.2917928099632263, + "median": 0.27318641543388367, + "p05": 0.07606390863656998, + "p95": 0.6153190732002258 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.43774232268333435, + "group": 1, + "latest_source_mass": 0.170285165309906, + "layer": 2, + "max_source_mass": 0.829714834690094, + "mixer_index": 3, + "normalized_entropy": 0.6315286781224629, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.829714834690094, + "median": 0.85542893409729, + "p05": 0.687734842300415, + "p95": 0.9260365962982178 + }, + { + "label": "current_group_partial", + "mean": 0.170285165309906, + "median": 0.14457105100154877, + "p05": 0.07396337389945984, + "p95": 0.31226515769958496 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6749181151390076, + "group": 1, + "latest_source_mass": 0.45927006006240845, + "layer": 3, + "max_source_mass": 0.5407299995422363, + "mixer_index": 4, + "normalized_entropy": 0.9737010177171727, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5407299995422363, + "median": 0.5310744643211365, + "p05": 0.4232047200202942, + "p95": 0.6916534900665283 + }, + { + "label": "current_group_partial", + "mean": 0.45927006006240845, + "median": 0.4689255356788635, + "p05": 0.3083464205265045, + "p95": 0.576795220375061 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6663151383399963, + "group": 1, + "latest_source_mass": 0.49885618686676025, + "layer": 3, + "max_source_mass": 0.5011438131332397, + "mixer_index": 5, + "normalized_entropy": 0.9612895457523564, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5011438131332397, + "median": 0.4804786741733551, + "p05": 0.3554112911224365, + "p95": 0.7207092642784119 + }, + { + "label": "current_group_partial", + "mean": 0.49885618686676025, + "median": 0.5195213556289673, + "p05": 0.27929070591926575, + "p95": 0.6445887684822083 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6865031719207764, + "group": 1, + "latest_source_mass": 0.4691343307495117, + "layer": 4, + "max_source_mass": 0.5308656692504883, + "mixer_index": 6, + "normalized_entropy": 0.9904147216846476, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5308656692504883, + "median": 0.5278033018112183, + "p05": 0.45786383748054504, + "p95": 0.6093498468399048 + }, + { + "label": "current_group_partial", + "mean": 0.4691343307495117, + "median": 0.47219666838645935, + "p05": 0.3906501829624176, + "p95": 0.5421361327171326 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4133813679218292, + "group": 1, + "latest_source_mass": 0.15208598971366882, + "layer": 4, + "max_source_mass": 0.8479140400886536, + "mixer_index": 7, + "normalized_entropy": 0.5963832494967191, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8479140400886536, + "median": 0.8514594435691833, + "p05": 0.7530128359794617, + "p95": 0.9402707815170288 + }, + { + "label": "current_group_partial", + "mean": 0.15208598971366882, + "median": 0.14854055643081665, + "p05": 0.05972917377948761, + "p95": 0.24698717892169952 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6850745677947998, + "group": 2, + "latest_source_mass": 0.4601486325263977, + "layer": 5, + "max_source_mass": 0.5398513674736023, + "mixer_index": 8, + "normalized_entropy": 0.9883536815967077, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5398513674736023, + "median": 0.5387102365493774, + "p05": 0.46027374267578125, + "p95": 0.623892605304718 + }, + { + "label": "completed_group_1", + "mean": 0.4601486325263977, + "median": 0.46128979325294495, + "p05": 0.376107394695282, + "p95": 0.5397262573242188 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074119567871094, + "group": 2, + "latest_source_mass": 0.4627366364002228, + "layer": 5, + "max_source_mass": 0.4627366364002228, + "mixer_index": 9, + "normalized_entropy": 0.9169858804405273, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32744842767715454, + "median": 0.30657994747161865, + "p05": 0.16577640175819397, + "p95": 0.5453717708587646 + }, + { + "label": "completed_group_1", + "mean": 0.20981493592262268, + "median": 0.2177819311618805, + "p05": 0.14342713356018066, + "p95": 0.2433384209871292 + }, + { + "label": "current_group_partial", + "mean": 0.4627366364002228, + "median": 0.4847465753555298, + "p05": 0.27109938859939575, + "p95": 0.6081984639167786 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0489801168441772, + "group": 2, + "latest_source_mass": 0.4763939380645752, + "layer": 6, + "max_source_mass": 0.4763939380645752, + "mixer_index": 10, + "normalized_entropy": 0.9548228503031734, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28095996379852295, + "median": 0.2765299677848816, + "p05": 0.23549969494342804, + "p95": 0.3424558639526367 + }, + { + "label": "completed_group_1", + "mean": 0.24264612793922424, + "median": 0.23937124013900757, + "p05": 0.21875721216201782, + "p95": 0.2814398407936096 + }, + { + "label": "current_group_partial", + "mean": 0.4763939380645752, + "median": 0.48882943391799927, + "p05": 0.38478636741638184, + "p95": 0.528400719165802 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.8527575731277466, + "group": 2, + "latest_source_mass": 0.6454645395278931, + "layer": 6, + "max_source_mass": 0.6454645395278931, + "mixer_index": 11, + "normalized_entropy": 0.7762133938639787, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20522089302539825, + "median": 0.1591135561466217, + "p05": 0.11052663624286652, + "p95": 0.47355735301971436 + }, + { + "label": "completed_group_1", + "mean": 0.1493145078420639, + "median": 0.15203402936458588, + "p05": 0.10004506260156631, + "p95": 0.18058395385742188 + }, + { + "label": "current_group_partial", + "mean": 0.6454645395278931, + "median": 0.68653404712677, + "p05": 0.4207671880722046, + "p95": 0.7333818674087524 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.049942135810852, + "group": 2, + "latest_source_mass": 0.4395729899406433, + "layer": 7, + "max_source_mass": 0.4395729899406433, + "mixer_index": 12, + "normalized_entropy": 0.9556985177033998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35788866877555847, + "median": 0.36233147978782654, + "p05": 0.30507102608680725, + "p95": 0.3985886573791504 + }, + { + "label": "completed_group_1", + "mean": 0.20253834128379822, + "median": 0.19928529858589172, + "p05": 0.180739626288414, + "p95": 0.23214691877365112 + }, + { + "label": "current_group_partial", + "mean": 0.4395729899406433, + "median": 0.43975552916526794, + "p05": 0.3959323763847351, + "p95": 0.4790603220462799 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9791408181190491, + "group": 2, + "latest_source_mass": 0.5214842557907104, + "layer": 7, + "max_source_mass": 0.5214842557907104, + "mixer_index": 13, + "normalized_entropy": 0.8912523810434521, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31465935707092285, + "median": 0.28992873430252075, + "p05": 0.20534247159957886, + "p95": 0.507034420967102 + }, + { + "label": "completed_group_1", + "mean": 0.1638563871383667, + "median": 0.1667816936969757, + "p05": 0.11925681680440903, + "p95": 0.19515612721443176 + }, + { + "label": "current_group_partial", + "mean": 0.5214842557907104, + "median": 0.5455081462860107, + "p05": 0.35950881242752075, + "p95": 0.6097541451454163 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0485806465148926, + "group": 2, + "latest_source_mass": 0.4457142949104309, + "layer": 8, + "max_source_mass": 0.4457142949104309, + "mixer_index": 14, + "normalized_entropy": 0.9544592367395849, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.346283882856369, + "median": 0.353412926197052, + "p05": 0.26436588168144226, + "p95": 0.4084087312221527 + }, + { + "label": "completed_group_1", + "mean": 0.20800183713436127, + "median": 0.20826736092567444, + "p05": 0.18089595437049866, + "p95": 0.23509056866168976 + }, + { + "label": "current_group_partial", + "mean": 0.4457142949104309, + "median": 0.4416901469230652, + "p05": 0.3883572816848755, + "p95": 0.5136083364486694 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9801348447799683, + "group": 2, + "latest_source_mass": 0.543371319770813, + "layer": 8, + "max_source_mass": 0.543371319770813, + "mixer_index": 15, + "normalized_entropy": 0.8921571831025336, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2748759388923645, + "median": 0.2577607035636902, + "p05": 0.17991375923156738, + "p95": 0.41774845123291016 + }, + { + "label": "completed_group_1", + "mean": 0.1817527413368225, + "median": 0.17968356609344482, + "p05": 0.130966454744339, + "p95": 0.23766928911209106 + }, + { + "label": "current_group_partial", + "mean": 0.543371319770813, + "median": 0.5535831451416016, + "p05": 0.4452384412288666, + "p95": 0.6142411828041077 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0688585042953491, + "group": 3, + "latest_source_mass": 0.36256325244903564, + "layer": 9, + "max_source_mass": 0.4051227569580078, + "mixer_index": 16, + "normalized_entropy": 0.9729169383233166, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4051227569580078, + "median": 0.4008670449256897, + "p05": 0.3286888897418976, + "p95": 0.48389261960983276 + }, + { + "label": "completed_group_1", + "mean": 0.23231396079063416, + "median": 0.23353952169418335, + "p05": 0.2008797824382782, + "p95": 0.2593720257282257 + }, + { + "label": "completed_group_2", + "mean": 0.36256325244903564, + "median": 0.3638863265514374, + "p05": 0.3081618845462799, + "p95": 0.4163197875022888 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2915732860565186, + "group": 3, + "latest_source_mass": 0.36661261320114136, + "layer": 9, + "max_source_mass": 0.36661261320114136, + "mixer_index": 17, + "normalized_entropy": 0.931673187369201, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.26621371507644653, + "median": 0.23494771122932434, + "p05": 0.16815711557865143, + "p95": 0.4694361090660095 + }, + { + "label": "completed_group_1", + "mean": 0.1108347550034523, + "median": 0.11330658197402954, + "p05": 0.06620892137289047, + "p95": 0.13892118632793427 + }, + { + "label": "completed_group_2", + "mean": 0.2563389539718628, + "median": 0.26728466153144836, + "p05": 0.16010139882564545, + "p95": 0.3091832995414734 + }, + { + "label": "current_group_partial", + "mean": 0.36661261320114136, + "median": 0.36950358748435974, + "p05": 0.29210716485977173, + "p95": 0.42882072925567627 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3568191528320312, + "group": 3, + "latest_source_mass": 0.30908840894699097, + "layer": 10, + "max_source_mass": 0.30908840894699097, + "mixer_index": 18, + "normalized_entropy": 0.978738131586968, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.294538676738739, + "median": 0.2961598038673401, + "p05": 0.2467859834432602, + "p95": 0.34424129128456116 + }, + { + "label": "completed_group_1", + "mean": 0.1710844486951828, + "median": 0.17187124490737915, + "p05": 0.1508811116218567, + "p95": 0.1898101568222046 + }, + { + "label": "completed_group_2", + "mean": 0.2252884805202484, + "median": 0.22875282168388367, + "p05": 0.18443678319454193, + "p95": 0.25181543827056885 + }, + { + "label": "current_group_partial", + "mean": 0.30908840894699097, + "median": 0.3080518841743469, + "p05": 0.2719307243824005, + "p95": 0.3494151830673218 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.320021152496338, + "group": 3, + "latest_source_mass": 0.3312946557998657, + "layer": 10, + "max_source_mass": 0.3312946557998657, + "mixer_index": 19, + "normalized_entropy": 0.9521939852875004, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30465462803840637, + "median": 0.311603307723999, + "p05": 0.21955373883247375, + "p95": 0.40013614296913147 + }, + { + "label": "completed_group_1", + "mean": 0.12911874055862427, + "median": 0.12981033325195312, + "p05": 0.09804825484752655, + "p95": 0.15542283654212952 + }, + { + "label": "completed_group_2", + "mean": 0.23493197560310364, + "median": 0.24103470146656036, + "p05": 0.16810482740402222, + "p95": 0.28502923250198364 + }, + { + "label": "current_group_partial", + "mean": 0.3312946557998657, + "median": 0.325168639421463, + "p05": 0.2647673785686493, + "p95": 0.41786038875579834 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3578683137893677, + "group": 3, + "latest_source_mass": 0.30723780393600464, + "layer": 11, + "max_source_mass": 0.30723780393600464, + "mixer_index": 20, + "normalized_entropy": 0.9794949412420898, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29558902978897095, + "median": 0.2938143014907837, + "p05": 0.24335844814777374, + "p95": 0.3542291522026062 + }, + { + "label": "completed_group_1", + "mean": 0.17812997102737427, + "median": 0.17726755142211914, + "p05": 0.15839283168315887, + "p95": 0.20142368972301483 + }, + { + "label": "completed_group_2", + "mean": 0.21904319524765015, + "median": 0.2218058705329895, + "p05": 0.18076662719249725, + "p95": 0.24698719382286072 + }, + { + "label": "current_group_partial", + "mean": 0.30723780393600464, + "median": 0.30536961555480957, + "p05": 0.26194074749946594, + "p95": 0.35952210426330566 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.258887767791748, + "group": 3, + "latest_source_mass": 0.34303945302963257, + "layer": 11, + "max_source_mass": 0.35744374990463257, + "mixer_index": 21, + "normalized_entropy": 0.9080955698144659, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35744374990463257, + "median": 0.359974205493927, + "p05": 0.17910680174827576, + "p95": 0.5207242965698242 + }, + { + "label": "completed_group_1", + "mean": 0.12067858874797821, + "median": 0.11769672483205795, + "p05": 0.08151324093341827, + "p95": 0.16476252675056458 + }, + { + "label": "completed_group_2", + "mean": 0.17883820831775665, + "median": 0.17648206651210785, + "p05": 0.0995911955833435, + "p95": 0.2739725708961487 + }, + { + "label": "current_group_partial", + "mean": 0.34303945302963257, + "median": 0.33267199993133545, + "p05": 0.22800606489181519, + "p95": 0.48691368103027344 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3539760112762451, + "group": 3, + "latest_source_mass": 0.3175881803035736, + "layer": 12, + "max_source_mass": 0.3175881803035736, + "mixer_index": 22, + "normalized_entropy": 0.976687238475429, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2736307978630066, + "median": 0.27999642491340637, + "p05": 0.18969643115997314, + "p95": 0.3464604914188385 + }, + { + "label": "completed_group_1", + "mean": 0.1722733974456787, + "median": 0.17083553969860077, + "p05": 0.1506597101688385, + "p95": 0.1988438367843628 + }, + { + "label": "completed_group_2", + "mean": 0.2365075945854187, + "median": 0.24049004912376404, + "p05": 0.19099700450897217, + "p95": 0.26942259073257446 + }, + { + "label": "current_group_partial", + "mean": 0.3175881803035736, + "median": 0.3092706799507141, + "p05": 0.268545538187027, + "p95": 0.3924510180950165 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3203014135360718, + "group": 3, + "latest_source_mass": 0.3395293354988098, + "layer": 12, + "max_source_mass": 0.3395293354988098, + "mixer_index": 23, + "normalized_entropy": 0.9523961508935896, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28207293152809143, + "median": 0.29242122173309326, + "p05": 0.1354818046092987, + "p95": 0.4195851981639862 + }, + { + "label": "completed_group_1", + "mean": 0.15420743823051453, + "median": 0.15396761894226074, + "p05": 0.11267110705375671, + "p95": 0.1951446533203125 + }, + { + "label": "completed_group_2", + "mean": 0.22419027984142303, + "median": 0.22542357444763184, + "p05": 0.1655150204896927, + "p95": 0.2747117578983307 + }, + { + "label": "current_group_partial", + "mean": 0.3395293354988098, + "median": 0.32681703567504883, + "p05": 0.2524653375148773, + "p95": 0.46652013063430786 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3734662532806396, + "group": 4, + "latest_source_mass": 0.283242404460907, + "layer": 13, + "max_source_mass": 0.283242404460907, + "mixer_index": 24, + "normalized_entropy": 0.9907464762181619, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2766023874282837, + "median": 0.282479852437973, + "p05": 0.22528076171875, + "p95": 0.3097468614578247 + }, + { + "label": "completed_group_1", + "mean": 0.21176087856292725, + "median": 0.2058793306350708, + "p05": 0.18811854720115662, + "p95": 0.2617422640323639 + }, + { + "label": "completed_group_2", + "mean": 0.22839435935020447, + "median": 0.22951346635818481, + "p05": 0.20004916191101074, + "p95": 0.2551718056201935 + }, + { + "label": "completed_group_3", + "mean": 0.283242404460907, + "median": 0.2849352955818176, + "p05": 0.2205706238746643, + "p95": 0.34444743394851685 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5325813293457031, + "group": 4, + "latest_source_mass": 0.27591708302497864, + "layer": 13, + "max_source_mass": 0.27591708302497864, + "mixer_index": 25, + "normalized_entropy": 0.9522463199762954, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17451074719429016, + "median": 0.16826045513153076, + "p05": 0.08149556815624237, + "p95": 0.3126368522644043 + }, + { + "label": "completed_group_1", + "mean": 0.10643941164016724, + "median": 0.10269765555858612, + "p05": 0.08178127557039261, + "p95": 0.13947664201259613 + }, + { + "label": "completed_group_2", + "mean": 0.1691833734512329, + "median": 0.17118802666664124, + "p05": 0.12382284551858902, + "p95": 0.2069271057844162 + }, + { + "label": "completed_group_3", + "mean": 0.27394935488700867, + "median": 0.27278125286102295, + "p05": 0.2122008204460144, + "p95": 0.33945947885513306 + }, + { + "label": "current_group_partial", + "mean": 0.27591708302497864, + "median": 0.27714359760284424, + "p05": 0.21523593366146088, + "p95": 0.33170604705810547 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.560360074043274, + "group": 4, + "latest_source_mass": 0.3255118429660797, + "layer": 14, + "max_source_mass": 0.3255118429660797, + "mixer_index": 26, + "normalized_entropy": 0.9695062244951087, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19377154111862183, + "median": 0.19472965598106384, + "p05": 0.17005598545074463, + "p95": 0.22297024726867676 + }, + { + "label": "completed_group_1", + "mean": 0.17211595177650452, + "median": 0.17136064171791077, + "p05": 0.1543424427509308, + "p95": 0.1928609162569046 + }, + { + "label": "completed_group_2", + "mean": 0.14605136215686798, + "median": 0.14629822969436646, + "p05": 0.13246256113052368, + "p95": 0.15914371609687805 + }, + { + "label": "completed_group_3", + "mean": 0.16254930198192596, + "median": 0.16272446513175964, + "p05": 0.14075018465518951, + "p95": 0.18463709950447083 + }, + { + "label": "current_group_partial", + "mean": 0.3255118429660797, + "median": 0.3261105418205261, + "p05": 0.3035835921764374, + "p95": 0.3457673192024231 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.540909767150879, + "group": 4, + "latest_source_mass": 0.2995486855506897, + "layer": 14, + "max_source_mass": 0.2995486855506897, + "mixer_index": 27, + "normalized_entropy": 0.9574210693349581, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21921303868293762, + "median": 0.21722173690795898, + "p05": 0.1413341462612152, + "p95": 0.2906968593597412 + }, + { + "label": "completed_group_1", + "mean": 0.10205383598804474, + "median": 0.09984464198350906, + "p05": 0.07455356419086456, + "p95": 0.13362759351730347 + }, + { + "label": "completed_group_2", + "mean": 0.16381604969501495, + "median": 0.16600549221038818, + "p05": 0.11865930259227753, + "p95": 0.2019941210746765 + }, + { + "label": "completed_group_3", + "mean": 0.215368390083313, + "median": 0.2136734575033188, + "p05": 0.1728287935256958, + "p95": 0.2628125846385956 + }, + { + "label": "current_group_partial", + "mean": 0.2995486855506897, + "median": 0.2982075810432434, + "p05": 0.23863884806632996, + "p95": 0.3662579655647278 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5142419338226318, + "group": 4, + "latest_source_mass": 0.369165301322937, + "layer": 15, + "max_source_mass": 0.369165301322937, + "mixer_index": 28, + "normalized_entropy": 0.940851412859105, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2000826597213745, + "median": 0.20369741320610046, + "p05": 0.15802279114723206, + "p95": 0.24072687327861786 + }, + { + "label": "completed_group_1", + "mean": 0.15480534732341766, + "median": 0.15228059887886047, + "p05": 0.13543802499771118, + "p95": 0.1815909892320633 + }, + { + "label": "completed_group_2", + "mean": 0.10827909409999847, + "median": 0.10761409997940063, + "p05": 0.09670238941907883, + "p95": 0.12229038774967194 + }, + { + "label": "completed_group_3", + "mean": 0.16766759753227234, + "median": 0.16637766361236572, + "p05": 0.1398160457611084, + "p95": 0.20042110979557037 + }, + { + "label": "current_group_partial", + "mean": 0.369165301322937, + "median": 0.3697334825992584, + "p05": 0.3279486298561096, + "p95": 0.40613043308258057 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5333527326583862, + "group": 4, + "latest_source_mass": 0.29974091053009033, + "layer": 15, + "max_source_mass": 0.29974091053009033, + "mixer_index": 29, + "normalized_entropy": 0.9527256198031004, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21770668029785156, + "median": 0.19699469208717346, + "p05": 0.1311802864074707, + "p95": 0.3720983564853668 + }, + { + "label": "completed_group_1", + "mean": 0.10525081306695938, + "median": 0.10493603348731995, + "p05": 0.08330865204334259, + "p95": 0.12839283049106598 + }, + { + "label": "completed_group_2", + "mean": 0.15970060229301453, + "median": 0.16236470639705658, + "p05": 0.11309029906988144, + "p95": 0.19861777126789093 + }, + { + "label": "completed_group_3", + "mean": 0.217601016163826, + "median": 0.21904586255550385, + "p05": 0.16268108785152435, + "p95": 0.2699928283691406 + }, + { + "label": "current_group_partial", + "mean": 0.29974091053009033, + "median": 0.30079299211502075, + "p05": 0.22156940400600433, + "p95": 0.37281978130340576 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.512608289718628, + "group": 4, + "latest_source_mass": 0.35851576924324036, + "layer": 16, + "max_source_mass": 0.35851576924324036, + "mixer_index": 30, + "normalized_entropy": 0.9398363727066501, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21392285823822021, + "median": 0.22146853804588318, + "p05": 0.1518818885087967, + "p95": 0.25874194502830505 + }, + { + "label": "completed_group_1", + "mean": 0.1860550194978714, + "median": 0.1838144212961197, + "p05": 0.1660098135471344, + "p95": 0.21311265230178833 + }, + { + "label": "completed_group_2", + "mean": 0.10291537642478943, + "median": 0.1024642288684845, + "p05": 0.09207107126712799, + "p95": 0.11504144221544266 + }, + { + "label": "completed_group_3", + "mean": 0.1385909616947174, + "median": 0.13646918535232544, + "p05": 0.11220520734786987, + "p95": 0.1728362739086151 + }, + { + "label": "current_group_partial", + "mean": 0.35851576924324036, + "median": 0.3558155298233032, + "p05": 0.31809645891189575, + "p95": 0.40622422099113464 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5262645483016968, + "group": 4, + "latest_source_mass": 0.3142552077770233, + "layer": 16, + "max_source_mass": 0.3142552077770233, + "mixer_index": 31, + "normalized_entropy": 0.9483214832396903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20246261358261108, + "median": 0.18459102511405945, + "p05": 0.12276223301887512, + "p95": 0.38892465829849243 + }, + { + "label": "completed_group_1", + "mean": 0.10530521720647812, + "median": 0.10532709956169128, + "p05": 0.0785030871629715, + "p95": 0.1342671513557434 + }, + { + "label": "completed_group_2", + "mean": 0.16650916635990143, + "median": 0.17111444473266602, + "p05": 0.1085265576839447, + "p95": 0.2094469517469406 + }, + { + "label": "completed_group_3", + "mean": 0.21146778762340546, + "median": 0.2099919468164444, + "p05": 0.15801574289798737, + "p95": 0.26882117986679077 + }, + { + "label": "current_group_partial", + "mean": 0.3142552077770233, + "median": 0.31609731912612915, + "p05": 0.22627028822898865, + "p95": 0.397154837846756 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5794764757156372, + "group": 5, + "latest_source_mass": 0.2760102152824402, + "layer": 17, + "max_source_mass": 0.2760102152824402, + "mixer_index": 32, + "normalized_entropy": 0.9813839126772218, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2210189402103424, + "median": 0.22435861825942993, + "p05": 0.17291823029518127, + "p95": 0.262204647064209 + }, + { + "label": "completed_group_1", + "mean": 0.18839938938617706, + "median": 0.18652752041816711, + "p05": 0.16997642815113068, + "p95": 0.2126670628786087 + }, + { + "label": "completed_group_2", + "mean": 0.14016608893871307, + "median": 0.14067986607551575, + "p05": 0.1265786588191986, + "p95": 0.15091627836227417 + }, + { + "label": "completed_group_3", + "mean": 0.17440536618232727, + "median": 0.17192018032073975, + "p05": 0.15062645077705383, + "p95": 0.20691406726837158 + }, + { + "label": "completed_group_4", + "mean": 0.2760102152824402, + "median": 0.27511972188949585, + "p05": 0.24412119388580322, + "p95": 0.30901142954826355 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6602650880813599, + "group": 5, + "latest_source_mass": 0.3575916588306427, + "layer": 17, + "max_source_mass": 0.3575916588306427, + "mixer_index": 33, + "normalized_entropy": 0.9266115885502495, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12695768475532532, + "median": 0.11564654856920242, + "p05": 0.07159632444381714, + "p95": 0.21949690580368042 + }, + { + "label": "completed_group_1", + "mean": 0.08036886155605316, + "median": 0.07964898645877838, + "p05": 0.06650256365537643, + "p95": 0.09638062119483948 + }, + { + "label": "completed_group_2", + "mean": 0.12672865390777588, + "median": 0.1287270039319992, + "p05": 0.09939455986022949, + "p95": 0.14712141454219818 + }, + { + "label": "completed_group_3", + "mean": 0.15391482412815094, + "median": 0.15372109413146973, + "p05": 0.13333335518836975, + "p95": 0.17581847310066223 + }, + { + "label": "completed_group_4", + "mean": 0.154438316822052, + "median": 0.15402798354625702, + "p05": 0.12481103092432022, + "p95": 0.18498417735099792 + }, + { + "label": "current_group_partial", + "mean": 0.3575916588306427, + "median": 0.36001941561698914, + "p05": 0.3110165596008301, + "p95": 0.39699214696884155 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5925545692443848, + "group": 5, + "latest_source_mass": 0.41908538341522217, + "layer": 18, + "max_source_mass": 0.41908538341522217, + "mixer_index": 34, + "normalized_entropy": 0.8888216284580353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14075040817260742, + "median": 0.1455841362476349, + "p05": 0.09744156897068024, + "p95": 0.1818566620349884 + }, + { + "label": "completed_group_1", + "mean": 0.11028306186199188, + "median": 0.10887216031551361, + "p05": 0.09629634767770767, + "p95": 0.12874113023281097 + }, + { + "label": "completed_group_2", + "mean": 0.07537610828876495, + "median": 0.07517752051353455, + "p05": 0.0666593387722969, + "p95": 0.08491044491529465 + }, + { + "label": "completed_group_3", + "mean": 0.10244421660900116, + "median": 0.10141433775424957, + "p05": 0.08653581142425537, + "p95": 0.12175601720809937 + }, + { + "label": "completed_group_4", + "mean": 0.1520608365535736, + "median": 0.1513972282409668, + "p05": 0.1301831305027008, + "p95": 0.17589205503463745 + }, + { + "label": "current_group_partial", + "mean": 0.41908538341522217, + "median": 0.4191858768463135, + "p05": 0.3649080991744995, + "p95": 0.4721684455871582 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5918848514556885, + "group": 5, + "latest_source_mass": 0.4151553511619568, + "layer": 18, + "max_source_mass": 0.4151553511619568, + "mixer_index": 35, + "normalized_entropy": 0.8884478518433735, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11773698031902313, + "median": 0.10857899487018585, + "p05": 0.06324320286512375, + "p95": 0.22339528799057007 + }, + { + "label": "completed_group_1", + "mean": 0.07344591617584229, + "median": 0.07228962332010269, + "p05": 0.05982384458184242, + "p95": 0.09060496836900711 + }, + { + "label": "completed_group_2", + "mean": 0.11976176500320435, + "median": 0.12151201069355011, + "p05": 0.08953659236431122, + "p95": 0.14441059529781342 + }, + { + "label": "completed_group_3", + "mean": 0.14735516905784607, + "median": 0.14620742201805115, + "p05": 0.11767779290676117, + "p95": 0.17902395129203796 + }, + { + "label": "completed_group_4", + "mean": 0.126544788479805, + "median": 0.12564796209335327, + "p05": 0.09858805686235428, + "p95": 0.1567486822605133 + }, + { + "label": "current_group_partial", + "mean": 0.4151553511619568, + "median": 0.4184337854385376, + "p05": 0.3375611901283264, + "p95": 0.4846380352973938 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6208878755569458, + "group": 5, + "latest_source_mass": 0.3991435170173645, + "layer": 19, + "max_source_mass": 0.3991435170173645, + "mixer_index": 36, + "normalized_entropy": 0.9046347477964072, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14785674214363098, + "median": 0.15170978009700775, + "p05": 0.09146977216005325, + "p95": 0.20172694325447083 + }, + { + "label": "completed_group_1", + "mean": 0.12492769956588745, + "median": 0.12345598638057709, + "p05": 0.10790181905031204, + "p95": 0.14623555541038513 + }, + { + "label": "completed_group_2", + "mean": 0.08781818300485611, + "median": 0.08839264512062073, + "p05": 0.07453933358192444, + "p95": 0.09894395619630814 + }, + { + "label": "completed_group_3", + "mean": 0.10019385814666748, + "median": 0.09901996701955795, + "p05": 0.08430047333240509, + "p95": 0.12013883143663406 + }, + { + "label": "completed_group_4", + "mean": 0.14006000757217407, + "median": 0.13918349146842957, + "p05": 0.11816112697124481, + "p95": 0.16475799679756165 + }, + { + "label": "current_group_partial", + "mean": 0.3991435170173645, + "median": 0.39726996421813965, + "p05": 0.34116071462631226, + "p95": 0.4634247422218323 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5470075607299805, + "group": 5, + "latest_source_mass": 0.44330859184265137, + "layer": 19, + "max_source_mass": 0.44330859184265137, + "mixer_index": 37, + "normalized_entropy": 0.8634013589985261, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12043030560016632, + "median": 0.1012125313282013, + "p05": 0.061379045248031616, + "p95": 0.25343188643455505 + }, + { + "label": "completed_group_1", + "mean": 0.06855648756027222, + "median": 0.06761305779218674, + "p05": 0.05337192490696907, + "p95": 0.08790726959705353 + }, + { + "label": "completed_group_2", + "mean": 0.11753211170434952, + "median": 0.1186976358294487, + "p05": 0.08412563800811768, + "p95": 0.14895018935203552 + }, + { + "label": "completed_group_3", + "mean": 0.1339646875858307, + "median": 0.13253431022167206, + "p05": 0.09857159107923508, + "p95": 0.1730121374130249 + }, + { + "label": "completed_group_4", + "mean": 0.1162077933549881, + "median": 0.11592038720846176, + "p05": 0.08609852194786072, + "p95": 0.14767511188983917 + }, + { + "label": "current_group_partial", + "mean": 0.44330859184265137, + "median": 0.4434061646461487, + "p05": 0.3389148414134979, + "p95": 0.5490255355834961 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.683492660522461, + "group": 5, + "latest_source_mass": 0.3365415334701538, + "layer": 20, + "max_source_mass": 0.3365415334701538, + "mixer_index": 38, + "normalized_entropy": 0.9395751435586169, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1658952683210373, + "median": 0.1726911962032318, + "p05": 0.10183650255203247, + "p95": 0.22522984445095062 + }, + { + "label": "completed_group_1", + "mean": 0.1435280442237854, + "median": 0.14210543036460876, + "p05": 0.12450659275054932, + "p95": 0.16782894730567932 + }, + { + "label": "completed_group_2", + "mean": 0.09461991488933563, + "median": 0.0948714017868042, + "p05": 0.08168491721153259, + "p95": 0.10723742842674255 + }, + { + "label": "completed_group_3", + "mean": 0.10784181207418442, + "median": 0.10693192481994629, + "p05": 0.08930590003728867, + "p95": 0.12958656251430511 + }, + { + "label": "completed_group_4", + "mean": 0.15157344937324524, + "median": 0.15053030848503113, + "p05": 0.128616601228714, + "p95": 0.17799487709999084 + }, + { + "label": "current_group_partial", + "mean": 0.3365415334701538, + "median": 0.33349889516830444, + "p05": 0.2778944969177246, + "p95": 0.40724658966064453 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5540567636489868, + "group": 5, + "latest_source_mass": 0.42266085743904114, + "layer": 20, + "max_source_mass": 0.42266085743904114, + "mixer_index": 39, + "normalized_entropy": 0.8673355940563396, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12987135350704193, + "median": 0.1083056628704071, + "p05": 0.03783264383673668, + "p95": 0.3301126956939697 + }, + { + "label": "completed_group_1", + "mean": 0.07178181409835815, + "median": 0.07081792503595352, + "p05": 0.05334944650530815, + "p95": 0.09311570972204208 + }, + { + "label": "completed_group_2", + "mean": 0.12132491171360016, + "median": 0.12282539904117584, + "p05": 0.07926258444786072, + "p95": 0.15763431787490845 + }, + { + "label": "completed_group_3", + "mean": 0.1441602110862732, + "median": 0.14273977279663086, + "p05": 0.09847753494977951, + "p95": 0.19304539263248444 + }, + { + "label": "completed_group_4", + "mean": 0.11020088195800781, + "median": 0.10934801399707794, + "p05": 0.07139573246240616, + "p95": 0.1517774760723114 + }, + { + "label": "current_group_partial", + "mean": 0.42266085743904114, + "median": 0.42266106605529785, + "p05": 0.28481048345565796, + "p95": 0.5570361018180847 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7208337783813477, + "group": 6, + "latest_source_mass": 0.27070939540863037, + "layer": 21, + "max_source_mass": 0.27070939540863037, + "mixer_index": 40, + "normalized_entropy": 0.9604156182429642, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18048501014709473, + "median": 0.18919861316680908, + "p05": 0.11199048161506653, + "p95": 0.23807701468467712 + }, + { + "label": "completed_group_1", + "mean": 0.1685236394405365, + "median": 0.1669977903366089, + "p05": 0.14678362011909485, + "p95": 0.19381441175937653 + }, + { + "label": "completed_group_2", + "mean": 0.08726631104946136, + "median": 0.08729851245880127, + "p05": 0.07629868388175964, + "p95": 0.0980045348405838 + }, + { + "label": "completed_group_3", + "mean": 0.11160479485988617, + "median": 0.1099197044968605, + "p05": 0.09190923720598221, + "p95": 0.13723208010196686 + }, + { + "label": "completed_group_4", + "mean": 0.18141084909439087, + "median": 0.18020854890346527, + "p05": 0.15656134486198425, + "p95": 0.2108723670244217 + }, + { + "label": "completed_group_5", + "mean": 0.27070939540863037, + "median": 0.2684924602508545, + "p05": 0.2159801721572876, + "p95": 0.33192309737205505 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6729483604431152, + "group": 6, + "latest_source_mass": 0.3803577423095703, + "layer": 21, + "max_source_mass": 0.3803577423095703, + "mixer_index": 41, + "normalized_entropy": 0.8597253893019091, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09229261428117752, + "median": 0.07798201590776443, + "p05": 0.04784023016691208, + "p95": 0.20285779237747192 + }, + { + "label": "completed_group_1", + "mean": 0.051420148462057114, + "median": 0.05041736364364624, + "p05": 0.040625106543302536, + "p95": 0.06547587364912033 + }, + { + "label": "completed_group_2", + "mean": 0.08079522103071213, + "median": 0.08136115223169327, + "p05": 0.05947277322411537, + "p95": 0.10017222166061401 + }, + { + "label": "completed_group_3", + "mean": 0.09550285339355469, + "median": 0.09445887804031372, + "p05": 0.07334083318710327, + "p95": 0.12144046276807785 + }, + { + "label": "completed_group_4", + "mean": 0.07590493559837341, + "median": 0.07492390275001526, + "p05": 0.055510398000478745, + "p95": 0.10084333270788193 + }, + { + "label": "completed_group_5", + "mean": 0.22372648119926453, + "median": 0.22069036960601807, + "p05": 0.1407623589038849, + "p95": 0.32290777564048767 + }, + { + "label": "current_group_partial", + "mean": 0.3803577423095703, + "median": 0.37903881072998047, + "p05": 0.2855539321899414, + "p95": 0.476589173078537 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.710235595703125, + "group": 6, + "latest_source_mass": 0.4011716842651367, + "layer": 22, + "max_source_mass": 0.4011716842651367, + "mixer_index": 42, + "normalized_entropy": 0.8788872376935791, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11657392978668213, + "median": 0.11877645552158356, + "p05": 0.06819256395101547, + "p95": 0.1684979349374771 + }, + { + "label": "completed_group_1", + "mean": 0.10617930442094803, + "median": 0.10509790480136871, + "p05": 0.090352863073349, + "p95": 0.12587188184261322 + }, + { + "label": "completed_group_2", + "mean": 0.061112720519304276, + "median": 0.061012350022792816, + "p05": 0.05164862796664238, + "p95": 0.07133162766695023 + }, + { + "label": "completed_group_3", + "mean": 0.06796726584434509, + "median": 0.06712733209133148, + "p05": 0.05444999784231186, + "p95": 0.08481802046298981 + }, + { + "label": "completed_group_4", + "mean": 0.103337861597538, + "median": 0.10192783921957016, + "p05": 0.0832459107041359, + "p95": 0.12805812060832977 + }, + { + "label": "completed_group_5", + "mean": 0.14365722239017487, + "median": 0.14159655570983887, + "p05": 0.10498540848493576, + "p95": 0.19020715355873108 + }, + { + "label": "current_group_partial", + "mean": 0.4011716842651367, + "median": 0.40084803104400635, + "p05": 0.32083970308303833, + "p95": 0.47994357347488403 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5640671253204346, + "group": 6, + "latest_source_mass": 0.4689176678657532, + "layer": 22, + "max_source_mass": 0.4689176678657532, + "mixer_index": 43, + "normalized_entropy": 0.8037715030571925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.090422622859478, + "median": 0.0774901732802391, + "p05": 0.04492149129509926, + "p95": 0.1988787055015564 + }, + { + "label": "completed_group_1", + "mean": 0.04561382159590721, + "median": 0.044634923338890076, + "p05": 0.03501313924789429, + "p95": 0.05900419130921364 + }, + { + "label": "completed_group_2", + "mean": 0.07565715163946152, + "median": 0.07550699263811111, + "p05": 0.053458306938409805, + "p95": 0.09821777045726776 + }, + { + "label": "completed_group_3", + "mean": 0.08878682553768158, + "median": 0.08692863583564758, + "p05": 0.06425619125366211, + "p95": 0.11807316541671753 + }, + { + "label": "completed_group_4", + "mean": 0.06555216014385223, + "median": 0.06414324045181274, + "p05": 0.04460844025015831, + "p95": 0.09090004861354828 + }, + { + "label": "completed_group_5", + "mean": 0.165049746632576, + "median": 0.15503892302513123, + "p05": 0.08368627727031708, + "p95": 0.2792147397994995 + }, + { + "label": "current_group_partial", + "mean": 0.4689176678657532, + "median": 0.46879667043685913, + "p05": 0.3151778280735016, + "p95": 0.6170428991317749 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7556228637695312, + "group": 6, + "latest_source_mass": 0.3706088662147522, + "layer": 23, + "max_source_mass": 0.3706088662147522, + "mixer_index": 44, + "normalized_entropy": 0.9022116795175967, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13281464576721191, + "median": 0.13658174872398376, + "p05": 0.08060654997825623, + "p95": 0.17402096092700958 + }, + { + "label": "completed_group_1", + "mean": 0.12175846099853516, + "median": 0.12066285312175751, + "p05": 0.10334118455648422, + "p95": 0.14384035766124725 + }, + { + "label": "completed_group_2", + "mean": 0.07235272228717804, + "median": 0.07218228280544281, + "p05": 0.059715576469898224, + "p95": 0.08584222197532654 + }, + { + "label": "completed_group_3", + "mean": 0.0707377940416336, + "median": 0.06942892074584961, + "p05": 0.05500924959778786, + "p95": 0.09052760899066925 + }, + { + "label": "completed_group_4", + "mean": 0.10551950335502625, + "median": 0.103996641933918, + "p05": 0.08408617228269577, + "p95": 0.13205930590629578 + }, + { + "label": "completed_group_5", + "mean": 0.12620802223682404, + "median": 0.12379512935876846, + "p05": 0.08937962353229523, + "p95": 0.17139491438865662 + }, + { + "label": "current_group_partial", + "mean": 0.3706088662147522, + "median": 0.3705657124519348, + "p05": 0.282781720161438, + "p95": 0.4561466872692108 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.594892978668213, + "group": 6, + "latest_source_mass": 0.4636557698249817, + "layer": 23, + "max_source_mass": 0.4636557698249817, + "mixer_index": 45, + "normalized_entropy": 0.8196128579947488, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10721131414175034, + "median": 0.10240983963012695, + "p05": 0.05563424155116081, + "p95": 0.18401460349559784 + }, + { + "label": "completed_group_1", + "mean": 0.053925663232803345, + "median": 0.0527028888463974, + "p05": 0.04038037359714508, + "p95": 0.07054222375154495 + }, + { + "label": "completed_group_2", + "mean": 0.07673212885856628, + "median": 0.07705660909414291, + "p05": 0.05211106687784195, + "p95": 0.10031480342149734 + }, + { + "label": "completed_group_3", + "mean": 0.08865120261907578, + "median": 0.08699032664299011, + "p05": 0.06242028623819351, + "p95": 0.11899784207344055 + }, + { + "label": "completed_group_4", + "mean": 0.06888366490602493, + "median": 0.06751501560211182, + "p05": 0.04784276708960533, + "p95": 0.09609999507665634 + }, + { + "label": "completed_group_5", + "mean": 0.14094024896621704, + "median": 0.13063763082027435, + "p05": 0.07297889143228531, + "p95": 0.24012833833694458 + }, + { + "label": "current_group_partial", + "mean": 0.4636557698249817, + "median": 0.46312135457992554, + "p05": 0.31259939074516296, + "p95": 0.6109440326690674 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7775511741638184, + "group": 6, + "latest_source_mass": 0.3569435477256775, + "layer": 24, + "max_source_mass": 0.3569435477256775, + "mixer_index": 46, + "normalized_entropy": 0.9134806018801903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13637229800224304, + "median": 0.13456633687019348, + "p05": 0.09586701542139053, + "p95": 0.18876218795776367 + }, + { + "label": "completed_group_1", + "mean": 0.10478455573320389, + "median": 0.10326151549816132, + "p05": 0.0880400538444519, + "p95": 0.12566551566123962 + }, + { + "label": "completed_group_2", + "mean": 0.07710614800453186, + "median": 0.07678486406803131, + "p05": 0.06292064487934113, + "p95": 0.09218747913837433 + }, + { + "label": "completed_group_3", + "mean": 0.08548048138618469, + "median": 0.08430111408233643, + "p05": 0.0665682703256607, + "p95": 0.10857507586479187 + }, + { + "label": "completed_group_4", + "mean": 0.11000140011310577, + "median": 0.10904969274997711, + "p05": 0.0881001353263855, + "p95": 0.13561299443244934 + }, + { + "label": "completed_group_5", + "mean": 0.12931162118911743, + "median": 0.12692990899085999, + "p05": 0.08648453652858734, + "p95": 0.1814313679933548 + }, + { + "label": "current_group_partial", + "mean": 0.3569435477256775, + "median": 0.3568228483200073, + "p05": 0.26841166615486145, + "p95": 0.4493965804576874 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.654036045074463, + "group": 6, + "latest_source_mass": 0.4265251159667969, + "layer": 24, + "max_source_mass": 0.4265251159667969, + "mixer_index": 47, + "normalized_entropy": 0.8500063817835848, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10232305526733398, + "median": 0.08623291552066803, + "p05": 0.04679499566555023, + "p95": 0.23077142238616943 + }, + { + "label": "completed_group_1", + "mean": 0.061968062072992325, + "median": 0.060845747590065, + "p05": 0.0479469932615757, + "p95": 0.07941868156194687 + }, + { + "label": "completed_group_2", + "mean": 0.08900615572929382, + "median": 0.08864577114582062, + "p05": 0.0641680359840393, + "p95": 0.11413820832967758 + }, + { + "label": "completed_group_3", + "mean": 0.1005433052778244, + "median": 0.09813467413187027, + "p05": 0.0731433779001236, + "p95": 0.1343638002872467 + }, + { + "label": "completed_group_4", + "mean": 0.07864196598529816, + "median": 0.07639941573143005, + "p05": 0.05487418174743652, + "p95": 0.10941869765520096 + }, + { + "label": "completed_group_5", + "mean": 0.14099235832691193, + "median": 0.12885531783103943, + "p05": 0.07315157353878021, + "p95": 0.24481485784053802 + }, + { + "label": "current_group_partial", + "mean": 0.4265251159667969, + "median": 0.42903727293014526, + "p05": 0.2803359627723694, + "p95": 0.5635722875595093 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.84956693649292, + "group": 7, + "latest_source_mass": 0.28775152564048767, + "layer": 25, + "max_source_mass": 0.28775152564048767, + "mixer_index": 48, + "normalized_entropy": 0.9504893827656096, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14851760864257812, + "median": 0.14813844859600067, + "p05": 0.10312555730342865, + "p95": 0.20065288245677948 + }, + { + "label": "completed_group_1", + "mean": 0.11630287766456604, + "median": 0.11500488221645355, + "p05": 0.09825178980827332, + "p95": 0.13750022649765015 + }, + { + "label": "completed_group_2", + "mean": 0.08512166142463684, + "median": 0.08458003401756287, + "p05": 0.0710662305355072, + "p95": 0.10092862695455551 + }, + { + "label": "completed_group_3", + "mean": 0.09215595573186874, + "median": 0.09079311043024063, + "p05": 0.07304700464010239, + "p95": 0.11515989899635315 + }, + { + "label": "completed_group_4", + "mean": 0.1263950914144516, + "median": 0.12563186883926392, + "p05": 0.10441911220550537, + "p95": 0.1519687920808792 + }, + { + "label": "completed_group_5", + "mean": 0.14375528693199158, + "median": 0.1422564834356308, + "p05": 0.10313796997070312, + "p95": 0.1929876059293747 + }, + { + "label": "completed_group_6", + "mean": 0.28775152564048767, + "median": 0.28676605224609375, + "p05": 0.21690022945404053, + "p95": 0.3624127507209778 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7161309719085693, + "group": 7, + "latest_source_mass": 0.32536065578460693, + "layer": 25, + "max_source_mass": 0.32536065578460693, + "mixer_index": 49, + "normalized_entropy": 0.8252845475628168, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0837959572672844, + "median": 0.0849243700504303, + "p05": 0.0271806251257658, + "p95": 0.15180703997612 + }, + { + "label": "completed_group_1", + "mean": 0.04434927552938461, + "median": 0.04302988201379776, + "p05": 0.03217232972383499, + "p95": 0.06002792343497276 + }, + { + "label": "completed_group_2", + "mean": 0.056708935648202896, + "median": 0.05563671886920929, + "p05": 0.03992946073412895, + "p95": 0.07731831818819046 + }, + { + "label": "completed_group_3", + "mean": 0.06495873630046844, + "median": 0.06223570555448532, + "p05": 0.04220382124185562, + "p95": 0.0949486792087555 + }, + { + "label": "completed_group_4", + "mean": 0.04885463789105415, + "median": 0.046898528933525085, + "p05": 0.030923232436180115, + "p95": 0.0743112787604332 + }, + { + "label": "completed_group_5", + "mean": 0.09997115284204483, + "median": 0.08650931715965271, + "p05": 0.04111029952764511, + "p95": 0.20292627811431885 + }, + { + "label": "completed_group_6", + "mean": 0.2760006785392761, + "median": 0.27142390608787537, + "p05": 0.13761089742183685, + "p95": 0.42605364322662354 + }, + { + "label": "current_group_partial", + "mean": 0.32536065578460693, + "median": 0.3348004221916199, + "p05": 0.10654139518737793, + "p95": 0.5236068964004517 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9285151958465576, + "group": 7, + "latest_source_mass": 0.2804902195930481, + "layer": 26, + "max_source_mass": 0.2804902195930481, + "mixer_index": 50, + "normalized_entropy": 0.9274197697756124, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10564712435007095, + "median": 0.10497278720140457, + "p05": 0.04608207941055298, + "p95": 0.16350959241390228 + }, + { + "label": "completed_group_1", + "mean": 0.09173665940761566, + "median": 0.09041133522987366, + "p05": 0.07509034872055054, + "p95": 0.11238400638103485 + }, + { + "label": "completed_group_2", + "mean": 0.07135222852230072, + "median": 0.06981296837329865, + "p05": 0.0522611029446125, + "p95": 0.09555281698703766 + }, + { + "label": "completed_group_3", + "mean": 0.07045470923185349, + "median": 0.06927657127380371, + "p05": 0.055566973984241486, + "p95": 0.08908769488334656 + }, + { + "label": "completed_group_4", + "mean": 0.09785081446170807, + "median": 0.0971219539642334, + "p05": 0.07590530812740326, + "p95": 0.12285276502370834 + }, + { + "label": "completed_group_5", + "mean": 0.10425601899623871, + "median": 0.10151226818561554, + "p05": 0.06786561012268066, + "p95": 0.15156632661819458 + }, + { + "label": "completed_group_6", + "mean": 0.1782122254371643, + "median": 0.1762663722038269, + "p05": 0.11771149188280106, + "p95": 0.24932436645030975 + }, + { + "label": "current_group_partial", + "mean": 0.2804902195930481, + "median": 0.2783043086528778, + "p05": 0.14811429381370544, + "p95": 0.4168073534965515 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7219383716583252, + "group": 7, + "latest_source_mass": 0.3661728501319885, + "layer": 26, + "max_source_mass": 0.3661728501319885, + "mixer_index": 51, + "normalized_entropy": 0.8280773165026276, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0923302173614502, + "median": 0.08522969484329224, + "p05": 0.056062981486320496, + "p95": 0.1509525179862976 + }, + { + "label": "completed_group_1", + "mean": 0.04646994173526764, + "median": 0.04543843865394592, + "p05": 0.032575178891420364, + "p95": 0.06306323409080505 + }, + { + "label": "completed_group_2", + "mean": 0.05750556290149689, + "median": 0.05649096518754959, + "p05": 0.03895717114210129, + "p95": 0.07940060645341873 + }, + { + "label": "completed_group_3", + "mean": 0.06549686938524246, + "median": 0.062135741114616394, + "p05": 0.04169235751032829, + "p95": 0.09948549419641495 + }, + { + "label": "completed_group_4", + "mean": 0.054404959082603455, + "median": 0.05241036415100098, + "p05": 0.03378590941429138, + "p95": 0.08157843351364136 + }, + { + "label": "completed_group_5", + "mean": 0.08427833020687103, + "median": 0.07576382160186768, + "p05": 0.041103728115558624, + "p95": 0.15204203128814697 + }, + { + "label": "completed_group_6", + "mean": 0.2333412617444992, + "median": 0.22215056419372559, + "p05": 0.10060560703277588, + "p95": 0.3983197808265686 + }, + { + "label": "current_group_partial", + "mean": 0.3661728501319885, + "median": 0.3673480749130249, + "p05": 0.16410952806472778, + "p95": 0.5635213255882263 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9470824003219604, + "group": 7, + "latest_source_mass": 0.26050227880477905, + "layer": 27, + "max_source_mass": 0.26050227880477905, + "mixer_index": 52, + "normalized_entropy": 0.9363487077155573, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10804469138383865, + "median": 0.10379163920879364, + "p05": 0.05421917140483856, + "p95": 0.17028658092021942 + }, + { + "label": "completed_group_1", + "mean": 0.07850409299135208, + "median": 0.0776081532239914, + "p05": 0.063654825091362, + "p95": 0.09574449062347412 + }, + { + "label": "completed_group_2", + "mean": 0.07298290729522705, + "median": 0.07200999557971954, + "p05": 0.05533483251929283, + "p95": 0.09299935400485992 + }, + { + "label": "completed_group_3", + "mean": 0.08511961251497269, + "median": 0.08372659981250763, + "p05": 0.06449062377214432, + "p95": 0.11086944490671158 + }, + { + "label": "completed_group_4", + "mean": 0.09968686103820801, + "median": 0.09877584874629974, + "p05": 0.07606595754623413, + "p95": 0.1266063004732132 + }, + { + "label": "completed_group_5", + "mean": 0.11502514779567719, + "median": 0.11171717941761017, + "p05": 0.07302433997392654, + "p95": 0.1681303083896637 + }, + { + "label": "completed_group_6", + "mean": 0.18013443052768707, + "median": 0.1765173077583313, + "p05": 0.11514104157686234, + "p95": 0.2599799633026123 + }, + { + "label": "current_group_partial", + "mean": 0.26050227880477905, + "median": 0.2575443983078003, + "p05": 0.13687226176261902, + "p95": 0.39756521582603455 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7855088710784912, + "group": 7, + "latest_source_mass": 0.28804832696914673, + "layer": 27, + "max_source_mass": 0.28804832696914673, + "mixer_index": 53, + "normalized_entropy": 0.858648264589397, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08493785560131073, + "median": 0.05951457470655441, + "p05": 0.02441210113465786, + "p95": 0.22703105211257935 + }, + { + "label": "completed_group_1", + "mean": 0.05972691997885704, + "median": 0.058127306401729584, + "p05": 0.03591413050889969, + "p95": 0.08903861790895462 + }, + { + "label": "completed_group_2", + "mean": 0.07017223536968231, + "median": 0.06906847655773163, + "p05": 0.04151676222681999, + "p95": 0.10273467004299164 + }, + { + "label": "completed_group_3", + "mean": 0.07963316142559052, + "median": 0.07547932863235474, + "p05": 0.042773112654685974, + "p95": 0.12874338030815125 + }, + { + "label": "completed_group_4", + "mean": 0.0645163506269455, + "median": 0.061876002699136734, + "p05": 0.0366181954741478, + "p95": 0.10098200291395187 + }, + { + "label": "completed_group_5", + "mean": 0.11586236953735352, + "median": 0.10182450711727142, + "p05": 0.048887670040130615, + "p95": 0.2313569337129593 + }, + { + "label": "completed_group_6", + "mean": 0.23710274696350098, + "median": 0.22610372304916382, + "p05": 0.10011275857686996, + "p95": 0.4084784686565399 + }, + { + "label": "current_group_partial", + "mean": 0.28804832696914673, + "median": 0.27438053488731384, + "p05": 0.07688213139772415, + "p95": 0.5529112815856934 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.973182201385498, + "group": 7, + "latest_source_mass": 0.23531918227672577, + "layer": 28, + "max_source_mass": 0.23531918227672577, + "mixer_index": 54, + "normalized_entropy": 0.9489000589030754, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.104981929063797, + "median": 0.09386023879051208, + "p05": 0.04212673753499985, + "p95": 0.18614700436592102 + }, + { + "label": "completed_group_1", + "mean": 0.0840599462389946, + "median": 0.08250434696674347, + "p05": 0.05170809105038643, + "p95": 0.12236068397760391 + }, + { + "label": "completed_group_2", + "mean": 0.09310521930456161, + "median": 0.0931512713432312, + "p05": 0.06374019384384155, + "p95": 0.12329407781362534 + }, + { + "label": "completed_group_3", + "mean": 0.11566174775362015, + "median": 0.11439734697341919, + "p05": 0.07886623591184616, + "p95": 0.1571493148803711 + }, + { + "label": "completed_group_4", + "mean": 0.0963074117898941, + "median": 0.09477876126766205, + "p05": 0.06686149537563324, + "p95": 0.13022682070732117 + }, + { + "label": "completed_group_5", + "mean": 0.1096896305680275, + "median": 0.10702179372310638, + "p05": 0.06579374521970749, + "p95": 0.1643676459789276 + }, + { + "label": "completed_group_6", + "mean": 0.16087493300437927, + "median": 0.15455034375190735, + "p05": 0.09876372665166855, + "p95": 0.24324838817119598 + }, + { + "label": "current_group_partial", + "mean": 0.23531918227672577, + "median": 0.22382870316505432, + "p05": 0.11656329780817032, + "p95": 0.38286590576171875 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.802741289138794, + "group": 7, + "latest_source_mass": 0.2497689425945282, + "layer": 28, + "max_source_mass": 0.2913346588611603, + "mixer_index": 55, + "normalized_entropy": 0.8669353059487718, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08190082013607025, + "median": 0.06415966153144836, + "p05": 0.039168842136859894, + "p95": 0.23471535742282867 + }, + { + "label": "completed_group_1", + "mean": 0.05632811039686203, + "median": 0.055233441293239594, + "p05": 0.03936203569173813, + "p95": 0.076178178191185 + }, + { + "label": "completed_group_2", + "mean": 0.07102417945861816, + "median": 0.06932097673416138, + "p05": 0.04159427434206009, + "p95": 0.1046731173992157 + }, + { + "label": "completed_group_3", + "mean": 0.08421063423156738, + "median": 0.0812690332531929, + "p05": 0.0486646331846714, + "p95": 0.12889066338539124 + }, + { + "label": "completed_group_4", + "mean": 0.06341387331485748, + "median": 0.061601825058460236, + "p05": 0.03807957097887993, + "p95": 0.09577237069606781 + }, + { + "label": "completed_group_5", + "mean": 0.10201875865459442, + "median": 0.09330011904239655, + "p05": 0.05116701498627663, + "p95": 0.18108239769935608 + }, + { + "label": "completed_group_6", + "mean": 0.2913346588611603, + "median": 0.28823426365852356, + "p05": 0.13295966386795044, + "p95": 0.4586564898490906 + }, + { + "label": "current_group_partial", + "mean": 0.2497689425945282, + "median": 0.234481081366539, + "p05": 0.09318265318870544, + "p95": 0.4530518054962158 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.945815086364746, + "group": 8, + "latest_source_mass": 0.153213769197464, + "layer": 29, + "max_source_mass": 0.2451319396495819, + "mixer_index": 56, + "normalized_entropy": 0.9357392585284499, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1064736545085907, + "median": 0.0881282389163971, + "p05": 0.029344500973820686, + "p95": 0.23970629274845123 + }, + { + "label": "completed_group_1", + "mean": 0.07925166189670563, + "median": 0.07691921293735504, + "p05": 0.062442928552627563, + "p95": 0.1030145138502121 + }, + { + "label": "completed_group_2", + "mean": 0.07823626697063446, + "median": 0.07715678215026855, + "p05": 0.059018202126026154, + "p95": 0.10055352747440338 + }, + { + "label": "completed_group_3", + "mean": 0.09447507560253143, + "median": 0.09318602085113525, + "p05": 0.0684189647436142, + "p95": 0.12381350249052048 + }, + { + "label": "completed_group_4", + "mean": 0.09724453091621399, + "median": 0.09628183394670486, + "p05": 0.0699525699019432, + "p95": 0.12776269018650055 + }, + { + "label": "completed_group_5", + "mean": 0.1459731012582779, + "median": 0.14035329222679138, + "p05": 0.08667999505996704, + "p95": 0.2258891463279724 + }, + { + "label": "completed_group_6", + "mean": 0.2451319396495819, + "median": 0.24505369365215302, + "p05": 0.16207969188690186, + "p95": 0.3307104706764221 + }, + { + "label": "completed_group_7", + "mean": 0.153213769197464, + "median": 0.13417893648147583, + "p05": 0.05588865652680397, + "p95": 0.30438560247421265 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8717057704925537, + "group": 8, + "latest_source_mass": 0.3048541843891144, + "layer": 29, + "max_source_mass": 0.3048541843891144, + "mixer_index": 57, + "normalized_entropy": 0.8518500065030654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07336360216140747, + "median": 0.0611552819609642, + "p05": 0.024239415302872658, + "p95": 0.16916418075561523 + }, + { + "label": "completed_group_1", + "mean": 0.05720400810241699, + "median": 0.05402737483382225, + "p05": 0.0314149409532547, + "p95": 0.09488057345151901 + }, + { + "label": "completed_group_2", + "mean": 0.057025715708732605, + "median": 0.05547948181629181, + "p05": 0.03480810672044754, + "p95": 0.08415934443473816 + }, + { + "label": "completed_group_3", + "mean": 0.06655597686767578, + "median": 0.06265220046043396, + "p05": 0.035200610756874084, + "p95": 0.10949824750423431 + }, + { + "label": "completed_group_4", + "mean": 0.05032068490982056, + "median": 0.047904569655656815, + "p05": 0.02614513412117958, + "p95": 0.08244554698467255 + }, + { + "label": "completed_group_5", + "mean": 0.08329877257347107, + "median": 0.0731179416179657, + "p05": 0.02939312346279621, + "p95": 0.17248544096946716 + }, + { + "label": "completed_group_6", + "mean": 0.1901058554649353, + "median": 0.18234091997146606, + "p05": 0.08719424903392792, + "p95": 0.317789226770401 + }, + { + "label": "completed_group_7", + "mean": 0.11727118492126465, + "median": 0.09272899478673935, + "p05": 0.031939513981342316, + "p95": 0.2891285717487335 + }, + { + "label": "current_group_partial", + "mean": 0.3048541843891144, + "median": 0.29432767629623413, + "p05": 0.07913826406002045, + "p95": 0.5632759928703308 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.09096622467041, + "group": 8, + "latest_source_mass": 0.13795366883277893, + "layer": 30, + "max_source_mass": 0.17320571839809418, + "mixer_index": 58, + "normalized_entropy": 0.951639739623416, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10326676070690155, + "median": 0.09009090811014175, + "p05": 0.01757553033530712, + "p95": 0.2674753665924072 + }, + { + "label": "completed_group_1", + "mean": 0.08232800662517548, + "median": 0.08093428611755371, + "p05": 0.06430500745773315, + "p95": 0.10457758605480194 + }, + { + "label": "completed_group_2", + "mean": 0.07164021581411362, + "median": 0.0711187869310379, + "p05": 0.05190654471516609, + "p95": 0.09468771517276764 + }, + { + "label": "completed_group_3", + "mean": 0.08822821825742722, + "median": 0.08667590469121933, + "p05": 0.06148773431777954, + "p95": 0.12025904655456543 + }, + { + "label": "completed_group_4", + "mean": 0.10803373157978058, + "median": 0.10627314448356628, + "p05": 0.07042569667100906, + "p95": 0.1514367014169693 + }, + { + "label": "completed_group_5", + "mean": 0.13578416407108307, + "median": 0.12929600477218628, + "p05": 0.08227657526731491, + "p95": 0.21117588877677917 + }, + { + "label": "completed_group_6", + "mean": 0.17320571839809418, + "median": 0.1694163680076599, + "p05": 0.11322733014822006, + "p95": 0.24482715129852295 + }, + { + "label": "completed_group_7", + "mean": 0.09955951571464539, + "median": 0.08973086625337601, + "p05": 0.037010837346315384, + "p95": 0.1912725567817688 + }, + { + "label": "current_group_partial", + "mean": 0.13795366883277893, + "median": 0.1256815791130066, + "p05": 0.053015124052762985, + "p95": 0.2601332366466522 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.96881103515625, + "group": 8, + "latest_source_mass": 0.21570813655853271, + "layer": 30, + "max_source_mass": 0.21570813655853271, + "mixer_index": 59, + "normalized_entropy": 0.896044517007504, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08843819797039032, + "median": 0.08496083319187164, + "p05": 0.03690534085035324, + "p95": 0.1522325873374939 + }, + { + "label": "completed_group_1", + "mean": 0.06052382290363312, + "median": 0.058369796723127365, + "p05": 0.03549504280090332, + "p95": 0.09154202789068222 + }, + { + "label": "completed_group_2", + "mean": 0.0642135739326477, + "median": 0.06355445086956024, + "p05": 0.03948400169610977, + "p95": 0.09175601601600647 + }, + { + "label": "completed_group_3", + "mean": 0.07744920998811722, + "median": 0.07566337287425995, + "p05": 0.04564609378576279, + "p95": 0.11733324825763702 + }, + { + "label": "completed_group_4", + "mean": 0.051616813987493515, + "median": 0.049985647201538086, + "p05": 0.03238597884774208, + "p95": 0.07699409872293472 + }, + { + "label": "completed_group_5", + "mean": 0.06631231307983398, + "median": 0.061356961727142334, + "p05": 0.030705859884619713, + "p95": 0.1188126802444458 + }, + { + "label": "completed_group_6", + "mean": 0.16417551040649414, + "median": 0.15454982221126556, + "p05": 0.07227650284767151, + "p95": 0.28998225927352905 + }, + { + "label": "completed_group_7", + "mean": 0.2115624099969864, + "median": 0.19997704029083252, + "p05": 0.09131578356027603, + "p95": 0.3709705173969269 + }, + { + "label": "current_group_partial", + "mean": 0.21570813655853271, + "median": 0.2005728930234909, + "p05": 0.06839637458324432, + "p95": 0.4193759858608246 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.124063014984131, + "group": 8, + "latest_source_mass": 0.1258423924446106, + "layer": 31, + "max_source_mass": 0.17017140984535217, + "mixer_index": 60, + "normalized_entropy": 0.9667027380329118, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09220977127552032, + "median": 0.09152225404977798, + "p05": 0.03251190483570099, + "p95": 0.17071731388568878 + }, + { + "label": "completed_group_1", + "mean": 0.06839152425527573, + "median": 0.06702418625354767, + "p05": 0.04446800425648689, + "p95": 0.0977444052696228 + }, + { + "label": "completed_group_2", + "mean": 0.10039852559566498, + "median": 0.09736946225166321, + "p05": 0.06476961821317673, + "p95": 0.14545989036560059 + }, + { + "label": "completed_group_3", + "mean": 0.10902463644742966, + "median": 0.10749617218971252, + "p05": 0.07769118249416351, + "p95": 0.14509552717208862 + }, + { + "label": "completed_group_4", + "mean": 0.0849665105342865, + "median": 0.08355648815631866, + "p05": 0.059977494180202484, + "p95": 0.11480176448822021 + }, + { + "label": "completed_group_5", + "mean": 0.12137698382139206, + "median": 0.11852307617664337, + "p05": 0.07902064919471741, + "p95": 0.17207489907741547 + }, + { + "label": "completed_group_6", + "mean": 0.17017140984535217, + "median": 0.16817378997802734, + "p05": 0.11521546542644501, + "p95": 0.23236754536628723 + }, + { + "label": "completed_group_7", + "mean": 0.12761825323104858, + "median": 0.1237221211194992, + "p05": 0.062459878623485565, + "p95": 0.2092883139848709 + }, + { + "label": "current_group_partial", + "mean": 0.1258423924446106, + "median": 0.11723431199789047, + "p05": 0.06504593789577484, + "p95": 0.21421310305595398 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9633058309555054, + "group": 8, + "latest_source_mass": 0.15763342380523682, + "layer": 31, + "max_source_mass": 0.23511825501918793, + "mixer_index": 61, + "normalized_entropy": 0.8935389906004497, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0846441388130188, + "median": 0.06999644637107849, + "p05": 0.04030589759349823, + "p95": 0.20743247866630554 + }, + { + "label": "completed_group_1", + "mean": 0.05872901901602745, + "median": 0.05653517693281174, + "p05": 0.03710630163550377, + "p95": 0.08772030472755432 + }, + { + "label": "completed_group_2", + "mean": 0.07210008800029755, + "median": 0.07183568179607391, + "p05": 0.03840115666389465, + "p95": 0.10702595114707947 + }, + { + "label": "completed_group_3", + "mean": 0.07792744785547256, + "median": 0.0762966051697731, + "p05": 0.04050692170858383, + "p95": 0.122304767370224 + }, + { + "label": "completed_group_4", + "mean": 0.059186771512031555, + "median": 0.057271093130111694, + "p05": 0.03537197411060333, + "p95": 0.08958204090595245 + }, + { + "label": "completed_group_5", + "mean": 0.07445863634347916, + "median": 0.06928195059299469, + "p05": 0.03154807910323143, + "p95": 0.1351260244846344 + }, + { + "label": "completed_group_6", + "mean": 0.23511825501918793, + "median": 0.22703276574611664, + "p05": 0.0953572615981102, + "p95": 0.3982555568218231 + }, + { + "label": "completed_group_7", + "mean": 0.18020221590995789, + "median": 0.16846294701099396, + "p05": 0.07464539259672165, + "p95": 0.324230432510376 + }, + { + "label": "current_group_partial", + "mean": 0.15763342380523682, + "median": 0.12027859687805176, + "p05": 0.036883872002363205, + "p95": 0.4093724191188812 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1238560676574707, + "group": 8, + "latest_source_mass": 0.11516246944665909, + "layer": 32, + "max_source_mass": 0.1766587197780609, + "mixer_index": 62, + "normalized_entropy": 0.966608552245626, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10231245309114456, + "median": 0.10458895564079285, + "p05": 0.025718798860907555, + "p95": 0.1875312179327011 + }, + { + "label": "completed_group_1", + "mean": 0.07575537264347076, + "median": 0.07447351515293121, + "p05": 0.05878028646111488, + "p95": 0.09651422500610352 + }, + { + "label": "completed_group_2", + "mean": 0.08897508680820465, + "median": 0.08751188218593597, + "p05": 0.06392349302768707, + "p95": 0.11800213158130646 + }, + { + "label": "completed_group_3", + "mean": 0.0967075377702713, + "median": 0.09579810500144958, + "p05": 0.06915527582168579, + "p95": 0.12801720201969147 + }, + { + "label": "completed_group_4", + "mean": 0.09801244735717773, + "median": 0.09654392302036285, + "p05": 0.07431379705667496, + "p95": 0.12754815816879272 + }, + { + "label": "completed_group_5", + "mean": 0.12438583374023438, + "median": 0.12010829150676727, + "p05": 0.08193086087703705, + "p95": 0.1824759691953659 + }, + { + "label": "completed_group_6", + "mean": 0.1766587197780609, + "median": 0.17520052194595337, + "p05": 0.12216726690530777, + "p95": 0.2341049611568451 + }, + { + "label": "completed_group_7", + "mean": 0.12203006446361542, + "median": 0.1200856938958168, + "p05": 0.05817953124642372, + "p95": 0.19647708535194397 + }, + { + "label": "current_group_partial", + "mean": 0.11516246944665909, + "median": 0.10430245846509933, + "p05": 0.05224763602018356, + "p95": 0.2294922173023224 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0835556983947754, + "group": 8, + "latest_source_mass": 0.09660109132528305, + "layer": 32, + "max_source_mass": 0.22511571645736694, + "mixer_index": 63, + "normalized_entropy": 0.9482670637704002, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09133295714855194, + "median": 0.08019149303436279, + "p05": 0.04441600665450096, + "p95": 0.19931957125663757 + }, + { + "label": "completed_group_1", + "mean": 0.07054752856492996, + "median": 0.06906121969223022, + "p05": 0.05099350959062576, + "p95": 0.09504306316375732 + }, + { + "label": "completed_group_2", + "mean": 0.09000501036643982, + "median": 0.0901019275188446, + "p05": 0.05998067185282707, + "p95": 0.11849517375230789 + }, + { + "label": "completed_group_3", + "mean": 0.10956105589866638, + "median": 0.1064089685678482, + "p05": 0.0759081020951271, + "p95": 0.1509535312652588 + }, + { + "label": "completed_group_4", + "mean": 0.07114207744598389, + "median": 0.06995616853237152, + "p05": 0.04991999268531799, + "p95": 0.09545011073350906 + }, + { + "label": "completed_group_5", + "mean": 0.11565887182950974, + "median": 0.11069735884666443, + "p05": 0.06978912651538849, + "p95": 0.17647549510002136 + }, + { + "label": "completed_group_6", + "mean": 0.22511571645736694, + "median": 0.22372552752494812, + "p05": 0.11895382404327393, + "p95": 0.33412501215934753 + }, + { + "label": "completed_group_7", + "mean": 0.13003569841384888, + "median": 0.12226496636867523, + "p05": 0.06884703785181046, + "p95": 0.21555939316749573 + }, + { + "label": "current_group_partial", + "mean": 0.09660109132528305, + "median": 0.09000182151794434, + "p05": 0.051288168877363205, + "p95": 0.16580010950565338 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.679484486579895, + "group": null, + "latest_source_mass": 0.2827349305152893, + "layer": null, + "max_source_mass": 0.32717907428741455, + "mixer_index": null, + "normalized_entropy": 0.7643663300981273, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08993907272815704, + "median": 0.09290459752082825, + "p05": 0.022419869899749756, + "p95": 0.15114322304725647 + }, + { + "label": "completed_group_1", + "mean": 0.043493133038282394, + "median": 0.042318835854530334, + "p05": 0.026918282732367516, + "p95": 0.06356139481067657 + }, + { + "label": "completed_group_2", + "mean": 0.04922422394156456, + "median": 0.047891765832901, + "p05": 0.03095712885260582, + "p95": 0.07266643643379211 + }, + { + "label": "completed_group_3", + "mean": 0.037924449890851974, + "median": 0.03562554344534874, + "p05": 0.020909149199724197, + "p95": 0.06224166974425316 + }, + { + "label": "completed_group_4", + "mean": 0.03939433768391609, + "median": 0.036868683993816376, + "p05": 0.020651251077651978, + "p95": 0.06597590446472168 + }, + { + "label": "completed_group_5", + "mean": 0.03898489475250244, + "median": 0.03362073004245758, + "p05": 0.014276490546762943, + "p95": 0.08282073587179184 + }, + { + "label": "completed_group_6", + "mean": 0.09112587571144104, + "median": 0.07136045396327972, + "p05": 0.02230823040008545, + "p95": 0.22286513447761536 + }, + { + "label": "completed_group_7", + "mean": 0.32717907428741455, + "median": 0.3213679790496826, + "p05": 0.08271751552820206, + "p95": 0.6128557920455933 + }, + { + "label": "completed_group_8", + "mean": 0.2827349305152893, + "median": 0.2627606987953186, + "p05": 0.05262754485011101, + "p95": 0.571351170539856 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "optimizer_state_before": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.08626964688301086, + 0.16337208449840546, + 0.15584443509578705, + 0.04305505007505417, + 0.1360139101743698, + 0.03393339365720749, + 0.03730965405702591, + 0.037494905292987823, + 0.05234000086784363, + 0.04223497211933136, + 0.041619230061769485, + 0.05722156912088394, + 0.062187470495700836, + 0.0369485504925251, + 0.04293901473283768, + 0.045369699597358704, + 0.04149825870990753, + 0.02440357394516468, + 0.026009807363152504, + 0.03634416684508324, + 0.038355328142642975, + 0.02631118707358837, + 0.033921729773283005, + 0.02731483057141304, + 0.027557097375392914, + 0.024315863847732544, + 0.02051021158695221, + 0.018631720915436745, + 0.019497457891702652, + 0.021715763956308365, + 0.017114246264100075, + 0.020576251670718193 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.9093876062470372e-05, + "first_to_last_ratio": 1.111329265095317, + "imbalance_abs_log_ratio": 0.105556834971714, + "last_quartile_mean": 1.718111514037446e-05, + "mean": 2.2651056013955895e-05, + "normalized": [ + 0.6946213574335215, + 0.7243995962608978, + 0.7139231072861242, + 0.6747553054781453, + 1.0542057875714943, + 1.0725327169849952, + 0.9486239566978842, + 0.8605976469555432, + 0.996006301084049, + 0.884894598700678, + 0.8197105185472539, + 0.7590659103257471, + 0.7393158988625451, + 0.6909877240841125, + 0.6603942706848981, + 0.6306792317657164, + 1.5905133022939153, + 1.481373083704922, + 1.3371031921307712, + 1.1787516061453995, + 2.2793598584591077, + 2.032248538637951, + 1.6978050641016875, + 1.4100293011904417, + 1.3387275181066927, + 1.1259668340287685, + 0.864980925975849, + 0.7178664763990735, + 0.7428917127279643, + 0.519872888532167, + 0.4082698409921403, + 0.34952592784954284 + ], + "peak_layer": 21, + "peak_normalized": 2.2793598584591077, + "population_cv": 0.44327592901215446, + "rest_mean": 1.9498221734937505e-05, + "spike_contrast": 2.034870751806145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.96763611206552e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.5733907275716774e-05, + 1.640841583139263e-05, + 1.6171112292795442e-05, + 1.528392022009939e-05, + 2.3878874344518408e-05, + 2.4293998649227433e-05, + 2.1487334379344247e-05, + 1.949344550666865e-05, + 2.2560594516107813e-05, + 2.004379712161608e-05, + 1.856730887084268e-05, + 1.719364445307292e-05, + 1.674628583714366e-05, + 1.565160164318513e-05, + 1.4958627616579179e-05, + 1.4285550605563913e-05, + 3.602680590120144e-05, + 3.355466469656676e-05, + 3.028679930139333e-05, + 2.6699968657339923e-05, + 5.162990782991983e-05, + 4.603257548296824e-05, + 3.8457077607745305e-05, + 3.193865268258378e-05, + 3.0323592000058852e-05, + 2.5504337827442214e-05, + 1.9592731405282393e-05, + 1.626043376745656e-05, + 1.682728179730475e-05, + 1.1775669918279164e-05, + 9.247743037121836e-06, + 7.917131370049901e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 7.440131457769894e-05, + "first_to_last_ratio": 1.0844939416800765, + "imbalance_abs_log_ratio": 0.08111346498986782, + "last_quartile_mean": 6.860463827251806e-05, + "mean": 8.996004032724159e-05, + "normalized": [ + 0.6841628567365404, + 0.7116686494012134, + 0.701182895977615, + 0.6656603296641571, + 1.0335676755092185, + 1.041234037061157, + 0.9271396156883384, + 0.8517717767883587, + 0.9923869547202551, + 0.8818209991609433, + 0.8176514654614152, + 0.7595549596016796, + 0.7388243939285175, + 0.6938730529537547, + 0.6653597800157137, + 0.6317049728603569, + 1.6039855928511222, + 1.4970751654028693, + 1.3561240124333966, + 1.1902182609760135, + 2.2883969934707804, + 2.0368236734227096, + 1.7055000379674186, + 1.4234130023737248, + 1.3429852370592492, + 1.129666888881199, + 0.8659251903055961, + 0.7211434042764958, + 0.7528108713654569, + 0.5246536122144689, + 0.41052437205164627, + 0.35318926941861845 + ], + "peak_layer": 21, + "peak_normalized": 2.2883969934707804, + "population_cv": 0.4471671640899957, + "rest_mean": 7.730859686961156e-05, + "spike_contrast": 2.047351024432569, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015827783499844372, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.154731818241999e-05, + 6.402174039976671e-05, + 6.307844159891829e-05, + 5.988283010083251e-05, + 9.297978976974264e-05, + 9.366945596411824e-05, + 8.340551721630618e-05, + 7.662542338948697e-05, + 8.927517046686262e-05, + 7.932865264592692e-05, + 7.355595880653709e-05, + 6.832959479652345e-05, + 6.646467227255926e-05, + 6.2420847825706e-05, + 5.98557926423382e-05, + 5.682820483343676e-05, + 0.00014429460861720145, + 0.000134676942252554, + 0.00012199697084724903, + 0.00010707208275562152, + 0.00020586428581736982, + 0.0001832327398005873, + 0.00015342685219366103, + 0.0001280502910958603, + 0.00012081500608474016, + 0.00010162487888010219, + 7.789866504026577e-05, + 6.487408973043784e-05, + 6.772289634682238e-05, + 4.7197860112646595e-05, + 3.6930789065081626e-05, + 3.177292092004791e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06597983045503497, + "first_to_last_ratio": 1.0844939725271123, + "imbalance_abs_log_ratio": 0.08111349343358171, + "last_quartile_mean": 0.060839278157800436, + "mean": 0.07977746380493045, + "normalized": [ + 0.6841628143469025, + 0.7116686398363108, + 0.7011829077449026, + 0.665660355693298, + 1.0335676288784994, + 1.0412340885708804, + 0.9271396405289938, + 0.8517717921686788, + 0.992386966395131, + 0.8818210566482357, + 0.817651474996915, + 0.7595549913352903, + 0.7388243852399965, + 0.6938730655814884, + 0.6653597733921379, + 0.6317049688753786, + 1.6039856083090063, + 1.4970750632560381, + 1.3561239578388844, + 1.1902182174623175, + 2.288397078979301, + 2.0368237484113063, + 1.705499960924184, + 1.4234131140142534, + 1.3429851911909423, + 1.1296668304704789, + 0.8659251700081085, + 0.7211434030260389, + 0.7528108714926084, + 0.5246535886165229, + 0.4105243987333961, + 0.3531892470335724 + ], + "peak_layer": 21, + "peak_normalized": 2.288397078979301, + "population_cv": 0.4471671702125523, + "rest_mean": 0.06855803672914151, + "spike_contrast": 2.0473510723291724, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14036237001419066, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.054580774158239365, + 0.05677511915564537, + 0.05593859404325485, + 0.053104694932699203, + 0.08245540410280228, + 0.08306701481342316, + 0.07396484911441803, + 0.06795219331979752, + 0.0791701152920723, + 0.07034944742918015, + 0.06523016095161438, + 0.06059537082910538, + 0.05894153565168381, + 0.055355433374643326, + 0.053080715239048004, + 0.050395820289850235, + 0.1279619038105011, + 0.11943285167217255, + 0.10818812996149063, + 0.09495259076356888, + 0.18256251513957977, + 0.16249263286590576, + 0.1360604614019394, + 0.11355628818273544, + 0.107139952480793, + 0.09012195467948914, + 0.06908131390810013, + 0.05753099173307419, + 0.06005734205245972, + 0.04185553267598152, + 0.03275059536099434, + 0.02817654237151146 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.814465545952771e-06, + "first_to_last_ratio": 0.6078152624962909, + "imbalance_abs_log_ratio": 0.49788428776455546, + "last_quartile_mean": 4.630462115073897e-06, + "mean": 4.80770270883113e-06, + "normalized": [ + 0.6119239482375798, + 0.5829506456587372, + 0.5668818808266815, + 0.5828449444131751, + 0.6017200654914986, + 0.5893976649400692, + 0.5746270627431066, + 0.5729143242160235, + 0.7060481880273346, + 0.6743671364044601, + 0.6598854036517451, + 0.6543712037742013, + 0.5975955414079575, + 0.5980430336609813, + 0.6043178095703504, + 0.6380156504172737, + 1.4746814450880759, + 1.445384559546309, + 1.4289995895563883, + 1.3562595393822299, + 2.503099703805365, + 2.3685351606296248, + 2.1182696084190358, + 1.7837936000863488, + 1.7255397755590118, + 1.469090393074497, + 1.1365579634387313, + 0.9131688091869705, + 0.9186131563845612, + 0.6247692270757808, + 0.4920086991179225, + 0.42532426620797215 + ], + "peak_layer": 21, + "peak_normalized": 2.503099703805365, + "population_cv": 0.5770470220283189, + "rest_mean": 3.82849157176679e-06, + "spike_contrast": 2.6369244021398415, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0095442848978565e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.9419484235404525e-06, + 2.802653398248367e-06, + 2.725399554037722e-06, + 2.8021452180837514e-06, + 2.8928911888215225e-06, + 2.833648750311113e-06, + 2.762636086117709e-06, + 2.7544017484615324e-06, + 3.3944697861443274e-06, + 3.242156708438415e-06, + 3.1725328426546184e-06, + 3.146022208966315e-06, + 2.873061703212443e-06, + 2.8752131129294867e-06, + 2.9053803700662684e-06, + 3.067389570787782e-06, + 7.089829978212947e-06, + 6.948979262233479e-06, + 6.87020519762882e-06, + 6.520492661366006e-06, + 1.2034159226459451e-05, + 1.1387212907720823e-05, + 1.0184010534430854e-05, + 8.575949323130772e-06, + 8.295882253150921e-06, + 7.062949862302048e-06, + 5.464232799567981e-06, + 4.390244157548295e-06, + 4.416418960317969e-06, + 3.0037047054065624e-06, + 2.3654315555177163e-06, + 2.04483262677968e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.492651735039544e-05, + "first_to_last_ratio": 1.2798105542794473, + "imbalance_abs_log_ratio": 0.24671206250860178, + "last_quartile_mean": 4.2917693690469605e-05, + "mean": 6.205384471513753e-05, + "normalized": [ + 0.760033268918417, + 0.780541814588071, + 0.7698447676581547, + 0.7372284537013999, + 1.0777402293794973, + 1.0696059665333593, + 0.9790150608789511, + 0.9071332880771722, + 1.0567179426445976, + 0.9392066065134083, + 0.8680723172882451, + 0.7984285396279348, + 0.8110573186244773, + 0.7570450933968408, + 0.7288656660723876, + 0.6907977052301831, + 1.5596824638835058, + 1.458709216541477, + 1.3045158420583727, + 1.1442588808017629, + 2.349062876038457, + 2.014830196908582, + 1.6040176787890326, + 1.3006270514227598, + 1.2844253595561577, + 1.048991483387396, + 0.8027663480449829, + 0.6725642064369326, + 0.6175576814869903, + 0.43627687990216457, + 0.3538390854052576, + 0.3165407102030714 + ], + "peak_layer": 21, + "peak_normalized": 2.349062876038457, + "population_cv": 0.43649449327625967, + "rest_mean": 5.3888103071518276e-05, + "spike_contrast": 1.9698011906228117, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001061488495906815, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.7162986447801813e-05, + 4.843562055611983e-05, + 4.777182766702026e-05, + 4.574785998556763e-05, + 6.687792483717203e-05, + 6.637316255364567e-05, + 6.075164856156334e-05, + 5.6291108194272965e-05, + 6.557341112056747e-05, + 5.8281380916014314e-05, + 5.3867224778514355e-05, + 4.95455606142059e-05, + 5.032922490499914e-05, + 4.697755866800435e-05, + 4.5228916860651225e-05, + 4.2866653529927135e-05, + 9.678429341875017e-05, + 9.051851520780474e-05, + 8.095022349152714e-05, + 7.100566290318966e-05, + 0.00014576838293578476, + 0.0001250279601663351, + 9.953546395990998e-05, + 8.070890908129513e-05, + 7.97035318100825e-05, + 6.509395461762324e-05, + 4.981473830412142e-05, + 4.173519482719712e-05, + 3.8321828469634056e-05, + 2.7072657758253627e-05, + 2.195707565988414e-05, + 1.9642568076960742e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.069607030032785e-05, + "first_to_last_ratio": 1.664272059731362, + "imbalance_abs_log_ratio": 0.5093878264694807, + "last_quartile_mean": 2.4452775051031495e-05, + "mean": 4.162627476489433e-05, + "normalized": [ + 0.8702598591224926, + 0.8916298939131501, + 0.8800748035118038, + 0.8469315373702728, + 1.1616929340311393, + 1.145619284586338, + 1.0550776811004199, + 0.969941446038908, + 1.1341420642114457, + 0.9958805548845696, + 0.9196031496339117, + 0.8483230598856473, + 0.9057150173064304, + 0.8369896936785922, + 0.8135149832683363, + 0.764963852038374, + 1.5382361125333859, + 1.4503807190733784, + 1.2717075500919934, + 1.1291961376827389, + 2.4272281265268307, + 1.9471701631729643, + 1.4094979938267411, + 1.0867348620232957, + 1.187654328911308, + 0.9474334251489733, + 0.7362053413370422, + 0.6329446172788022, + 0.43206791581619797, + 0.292249639582794, + 0.24209900469777865, + 0.22883424771394378 + ], + "peak_layer": 21, + "peak_normalized": 2.4272281265268307, + "population_cv": 0.44080465046483597, + "rest_mean": 3.6911273599360825e-05, + "spike_contrast": 1.817528210674827, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.708728105877526e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.622567601269111e-05, + 3.711523095262237e-05, + 3.663423558464274e-05, + 3.525460488162935e-05, + 4.835694926441647e-05, + 4.7687863116152585e-05, + 4.391895345179364e-05, + 4.037504913867451e-05, + 4.721010918729007e-05, + 4.1454797610640526e-05, + 3.827965338132344e-05, + 3.531252878019586e-05, + 3.77015421690885e-05, + 3.484076296444982e-05, + 3.386359821888618e-05, + 3.184259549016133e-05, + 6.403103907359764e-05, + 6.037394632585347e-05, + 5.293644790071994e-05, + 4.700422869063914e-05, + 0.00010103646491188556, + 8.105344022624195e-05, + 5.867215077159926e-05, + 4.5236723963171244e-05, + 4.943762542097829e-05, + 3.943812407669611e-05, + 3.064548582187854e-05, + 2.634712654980831e-05, + 1.798537778086029e-05, + 1.216526379721472e-05, + 1.0077679689857177e-05, + 9.525517270958517e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00014326406108011724, + "first_to_last_ratio": 0.9675119405224295, + "imbalance_abs_log_ratio": 0.03302751252910728, + "last_quartile_mean": 0.00014807472143729683, + "mean": 0.0001869507727860764, + "normalized": [ + 0.6324253729661448, + 0.6595289911573035, + 0.6411507649026799, + 0.6215886971725172, + 0.9471130335994487, + 0.9408601337761202, + 0.8694437233530738, + 0.8184469243209181, + 0.9716541786960263, + 0.8739107809137097, + 0.8227884288340134, + 0.7600796243144189, + 0.7142362602596459, + 0.6659890963200943, + 0.6364485580818953, + 0.6213951524405211, + 1.57652704046099, + 1.4727978057853581, + 1.3754263968525124, + 1.1477387862326969, + 2.376960332745933, + 2.1664621252591743, + 1.8156424692300108, + 1.5349698379454737, + 1.4342006350688952, + 1.1801289816751488, + 0.9093707600442614, + 0.7581683846806241, + 0.752048199021001, + 0.5241419408435032, + 0.417566045735057, + 0.36079053731082966 + ], + "peak_layer": 21, + "peak_normalized": 2.376960332745933, + "population_cv": 0.4810121978731533, + "rest_mean": 0.00015698162638323588, + "spike_contrast": 2.2218151983591756, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00034878416336141526, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011823241220554337, + 0.00012329945457167923, + 0.00011986363097094, + 0.00011620648729149252, + 0.00017706351354718208, + 0.00017589452909305692, + 0.0001625431759748608, + 0.00015300928498618305, + 0.0001816514995880425, + 0.00016337829583790153, + 0.00015382093260996044, + 0.00014209747314453125, + 0.000133527020807378, + 0.0001245071762241423, + 0.00011898454977199435, + 0.00011617030395427719, + 0.000294732948532328, + 0.0002753406879492104, + 0.0002571370278019458, + 0.00021457065304275602, + 0.0004443745710887015, + 0.0004050217685289681, + 0.0003394357627257705, + 0.0002869637974072248, + 0.0002681249170564115, + 0.0002206260251114145, + 0.00017000756633933634, + 0.00014174016541801393, + 0.00014059599197935313, + 9.798874089028686e-05, + 7.806429493939504e-05, + 6.745006976416335e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13547764718532562, + 0.039667773991823196, + 0.026703152805566788, + 0.05304057523608208, + 0.016798952594399452, + 0.00753376679494977, + 0.01294450368732214, + 0.03526800498366356, + 0.030294200405478477, + 0.034149833023548126, + 0.04367009550333023, + 0.05930330231785774, + 0.04527241364121437, + 0.04281846433877945, + 0.04812675341963768, + 0.06094835698604584, + 0.038031257688999176, + 0.040106989443302155, + 0.04961257055401802, + 0.07085621356964111, + 0.03753058984875679, + 0.04544757679104805, + 0.05368982255458832, + 0.06174425780773163, + 0.05051252990961075, + 0.054937634617090225, + 0.05777662619948387, + 0.07133159041404724, + 0.0636468306183815, + 0.09261580556631088, + 0.1020733118057251, + 0.12878532707691193 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8983760583068943e-05, + "first_to_last_ratio": 1.3500112943631184, + "imbalance_abs_log_ratio": 0.3001129586102442, + "last_quartile_mean": 1.4061927231523441e-05, + "mean": 2.0583363209425443e-05, + "normalized": [ + 0.7641682400188751, + 0.7964973937008093, + 0.7903246216156576, + 0.7412309789481601, + 1.1595280618590635, + 1.1689606938823263, + 1.0270355404284468, + 0.9305474685745297, + 1.009443360915331, + 0.92439237085469, + 0.8548158178555026, + 0.798458806391288, + 0.7719145608477377, + 0.7216837495160779, + 0.7013551805794737, + 0.6759138154313806, + 1.6233726027813016, + 1.4876771328688778, + 1.307017635744062, + 1.1529668955961847, + 2.3147875877516704, + 1.9234371956824674, + 1.5697411551918221, + 1.319372635699807, + 1.2524447630862485, + 0.9556486905621968, + 0.7985175736559498, + 0.6718204766110775, + 0.5751452774144216, + 0.45005871272267506, + 0.38512561316412136, + 0.3765953900477665 + ], + "peak_layer": 21, + "peak_normalized": 2.3147875877516704, + "population_cv": 0.4284577654113479, + "rest_mean": 1.8006796246347086e-05, + "spike_contrast": 1.9157669325572948, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4496824810048565e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.5729152437415905e-05, + 1.639459514990449e-05, + 1.626753874006681e-05, + 1.5257026461767964e-05, + 2.3866987248766236e-05, + 2.4061142539721914e-05, + 2.1139845557627268e-05, + 1.9153796529280953e-05, + 2.0777739337063394e-05, + 1.9027103917323984e-05, + 1.7594984456081875e-05, + 1.643496761971619e-05, + 1.588859777257312e-05, + 1.4854678738629445e-05, + 1.4436248420679476e-05, + 1.391257956129266e-05, + 3.341446790727787e-05, + 3.062139876419678e-05, + 2.690281871764455e-05, + 2.3731936380499974e-05, + 4.76461136713624e-05, + 3.9590806409250945e-05, + 3.2310552342096344e-05, + 2.7157126169186085e-05, + 2.5779525458347052e-05, + 1.9670464098453522e-05, + 1.6436177247669548e-05, + 1.3828324881615117e-05, + 1.1838424143206794e-05, + 9.263721949537285e-06, + 7.92718037700979e-06, + 7.751599696348421e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 7.388659878415638e-05, + "first_to_last_ratio": 1.3167747149325586, + "imbalance_abs_log_ratio": 0.2751853488521164, + "last_quartile_mean": 5.611179949482903e-05, + "mean": 8.168216641024628e-05, + "normalized": [ + 0.7533112744592003, + 0.7827721531209632, + 0.7759215503905756, + 0.7316303776947085, + 1.1365209343706772, + 1.135346283191908, + 1.004240264612514, + 0.9167546285036348, + 1.0078386860630901, + 0.9220221645206613, + 0.8550885147558498, + 0.8013979502567415, + 0.7721731241940338, + 0.7274558542877592, + 0.7065441772286032, + 0.6779218197522006, + 1.6398171430789432, + 1.5049144188657668, + 1.3267658786647274, + 1.1597753221298892, + 2.322010757724194, + 1.9301972923447444, + 1.5840545712188465, + 1.3299018151740607, + 1.2567171362804614, + 0.9579514188552475, + 0.8014611945379978, + 0.6752124703737634, + 0.5811954370019358, + 0.4526517471161714, + 0.3896333192051585, + 0.38080032002497083 + ], + "peak_layer": 21, + "peak_normalized": 2.322010757724194, + "population_cv": 0.4315619777351012, + "rest_mean": 7.13270411425684e-05, + "spike_contrast": 1.929139925777552, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001375998428557068, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.153209687909111e-05, + 6.39385252725333e-05, + 6.337895320029929e-05, + 5.976115426165052e-05, + 9.283349208999425e-05, + 9.273754403693601e-05, + 8.202852040994912e-05, + 7.48825041227974e-05, + 8.232244726968929e-05, + 7.531276787631214e-05, + 6.984548235777766e-05, + 6.545992073370144e-05, + 6.307277362793684e-05, + 5.942017014604062e-05, + 5.77120590605773e-05, + 5.5374122894136235e-05, + 0.00013394381676334888, + 0.00012292466999497265, + 0.00010837311128852889, + 9.473296086071059e-05, + 0.0001896668691188097, + 0.0001576626964379102, + 0.00012938900908920914, + 0.00010862926137633622, + 0.0001026513782562688, + 7.824754720786586e-05, + 6.546508666360751e-05, + 5.515281736734323e-05, + 4.747330240206793e-05, + 3.697357533383183e-05, + 3.182609361829236e-05, + 3.1104595109354705e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06552337622269988, + "first_to_last_ratio": 1.3167747352194785, + "imbalance_abs_log_ratio": 0.2751853642586392, + "last_quartile_mean": 0.0497605053242296, + "mean": 0.07243656268110499, + "normalized": [ + 0.7533112993917941, + 0.7827721634134585, + 0.7759215519760847, + 0.7316303564434335, + 1.1365209991517615, + 1.1353462736009083, + 1.004240278158844, + 0.9167546473202071, + 1.0078387172969752, + 0.9220222438210248, + 0.8550884472945091, + 0.8013979045056098, + 0.7721731478740148, + 0.7274558173306654, + 0.7065441905330453, + 0.6779218137445964, + 1.6398170980615752, + 1.5049143559021794, + 1.3267658529793183, + 1.1597753399419397, + 2.322010804717546, + 1.9301971994356375, + 1.584054611353896, + 1.3299018487169911, + 1.2567171977522282, + 0.9579514011929847, + 0.8014611613319625, + 0.6752125187290771, + 0.5811954405704202, + 0.4526517065938406, + 0.3896333019053031, + 0.3808003089581684 + ], + "peak_layer": 21, + "peak_normalized": 2.322010804717546, + "population_cv": 0.43156197832928544, + "rest_mean": 0.0632535337160031, + "spike_contrast": 1.9291399535166673, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12202491909265518, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.054567281156778336, + 0.05670132488012314, + 0.05620509013533592, + 0.052996788173913956, + 0.08232567459344864, + 0.08224058151245117, + 0.07274371385574341, + 0.06640655547380447, + 0.07300437241792679, + 0.06678812205791473, + 0.06193966791033745, + 0.05805050954222679, + 0.05593356862664223, + 0.052694398909807205, + 0.05117963254451752, + 0.04910632595419884, + 0.11878271400928497, + 0.10901082307100296, + 0.09610635787248611, + 0.08401013910770416, + 0.16819848120212555, + 0.13981685042381287, + 0.11474347114562988, + 0.09633351862430573, + 0.09103227406740189, + 0.06939070671796799, + 0.0580550916492939, + 0.048910073935985565, + 0.04209979996085167, + 0.032788533717393875, + 0.02822369709610939, + 0.027583865448832512 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.7909886171073595e-06, + "first_to_last_ratio": 0.73274866566625, + "imbalance_abs_log_ratio": 0.3109525203873344, + "last_quartile_mean": 3.8089303302513144e-06, + "mean": 4.300410559210377e-06, + "normalized": [ + 0.6752193249648764, + 0.648060123214754, + 0.6340762852531016, + 0.648956154331942, + 0.6688990988684992, + 0.6540429695649356, + 0.6333917443388257, + 0.6293955312096993, + 0.7505991389397793, + 0.716809349220506, + 0.6944304605268068, + 0.6987346033913733, + 0.6285871627816183, + 0.6319728565674881, + 0.6466036959323995, + 0.7005312655376627, + 1.4911248937697517, + 1.437684597591998, + 1.3858356594460568, + 1.3108305532381637, + 2.5649491626046244, + 2.293141742331734, + 1.9862148934338621, + 1.784203258435835, + 1.6977937545261073, + 1.262754388821972, + 1.06360231285752, + 0.8741748207857217, + 0.7144275994278602, + 0.5384026644950104, + 0.4702436547730354, + 0.4643062788164799 + ], + "peak_layer": 21, + "peak_normalized": 2.5649491626046244, + "population_cv": 0.5484988453962795, + "rest_mean": 3.4520665276768676e-06, + "spike_contrast": 2.5727975571398605, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.881468329491326e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.9037203148618573e-06, + 2.786924596875906e-06, + 2.726788352447329e-06, + 2.7907778985536424e-06, + 2.8765407478203997e-06, + 2.8126532924943604e-06, + 2.7238445454713656e-06, + 2.706659188334015e-06, + 3.2278844628308434e-06, + 3.0825744943285827e-06, + 2.986336085086805e-06, + 3.0048456665099366e-06, + 2.7031828722101636e-06, + 2.717742745517171e-06, + 2.7806613616121467e-06, + 3.012572051375173e-06, + 6.412449238268891e-06, + 6.18263402429875e-06, + 5.959662303212099e-06, + 5.637109552480979e-06, + 1.103033446270274e-05, + 9.86145096248947e-06, + 8.541539500583895e-06, + 7.672806532355025e-06, + 7.301210189325502e-06, + 5.430362307379255e-06, + 4.573926617013058e-06, + 3.7593106299027568e-06, + 3.0723319923708914e-06, + 2.3153525035013445e-06, + 2.0222407783876406e-06, + 1.9967076241300674e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.4603370244876714e-05, + "first_to_last_ratio": 1.5303484438582295, + "imbalance_abs_log_ratio": 0.42549545055909027, + "last_quartile_mean": 3.5680351402334054e-05, + "mean": 5.699801209857469e-05, + "normalized": [ + 0.8264909998027334, + 0.8487922685363437, + 0.841606882150277, + 0.8012941819001962, + 1.1726168644790063, + 1.1546993854462606, + 1.050180566275997, + 0.9682170449733281, + 1.0710784836103833, + 0.9771184045421, + 0.8995231631213109, + 0.8346230595845928, + 0.8429727036171436, + 0.7876480304240737, + 0.76714462416435, + 0.7352965831476843, + 1.5890796305970005, + 1.4640441930989896, + 1.2783685546924959, + 1.1236995336661568, + 2.371393623645145, + 1.8882377423241299, + 1.482591511200619, + 1.2153386927849252, + 1.1838783972428712, + 0.8905908477409856, + 0.7420819055609336, + 0.634319683256865, + 0.48200464426330825, + 0.3870264062137896, + 0.3477017707900011, + 0.34033961714600236 + ], + "peak_layer": 21, + "peak_normalized": 2.371393623645145, + "population_cv": 0.4245100179440155, + "rest_mean": 5.036631457052728e-05, + "spike_contrast": 1.8426835384207285, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.28091787500307e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.7108344006119296e-05, + 4.8379471991211176e-05, + 4.796991925104521e-05, + 4.567217547446489e-05, + 6.683683022856712e-05, + 6.581556954188272e-05, + 5.98582046222873e-05, + 5.518644684343599e-05, + 6.104934436734766e-05, + 5.569380664383061e-05, + 5.1271032134536654e-05, + 4.7571855247952044e-05, + 4.804776835953817e-05, + 4.489437196752988e-05, + 4.372571856947616e-05, + 4.191044354229234e-05, + 9.057438001036644e-05, + 8.344760863110423e-05, + 7.286446634680033e-05, + 6.404863961506635e-05, + 0.00013516472245100886, + 0.00010762579768197611, + 8.450476889265701e-05, + 6.927188951522112e-05, + 6.747871520929039e-05, + 5.076190791442059e-05, + 4.2297193431295455e-05, + 3.615496098063886e-05, + 2.7473306545289233e-05, + 2.205973578384146e-05, + 1.9818309738184325e-05, + 1.939868161571212e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.048281562063494e-05, + "first_to_last_ratio": 1.9630119222573643, + "imbalance_abs_log_ratio": 0.6744799887640945, + "last_quartile_mean": 2.062280680092954e-05, + "mean": 3.8703238459447675e-05, + "normalized": [ + 0.9360916836648231, + 0.957580281730348, + 0.9482712191433338, + 0.910353586321295, + 1.2460232346920597, + 1.2224003463169428, + 1.1176779980540072, + 1.0294421165646366, + 1.1460215914378586, + 1.0320514664718978, + 0.9473985532737729, + 0.8788138212913126, + 0.9341792256960503, + 0.8671852947881752, + 0.8459732351746374, + 0.8066658603360692, + 1.5684062538088193, + 1.4484147907409404, + 1.2467146748187818, + 1.105475527367378, + 2.4247996872064124, + 1.8050505092184956, + 1.2955371531304132, + 1.0167160839549727, + 1.0675715240747539, + 0.8180409715888471, + 0.6887416678917638, + 0.6147875592971825, + 0.3233626767922262, + 0.26293806156630717, + 0.24647344903818308, + 0.24083989454730284 + ], + "peak_layer": 21, + "peak_normalized": 2.4247996872064124, + "population_cv": 0.4349671070041115, + "rest_mean": 3.4962391341482374e-05, + "spike_contrast": 1.684776430797849, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.89038128964603e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.622977965278551e-05, + 3.7061457987874746e-05, + 3.670116711873561e-05, + 3.5233631933806464e-05, + 4.822513437829912e-05, + 4.7310852096416056e-05, + 4.3257758079562336e-05, + 3.984274371759966e-05, + 4.435474693309516e-05, + 3.9943734009284526e-05, + 3.666739212349057e-05, + 3.4012940886896104e-05, + 3.615576133597642e-05, + 3.3562879252713174e-05, + 3.27419038512744e-05, + 3.12205811496824e-05, + 6.070240124245174e-05, + 5.605834303423762e-05, + 4.8251895350404084e-05, + 4.2785482946783304e-05, + 9.384760051034391e-05, + 6.986130028963089e-05, + 5.014148337068036e-05, + 3.935020504286513e-05, + 4.1318475268781185e-05, + 3.166083479300141e-05, + 2.665653300937265e-05, + 2.3794269509380683e-05, + 1.2515182788774837e-05, + 1.017655449686572e-05, + 9.539320672047324e-06, + 9.321283869212493e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00014194276809575967, + "first_to_last_ratio": 1.1782647028111788, + "imbalance_abs_log_ratio": 0.16404276525959405, + "last_quartile_mean": 0.00012046764004480792, + "mean": 0.00016912808678171132, + "normalized": [ + 0.6984404643238884, + 0.7279605452185844, + 0.716541899087504, + 0.6844000690950613, + 1.0470185740245912, + 1.0262206207114242, + 0.9392540256525687, + 0.8742594105695676, + 1.0007921238341297, + 0.9198815102082726, + 0.860099074470987, + 0.8041171467379208, + 0.7517592573362873, + 0.7071749387596568, + 0.6728971890246059, + 0.6653759328341454, + 1.636701579189545, + 1.4903312139370086, + 1.3329613072711604, + 1.154725819226416, + 2.4301173158726157, + 2.015490849459162, + 1.6993053786715127, + 1.4458823665938263, + 1.3104146608182012, + 1.004154426063848, + 0.8373805185074344, + 0.7042667168313188, + 0.5859481462881533, + 0.463333171612487, + 0.3998633715178443, + 0.3929303762502709 + ], + "peak_layer": 21, + "peak_normalized": 2.4301173158726157, + "population_cv": 0.4589844309602813, + "rest_mean": 0.00014469089122333875, + "spike_contrast": 2.0809115228419945, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00030108894279692324, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00011812589946202934, + 0.00012311857426539063, + 0.00012118736049160361, + 0.00011575127427931875, + 0.0001770802482496947, + 0.0001735627301968634, + 0.00015885423636063933, + 0.0001478618214605376, + 0.00016926205717027187, + 0.0001555777998873964, + 0.00014546691090799868, + 0.00013599879457615316, + 0.00012714360491372645, + 0.00011960314441239461, + 0.00011380581418052316, + 0.00011253375851083547, + 0.0002768122067209333, + 0.00025205686688423157, + 0.00022544119565282017, + 0.00019529656856320798, + 0.0004110010922886431, + 0.0003408761112950742, + 0.0002874002675525844, + 0.0002445393183734268, + 0.00022162792447488755, + 0.000169830716913566, + 0.00014162456500343978, + 0.0001191112824017182, + 9.910028893500566e-05, + 7.836265285732225e-05, + 6.762812699889764e-05, + 6.64555627736263e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.08626964688301086, + 0.27676042914390564, + 0.41335058212280273, + 0.43042075634002686, + 0.1360139101743698, + 0.15445689857006073, + 0.1670466512441635, + 0.17834730446338654, + 0.05234000086784363, + 0.08269349485635757, + 0.11548293381929398, + 0.16410532593727112, + 0.062187470495700836, + 0.0988159254193306, + 0.13199056684970856, + 0.17199495434761047, + 0.04149825870990753, + 0.06434677541255951, + 0.08728200942277908, + 0.11647984385490417, + 0.038355328142642975, + 0.06353354454040527, + 0.09196189045906067, + 0.1169828325510025, + 0.027557097375392914, + 0.06579430401325226, + 0.09665275365114212, + 0.12799666821956635, + 0.019497457891702652, + 0.07176875323057175, + 0.14131556451320648, + 0.22250843048095703 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.9093950641035917e-05, + "first_to_last_ratio": 1.1113330910687114, + "imbalance_abs_log_ratio": 0.10556027766590795, + "last_quartile_mean": 1.71811230984531e-05, + "mean": 2.265107818288925e-05, + "normalized": [ + 0.6945983528743319, + 0.724410451166159, + 0.7139225691689148, + 0.6747768091993389, + 1.054240973252137, + 1.0725014726920594, + 0.9486330663611198, + 0.8605955198001844, + 0.995990630507342, + 0.8848804020515041, + 0.8196954220375458, + 0.7590673356456394, + 0.7393107585217125, + 0.6909917857854257, + 0.6603937849571249, + 0.6306897768726338, + 1.5905019484551022, + 1.4813909070064188, + 1.337087187717671, + 1.1787423417035985, + 2.2793733673501118, + 2.0322272765048472, + 1.6978229967971588, + 1.4100558672312264, + 1.3387234775131232, + 1.1259802671905756, + 0.8649860219589377, + 0.7178508371275001, + 0.7428919493063704, + 0.5198747888679629, + 0.4082655064793689, + 0.3495261478968529 + ], + "peak_layer": 21, + "peak_normalized": 2.2793733673501118, + "population_cv": 0.44327706814451906, + "rest_mean": 1.9498213381060243e-05, + "spike_contrast": 2.034881213850395, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.9676548112765886e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.573340159666259e-05, + 1.640867776586674e-05, + 1.617111593077425e-05, + 1.5284422261174768e-05, + 2.3879694708739407e-05, + 2.42933147092117e-05, + 2.148756175301969e-05, + 1.9493416402838193e-05, + 2.2560261641046964e-05, + 2.0043495169375092e-05, + 1.856698509072885e-05, + 1.7193693565786816e-05, + 1.6746185792726465e-05, + 1.565170896355994e-05, + 1.4958631254557986e-05, + 1.4285803445091005e-05, + 3.602658398449421e-05, + 3.355510125402361e-05, + 3.028646642633248e-05, + 2.6699784939410165e-05, + 5.1630264351842925e-05, + 4.603213892551139e-05, + 3.845752144115977e-05, + 3.193928569089621e-05, + 3.032353015441913e-05, + 2.5504667064524256e-05, + 1.9592866010498255e-05, + 1.6260095435427502e-05, + 1.6827303625177592e-05, + 1.177572448796127e-05, + 9.247653906641062e-06, + 7.917144102975726e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 7.440197032337892e-05, + "first_to_last_ratio": 1.0845009049103946, + "imbalance_abs_log_ratio": 0.08111988568775846, + "last_quartile_mean": 6.860480243631173e-05, + "mean": 8.99602513300124e-05, + "normalized": [ + 0.6841490391911266, + 0.7116943175111012, + 0.70118634676606, + 0.6656715473421221, + 1.0336051249539524, + 1.0412072500471912, + 0.9271427791318646, + 0.8517742273313543, + 0.9923652968187024, + 0.8818152912537373, + 0.8176321585165575, + 0.7595628027406727, + 0.7388217713310421, + 0.693873609218528, + 0.665364325823853, + 0.6317142481886351, + 1.6039671105807234, + 1.4970661541772026, + 1.3561211551482075, + 1.1902099694866315, + 2.2884215514970006, + 2.0367978673096307, + 1.7055055815020073, + 1.4234313395007192, + 1.3429770725313894, + 1.12966876849781, + 0.8659356147413246, + 0.7211347571729164, + 0.7528133113832752, + 0.5246594990466679, + 0.4105183137414301, + 0.3531917975165635 + ], + "peak_layer": 21, + "peak_normalized": 2.2884215514970006, + "population_cv": 0.44716612344844425, + "rest_mean": 7.73087299926879e-05, + "spike_contrast": 2.0473556681960123, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00015827846655156464, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.154621951282024e-05, + 6.402419967344031e-05, + 6.307889998424798e-05, + 5.988397970213555e-05, + 9.298337681684643e-05, + 9.366726590087637e-05, + 8.340599742950872e-05, + 7.662582356715575e-05, + 8.927343151299283e-05, + 7.932832522783428e-05, + 7.355439447565004e-05, + 6.833046063547954e-05, + 6.64645922370255e-05, + 6.242104427656159e-05, + 5.985634197713807e-05, + 5.6829172535799444e-05, + 0.00014429328439291567, + 0.00013467644748743623, + 0.00012199699995107949, + 0.00010707158799050376, + 0.00020586697792168707, + 0.0001832308480516076, + 0.0001534277107566595, + 0.0001280522410525009, + 0.00012081455497536808, + 0.00010162528633372858, + 7.789978553773835e-05, + 6.487346399808303e-05, + 6.772327469661832e-05, + 4.719850039691664e-05, + 3.693033067975193e-05, + 3.17732228722889e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06598041160032153, + "first_to_last_ratio": 1.0845009182393224, + "imbalance_abs_log_ratio": 0.08111989797813793, + "last_quartile_mean": 0.06083942437544465, + "mean": 0.07977765158284456, + "normalized": [ + 0.6841490163544541, + 0.7116942818442077, + 0.7011863471790457, + 0.6656715368689867, + 1.0336051678040334, + 1.0412072624828899, + 0.9271427815934556, + 0.8517741767131453, + 0.992365298440462, + 0.8818153387666878, + 0.8176321795570402, + 0.7595627761828878, + 0.7388218056939301, + 0.6938736270746664, + 0.665364324455644, + 0.6317142220498281, + 1.6039670769938226, + 1.497066216157726, + 1.3561211394078925, + 1.190209999241352, + 2.2884215780128287, + 2.0367978476522746, + 1.7055056593265296, + 1.423431337144962, + 1.3429770803514192, + 1.1296687476987397, + 0.8659355529359181, + 0.721134747932228, + 0.7528132554883469, + 0.5246594981795372, + 0.41051829590461564, + 0.35319182451044234 + ], + "peak_layer": 21, + "peak_normalized": 2.2884215780128287, + "population_cv": 0.4471661317726333, + "rest_mean": 0.06855815552450993, + "spike_contrast": 2.047355697130315, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14036293029785157, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.054579801857471466, + 0.05677729845046997, + 0.055939000099897385, + 0.053105711936950684, + 0.08245859295129776, + 0.08306507021188736, + 0.07396527379751205, + 0.06795254349708557, + 0.07916857302188873, + 0.07034915685653687, + 0.06522877514362335, + 0.06059613451361656, + 0.058941468596458435, + 0.055355608463287354, + 0.0530812032520771, + 0.05039667710661888, + 0.12796072661876678, + 0.11943242698907852, + 0.10818815976381302, + 0.09495215862989426, + 0.1825648993253708, + 0.16249094903469086, + 0.13606123626232147, + 0.11355800926685333, + 0.10713955760002136, + 0.09012231975793839, + 0.06908230483531952, + 0.05753043666481972, + 0.06005767360329628, + 0.041856102645397186, + 0.03275018557906151, + 0.028176814317703247 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.8144915802386095e-06, + "first_to_last_ratio": 0.6078045854813123, + "imbalance_abs_log_ratio": 0.49790185413614213, + "last_quartile_mean": 4.630586289522398e-06, + "mean": 4.807722199018372e-06, + "normalized": [ + 0.6119714094009943, + 0.5829248721673723, + 0.5668443018259814, + 0.5828395075254097, + 0.6017526232967554, + 0.589367845371327, + 0.5746677702647779, + 0.5729165418286368, + 0.7059387263625273, + 0.6743568356158998, + 0.6598434749727681, + 0.6543816039835032, + 0.5975752891719487, + 0.5980213135145247, + 0.6043413237993088, + 0.6380324542505855, + 1.4745083318439967, + 1.4453097462314772, + 1.4289980528954762, + 1.3562696480279695, + 2.5032567859750894, + 2.3684498892562993, + 2.118380011365409, + 1.783803961861413, + 1.725559075482424, + 1.4691197183784992, + 1.1366835074286532, + 0.9131953750566115, + 0.9185830426535001, + 0.624807082892335, + 0.49197761908752924, + 0.4253222582109956 + ], + "peak_layer": 21, + "peak_normalized": 2.5032567859750894, + "population_cv": 0.5770592537991902, + "rest_mean": 3.8284693649378214e-06, + "spike_contrast": 2.6370036013641465, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0095687503053342e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.9421885301417205e-06, + 2.8025408482790226e-06, + 2.725229933275841e-06, + 2.8021304387948476e-06, + 2.8930594453413505e-06, + 2.833516873579356e-06, + 2.7628429961623624e-06, + 2.754423576334375e-06, + 3.3939572858798783e-06, + 3.242120328650344e-06, + 3.1723441225040006e-06, + 3.146084964100737e-06, + 2.8729759833368007e-06, + 2.875120344469906e-06, + 2.9055051982140867e-06, + 3.0674827939947136e-06, + 7.089026439643931e-06, + 6.948647751414683e-06, + 6.87022566125961e-06, + 6.520567694678903e-06, + 1.2034963219775818e-05, + 1.1386849109840114e-05, + 1.018458260659827e-05, + 8.576033906138036e-06, + 8.296008672914468e-06, + 7.063119483063929e-06, + 5.4648585319228005e-06, + 4.390389676700579e-06, + 4.416292085807072e-06, + 3.003898882525391e-06, + 2.3652917207073187e-06, + 2.0448312625376275e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.492669151863083e-05, + "first_to_last_ratio": 1.2798129309515296, + "imbalance_abs_log_ratio": 0.2467139195567934, + "last_quartile_mean": 4.2917750079141115e-05, + "mean": 6.205391622415846e-05, + "normalized": [ + 0.7600355002625492, + 0.7805513505607012, + 0.7698486878525821, + 0.7372280145247141, + 1.0777481330957353, + 1.0696009818823022, + 0.9790057250388184, + 0.9071387502213032, + 1.0567092207752022, + 0.9392041171743681, + 0.8680643404431132, + 0.7984310198570167, + 0.8110583772742559, + 0.7570453348972137, + 0.7288688127245345, + 0.6908032407944877, + 1.5596744521873913, + 1.4587063630480184, + 1.3045232499410013, + 1.144251230574794, + 2.3490749428268014, + 2.0148170878788103, + 1.604017002890392, + 1.3006314152297547, + 1.2844195410928985, + 1.0489976614499437, + 0.8027713441974065, + 0.6725629037598078, + 0.6175554455540694, + 0.436277197915466, + 0.3538367723039011, + 0.316541781770646 + ], + "peak_layer": 21, + "peak_normalized": 2.3490749428268014, + "population_cv": 0.4364939701143597, + "rest_mean": 5.3888172462595525e-05, + "spike_contrast": 1.9698001933592688, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010614893253659829, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.716317926067859e-05, + 4.843626811634749e-05, + 4.777212598128244e-05, + 4.574788545141928e-05, + 6.687849236186594e-05, + 6.637292972300202e-05, + 6.075113924453035e-05, + 5.629151200992055e-05, + 6.557294545928016e-05, + 5.8281293604522943e-05, + 5.386679185903631e-05, + 4.954577161697671e-05, + 5.032934859627858e-05, + 4.6977627789601684e-05, + 4.522916424321011e-05, + 4.28670464316383e-05, + 9.678390779299662e-05, + 9.05184424482286e-05, + 8.095077646430582e-05, + 7.100527000147849e-05, + 0.00014576929970644414, + 0.0001250272907782346, + 9.953553671948612e-05, + 8.070927287917584e-05, + 7.970326259965077e-05, + 6.509441300295293e-05, + 4.9815105739980936e-05, + 4.1735162085387856e-05, + 3.832173388218507e-05, + 2.7072708689956926e-05, + 2.195695742557291e-05, + 1.9642657207441516e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.069824126418098e-05, + "first_to_last_ratio": 1.664243286286015, + "imbalance_abs_log_ratio": 0.5093705374127352, + "last_quartile_mean": 2.4454502295157e-05, + "mean": 4.162822952480383e-05, + "normalized": [ + 0.8703493828745057, + 0.8916901865290843, + 0.88010828499616, + 0.8470021437701022, + 1.1617085596346044, + 1.1457007720909245, + 1.0547659609018474, + 0.9699520930859898, + 1.1341637901024582, + 0.9959865521758748, + 0.9196947259511007, + 0.8482508896276112, + 0.9057174940795618, + 0.8369874449429452, + 0.8134828126487191, + 0.7649775699240196, + 1.5380296464446301, + 1.4500791010223821, + 1.2716272092758651, + 1.129435876960094, + 2.4274111082159515, + 1.9474132658179035, + 1.4092447881310306, + 1.0866205597228813, + 1.1876165623415504, + 0.9475667789345992, + 0.7362149039491379, + 0.6328866680951589, + 0.43209730937738905, + 0.2922508384567999, + 0.2421482427484381, + 0.22881847717067896 + ], + "peak_layer": 21, + "peak_normalized": 2.4274111082159515, + "population_cv": 0.44080997556292906, + "rest_mean": 3.691297485972798e-05, + "spike_contrast": 1.8175344840442327, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.709060471621342e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.623110387707129e-05, + 3.711948374984786e-05, + 3.663734969450161e-05, + 3.5259199648862705e-05, + 4.8359870561398566e-05, + 4.769349470734596e-05, + 4.3908039515372366e-05, + 4.037738835904747e-05, + 4.721323057310656e-05, + 4.146115679759532e-05, + 3.828526314464398e-05, + 3.531118272803724e-05, + 3.770341572817415e-05, + 3.484230546746403e-05, + 3.386384923942387e-05, + 3.184466186212376e-05, + 6.402545113814995e-05, + 6.036422564648092e-05, + 5.293558933772147e-05, + 4.7016415919642895e-05, + 0.00010104882676387206, + 8.10673664091155e-05, + 5.866436549695209e-05, + 4.523409006651491e-05, + 4.9438374844612554e-05, + 3.9445527363568544e-05, + 3.064732300117612e-05, + 2.6345951482653618e-05, + 1.798744597181212e-05, + 1.2165884982096031e-05, + 1.0080202628159896e-05, + 9.525308087177109e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00014326045857160352, + "first_to_last_ratio": 0.9674628975633102, + "imbalance_abs_log_ratio": 0.03307820358530557, + "last_quartile_mean": 0.0001480785040257615, + "mean": 0.00018694848358791205, + "normalized": [ + 0.6324557293416971, + 0.6595615864784095, + 0.6410845907888777, + 0.6214095723982178, + 0.9472452039473033, + 0.9409894253465454, + 0.8694511005112591, + 0.8182813410919756, + 0.9717054632953117, + 0.8737801260796588, + 0.8227731283696897, + 0.7600623105493365, + 0.714347909521877, + 0.6659205019824871, + 0.636328539521148, + 0.6214749962225501, + 1.5765359147034264, + 1.472710913100365, + 1.3755308859596458, + 1.1476507932250721, + 2.3770794208861905, + 2.166537848106843, + 1.8154778878942872, + 1.5349498698581336, + 1.4342364113292647, + 1.1800424750363023, + 0.9095504950044728, + 0.7581051224006025, + 0.7520703292100521, + 0.5242482655784406, + 0.4175420470120725, + 0.3608597952484843 + ], + "peak_layer": 21, + "peak_normalized": 2.3770794208861905, + "population_cv": 0.4810200206149476, + "rest_mean": 0.0001569793853882907, + "spike_contrast": 2.2218306754300956, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00034878161386586726, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011823663953691721, + 0.00012330403842497617, + 0.00011984979209955782, + 0.00011617157724685967, + 0.00017708605446387082, + 0.00017591654614079744, + 0.0001625425647944212, + 0.00015297645586542785, + 0.00018165886285714805, + 0.00016335186955984682, + 0.000153816188685596, + 0.00014209249638952315, + 0.0001335462584393099, + 0.00012449282803572714, + 0.00011896065552718937, + 0.00011618380813160911, + 0.0002947309985756874, + 0.00027532107196748257, + 0.00025715341325849295, + 0.00021455157548189163, + 0.0004443913931027055, + 0.0004050309653393924, + 0.0003394008381292224, + 0.00028695655055344105, + 0.00026812832220457494, + 0.0002206071512773633, + 0.00017003908578772098, + 0.0001417266030330211, + 0.00014059840759728104, + 9.800741827348247e-05, + 7.805885252309963e-05, + 6.746219150954857e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16424456238746643, + 0.29588937759399414, + 0.42337876558303833, + 0.45154979825019836, + 0.13891471922397614, + 0.1555756777524948, + 0.16914354264736176, + 0.1874145120382309, + 0.06211879849433899, + 0.09842939674854279, + 0.137006014585495, + 0.20118924975395203, + 0.08399304747581482, + 0.12138351052999496, + 0.1587214469909668, + 0.2071460783481598, + 0.0581100657582283, + 0.08358953893184662, + 0.10947012156248093, + 0.1472771316766739, + 0.05584336072206497, + 0.08318652212619781, + 0.11238116025924683, + 0.1379220336675644, + 0.05995604768395424, + 0.09392217546701431, + 0.1259489357471466, + 0.17086055874824524, + 0.06700707972049713, + 0.13971327245235443, + 0.2211977243423462, + 0.32473331689834595 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8983928498528257e-05, + "first_to_last_ratio": 1.3500192298694966, + "imbalance_abs_log_ratio": 0.3001188366966636, + "last_quartile_mean": 1.4061968954592885e-05, + "mean": 2.0583478459457183e-05, + "normalized": [ + 0.7642043470230905, + 0.7964604133404727, + 0.7903456474112416, + 0.7412458727049767, + 1.1595129974656138, + 1.169001073870473, + 1.0269918785986678, + 0.9305547186552634, + 1.0094506994739345, + 0.9243807439337586, + 0.8548226966273005, + 0.7984573403240672, + 0.7719215503109763, + 0.72169521786754, + 0.7013534186825957, + 0.6759055681347007, + 1.6233704062346581, + 1.4876698635939476, + 1.3070103175531984, + 1.1529797049092063, + 2.314780636153873, + 1.9234294306898572, + 1.5697558727461935, + 1.3193574716542222, + 1.2524561316942173, + 0.9556524419867579, + 0.7985100096377913, + 0.6718093801653432, + 0.5751411733750736, + 0.450054823019735, + 0.38512487072844825, + 0.3765932814328042 + ], + "peak_layer": 21, + "peak_normalized": 2.314780636153873, + "population_cv": 0.42845686821906764, + "rest_mean": 1.8006899962428077e-05, + "spike_contrast": 1.9157657573148825, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4497002343414354e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.5729983715573326e-05, + 1.6393925761803985e-05, + 1.6268062609015033e-05, + 1.5257418453984428e-05, + 2.3866810806794092e-05, + 2.4062108423095196e-05, + 2.1139065211173147e-05, + 1.9154053006786853e-05, + 2.0778006728505716e-05, + 1.9026971131097525e-05, + 1.7595224562683143e-05, + 1.643502946535591e-05, + 1.5888830603216775e-05, + 1.4854997971269768e-05, + 1.4436292985919863e-05, + 1.391248770232778e-05, + 3.3414609788451344e-05, + 3.0621420592069626e-05, + 2.690281871764455e-05, + 2.3732332920189947e-05, + 4.764623736264184e-05, + 3.9590868254890665e-05, + 3.231103619327769e-05, + 2.7156966098118573e-05, + 2.577990380814299e-05, + 1.9670651454362087e-05, + 1.6436113583040424e-05, + 1.3828173905494623e-05, + 1.1838405953312758e-05, + 9.26369375520153e-06, + 7.927209480840247e-06, + 7.751599696348421e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 7.388767562588328e-05, + "first_to_last_ratio": 1.316792081076884, + "imbalance_abs_log_ratio": 0.2751985371595874, + "last_quartile_mean": 5.611187725662603e-05, + "mean": 8.168265617314319e-05, + "normalized": [ + 0.7533431897136043, + 0.7827542764436777, + 0.7759396123840813, + 0.7316445186865006, + 1.1365162577009855, + 1.1353975532420753, + 1.004206629730762, + 0.9167575048500237, + 1.0078390566063014, + 0.9220130731102486, + 0.8550917608474935, + 0.8013922543773514, + 0.7721726808717908, + 0.7274660564313035, + 0.7065479131389877, + 0.6779112969812525, + 1.6398021444587536, + 1.5049103837665865, + 1.3267399301434915, + 1.1597912607176666, + 2.3220162536772593, + 1.9302044249737353, + 1.584049170842242, + 1.3298950882483613, + 1.2567261692224045, + 0.9579394397382498, + 0.801457903329196, + 0.6752085554658249, + 0.5811903488324587, + 0.45264756329860717, + 0.3896296913930569, + 0.38079803677566654 + ], + "peak_layer": 21, + "peak_normalized": 2.3220162536772593, + "population_cv": 0.4315618932878077, + "rest_mean": 7.132743997283763e-05, + "spike_contrast": 1.9291428895694742, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001376008236547932, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.153507274575531e-05, + 6.39374484308064e-05, + 6.338080856949091e-05, + 5.976266766083427e-05, + 9.283366671297699e-05, + 9.274228796130046e-05, + 8.202626486308873e-05, + 7.488318806281313e-05, + 8.232297113863751e-05, + 7.531247683800757e-05, + 6.984616629779339e-05, + 6.54598479741253e-05, + 6.30731155979447e-05, + 5.942135976511054e-05, + 5.771271025878377e-05, + 5.537359538720921e-05, + 0.00013394339475780725, + 0.00012292507744859904, + 0.00010837164154509082, + 9.473483078181744e-05, + 0.0001896684552775696, + 0.00015766422438900918, + 0.0001293893437832594, + 0.00010862936323974282, + 0.00010265273158438504, + 7.824703789083287e-05, + 6.546521035488695e-05, + 5.515282828127965e-05, + 4.7473171434830874e-05, + 3.69734552805312e-05, + 3.1825988116906956e-05, + 3.1104595109354705e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.06552432896569371, + "first_to_last_ratio": 1.3167920765276309, + "imbalance_abs_log_ratio": 0.2751985337047875, + "last_quartile_mean": 0.04976057354360819, + "mean": 0.07243699650280178, + "normalized": [ + 0.7533431474468802, + 0.7827543098589632, + 0.7759395847698287, + 0.731644488826528, + 1.136516249702268, + 1.1353975877134457, + 1.0042065955447128, + 0.9167574882575961, + 1.007839058472007, + 0.9220131219136145, + 0.8550918118318299, + 0.8013922821239102, + 0.7721726890345729, + 0.7274660147584042, + 0.706547878986726, + 0.6779113252026149, + 1.6398021344750369, + 1.5049103829869008, + 1.3267399072401411, + 1.1597912281377818, + 2.3220164409485586, + 1.9302043593714369, + 1.584049135911302, + 1.3298951182707694, + 1.2567262311511898, + 0.957939389851257, + 0.8014578528316956, + 0.6752085263417028, + 0.581190365550621, + 0.4526475557003154, + 0.3896297084240016, + 0.3807980283633875 + ], + "peak_layer": 21, + "peak_normalized": 2.3220164409485586, + "population_cv": 0.43156190791174726, + "rest_mean": 0.0632538867217523, + "spike_contrast": 1.9291429451165345, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1220257893204689, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05456991493701935, + 0.05670037120580673, + 0.056206732988357544, + 0.0529981292784214, + 0.08232582360506058, + 0.08224479109048843, + 0.07274170964956284, + 0.06640715897083282, + 0.0730048343539238, + 0.06678786128759384, + 0.061940282583236694, + 0.058050449937582016, + 0.0559338703751564, + 0.05269545316696167, + 0.05118020623922348, + 0.04910586029291153, + 0.11878234148025513, + 0.10901118814945221, + 0.09610505402088165, + 0.08401179313659668, + 0.16819989681243896, + 0.1398182064294815, + 0.11474376171827316, + 0.09633360803127289, + 0.09103347361087799, + 0.06939025223255157, + 0.058055199682712555, + 0.048910077661275864, + 0.04209968447685242, + 0.03278842940926552, + 0.028223605826497078, + 0.027583865448832512 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.7910509743378498e-06, + "first_to_last_ratio": 0.7327699142917456, + "imbalance_abs_log_ratio": 0.3109235222896129, + "last_quartile_mean": 3.8089049780865025e-06, + "mean": 4.300465420215005e-06, + "normalized": [ + 0.6752230832161266, + 0.6481028772772607, + 0.634134920654564, + 0.6489938741355444, + 0.6688274895794406, + 0.6540545586410917, + 0.6334055531219255, + 0.6293486411866674, + 0.7505868142147136, + 0.7168130527550195, + 0.6943818949036571, + 0.6987315584176189, + 0.6285804128249975, + 0.6319836168825069, + 0.6466025849172643, + 0.7005267172313684, + 1.4911705866773426, + 1.4376163460093667, + 1.3858563653332445, + 1.3109106922600342, + 2.564962545843758, + 2.293160284916099, + 1.9861975918307744, + 1.7842600166703042, + 1.697776325534204, + 1.2627187172796404, + 1.0635667497903984, + 0.874137655981081, + 0.714444551315396, + 0.5383938398368974, + 0.4702297250968762, + 0.4643003556648156 + ], + "peak_layer": 21, + "peak_normalized": 2.564962545843758, + "population_cv": 0.5485038394294482, + "rest_mean": 3.4521019727064953e-06, + "spike_contrast": 2.572817404289376, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.881628036760959e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.903773520301911e-06, + 2.7871440124727087e-06, + 2.7270752980257384e-06, + 2.790975713651278e-06, + 2.876269491025596e-06, + 2.8127390123700025e-06, + 2.723938678172999e-06, + 2.7064920686825644e-06, + 3.2278726393997204e-06, + 3.0826297461317154e-06, + 2.986165327456547e-06, + 3.0048709049879108e-06, + 2.7031883291783743e-06, + 2.7178236905456288e-06, + 2.7806920570583316e-06, + 3.012590923390235e-06, + 6.412727543647634e-06, + 6.1824193835491315e-06, + 5.9598273765004706e-06, + 5.637526101054391e-06, + 1.1030532732547726e-05, + 9.861656508292072e-06, + 8.541574061382562e-06, + 7.673148502362892e-06, + 7.301228379219538e-06, + 5.430278179119341e-06, + 4.573832029564073e-06, + 3.7591987620544387e-06, + 3.072444087592885e-06, + 2.315344090675353e-06, + 2.022206672336324e-06, + 1.9967076241300674e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.460357715492137e-05, + "first_to_last_ratio": 1.5303519803401489, + "imbalance_abs_log_ratio": 0.4254977614562244, + "last_quartile_mean": 3.568040415302676e-05, + "mean": 5.699818092352871e-05, + "normalized": [ + 0.826499466074168, + 0.8487872014190548, + 0.8416021554467742, + 0.8012967869639217, + 1.1726116041287642, + 1.1547121771623214, + 1.0501726049153397, + 0.9682225384122511, + 1.0710773535839004, + 0.9771144253342418, + 0.8995261154993631, + 0.8346162473012488, + 0.8429729513138341, + 0.7876515056477689, + 0.7671474580293269, + 0.735290511846575, + 1.5890744132353023, + 1.4640380695597026, + 1.278360428063473, + 1.1237055239693245, + 2.371402173328689, + 1.8882521909202439, + 1.4825951619642506, + 1.215335093029712, + 1.183884081647654, + 0.8905874439533303, + 0.742080218171765, + 0.6343188256622954, + 0.482002642160446, + 0.38702398334188143, + 0.3477000388311781, + 0.3403386090818978 + ], + "peak_layer": 21, + "peak_normalized": 2.371402173328689, + "population_cv": 0.42451190461357197, + "rest_mean": 5.0366403094678254e-05, + "spike_contrast": 1.8426922610466794, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.280978119932115e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.710896610049531e-05, + 4.837932647205889e-05, + 4.7969791921786964e-05, + 4.5672459236811846e-05, + 6.683672836516052e-05, + 6.581649358849972e-05, + 5.9857928135897964e-05, + 5.518692341865972e-05, + 6.104946078266948e-05, + 5.569374479819089e-05, + 5.127135227667168e-05, + 4.757160786539316e-05, + 4.804792479262687e-05, + 4.489470302360132e-05, + 4.3726009607780725e-05, + 4.191022162558511e-05, + 9.057435090653598e-05, + 8.344750676769763e-05, + 7.286421896424145e-05, + 6.404917075997218e-05, + 0.0001351656101178378, + 0.00010762694000732154, + 8.45052272779867e-05, + 6.927188951522112e-05, + 6.74792390782386e-05, + 5.0761864258674905e-05, + 4.229722253512591e-05, + 3.6155019188299775e-05, + 2.747327380347997e-05, + 2.205966302426532e-05, + 1.9818269720417447e-05, + 1.939868161571212e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.0483532302459935e-05, + "first_to_last_ratio": 1.963155318608148, + "imbalance_abs_log_ratio": 0.674553035245332, + "last_quartile_mean": 2.062166549876565e-05, + "mean": 3.870260115945712e-05, + "normalized": [ + 0.9359797302060725, + 0.9576124055199563, + 0.9480772177254478, + 0.9102648966159671, + 1.2467899109742078, + 1.222393779568341, + 1.1175155496691729, + 1.0294929073580932, + 1.1457881710656306, + 1.0321570072444493, + 0.9475098439722178, + 0.8789722977603025, + 0.9341127359239892, + 0.8671918665148908, + 0.8459313304801716, + 0.8065696353523856, + 1.568420236330532, + 1.4485623429640861, + 1.2469929473073715, + 1.1052688868439036, + 2.4248452552946893, + 1.805023081283152, + 1.2952633315442323, + 1.0166745468576837, + 1.067512212729438, + 0.8179083685748635, + 0.6887736887471813, + 0.614681124858885, + 0.32340289834365593, + 0.26296753580630333, + 0.24650039620284458, + 0.2408438603598819 + ], + "peak_layer": 21, + "peak_normalized": 2.4248452552946893, + "population_cv": 0.43498346843587055, + "rest_mean": 3.496232669998857e-05, + "spike_contrast": 1.68467286934329, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.890008324058726e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.62248501915019e-05, + 3.706209099618718e-05, + 3.66930544259958e-05, + 3.522961924318224e-05, + 4.825401265406981e-05, + 4.730981891043484e-05, + 4.325075860833749e-05, + 3.984405338997021e-05, + 4.434498259797692e-05, + 3.9947160985320807e-05, + 3.667109558591619e-05, + 3.401851427042857e-05, + 3.6152592656435445e-05, + 3.356258093845099e-05, + 3.273974289186299e-05, + 3.121634290437214e-05, + 6.070194285712205e-05, + 5.6063130614347756e-05, + 4.826187068829313e-05, + 4.277678090147674e-05, + 9.384781878907233e-05, + 6.985908839851618e-05, + 5.0130060117226094e-05, + 3.934794949600473e-05, + 4.1315499402116984e-05, + 3.165518137393519e-05, + 2.6657333364710212e-05, + 2.378975841565989e-05, + 1.251653338840697e-05, + 1.0177527656196617e-05, + 9.540206519886851e-06, + 9.321283869212493e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001419489935869933, + "first_to_last_ratio": 1.1783352135158363, + "imbalance_abs_log_ratio": 0.16410260630786871, + "last_quartile_mean": 0.00012046571464452427, + "mean": 0.00016912645673983207, + "normalized": [ + 0.6986475868430325, + 0.7282565751265736, + 0.7166008603062816, + 0.6842111357537164, + 1.0469692964748558, + 1.026049824019131, + 0.9391127634694222, + 0.874606754649669, + 1.0007023053561546, + 0.9199914749419839, + 0.8601715511639205, + 0.8040475454029659, + 0.7517874109118121, + 0.7072705924944495, + 0.6728256776786633, + 0.6655104186910116, + 1.6362823272721128, + 1.4904042581473382, + 1.3331379776154995, + 1.1545215002514102, + 2.429646858429421, + 2.015441097306164, + 1.7000198983358237, + 1.4455290763603736, + 1.310596018230462, + 1.0039843631553074, + 0.8374679195648416, + 0.7043981788736101, + 0.5858096739376918, + 0.46318758062596044, + 0.3998773352945733, + 0.39293416331576775 + ], + "peak_layer": 21, + "peak_normalized": 2.429646858429421, + "population_cv": 0.4589394276046076, + "rest_mean": 0.000144689356535242, + "spike_contrast": 2.0809187700774854, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00030108679784461855, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.0001181597908725962, + 0.00012316745414864272, + 0.00012119616440031677, + 0.00011571820505196229, + 0.00017707020742818713, + 0.00017353217117488384, + 0.00015882881416473538, + 0.00014791914145462215, + 0.00016924523515626788, + 0.00015559489838778973, + 0.00014547776663675904, + 0.00013598571240436286, + 0.00012714714102912694, + 0.0001196181692648679, + 0.00011379262286936864, + 0.00011255541903665289, + 0.0002767386322375387, + 0.0002520667912904173, + 0.00022546890249941498, + 0.00019526013056747615, + 0.0004109175642952323, + 0.0003408644115552306, + 0.0002875183417927474, + 0.0002444772107992321, + 0.00022165646078065038, + 0.00016980031796265393, + 0.00014163798186928034, + 0.0001191323681268841, + 9.907591447699815e-05, + 7.833727431716397e-05, + 6.762983684893698e-05, + 6.64555627736263e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.062413111329078674, + 0.07552354782819748, + 0.14653873443603516, + 0.2105521410703659, + 0.2186809629201889, + 0.14272794127464294, + 0.1287408471107483, + 0.13253600895404816, + 0.13371841609477997, + 0.09585829079151154, + 0.09883332997560501, + 0.10020580142736435, + 0.11237040907144547, + 0.08538161963224411, + 0.08663177490234375, + 0.10358677804470062, + 0.10874549299478531, + 0.06467291712760925, + 0.07278503477573395, + 0.08346924930810928, + 0.09857872873544693, + 0.06093711405992508, + 0.06834813207387924, + 0.0680762529373169, + 0.07369214296340942, + 0.05633483827114105, + 0.05850831791758537, + 0.06479237228631973, + 0.06705968081951141, + 0.061747051775455475, + 0.0654391348361969, + 0.07205834239721298 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 9.513242389402876e-06, + "first_to_last_ratio": 1.2607108267154772, + "imbalance_abs_log_ratio": 0.2316757100785088, + "last_quartile_mean": 7.545935346797705e-06, + "mean": 1.3414615857243462e-05, + "normalized": [ + 0.54411236240655, + 0.5345088335287688, + 0.359970954850746, + 0.5314334804876196, + 0.3673058684703728, + 0.903301495529033, + 1.2731805653143804, + 1.1595459992487693, + 1.4831078521573031, + 1.119758551480115, + 0.913628335780701, + 0.9147234218106974, + 0.5358305370089425, + 0.8430737978428791, + 1.0000117212452202, + 0.9024986222769764, + 1.0159470806684705, + 1.855308691450899, + 1.5791792776893256, + 1.8749806454984184, + 1.9537650071631565, + 2.5528608648736655, + 2.0787080043386754, + 1.2031304503621438, + 1.0522901465907954, + 0.940487300390985, + 0.5556490720369505, + 0.5996248916562782, + 0.3826096482804215, + 0.4666405513855537, + 0.25941785990949606, + 0.24340810826569018 + ], + "peak_layer": 22, + "peak_normalized": 2.5528608648736655, + "population_cv": 0.575130971839342, + "rest_mean": 1.1506384529032706e-05, + "spike_contrast": 2.0613829626268805, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.3719065029581544e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.299058324861107e-06, + 7.1702306740917265e-06, + 4.828872079087887e-06, + 7.1289759944193065e-06, + 4.927267127641244e-06, + 1.21174425657955e-05, + 1.7079228200600483e-05, + 1.5554864148725756e-05, + 1.989532211155165e-05, + 1.502113082096912e-05, + 1.2255973160790745e-05, + 1.2270663319213782e-05, + 7.187960818555439e-06, + 1.1309511137369554e-05, + 1.3414773093245458e-05, + 1.2106672329537105e-05, + 1.3628539818455465e-05, + 2.4888253392418846e-05, + 2.1184083379921503e-05, + 2.5152145099127665e-05, + 2.6209007046418265e-05, + 3.424564783927053e-05, + 2.7885069357580505e-05, + 1.6139532817760482e-05, + 1.4116068086877931e-05, + 1.2616275853361003e-05, + 7.453818852809491e-06, + 8.043737580010202e-06, + 5.1325614549568854e-06, + 6.259803740249481e-06, + 3.4799909371940885e-06, + 3.265226268922561e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.6330423654362676e-05, + "first_to_last_ratio": 1.2225177619555718, + "imbalance_abs_log_ratio": 0.20091247146835384, + "last_quartile_mean": 2.9717706183873815e-05, + "mean": 5.264320969899927e-05, + "normalized": [ + 0.5035242395982877, + 0.5277848791426298, + 0.35262352810018, + 0.5210210629397536, + 0.3595263516776504, + 0.8771435825340668, + 1.2750208353143138, + 1.1043598609817844, + 1.421449261823989, + 1.1627357712534139, + 0.9147350742210957, + 0.9225896301191776, + 0.5357176288484936, + 0.8409556403265339, + 1.0036519032774602, + 0.925597482954976, + 1.0604518437864157, + 1.8336332517284446, + 1.6012013115443406, + 1.8626503069202516, + 1.929299073680775, + 2.5897943935525722, + 2.1504101537648252, + 1.2080295776025163, + 1.0642429477202255, + 0.944975242275414, + 0.5665571215143791, + 0.570934005219875, + 0.3855916338171456, + 0.47604359079756364, + 0.2627082854000429, + 0.24504052756140657 + ], + "peak_layer": 22, + "peak_normalized": 2.5897943935525722, + "population_cv": 0.5809548582160158, + "rest_mean": 4.4957737541322046e-05, + "spike_contrast": 2.094072444457974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.414475935045629e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.6507132133701816e-05, + 2.778429006866645e-05, + 1.8563234334578738e-05, + 2.7428221073932946e-05, + 1.8926621123682708e-05, + 4.617565355147235e-05, + 6.712118920404464e-05, + 5.813704774482176e-05, + 7.482965156668797e-05, + 6.121014303062111e-05, + 4.81545903312508e-05, + 4.856807936448604e-05, + 2.8201895474921912e-05, + 4.427060412126593e-05, + 5.2835457609035075e-05, + 4.8726422392064705e-05, + 5.58255887881387e-05, + 9.652833978179842e-05, + 8.429237641394138e-05, + 9.805589070310816e-05, + 0.00010156449570786208, + 0.00013633508933708072, + 0.00011320449266349897, + 6.359455437632278e-05, + 5.602516466751695e-05, + 4.974652983946726e-05, + 2.982538535434287e-05, + 3.0055798561079428e-05, + 2.0298781237215735e-05, + 2.5060462576220743e-05, + 1.3829807357979007e-05, + 1.2899719877168536e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.032218182226642966, + "first_to_last_ratio": 1.2225177500182758, + "imbalance_abs_log_ratio": 0.20091246170383634, + "last_quartile_mean": 0.026353958644904196, + "mean": 0.046684524888405576, + "normalized": [ + 0.5035242503965048, + 0.5277848868035033, + 0.3526235212534414, + 0.5210210441313687, + 0.3595263305975145, + 0.8771435091867963, + 1.2750207710793637, + 1.1043598828940213, + 1.4214492710356383, + 1.162735779950175, + 0.9147351310550158, + 0.9225896406648795, + 0.5357176374824659, + 0.8409556781269271, + 1.003651856889631, + 0.9255975130347041, + 1.0604518407651282, + 1.8336332774847264, + 1.601201364484995, + 1.8626502208000761, + 1.9292991633227214, + 2.589794548624206, + 2.150410100335512, + 1.2080294989428133, + 1.0642429217970695, + 0.9449752045153401, + 0.5665571468541322, + 0.5709340184911269, + 0.38559161860900615, + 0.4760435610967507, + 0.2627082741971009, + 0.2450405350973446 + ], + "peak_layer": 22, + "peak_normalized": 2.589794548624206, + "population_cv": 0.5809548711713294, + "rest_mean": 0.03986897116044053, + "spike_contrast": 2.094072472636495, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08348851501941681, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.02350679039955139, + 0.02463938668370247, + 0.016462061554193497, + 0.02432361990213394, + 0.016784315928816795, + 0.040949027985334396, + 0.05952373892068863, + 0.05155651643872261, + 0.06635968387126923, + 0.05428176745772362, + 0.04270397499203682, + 0.043070659041404724, + 0.025009723380208015, + 0.039259616285562515, + 0.04685501009225845, + 0.04321108013391495, + 0.049506690353155136, + 0.0856022983789444, + 0.07475132495164871, + 0.0869569405913353, + 0.09006841480731964, + 0.12090332806110382, + 0.10039087384939194, + 0.05639628320932388, + 0.04968367516994476, + 0.04411571845412254, + 0.026449451223015785, + 0.02665378339588642, + 0.018001161515712738, + 0.02222386747598648, + 0.012264410965144634, + 0.011439600959420204 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.2802849315439744e-06, + "first_to_last_ratio": 0.4283455786552265, + "imbalance_abs_log_ratio": 0.8478249824294246, + "last_quartile_mean": 5.323470219309456e-06, + "mean": 4.877433806171894e-06, + "normalized": [ + 1.4839433033627585, + 0.703687558597217, + 0.19787363001413222, + 0.2183139267711944, + 0.1618540975898947, + 0.2679413581551488, + 0.34335611131172533, + 0.36316882023729685, + 0.49344402674477783, + 0.6027840686307443, + 0.5647136700772823, + 0.7110849136409753, + 0.4430254500989309, + 0.5976516238780146, + 0.6050388629285002, + 0.6350381439922589, + 0.9181070377370916, + 1.494677817366969, + 1.5696877676314565, + 1.6091270885003666, + 1.7368727838973916, + 3.0783329280416862, + 2.262173099042413, + 2.2065099626031874, + 1.6586180705188411, + 1.7272863652034096, + 1.2173480016488631, + 1.3531761425833577, + 0.9843111324014115, + 0.8080573379451803, + 0.5264086923618475, + 0.4563862064856749 + ], + "peak_layer": 22, + "peak_normalized": 3.0783329280416862, + "population_cv": 0.6993295019446811, + "rest_mean": 3.8039455182149175e-06, + "spike_contrast": 2.806105005980394, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0674270561139565e-05, + "top_five_layers": [ + 22, + 23, + 24, + 21, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 7.237835234263912e-06, + 3.4321894872846315e-06, + 9.65115532380878e-07, + 1.0648117267919588e-06, + 7.894326472523971e-07, + 1.306866238337534e-06, + 1.6746967048675288e-06, + 1.771331881172955e-06, + 2.4067405774985673e-06, + 2.940039394161431e-06, + 2.7543535452423384e-06, + 3.4682695968513144e-06, + 2.160827307307045e-06, + 2.9150062346161576e-06, + 2.9510370040952694e-06, + 3.097356511716498e-06, + 4.478006303543225e-06, + 7.290192115760874e-06, + 7.656048182980157e-06, + 7.84841085987864e-06, + 8.471482033201028e-06, + 1.5014365089882631e-05, + 1.1033599548682105e-05, + 1.0762106285255868e-05, + 8.089799848676194e-06, + 8.424724910582881e-06, + 5.9375342971179634e-06, + 6.600027063541347e-06, + 4.800912392965984e-06, + 3.941246177419089e-06, + 2.5675235519884154e-06, + 2.225993512183777e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.354378420932335e-05, + "first_to_last_ratio": 1.2464041441561005, + "imbalance_abs_log_ratio": 0.22026272102941338, + "last_quartile_mean": 1.888936611749159e-05, + "mean": 3.545669885340885e-05, + "normalized": [ + 0.4191906097330923, + 0.49348811099284096, + 0.38372804382004466, + 0.5201360165053419, + 0.3641940443046511, + 0.8688909640601193, + 1.2049617168759028, + 1.0575314684579886, + 1.2475942615588305, + 1.1749760705863097, + 0.940415402502436, + 0.897650499398874, + 0.5756566289364113, + 0.8922773631416766, + 1.048544229093483, + 0.9399393226018595, + 1.0859580556766202, + 1.9234179989731328, + 1.6483168700797706, + 1.8071073699183404, + 1.9702308124727184, + 2.7499314904694696, + 2.243433777969385, + 1.2804717854341459, + 1.0657725757024525, + 0.9763545100314811, + 0.5574845360072901, + 0.4565738604950286, + 0.3799339743020443, + 0.40651690619449354, + 0.21895640546763065, + 0.2003643182361338 + ], + "peak_layer": 22, + "peak_normalized": 2.7499314904694696, + "population_cv": 0.6127980670514929, + "rest_mean": 2.9796968680818315e-05, + "spike_contrast": 2.215636177377244, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.601924178539775e-05, + "top_five_layers": [ + 22, + 23, + 21, + 18, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.4863115211483091e-05, + 1.7497459339210764e-05, + 1.3605729691335e-05, + 1.8442306100041606e-05, + 1.2913118553115055e-05, + 3.0808005249127746e-05, + 4.272396472515538e-05, + 3.7496574805118144e-05, + 4.423557402333245e-05, + 4.1660772694740444e-05, + 3.334402572363615e-05, + 3.182772343279794e-05, + 2.0410883735166863e-05, + 3.163720975862816e-05, + 3.7177916965447366e-05, + 3.3327145501971245e-05, + 3.8504487747559324e-05, + 6.819805275881663e-05, + 5.844387487741187e-05, + 6.40740618109703e-05, + 6.985788058955222e-05, + 9.750349272508174e-05, + 7.954475586302578e-05, + 4.5401302486425266e-05, + 3.778877726290375e-05, + 3.461830783635378e-05, + 1.976656130864285e-05, + 1.6188601875910535e-05, + 1.3471204511006363e-05, + 1.4413747521757614e-05, + 7.763471330690663e-06, + 7.104257292667171e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.6053550893957436e-05, + "first_to_last_ratio": 1.291144701278337, + "imbalance_abs_log_ratio": 0.2555291902297627, + "last_quartile_mean": 1.2433579968273989e-05, + "mean": 2.3879894939682345e-05, + "normalized": [ + 0.381944208893466, + 0.5064170283475032, + 0.4207326908745865, + 0.5077824951609179, + 0.39135241691859496, + 0.887743098047013, + 1.221771198045867, + 1.0603545260365184, + 1.112358047025732, + 1.1817339073136295, + 0.9153849269316471, + 0.8293975374829873, + 0.6253146359858851, + 0.9134444345143908, + 1.0884909424487381, + 0.93703564258362, + 1.0885287239694037, + 2.0378246223538903, + 1.6522638992705716, + 1.6683493816940003, + 1.9552451870072145, + 2.8560629763771863, + 2.298629545440282, + 1.2964661784710367, + 1.0642451847684595, + 1.0310733143136432, + 0.5500502286297949, + 0.40889324961186074, + 0.38796838095670966, + 0.3474524989052327, + 0.1966278305110652, + 0.17906106110855208 + ], + "peak_layer": 22, + "peak_normalized": 2.8560629763771863, + "population_cv": 0.6286511089273724, + "rest_mean": 1.9925871600679346e-05, + "spike_contrast": 2.2699946018299357, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.523162097029854e-05, + "top_five_layers": [ + 22, + 23, + 18, + 21, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.120787581196055e-06, + 1.2093185432604514e-05, + 1.0047052455774974e-05, + 1.2125792636652477e-05, + 9.34545460040681e-06, + 2.1199211914790794e-05, + 2.9175767849665135e-05, + 2.5321154680568725e-05, + 2.656299329828471e-05, + 2.8219681553309783e-05, + 2.1859295884496532e-05, + 1.9805926058324985e-05, + 1.4932447811588645e-05, + 2.18129571294412e-05, + 2.5993049348471686e-05, + 2.2376312699634582e-05, + 2.5993951567215845e-05, + 4.8663037887308747e-05, + 3.945588832721114e-05, + 3.984000795753673e-05, + 4.669104964705184e-05, + 6.820248381700367e-05, + 5.489103205036372e-05, + 3.0959476134739816e-05, + 2.5414063202333637e-05, + 2.4621922420919873e-05, + 1.3135141671227757e-05, + 9.764327842276543e-06, + 9.264644177164882e-06, + 8.297129170387052e-06, + 4.695451934821904e-06, + 4.275959327060264e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.614753056055633e-05, + "first_to_last_ratio": 1.1762933160121276, + "imbalance_abs_log_ratio": 0.16236823675450227, + "last_quartile_mean": 5.623387437481142e-05, + "mean": 0.00010358891012174354, + "normalized": [ + 0.42097325737912505, + 0.48924702737687037, + 0.34042189089398245, + 0.5266302780781597, + 0.33345251919713514, + 0.8199682417706958, + 1.1522135672308953, + 1.025557472234399, + 1.3534387224479996, + 1.1847086209851718, + 0.9533429560557738, + 0.955796607001976, + 0.5321117819138612, + 0.8503724980636721, + 0.997244364596572, + 0.928898529620989, + 1.0923838319024162, + 1.8584595513189417, + 1.6516870083000426, + 1.9684072477501107, + 1.9630027958449041, + 2.688216127574533, + 2.254735452453152, + 1.3158806474486227, + 1.0888392323833342, + 0.9550626820961028, + 0.5732683646881589, + 0.4722751280665548, + 0.3639803606056138, + 0.4532808789761308, + 0.2263430485366378, + 0.20979930720746648 + ], + "peak_layer": 22, + "peak_normalized": 2.688216127574533, + "population_cv": 0.6154722710504141, + "rest_mean": 8.705046389810309e-05, + "spike_contrast": 2.2159160456077167, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019289651972940192, + "top_five_layers": [ + 22, + 23, + 20, + 21, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.3608160922303796e-05, + 5.0680566346272826e-05, + 3.526393265929073e-05, + 5.4553056543227285e-05, + 3.4541983040980995e-05, + 8.493961649946868e-05, + 0.00011935654765693471, + 0.00010623638081597164, + 0.00014020124217495322, + 0.00012272267485968769, + 9.875575779005885e-05, + 9.900992881739512e-05, + 5.5120879551395774e-05, + 8.808916027192026e-05, + 0.00010330345685360953, + 9.622358629712835e-05, + 0.0001131588505813852, + 0.00019251579942647368, + 0.00017109645705204457, + 0.0002039051614701748, + 0.00020334532018750906, + 0.00027846937882713974, + 0.0002335655881324783, + 0.00013631064211949706, + 0.00011279166938038543, + 9.893390233628452e-05, + 5.938424510532059e-05, + 4.8922465794021264e-05, + 3.770432886085473e-05, + 4.6954872232163325e-05, + 2.344662971154321e-05, + 2.1732881577918306e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05931045487523079, + 0.07934419065713882, + 0.214645653963089, + 0.09514089673757553, + 0.12821291387081146, + 0.1321839988231659, + 0.12568901479244232, + 0.1335759311914444, + 0.07788944244384766, + 0.08352623134851456, + 0.07934392243623734, + 0.09343641996383667, + 0.07063088566064835, + 0.06870647519826889, + 0.07622843235731125, + 0.08666495233774185, + 0.057347409427165985, + 0.05777324363589287, + 0.06351501494646072, + 0.07469023764133453, + 0.051319729536771774, + 0.05102822557091713, + 0.05902284011244774, + 0.06656526774168015, + 0.050236333161592484, + 0.05184732377529144, + 0.061248794198036194, + 0.06608261168003082, + 0.048912134021520615, + 0.062179163098335266, + 0.07131517678499222, + 0.06802479177713394 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.9132973458654305e-06, + "first_to_last_ratio": 0.18177325947364456, + "imbalance_abs_log_ratio": 1.7049951956609164, + "last_quartile_mean": 2.702981373658986e-05, + "mean": 1.9435898153119524e-05, + "normalized": [ + 0.5827233187547133, + 0.21785343322460332, + 0.09025019230653299, + 0.27877726302073735, + 0.20197289680560415, + 0.11728555064621506, + 0.16965705081784332, + 0.36384007719584965, + 0.6141661884449419, + 0.4558215636950886, + 0.47742147413318603, + 0.4568805250530343, + 0.5629610314949864, + 0.5194178319866835, + 0.4756935842462155, + 0.4665970921430879, + 1.370025243738621, + 1.38943151782949, + 1.4294802852403141, + 1.357602686438849, + 2.472173788023745, + 2.540944975389688, + 2.27148212683372, + 1.9918126315455524, + 2.108389723940819, + 1.909399326521429, + 1.5199007981214199, + 1.3824424664562098, + 1.4431666705716988, + 1.096865155373051, + 0.8791837858211463, + 0.7863797441849225 + ], + "peak_layer": 22, + "peak_normalized": 2.540944975389688, + "population_cv": 0.7314263156880372, + "rest_mean": 1.4839809797127532e-05, + "spike_contrast": 2.982165936118834, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.425477527547628e-05, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1325751074764412e-05, + 4.234177140460815e-06, + 1.7540935459692264e-06, + 5.418286491476465e-06, + 3.925524652004242e-06, + 2.2795500171923777e-06, + 3.2974371606542263e-06, + 7.07155868440168e-06, + 1.1936871487705503e-05, + 8.859301487973426e-06, + 9.279115147364791e-06, + 8.879883353074547e-06, + 1.0941653272311669e-05, + 1.0095352081407327e-05, + 9.245532055501826e-06, + 9.068733561434783e-06, + 2.662767110450659e-05, + 2.7004849471268244e-05, + 2.778323323582299e-05, + 2.638622754602693e-05, + 4.804891796084121e-05, + 4.9385547754354775e-05, + 4.414829527377151e-05, + 3.871266744681634e-05, + 4.097844794159755e-05, + 3.711089084390551e-05, + 2.9540637115132995e-05, + 2.6869010980590247e-05, + 2.8049240427208133e-05, + 2.1318559447536245e-05, + 1.708772651909385e-05, + 1.528399661765434e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.9200162171273405e-05, + "first_to_last_ratio": 0.17782955919377144, + "imbalance_abs_log_ratio": 1.7269297200365261, + "last_quartile_mean": 0.00010796946389746154, + "mean": 7.753401975207908e-05, + "normalized": [ + 0.5802604748565295, + 0.21657891030507218, + 0.08122798291669481, + 0.27804637394409343, + 0.19800644999728254, + 0.11583331846598444, + 0.16581158447272498, + 0.34531750415538864, + 0.6119033375078596, + 0.4582687548340118, + 0.4778600356941264, + 0.4551699938366303, + 0.5608630718384122, + 0.5132558797569152, + 0.4786206261633362, + 0.46421909665654804, + 1.3655188026701754, + 1.3920864482886812, + 1.4446997895948175, + 1.3618673115230535, + 2.481688366104822, + 2.532895764609021, + 2.2670762930167974, + 2.0125790621760973, + 2.1041793043599823, + 1.9172636100383544, + 1.5214473562286417, + 1.3815731267036704, + 1.4472156032763779, + 1.1033023996856395, + 0.8759464714083287, + 0.7894168949139294 + ], + "peak_layer": 22, + "peak_normalized": 2.532895764609021, + "population_cv": 0.7341542937111465, + "rest_mean": 5.916012609035264e-05, + "spike_contrast": 2.9877056931125323, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017675304552540184, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.498992711887695e-05, + 1.6792233509477228e-05, + 6.297932031884557e-06, + 2.1558053049375303e-05, + 1.535223600512836e-05, + 8.981022801890504e-06, + 1.2856038665631786e-05, + 2.6773854187922552e-05, + 4.7443325456697494e-05, + 3.5531418689060956e-05, + 3.705040944623761e-05, + 3.5291159292683005e-05, + 4.34859684901312e-05, + 3.9794791518943384e-05, + 3.7109381082700565e-05, + 3.59927726094611e-05, + 0.00010587416181806475, + 0.00010793405817821622, + 0.00011201338202226907, + 0.00010559104703133926, + 0.00019241527479607612, + 0.00019638559024315327, + 0.00017577553808223456, + 0.0001560433447593823, + 0.0001631454797461629, + 0.0001486531546106562, + 0.00011796392936958, + 0.00010711891809478402, + 0.00011220844316994771, + 8.554347004974261e-05, + 6.791565101593733e-05, + 6.120666512288153e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.0170268957153894, + "first_to_last_ratio": 0.17782955808409184, + "imbalance_abs_log_ratio": 1.7269297262766563, + "last_quartile_mean": 0.09574840031564236, + "mean": 0.06875794463849161, + "normalized": [ + 0.5802604568764914, + 0.21657890183963943, + 0.08122798509807252, + 0.27804637670503696, + 0.19800643904545134, + 0.115833321551524, + 0.16581158131171497, + 0.34531750831052266, + 0.6119033445588002, + 0.45826877711162134, + 0.4778600229397992, + 0.4551700187870312, + 0.5608630663003571, + 0.5132558897875106, + 0.4786205986883147, + 0.4642191256054569, + 1.3655187793056378, + 1.3920863761172964, + 1.4446998184613569, + 1.361867387227318, + 2.481688342051939, + 2.532895819110775, + 2.267076382937339, + 2.0125790037035514, + 2.1041792403394113, + 1.9172635512087937, + 1.5214474269881526, + 1.3815731149301655, + 1.4472156023555418, + 1.1033024068469601, + 0.8759464966522335, + 0.7894168372461831 + ], + "peak_layer": 22, + "peak_normalized": 2.532895819110775, + "population_cv": 0.7341542943028188, + "rest_mean": 0.05246379186778709, + "spike_contrast": 2.9877056922478897, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15674636960029603, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.03989751636981964, + 0.014891520142555237, + 0.005585069302469492, + 0.019117897376418114, + 0.013614515773952007, + 0.007964461110532284, + 0.011400863528251648, + 0.023743322119116783, + 0.042073216289281845, + 0.03150961920619011, + 0.03285667300224304, + 0.03129655495285988, + 0.038563791662454605, + 0.035290420055389404, + 0.03290896862745285, + 0.03191875293850899, + 0.09389026463031769, + 0.09571699798107147, + 0.09933459013700485, + 0.09363920241594315, + 0.17063578963279724, + 0.17415671050548553, + 0.15587951242923737, + 0.13838079571723938, + 0.14467903971672058, + 0.13182710111141205, + 0.1046115979552269, + 0.09499412775039673, + 0.09950757026672363, + 0.07586080580949783, + 0.06022828072309494, + 0.05427867919206619 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 9.967008836042623e-07, + "first_to_last_ratio": 0.13886164262645015, + "imbalance_abs_log_ratio": 1.974277218356026, + "last_quartile_mean": 7.177654424594948e-06, + "mean": 4.984633447335796e-06, + "normalized": [ + 0.5560650351741472, + 0.20102122926862334, + 0.07002478683248901, + 0.2844055838743635, + 0.12143455655713824, + 0.09092975294696704, + 0.1066409068927962, + 0.16911574529539783, + 0.4501637856763599, + 0.3830962086593998, + 0.45012441999702146, + 0.4287715723483553, + 0.5612487151278214, + 0.5091276890001657, + 0.4770758153323667, + 0.44669518124695695, + 1.3364518589018204, + 1.4488775934268328, + 1.4466091632605513, + 1.4235110430079823, + 2.502127463836705, + 2.5139466555424126, + 2.39841810266287, + 2.1044665924310153, + 2.138548056523478, + 2.0036229432372017, + 1.6440023639406895, + 1.4922570211503863, + 1.464470142868618, + 1.100253768757845, + 0.86132188499381, + 0.8151743612274127 + ], + "peak_layer": 22, + "peak_normalized": 2.5139466555424126, + "population_cv": 0.7762261395684433, + "rest_mean": 3.7555508019640576e-06, + "spike_contrast": 3.094534023149238, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1621679732343182e-05, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7717803732230095e-06, + 1.0020171430369373e-06, + 3.4904789458778396e-07, + 1.4176575859892182e-06, + 6.053067522771016e-07, + 4.532514878974325e-07, + 5.315658313520544e-07, + 8.429800004705612e-07, + 2.243901462861686e-06, + 1.9095941752311774e-06, + 2.2437052393797785e-06, + 2.1372691207943717e-06, + 2.7976191177003784e-06, + 2.5378149075550027e-06, + 2.3780480660207104e-06, + 2.226611741207307e-06, + 6.661722636636114e-06, + 7.2221237132907845e-06, + 7.210816420410993e-06, + 7.095680757629452e-06, + 1.2472188245737925e-05, + 1.253110258403467e-05, + 1.1955235095229e-05, + 1.0489994565432426e-05, + 1.0659878171281889e-05, + 9.987325938709546e-06, + 8.194749170797877e-06, + 7.438354259647895e-06, + 7.2998468567675445e-06, + 5.484361736307619e-06, + 4.293373876862461e-06, + 4.0633453863847535e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.2451597399376624e-05, + "first_to_last_ratio": 0.19401707978808486, + "imbalance_abs_log_ratio": 1.6398090836493477, + "last_quartile_mean": 6.417784152290551e-05, + "mean": 4.8993523961371466e-05, + "normalized": [ + 0.6223930566187945, + 0.21900570100977182, + 0.07494974477635315, + 0.2911875989654724, + 0.19630689183835176, + 0.10605512222211717, + 0.17316480318369307, + 0.3501196561590397, + 0.648450442529022, + 0.5057741493666605, + 0.5380805526513028, + 0.5110202884462354, + 0.636128316167611, + 0.591783883393767, + 0.5520148879226475, + 0.5333926201155943, + 1.382390430732233, + 1.3902405931289812, + 1.434940184120226, + 1.3725264922836518, + 2.63138898922869, + 2.6354613909169915, + 2.2184615366462355, + 1.9053627250852199, + 2.10966594886892, + 1.8441371017158037, + 1.4431334012868662, + 1.3129956485826366, + 1.2970260777727707, + 0.9656964298374909, + 0.7844827226787779, + 0.722262611748071 + ], + "peak_layer": 22, + "peak_normalized": 2.6354613909169915, + "population_cv": 0.7271026814282602, + "rest_mean": 3.7198168683971744e-05, + "spike_contrast": 3.029408340413438, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011268844245932996, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0493229132844135e-05, + 1.072986106009921e-05, + 3.672052116598934e-06, + 1.4266306607169099e-05, + 9.617766409064643e-06, + 5.196014171815477e-06, + 8.48395393404644e-06, + 1.715359576337505e-05, + 3.1769872293807566e-05, + 2.4779657906037755e-05, + 2.6362462449469604e-05, + 2.5036684746737592e-05, + 3.116616790066473e-05, + 2.8993577871005982e-05, + 2.7045154638472013e-05, + 2.613278411445208e-05, + 6.772817869205028e-05, + 6.811278581153601e-05, + 7.030277629382908e-05, + 6.724490958731622e-05, + 0.00012892101949546486, + 0.000129120540805161, + 0.0001086902484530583, + 9.335043432656676e-05, + 0.0001033599692163989, + 9.035077528096735e-05, + 7.070419087540358e-05, + 6.432828377000988e-05, + 6.354587821988389e-05, + 4.731287117465399e-05, + 3.8434573070844635e-05, + 3.5386190575081855e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.240943799364686e-06, + "first_to_last_ratio": 0.24534451042071564, + "imbalance_abs_log_ratio": 1.4050918911790493, + "last_quartile_mean": 3.3589273243705975e-05, + "mean": 2.9281088309573988e-05, + "normalized": [ + 0.697594278712878, + 0.24149914140180637, + 0.07261827844624033, + 0.3161533325536177, + 0.210190468546024, + 0.1180112097175629, + 0.20064229389807767, + 0.3948312980233614, + 0.7281804923556615, + 0.5807663232829994, + 0.6275700759618513, + 0.5956218452297064, + 0.7485593740948113, + 0.7082998262177285, + 0.6766344353399901, + 0.6275549804018643, + 1.4496279922272475, + 1.4564659702931255, + 1.4770283593746687, + 1.4496530893722464, + 2.8466255319856013, + 2.810910306759418, + 2.114703825619007, + 1.6732013114486135, + 2.0139259913891268, + 1.6968132521341137, + 1.3105709239526526, + 1.2133397387383178, + 1.0442280938677242, + 0.7369046078385154, + 0.6041734489015155, + 0.5570999019139249 + ], + "peak_layer": 21, + "peak_normalized": 2.8466255319856013, + "population_cv": 0.7213237800230021, + "rest_mean": 2.227600332473769e-05, + "spike_contrast": 3.0125936977736663, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.710854722769e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.0426319679245353e-05, + 7.071357686072588e-06, + 2.1263422240735963e-06, + 9.257313649868593e-06, + 6.154605671326863e-06, + 3.4554966532596154e-06, + 5.87502472626511e-06, + 1.1561090104805771e-05, + 2.132191730197519e-05, + 1.70054699992761e-05, + 1.8375934814685024e-05, + 1.7440455849282444e-05, + 2.1918633137829602e-05, + 2.0739789761137217e-05, + 1.981259265448898e-05, + 1.8375492800259963e-05, + 4.244668525643647e-05, + 4.2646908696042374e-05, + 4.324899782659486e-05, + 4.24474201281555e-05, + 8.335229358635843e-05, + 8.230651292251423e-05, + 6.192082946654409e-05, + 4.899315536022186e-05, + 5.896994480281137e-05, + 4.9684538680594414e-05, + 3.8374942960217595e-05, + 3.552790803951211e-05, + 3.057613503187895e-05, + 2.1577368897851557e-05, + 1.7690856111585163e-05, + 1.631249142519664e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.6764148262591334e-05, + "first_to_last_ratio": 0.15860381719921426, + "imbalance_abs_log_ratio": 1.8413459019801368, + "last_quartile_mean": 0.00023179863455879968, + "mean": 0.00016184205071567703, + "normalized": [ + 0.5514172677776774, + 0.18824962050737556, + 0.07412330605763978, + 0.2681691846889825, + 0.18477039778199453, + 0.09341254286112102, + 0.1477303241859572, + 0.3094126957660134, + 0.5998604868596944, + 0.4469180103423267, + 0.4751811779005959, + 0.4510677352664099, + 0.5417700474807862, + 0.5021116806130881, + 0.4526412355734704, + 0.45370919271044824, + 1.297304157819389, + 1.297997666840438, + 1.3953174020318846, + 1.2918577787279846, + 2.479178311227491, + 2.5452011253115043, + 2.3881988831304657, + 2.1063821008782244, + 2.2045217646011044, + 2.008161694111016, + 1.608368174721935, + 1.4511166900922579, + 1.4253132631711374, + 1.0766448367874268, + 0.8993268174418685, + 0.7845644267322914 + ], + "peak_layer": 22, + "peak_normalized": 2.5452011253115043, + "population_cv": 0.7597506786457792, + "rest_mean": 0.00012154048979735105, + "spike_contrast": 3.122173362204994, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037947047967463733, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.924250141717494e-05, + 3.046670462936163e-05, + 1.1996267858194187e-05, + 4.3401050788816065e-05, + 2.9903620088589378e-05, + 1.5118077499209903e-05, + 2.3908978619147092e-05, + 5.007598520023748e-05, + 9.708265133667737e-05, + 7.233012729557231e-05, + 7.690429629292339e-05, + 7.30017272871919e-05, + 8.768117550062016e-05, + 8.126278407871723e-05, + 7.325638580368832e-05, + 7.342922617681324e-05, + 0.00020995836530346423, + 0.00021007060422562063, + 0.000225821029744111, + 0.00020907691214233637, + 0.00040123530197888613, + 0.0004119205696042627, + 0.00038651100476272404, + 0.00034090119879692793, + 0.0003567843232303858, + 0.000325005006743595, + 0.0002603016037028283, + 0.0002348517009522766, + 0.00023067562142387033, + 0.00017424640827812254, + 0.0001455488963983953, + 0.00012697551574092358 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 1.8565786854442996, + "forward": { + "activation_sha256": { + "attention_branch_output": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", + "mlp_branch_output": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", + "post_attention_state": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", + "post_mlp_state": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", + "pre_attention_input": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", + "pre_mlp_input": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f" + }, + "logits_sha256": "a911233a29684ae3ced35122e9aee12a584fd2d6b7e0272c227ab94eb4e607ec", + "loss_nats": 1.2868822813034058, + "mixer_summary_sha256": "998a9ce6e3ee4cab0180aeb81cc12c18ccf6b1832b43c8f455133227a7e72a26" + }, + "loss_nats": 1.2868822813034058, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.21598131954669952, + "group": 1, + "latest_source_mass": 0.061274416744709015, + "layer": 1, + "max_source_mass": 0.9387255907058716, + "mixer_index": 1, + "normalized_entropy": 0.31159517863467795, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9387255907058716, + "median": 0.9528161883354187, + "p05": 0.8532108068466187, + "p95": 0.9815421104431152 + }, + { + "label": "current_group_partial", + "mean": 0.061274416744709015, + "median": 0.047183819115161896, + "p05": 0.018457919359207153, + "p95": 0.14678917825222015 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5377621054649353, + "group": 1, + "latest_source_mass": 0.2917928099632263, + "layer": 2, + "max_source_mass": 0.7082071900367737, + "mixer_index": 2, + "normalized_entropy": 0.7758267227322699, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7082071900367737, + "median": 0.7268136143684387, + "p05": 0.38468095660209656, + "p95": 0.923936128616333 + }, + { + "label": "current_group_partial", + "mean": 0.2917928099632263, + "median": 0.27318641543388367, + "p05": 0.07606390863656998, + "p95": 0.6153190732002258 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.43774232268333435, + "group": 1, + "latest_source_mass": 0.170285165309906, + "layer": 2, + "max_source_mass": 0.829714834690094, + "mixer_index": 3, + "normalized_entropy": 0.6315286781224629, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.829714834690094, + "median": 0.85542893409729, + "p05": 0.687734842300415, + "p95": 0.9260365962982178 + }, + { + "label": "current_group_partial", + "mean": 0.170285165309906, + "median": 0.14457105100154877, + "p05": 0.07396337389945984, + "p95": 0.31226515769958496 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6749181151390076, + "group": 1, + "latest_source_mass": 0.45927006006240845, + "layer": 3, + "max_source_mass": 0.5407299995422363, + "mixer_index": 4, + "normalized_entropy": 0.9737010177171727, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5407299995422363, + "median": 0.5310744643211365, + "p05": 0.4232047200202942, + "p95": 0.6916534900665283 + }, + { + "label": "current_group_partial", + "mean": 0.45927006006240845, + "median": 0.4689255356788635, + "p05": 0.3083464205265045, + "p95": 0.576795220375061 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6663151383399963, + "group": 1, + "latest_source_mass": 0.49885618686676025, + "layer": 3, + "max_source_mass": 0.5011438131332397, + "mixer_index": 5, + "normalized_entropy": 0.9612895457523564, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5011438131332397, + "median": 0.4804786741733551, + "p05": 0.3554112911224365, + "p95": 0.7207092642784119 + }, + { + "label": "current_group_partial", + "mean": 0.49885618686676025, + "median": 0.5195213556289673, + "p05": 0.27929070591926575, + "p95": 0.6445887684822083 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6865031719207764, + "group": 1, + "latest_source_mass": 0.4691343307495117, + "layer": 4, + "max_source_mass": 0.5308656692504883, + "mixer_index": 6, + "normalized_entropy": 0.9904147216846476, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5308656692504883, + "median": 0.5278033018112183, + "p05": 0.45786383748054504, + "p95": 0.6093498468399048 + }, + { + "label": "current_group_partial", + "mean": 0.4691343307495117, + "median": 0.47219666838645935, + "p05": 0.3906501829624176, + "p95": 0.5421361327171326 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4133813679218292, + "group": 1, + "latest_source_mass": 0.15208598971366882, + "layer": 4, + "max_source_mass": 0.8479140400886536, + "mixer_index": 7, + "normalized_entropy": 0.5963832494967191, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8479140400886536, + "median": 0.8514594435691833, + "p05": 0.7530128359794617, + "p95": 0.9402707815170288 + }, + { + "label": "current_group_partial", + "mean": 0.15208598971366882, + "median": 0.14854055643081665, + "p05": 0.05972917377948761, + "p95": 0.24698717892169952 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6850745677947998, + "group": 2, + "latest_source_mass": 0.4601486325263977, + "layer": 5, + "max_source_mass": 0.5398513674736023, + "mixer_index": 8, + "normalized_entropy": 0.9883536815967077, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5398513674736023, + "median": 0.5387102365493774, + "p05": 0.46027374267578125, + "p95": 0.623892605304718 + }, + { + "label": "completed_group_1", + "mean": 0.4601486325263977, + "median": 0.46128979325294495, + "p05": 0.376107394695282, + "p95": 0.5397262573242188 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074119567871094, + "group": 2, + "latest_source_mass": 0.4627366364002228, + "layer": 5, + "max_source_mass": 0.4627366364002228, + "mixer_index": 9, + "normalized_entropy": 0.9169858804405273, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32744842767715454, + "median": 0.30657994747161865, + "p05": 0.16577640175819397, + "p95": 0.5453717708587646 + }, + { + "label": "completed_group_1", + "mean": 0.20981493592262268, + "median": 0.2177819311618805, + "p05": 0.14342713356018066, + "p95": 0.2433384209871292 + }, + { + "label": "current_group_partial", + "mean": 0.4627366364002228, + "median": 0.4847465753555298, + "p05": 0.27109938859939575, + "p95": 0.6081984639167786 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0489801168441772, + "group": 2, + "latest_source_mass": 0.4763939380645752, + "layer": 6, + "max_source_mass": 0.4763939380645752, + "mixer_index": 10, + "normalized_entropy": 0.9548228503031734, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28095996379852295, + "median": 0.2765299677848816, + "p05": 0.23549969494342804, + "p95": 0.3424558639526367 + }, + { + "label": "completed_group_1", + "mean": 0.24264612793922424, + "median": 0.23937124013900757, + "p05": 0.21875721216201782, + "p95": 0.2814398407936096 + }, + { + "label": "current_group_partial", + "mean": 0.4763939380645752, + "median": 0.48882943391799927, + "p05": 0.38478636741638184, + "p95": 0.528400719165802 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.8527575731277466, + "group": 2, + "latest_source_mass": 0.6454645395278931, + "layer": 6, + "max_source_mass": 0.6454645395278931, + "mixer_index": 11, + "normalized_entropy": 0.7762133938639787, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20522089302539825, + "median": 0.1591135561466217, + "p05": 0.11052663624286652, + "p95": 0.47355735301971436 + }, + { + "label": "completed_group_1", + "mean": 0.1493145078420639, + "median": 0.15203402936458588, + "p05": 0.10004506260156631, + "p95": 0.18058395385742188 + }, + { + "label": "current_group_partial", + "mean": 0.6454645395278931, + "median": 0.68653404712677, + "p05": 0.4207671880722046, + "p95": 0.7333818674087524 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.049942135810852, + "group": 2, + "latest_source_mass": 0.4395729899406433, + "layer": 7, + "max_source_mass": 0.4395729899406433, + "mixer_index": 12, + "normalized_entropy": 0.9556985177033998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35788866877555847, + "median": 0.36233147978782654, + "p05": 0.30507102608680725, + "p95": 0.3985886573791504 + }, + { + "label": "completed_group_1", + "mean": 0.20253834128379822, + "median": 0.19928529858589172, + "p05": 0.180739626288414, + "p95": 0.23214691877365112 + }, + { + "label": "current_group_partial", + "mean": 0.4395729899406433, + "median": 0.43975552916526794, + "p05": 0.3959323763847351, + "p95": 0.4790603220462799 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9791408181190491, + "group": 2, + "latest_source_mass": 0.5214842557907104, + "layer": 7, + "max_source_mass": 0.5214842557907104, + "mixer_index": 13, + "normalized_entropy": 0.8912523810434521, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31465935707092285, + "median": 0.28992873430252075, + "p05": 0.20534247159957886, + "p95": 0.507034420967102 + }, + { + "label": "completed_group_1", + "mean": 0.1638563871383667, + "median": 0.1667816936969757, + "p05": 0.11925681680440903, + "p95": 0.19515612721443176 + }, + { + "label": "current_group_partial", + "mean": 0.5214842557907104, + "median": 0.5455081462860107, + "p05": 0.35950881242752075, + "p95": 0.6097541451454163 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0485806465148926, + "group": 2, + "latest_source_mass": 0.4457142949104309, + "layer": 8, + "max_source_mass": 0.4457142949104309, + "mixer_index": 14, + "normalized_entropy": 0.9544592367395849, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.346283882856369, + "median": 0.353412926197052, + "p05": 0.26436588168144226, + "p95": 0.4084087312221527 + }, + { + "label": "completed_group_1", + "mean": 0.20800183713436127, + "median": 0.20826736092567444, + "p05": 0.18089595437049866, + "p95": 0.23509056866168976 + }, + { + "label": "current_group_partial", + "mean": 0.4457142949104309, + "median": 0.4416901469230652, + "p05": 0.3883572816848755, + "p95": 0.5136083364486694 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9801348447799683, + "group": 2, + "latest_source_mass": 0.543371319770813, + "layer": 8, + "max_source_mass": 0.543371319770813, + "mixer_index": 15, + "normalized_entropy": 0.8921571831025336, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2748759388923645, + "median": 0.2577607035636902, + "p05": 0.17991375923156738, + "p95": 0.41774845123291016 + }, + { + "label": "completed_group_1", + "mean": 0.1817527413368225, + "median": 0.17968356609344482, + "p05": 0.130966454744339, + "p95": 0.23766928911209106 + }, + { + "label": "current_group_partial", + "mean": 0.543371319770813, + "median": 0.5535831451416016, + "p05": 0.4452384412288666, + "p95": 0.6142411828041077 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0688585042953491, + "group": 3, + "latest_source_mass": 0.36256325244903564, + "layer": 9, + "max_source_mass": 0.4051227569580078, + "mixer_index": 16, + "normalized_entropy": 0.9729169383233166, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4051227569580078, + "median": 0.4008670449256897, + "p05": 0.3286888897418976, + "p95": 0.48389261960983276 + }, + { + "label": "completed_group_1", + "mean": 0.23231396079063416, + "median": 0.23353952169418335, + "p05": 0.2008797824382782, + "p95": 0.2593720257282257 + }, + { + "label": "completed_group_2", + "mean": 0.36256325244903564, + "median": 0.3638863265514374, + "p05": 0.3081618845462799, + "p95": 0.4163197875022888 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2915732860565186, + "group": 3, + "latest_source_mass": 0.36661261320114136, + "layer": 9, + "max_source_mass": 0.36661261320114136, + "mixer_index": 17, + "normalized_entropy": 0.931673187369201, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.26621371507644653, + "median": 0.23494771122932434, + "p05": 0.16815711557865143, + "p95": 0.4694361090660095 + }, + { + "label": "completed_group_1", + "mean": 0.1108347550034523, + "median": 0.11330658197402954, + "p05": 0.06620892137289047, + "p95": 0.13892118632793427 + }, + { + "label": "completed_group_2", + "mean": 0.2563389539718628, + "median": 0.26728466153144836, + "p05": 0.16010139882564545, + "p95": 0.3091832995414734 + }, + { + "label": "current_group_partial", + "mean": 0.36661261320114136, + "median": 0.36950358748435974, + "p05": 0.29210716485977173, + "p95": 0.42882072925567627 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3568191528320312, + "group": 3, + "latest_source_mass": 0.30908840894699097, + "layer": 10, + "max_source_mass": 0.30908840894699097, + "mixer_index": 18, + "normalized_entropy": 0.978738131586968, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.294538676738739, + "median": 0.2961598038673401, + "p05": 0.2467859834432602, + "p95": 0.34424129128456116 + }, + { + "label": "completed_group_1", + "mean": 0.1710844486951828, + "median": 0.17187124490737915, + "p05": 0.1508811116218567, + "p95": 0.1898101568222046 + }, + { + "label": "completed_group_2", + "mean": 0.2252884805202484, + "median": 0.22875282168388367, + "p05": 0.18443678319454193, + "p95": 0.25181543827056885 + }, + { + "label": "current_group_partial", + "mean": 0.30908840894699097, + "median": 0.3080518841743469, + "p05": 0.2719307243824005, + "p95": 0.3494151830673218 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.320021152496338, + "group": 3, + "latest_source_mass": 0.3312946557998657, + "layer": 10, + "max_source_mass": 0.3312946557998657, + "mixer_index": 19, + "normalized_entropy": 0.9521939852875004, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30465462803840637, + "median": 0.311603307723999, + "p05": 0.21955373883247375, + "p95": 0.40013614296913147 + }, + { + "label": "completed_group_1", + "mean": 0.12911874055862427, + "median": 0.12981033325195312, + "p05": 0.09804825484752655, + "p95": 0.15542283654212952 + }, + { + "label": "completed_group_2", + "mean": 0.23493197560310364, + "median": 0.24103470146656036, + "p05": 0.16810482740402222, + "p95": 0.28502923250198364 + }, + { + "label": "current_group_partial", + "mean": 0.3312946557998657, + "median": 0.325168639421463, + "p05": 0.2647673785686493, + "p95": 0.41786038875579834 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3578683137893677, + "group": 3, + "latest_source_mass": 0.30723780393600464, + "layer": 11, + "max_source_mass": 0.30723780393600464, + "mixer_index": 20, + "normalized_entropy": 0.9794949412420898, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29558902978897095, + "median": 0.2938143014907837, + "p05": 0.24335844814777374, + "p95": 0.3542291522026062 + }, + { + "label": "completed_group_1", + "mean": 0.17812997102737427, + "median": 0.17726755142211914, + "p05": 0.15839283168315887, + "p95": 0.20142368972301483 + }, + { + "label": "completed_group_2", + "mean": 0.21904319524765015, + "median": 0.2218058705329895, + "p05": 0.18076662719249725, + "p95": 0.24698719382286072 + }, + { + "label": "current_group_partial", + "mean": 0.30723780393600464, + "median": 0.30536961555480957, + "p05": 0.26194074749946594, + "p95": 0.35952210426330566 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.258887767791748, + "group": 3, + "latest_source_mass": 0.34303945302963257, + "layer": 11, + "max_source_mass": 0.35744374990463257, + "mixer_index": 21, + "normalized_entropy": 0.9080955698144659, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35744374990463257, + "median": 0.359974205493927, + "p05": 0.17910680174827576, + "p95": 0.5207242965698242 + }, + { + "label": "completed_group_1", + "mean": 0.12067858874797821, + "median": 0.11769672483205795, + "p05": 0.08151324093341827, + "p95": 0.16476252675056458 + }, + { + "label": "completed_group_2", + "mean": 0.17883820831775665, + "median": 0.17648206651210785, + "p05": 0.0995911955833435, + "p95": 0.2739725708961487 + }, + { + "label": "current_group_partial", + "mean": 0.34303945302963257, + "median": 0.33267199993133545, + "p05": 0.22800606489181519, + "p95": 0.48691368103027344 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3539760112762451, + "group": 3, + "latest_source_mass": 0.3175881803035736, + "layer": 12, + "max_source_mass": 0.3175881803035736, + "mixer_index": 22, + "normalized_entropy": 0.976687238475429, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2736307978630066, + "median": 0.27999642491340637, + "p05": 0.18969643115997314, + "p95": 0.3464604914188385 + }, + { + "label": "completed_group_1", + "mean": 0.1722733974456787, + "median": 0.17083553969860077, + "p05": 0.1506597101688385, + "p95": 0.1988438367843628 + }, + { + "label": "completed_group_2", + "mean": 0.2365075945854187, + "median": 0.24049004912376404, + "p05": 0.19099700450897217, + "p95": 0.26942259073257446 + }, + { + "label": "current_group_partial", + "mean": 0.3175881803035736, + "median": 0.3092706799507141, + "p05": 0.268545538187027, + "p95": 0.3924510180950165 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3203014135360718, + "group": 3, + "latest_source_mass": 0.3395293354988098, + "layer": 12, + "max_source_mass": 0.3395293354988098, + "mixer_index": 23, + "normalized_entropy": 0.9523961508935896, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28207293152809143, + "median": 0.29242122173309326, + "p05": 0.1354818046092987, + "p95": 0.4195851981639862 + }, + { + "label": "completed_group_1", + "mean": 0.15420743823051453, + "median": 0.15396761894226074, + "p05": 0.11267110705375671, + "p95": 0.1951446533203125 + }, + { + "label": "completed_group_2", + "mean": 0.22419027984142303, + "median": 0.22542357444763184, + "p05": 0.1655150204896927, + "p95": 0.2747117578983307 + }, + { + "label": "current_group_partial", + "mean": 0.3395293354988098, + "median": 0.32681703567504883, + "p05": 0.2524653375148773, + "p95": 0.46652013063430786 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3734662532806396, + "group": 4, + "latest_source_mass": 0.283242404460907, + "layer": 13, + "max_source_mass": 0.283242404460907, + "mixer_index": 24, + "normalized_entropy": 0.9907464762181619, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2766023874282837, + "median": 0.282479852437973, + "p05": 0.22528076171875, + "p95": 0.3097468614578247 + }, + { + "label": "completed_group_1", + "mean": 0.21176087856292725, + "median": 0.2058793306350708, + "p05": 0.18811854720115662, + "p95": 0.2617422640323639 + }, + { + "label": "completed_group_2", + "mean": 0.22839435935020447, + "median": 0.22951346635818481, + "p05": 0.20004916191101074, + "p95": 0.2551718056201935 + }, + { + "label": "completed_group_3", + "mean": 0.283242404460907, + "median": 0.2849352955818176, + "p05": 0.2205706238746643, + "p95": 0.34444743394851685 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5325813293457031, + "group": 4, + "latest_source_mass": 0.27591708302497864, + "layer": 13, + "max_source_mass": 0.27591708302497864, + "mixer_index": 25, + "normalized_entropy": 0.9522463199762954, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17451074719429016, + "median": 0.16826045513153076, + "p05": 0.08149556815624237, + "p95": 0.3126368522644043 + }, + { + "label": "completed_group_1", + "mean": 0.10643941164016724, + "median": 0.10269765555858612, + "p05": 0.08178127557039261, + "p95": 0.13947664201259613 + }, + { + "label": "completed_group_2", + "mean": 0.1691833734512329, + "median": 0.17118802666664124, + "p05": 0.12382284551858902, + "p95": 0.2069271057844162 + }, + { + "label": "completed_group_3", + "mean": 0.27394935488700867, + "median": 0.27278125286102295, + "p05": 0.2122008204460144, + "p95": 0.33945947885513306 + }, + { + "label": "current_group_partial", + "mean": 0.27591708302497864, + "median": 0.27714359760284424, + "p05": 0.21523593366146088, + "p95": 0.33170604705810547 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.560360074043274, + "group": 4, + "latest_source_mass": 0.3255118429660797, + "layer": 14, + "max_source_mass": 0.3255118429660797, + "mixer_index": 26, + "normalized_entropy": 0.9695062244951087, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19377154111862183, + "median": 0.19472965598106384, + "p05": 0.17005598545074463, + "p95": 0.22297024726867676 + }, + { + "label": "completed_group_1", + "mean": 0.17211595177650452, + "median": 0.17136064171791077, + "p05": 0.1543424427509308, + "p95": 0.1928609162569046 + }, + { + "label": "completed_group_2", + "mean": 0.14605136215686798, + "median": 0.14629822969436646, + "p05": 0.13246256113052368, + "p95": 0.15914371609687805 + }, + { + "label": "completed_group_3", + "mean": 0.16254930198192596, + "median": 0.16272446513175964, + "p05": 0.14075018465518951, + "p95": 0.18463709950447083 + }, + { + "label": "current_group_partial", + "mean": 0.3255118429660797, + "median": 0.3261105418205261, + "p05": 0.3035835921764374, + "p95": 0.3457673192024231 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.540909767150879, + "group": 4, + "latest_source_mass": 0.2995486855506897, + "layer": 14, + "max_source_mass": 0.2995486855506897, + "mixer_index": 27, + "normalized_entropy": 0.9574210693349581, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21921303868293762, + "median": 0.21722173690795898, + "p05": 0.1413341462612152, + "p95": 0.2906968593597412 + }, + { + "label": "completed_group_1", + "mean": 0.10205383598804474, + "median": 0.09984464198350906, + "p05": 0.07455356419086456, + "p95": 0.13362759351730347 + }, + { + "label": "completed_group_2", + "mean": 0.16381604969501495, + "median": 0.16600549221038818, + "p05": 0.11865930259227753, + "p95": 0.2019941210746765 + }, + { + "label": "completed_group_3", + "mean": 0.215368390083313, + "median": 0.2136734575033188, + "p05": 0.1728287935256958, + "p95": 0.2628125846385956 + }, + { + "label": "current_group_partial", + "mean": 0.2995486855506897, + "median": 0.2982075810432434, + "p05": 0.23863884806632996, + "p95": 0.3662579655647278 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5142419338226318, + "group": 4, + "latest_source_mass": 0.369165301322937, + "layer": 15, + "max_source_mass": 0.369165301322937, + "mixer_index": 28, + "normalized_entropy": 0.940851412859105, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2000826597213745, + "median": 0.20369741320610046, + "p05": 0.15802279114723206, + "p95": 0.24072687327861786 + }, + { + "label": "completed_group_1", + "mean": 0.15480534732341766, + "median": 0.15228059887886047, + "p05": 0.13543802499771118, + "p95": 0.1815909892320633 + }, + { + "label": "completed_group_2", + "mean": 0.10827909409999847, + "median": 0.10761409997940063, + "p05": 0.09670238941907883, + "p95": 0.12229038774967194 + }, + { + "label": "completed_group_3", + "mean": 0.16766759753227234, + "median": 0.16637766361236572, + "p05": 0.1398160457611084, + "p95": 0.20042110979557037 + }, + { + "label": "current_group_partial", + "mean": 0.369165301322937, + "median": 0.3697334825992584, + "p05": 0.3279486298561096, + "p95": 0.40613043308258057 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5333527326583862, + "group": 4, + "latest_source_mass": 0.29974091053009033, + "layer": 15, + "max_source_mass": 0.29974091053009033, + "mixer_index": 29, + "normalized_entropy": 0.9527256198031004, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21770668029785156, + "median": 0.19699469208717346, + "p05": 0.1311802864074707, + "p95": 0.3720983564853668 + }, + { + "label": "completed_group_1", + "mean": 0.10525081306695938, + "median": 0.10493603348731995, + "p05": 0.08330865204334259, + "p95": 0.12839283049106598 + }, + { + "label": "completed_group_2", + "mean": 0.15970060229301453, + "median": 0.16236470639705658, + "p05": 0.11309029906988144, + "p95": 0.19861777126789093 + }, + { + "label": "completed_group_3", + "mean": 0.217601016163826, + "median": 0.21904586255550385, + "p05": 0.16268108785152435, + "p95": 0.2699928283691406 + }, + { + "label": "current_group_partial", + "mean": 0.29974091053009033, + "median": 0.30079299211502075, + "p05": 0.22156940400600433, + "p95": 0.37281978130340576 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.512608289718628, + "group": 4, + "latest_source_mass": 0.35851576924324036, + "layer": 16, + "max_source_mass": 0.35851576924324036, + "mixer_index": 30, + "normalized_entropy": 0.9398363727066501, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21392285823822021, + "median": 0.22146853804588318, + "p05": 0.1518818885087967, + "p95": 0.25874194502830505 + }, + { + "label": "completed_group_1", + "mean": 0.1860550194978714, + "median": 0.1838144212961197, + "p05": 0.1660098135471344, + "p95": 0.21311265230178833 + }, + { + "label": "completed_group_2", + "mean": 0.10291537642478943, + "median": 0.1024642288684845, + "p05": 0.09207107126712799, + "p95": 0.11504144221544266 + }, + { + "label": "completed_group_3", + "mean": 0.1385909616947174, + "median": 0.13646918535232544, + "p05": 0.11220520734786987, + "p95": 0.1728362739086151 + }, + { + "label": "current_group_partial", + "mean": 0.35851576924324036, + "median": 0.3558155298233032, + "p05": 0.31809645891189575, + "p95": 0.40622422099113464 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5262645483016968, + "group": 4, + "latest_source_mass": 0.3142552077770233, + "layer": 16, + "max_source_mass": 0.3142552077770233, + "mixer_index": 31, + "normalized_entropy": 0.9483214832396903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20246261358261108, + "median": 0.18459102511405945, + "p05": 0.12276223301887512, + "p95": 0.38892465829849243 + }, + { + "label": "completed_group_1", + "mean": 0.10530521720647812, + "median": 0.10532709956169128, + "p05": 0.0785030871629715, + "p95": 0.1342671513557434 + }, + { + "label": "completed_group_2", + "mean": 0.16650916635990143, + "median": 0.17111444473266602, + "p05": 0.1085265576839447, + "p95": 0.2094469517469406 + }, + { + "label": "completed_group_3", + "mean": 0.21146778762340546, + "median": 0.2099919468164444, + "p05": 0.15801574289798737, + "p95": 0.26882117986679077 + }, + { + "label": "current_group_partial", + "mean": 0.3142552077770233, + "median": 0.31609731912612915, + "p05": 0.22627028822898865, + "p95": 0.397154837846756 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5794764757156372, + "group": 5, + "latest_source_mass": 0.2760102152824402, + "layer": 17, + "max_source_mass": 0.2760102152824402, + "mixer_index": 32, + "normalized_entropy": 0.9813839126772218, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2210189402103424, + "median": 0.22435861825942993, + "p05": 0.17291823029518127, + "p95": 0.262204647064209 + }, + { + "label": "completed_group_1", + "mean": 0.18839938938617706, + "median": 0.18652752041816711, + "p05": 0.16997642815113068, + "p95": 0.2126670628786087 + }, + { + "label": "completed_group_2", + "mean": 0.14016608893871307, + "median": 0.14067986607551575, + "p05": 0.1265786588191986, + "p95": 0.15091627836227417 + }, + { + "label": "completed_group_3", + "mean": 0.17440536618232727, + "median": 0.17192018032073975, + "p05": 0.15062645077705383, + "p95": 0.20691406726837158 + }, + { + "label": "completed_group_4", + "mean": 0.2760102152824402, + "median": 0.27511972188949585, + "p05": 0.24412119388580322, + "p95": 0.30901142954826355 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6602650880813599, + "group": 5, + "latest_source_mass": 0.3575916588306427, + "layer": 17, + "max_source_mass": 0.3575916588306427, + "mixer_index": 33, + "normalized_entropy": 0.9266115885502495, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12695768475532532, + "median": 0.11564654856920242, + "p05": 0.07159632444381714, + "p95": 0.21949690580368042 + }, + { + "label": "completed_group_1", + "mean": 0.08036886155605316, + "median": 0.07964898645877838, + "p05": 0.06650256365537643, + "p95": 0.09638062119483948 + }, + { + "label": "completed_group_2", + "mean": 0.12672865390777588, + "median": 0.1287270039319992, + "p05": 0.09939455986022949, + "p95": 0.14712141454219818 + }, + { + "label": "completed_group_3", + "mean": 0.15391482412815094, + "median": 0.15372109413146973, + "p05": 0.13333335518836975, + "p95": 0.17581847310066223 + }, + { + "label": "completed_group_4", + "mean": 0.154438316822052, + "median": 0.15402798354625702, + "p05": 0.12481103092432022, + "p95": 0.18498417735099792 + }, + { + "label": "current_group_partial", + "mean": 0.3575916588306427, + "median": 0.36001941561698914, + "p05": 0.3110165596008301, + "p95": 0.39699214696884155 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5925545692443848, + "group": 5, + "latest_source_mass": 0.41908538341522217, + "layer": 18, + "max_source_mass": 0.41908538341522217, + "mixer_index": 34, + "normalized_entropy": 0.8888216284580353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14075040817260742, + "median": 0.1455841362476349, + "p05": 0.09744156897068024, + "p95": 0.1818566620349884 + }, + { + "label": "completed_group_1", + "mean": 0.11028306186199188, + "median": 0.10887216031551361, + "p05": 0.09629634767770767, + "p95": 0.12874113023281097 + }, + { + "label": "completed_group_2", + "mean": 0.07537610828876495, + "median": 0.07517752051353455, + "p05": 0.0666593387722969, + "p95": 0.08491044491529465 + }, + { + "label": "completed_group_3", + "mean": 0.10244421660900116, + "median": 0.10141433775424957, + "p05": 0.08653581142425537, + "p95": 0.12175601720809937 + }, + { + "label": "completed_group_4", + "mean": 0.1520608365535736, + "median": 0.1513972282409668, + "p05": 0.1301831305027008, + "p95": 0.17589205503463745 + }, + { + "label": "current_group_partial", + "mean": 0.41908538341522217, + "median": 0.4191858768463135, + "p05": 0.3649080991744995, + "p95": 0.4721684455871582 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5918848514556885, + "group": 5, + "latest_source_mass": 0.4151553511619568, + "layer": 18, + "max_source_mass": 0.4151553511619568, + "mixer_index": 35, + "normalized_entropy": 0.8884478518433735, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11773698031902313, + "median": 0.10857899487018585, + "p05": 0.06324320286512375, + "p95": 0.22339528799057007 + }, + { + "label": "completed_group_1", + "mean": 0.07344591617584229, + "median": 0.07228962332010269, + "p05": 0.05982384458184242, + "p95": 0.09060496836900711 + }, + { + "label": "completed_group_2", + "mean": 0.11976176500320435, + "median": 0.12151201069355011, + "p05": 0.08953659236431122, + "p95": 0.14441059529781342 + }, + { + "label": "completed_group_3", + "mean": 0.14735516905784607, + "median": 0.14620742201805115, + "p05": 0.11767779290676117, + "p95": 0.17902395129203796 + }, + { + "label": "completed_group_4", + "mean": 0.126544788479805, + "median": 0.12564796209335327, + "p05": 0.09858805686235428, + "p95": 0.1567486822605133 + }, + { + "label": "current_group_partial", + "mean": 0.4151553511619568, + "median": 0.4184337854385376, + "p05": 0.3375611901283264, + "p95": 0.4846380352973938 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6208878755569458, + "group": 5, + "latest_source_mass": 0.3991435170173645, + "layer": 19, + "max_source_mass": 0.3991435170173645, + "mixer_index": 36, + "normalized_entropy": 0.9046347477964072, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14785674214363098, + "median": 0.15170978009700775, + "p05": 0.09146977216005325, + "p95": 0.20172694325447083 + }, + { + "label": "completed_group_1", + "mean": 0.12492769956588745, + "median": 0.12345598638057709, + "p05": 0.10790181905031204, + "p95": 0.14623555541038513 + }, + { + "label": "completed_group_2", + "mean": 0.08781818300485611, + "median": 0.08839264512062073, + "p05": 0.07453933358192444, + "p95": 0.09894395619630814 + }, + { + "label": "completed_group_3", + "mean": 0.10019385814666748, + "median": 0.09901996701955795, + "p05": 0.08430047333240509, + "p95": 0.12013883143663406 + }, + { + "label": "completed_group_4", + "mean": 0.14006000757217407, + "median": 0.13918349146842957, + "p05": 0.11816112697124481, + "p95": 0.16475799679756165 + }, + { + "label": "current_group_partial", + "mean": 0.3991435170173645, + "median": 0.39726996421813965, + "p05": 0.34116071462631226, + "p95": 0.4634247422218323 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5470075607299805, + "group": 5, + "latest_source_mass": 0.44330859184265137, + "layer": 19, + "max_source_mass": 0.44330859184265137, + "mixer_index": 37, + "normalized_entropy": 0.8634013589985261, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12043030560016632, + "median": 0.1012125313282013, + "p05": 0.061379045248031616, + "p95": 0.25343188643455505 + }, + { + "label": "completed_group_1", + "mean": 0.06855648756027222, + "median": 0.06761305779218674, + "p05": 0.05337192490696907, + "p95": 0.08790726959705353 + }, + { + "label": "completed_group_2", + "mean": 0.11753211170434952, + "median": 0.1186976358294487, + "p05": 0.08412563800811768, + "p95": 0.14895018935203552 + }, + { + "label": "completed_group_3", + "mean": 0.1339646875858307, + "median": 0.13253431022167206, + "p05": 0.09857159107923508, + "p95": 0.1730121374130249 + }, + { + "label": "completed_group_4", + "mean": 0.1162077933549881, + "median": 0.11592038720846176, + "p05": 0.08609852194786072, + "p95": 0.14767511188983917 + }, + { + "label": "current_group_partial", + "mean": 0.44330859184265137, + "median": 0.4434061646461487, + "p05": 0.3389148414134979, + "p95": 0.5490255355834961 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.683492660522461, + "group": 5, + "latest_source_mass": 0.3365415334701538, + "layer": 20, + "max_source_mass": 0.3365415334701538, + "mixer_index": 38, + "normalized_entropy": 0.9395751435586169, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1658952683210373, + "median": 0.1726911962032318, + "p05": 0.10183650255203247, + "p95": 0.22522984445095062 + }, + { + "label": "completed_group_1", + "mean": 0.1435280442237854, + "median": 0.14210543036460876, + "p05": 0.12450659275054932, + "p95": 0.16782894730567932 + }, + { + "label": "completed_group_2", + "mean": 0.09461991488933563, + "median": 0.0948714017868042, + "p05": 0.08168491721153259, + "p95": 0.10723742842674255 + }, + { + "label": "completed_group_3", + "mean": 0.10784181207418442, + "median": 0.10693192481994629, + "p05": 0.08930590003728867, + "p95": 0.12958656251430511 + }, + { + "label": "completed_group_4", + "mean": 0.15157344937324524, + "median": 0.15053030848503113, + "p05": 0.128616601228714, + "p95": 0.17799487709999084 + }, + { + "label": "current_group_partial", + "mean": 0.3365415334701538, + "median": 0.33349889516830444, + "p05": 0.2778944969177246, + "p95": 0.40724658966064453 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5540567636489868, + "group": 5, + "latest_source_mass": 0.42266085743904114, + "layer": 20, + "max_source_mass": 0.42266085743904114, + "mixer_index": 39, + "normalized_entropy": 0.8673355940563396, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12987135350704193, + "median": 0.1083056628704071, + "p05": 0.03783264383673668, + "p95": 0.3301126956939697 + }, + { + "label": "completed_group_1", + "mean": 0.07178181409835815, + "median": 0.07081792503595352, + "p05": 0.05334944650530815, + "p95": 0.09311570972204208 + }, + { + "label": "completed_group_2", + "mean": 0.12132491171360016, + "median": 0.12282539904117584, + "p05": 0.07926258444786072, + "p95": 0.15763431787490845 + }, + { + "label": "completed_group_3", + "mean": 0.1441602110862732, + "median": 0.14273977279663086, + "p05": 0.09847753494977951, + "p95": 0.19304539263248444 + }, + { + "label": "completed_group_4", + "mean": 0.11020088195800781, + "median": 0.10934801399707794, + "p05": 0.07139573246240616, + "p95": 0.1517774760723114 + }, + { + "label": "current_group_partial", + "mean": 0.42266085743904114, + "median": 0.42266106605529785, + "p05": 0.28481048345565796, + "p95": 0.5570361018180847 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7208337783813477, + "group": 6, + "latest_source_mass": 0.27070939540863037, + "layer": 21, + "max_source_mass": 0.27070939540863037, + "mixer_index": 40, + "normalized_entropy": 0.9604156182429642, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18048501014709473, + "median": 0.18919861316680908, + "p05": 0.11199048161506653, + "p95": 0.23807701468467712 + }, + { + "label": "completed_group_1", + "mean": 0.1685236394405365, + "median": 0.1669977903366089, + "p05": 0.14678362011909485, + "p95": 0.19381441175937653 + }, + { + "label": "completed_group_2", + "mean": 0.08726631104946136, + "median": 0.08729851245880127, + "p05": 0.07629868388175964, + "p95": 0.0980045348405838 + }, + { + "label": "completed_group_3", + "mean": 0.11160479485988617, + "median": 0.1099197044968605, + "p05": 0.09190923720598221, + "p95": 0.13723208010196686 + }, + { + "label": "completed_group_4", + "mean": 0.18141084909439087, + "median": 0.18020854890346527, + "p05": 0.15656134486198425, + "p95": 0.2108723670244217 + }, + { + "label": "completed_group_5", + "mean": 0.27070939540863037, + "median": 0.2684924602508545, + "p05": 0.2159801721572876, + "p95": 0.33192309737205505 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6729483604431152, + "group": 6, + "latest_source_mass": 0.3803577423095703, + "layer": 21, + "max_source_mass": 0.3803577423095703, + "mixer_index": 41, + "normalized_entropy": 0.8597253893019091, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09229261428117752, + "median": 0.07798201590776443, + "p05": 0.04784023016691208, + "p95": 0.20285779237747192 + }, + { + "label": "completed_group_1", + "mean": 0.051420148462057114, + "median": 0.05041736364364624, + "p05": 0.040625106543302536, + "p95": 0.06547587364912033 + }, + { + "label": "completed_group_2", + "mean": 0.08079522103071213, + "median": 0.08136115223169327, + "p05": 0.05947277322411537, + "p95": 0.10017222166061401 + }, + { + "label": "completed_group_3", + "mean": 0.09550285339355469, + "median": 0.09445887804031372, + "p05": 0.07334083318710327, + "p95": 0.12144046276807785 + }, + { + "label": "completed_group_4", + "mean": 0.07590493559837341, + "median": 0.07492390275001526, + "p05": 0.055510398000478745, + "p95": 0.10084333270788193 + }, + { + "label": "completed_group_5", + "mean": 0.22372648119926453, + "median": 0.22069036960601807, + "p05": 0.1407623589038849, + "p95": 0.32290777564048767 + }, + { + "label": "current_group_partial", + "mean": 0.3803577423095703, + "median": 0.37903881072998047, + "p05": 0.2855539321899414, + "p95": 0.476589173078537 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.710235595703125, + "group": 6, + "latest_source_mass": 0.4011716842651367, + "layer": 22, + "max_source_mass": 0.4011716842651367, + "mixer_index": 42, + "normalized_entropy": 0.8788872376935791, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11657392978668213, + "median": 0.11877645552158356, + "p05": 0.06819256395101547, + "p95": 0.1684979349374771 + }, + { + "label": "completed_group_1", + "mean": 0.10617930442094803, + "median": 0.10509790480136871, + "p05": 0.090352863073349, + "p95": 0.12587188184261322 + }, + { + "label": "completed_group_2", + "mean": 0.061112720519304276, + "median": 0.061012350022792816, + "p05": 0.05164862796664238, + "p95": 0.07133162766695023 + }, + { + "label": "completed_group_3", + "mean": 0.06796726584434509, + "median": 0.06712733209133148, + "p05": 0.05444999784231186, + "p95": 0.08481802046298981 + }, + { + "label": "completed_group_4", + "mean": 0.103337861597538, + "median": 0.10192783921957016, + "p05": 0.0832459107041359, + "p95": 0.12805812060832977 + }, + { + "label": "completed_group_5", + "mean": 0.14365722239017487, + "median": 0.14159655570983887, + "p05": 0.10498540848493576, + "p95": 0.19020715355873108 + }, + { + "label": "current_group_partial", + "mean": 0.4011716842651367, + "median": 0.40084803104400635, + "p05": 0.32083970308303833, + "p95": 0.47994357347488403 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5640671253204346, + "group": 6, + "latest_source_mass": 0.4689176678657532, + "layer": 22, + "max_source_mass": 0.4689176678657532, + "mixer_index": 43, + "normalized_entropy": 0.8037715030571925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.090422622859478, + "median": 0.0774901732802391, + "p05": 0.04492149129509926, + "p95": 0.1988787055015564 + }, + { + "label": "completed_group_1", + "mean": 0.04561382159590721, + "median": 0.044634923338890076, + "p05": 0.03501313924789429, + "p95": 0.05900419130921364 + }, + { + "label": "completed_group_2", + "mean": 0.07565715163946152, + "median": 0.07550699263811111, + "p05": 0.053458306938409805, + "p95": 0.09821777045726776 + }, + { + "label": "completed_group_3", + "mean": 0.08878682553768158, + "median": 0.08692863583564758, + "p05": 0.06425619125366211, + "p95": 0.11807316541671753 + }, + { + "label": "completed_group_4", + "mean": 0.06555216014385223, + "median": 0.06414324045181274, + "p05": 0.04460844025015831, + "p95": 0.09090004861354828 + }, + { + "label": "completed_group_5", + "mean": 0.165049746632576, + "median": 0.15503892302513123, + "p05": 0.08368627727031708, + "p95": 0.2792147397994995 + }, + { + "label": "current_group_partial", + "mean": 0.4689176678657532, + "median": 0.46879667043685913, + "p05": 0.3151778280735016, + "p95": 0.6170428991317749 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7556228637695312, + "group": 6, + "latest_source_mass": 0.3706088662147522, + "layer": 23, + "max_source_mass": 0.3706088662147522, + "mixer_index": 44, + "normalized_entropy": 0.9022116795175967, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13281464576721191, + "median": 0.13658174872398376, + "p05": 0.08060654997825623, + "p95": 0.17402096092700958 + }, + { + "label": "completed_group_1", + "mean": 0.12175846099853516, + "median": 0.12066285312175751, + "p05": 0.10334118455648422, + "p95": 0.14384035766124725 + }, + { + "label": "completed_group_2", + "mean": 0.07235272228717804, + "median": 0.07218228280544281, + "p05": 0.059715576469898224, + "p95": 0.08584222197532654 + }, + { + "label": "completed_group_3", + "mean": 0.0707377940416336, + "median": 0.06942892074584961, + "p05": 0.05500924959778786, + "p95": 0.09052760899066925 + }, + { + "label": "completed_group_4", + "mean": 0.10551950335502625, + "median": 0.103996641933918, + "p05": 0.08408617228269577, + "p95": 0.13205930590629578 + }, + { + "label": "completed_group_5", + "mean": 0.12620802223682404, + "median": 0.12379512935876846, + "p05": 0.08937962353229523, + "p95": 0.17139491438865662 + }, + { + "label": "current_group_partial", + "mean": 0.3706088662147522, + "median": 0.3705657124519348, + "p05": 0.282781720161438, + "p95": 0.4561466872692108 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.594892978668213, + "group": 6, + "latest_source_mass": 0.4636557698249817, + "layer": 23, + "max_source_mass": 0.4636557698249817, + "mixer_index": 45, + "normalized_entropy": 0.8196128579947488, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10721131414175034, + "median": 0.10240983963012695, + "p05": 0.05563424155116081, + "p95": 0.18401460349559784 + }, + { + "label": "completed_group_1", + "mean": 0.053925663232803345, + "median": 0.0527028888463974, + "p05": 0.04038037359714508, + "p95": 0.07054222375154495 + }, + { + "label": "completed_group_2", + "mean": 0.07673212885856628, + "median": 0.07705660909414291, + "p05": 0.05211106687784195, + "p95": 0.10031480342149734 + }, + { + "label": "completed_group_3", + "mean": 0.08865120261907578, + "median": 0.08699032664299011, + "p05": 0.06242028623819351, + "p95": 0.11899784207344055 + }, + { + "label": "completed_group_4", + "mean": 0.06888366490602493, + "median": 0.06751501560211182, + "p05": 0.04784276708960533, + "p95": 0.09609999507665634 + }, + { + "label": "completed_group_5", + "mean": 0.14094024896621704, + "median": 0.13063763082027435, + "p05": 0.07297889143228531, + "p95": 0.24012833833694458 + }, + { + "label": "current_group_partial", + "mean": 0.4636557698249817, + "median": 0.46312135457992554, + "p05": 0.31259939074516296, + "p95": 0.6109440326690674 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7775511741638184, + "group": 6, + "latest_source_mass": 0.3569435477256775, + "layer": 24, + "max_source_mass": 0.3569435477256775, + "mixer_index": 46, + "normalized_entropy": 0.9134806018801903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13637229800224304, + "median": 0.13456633687019348, + "p05": 0.09586701542139053, + "p95": 0.18876218795776367 + }, + { + "label": "completed_group_1", + "mean": 0.10478455573320389, + "median": 0.10326151549816132, + "p05": 0.0880400538444519, + "p95": 0.12566551566123962 + }, + { + "label": "completed_group_2", + "mean": 0.07710614800453186, + "median": 0.07678486406803131, + "p05": 0.06292064487934113, + "p95": 0.09218747913837433 + }, + { + "label": "completed_group_3", + "mean": 0.08548048138618469, + "median": 0.08430111408233643, + "p05": 0.0665682703256607, + "p95": 0.10857507586479187 + }, + { + "label": "completed_group_4", + "mean": 0.11000140011310577, + "median": 0.10904969274997711, + "p05": 0.0881001353263855, + "p95": 0.13561299443244934 + }, + { + "label": "completed_group_5", + "mean": 0.12931162118911743, + "median": 0.12692990899085999, + "p05": 0.08648453652858734, + "p95": 0.1814313679933548 + }, + { + "label": "current_group_partial", + "mean": 0.3569435477256775, + "median": 0.3568228483200073, + "p05": 0.26841166615486145, + "p95": 0.4493965804576874 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.654036045074463, + "group": 6, + "latest_source_mass": 0.4265251159667969, + "layer": 24, + "max_source_mass": 0.4265251159667969, + "mixer_index": 47, + "normalized_entropy": 0.8500063817835848, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10232305526733398, + "median": 0.08623291552066803, + "p05": 0.04679499566555023, + "p95": 0.23077142238616943 + }, + { + "label": "completed_group_1", + "mean": 0.061968062072992325, + "median": 0.060845747590065, + "p05": 0.0479469932615757, + "p95": 0.07941868156194687 + }, + { + "label": "completed_group_2", + "mean": 0.08900615572929382, + "median": 0.08864577114582062, + "p05": 0.0641680359840393, + "p95": 0.11413820832967758 + }, + { + "label": "completed_group_3", + "mean": 0.1005433052778244, + "median": 0.09813467413187027, + "p05": 0.0731433779001236, + "p95": 0.1343638002872467 + }, + { + "label": "completed_group_4", + "mean": 0.07864196598529816, + "median": 0.07639941573143005, + "p05": 0.05487418174743652, + "p95": 0.10941869765520096 + }, + { + "label": "completed_group_5", + "mean": 0.14099235832691193, + "median": 0.12885531783103943, + "p05": 0.07315157353878021, + "p95": 0.24481485784053802 + }, + { + "label": "current_group_partial", + "mean": 0.4265251159667969, + "median": 0.42903727293014526, + "p05": 0.2803359627723694, + "p95": 0.5635722875595093 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.84956693649292, + "group": 7, + "latest_source_mass": 0.28775152564048767, + "layer": 25, + "max_source_mass": 0.28775152564048767, + "mixer_index": 48, + "normalized_entropy": 0.9504893827656096, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14851760864257812, + "median": 0.14813844859600067, + "p05": 0.10312555730342865, + "p95": 0.20065288245677948 + }, + { + "label": "completed_group_1", + "mean": 0.11630287766456604, + "median": 0.11500488221645355, + "p05": 0.09825178980827332, + "p95": 0.13750022649765015 + }, + { + "label": "completed_group_2", + "mean": 0.08512166142463684, + "median": 0.08458003401756287, + "p05": 0.0710662305355072, + "p95": 0.10092862695455551 + }, + { + "label": "completed_group_3", + "mean": 0.09215595573186874, + "median": 0.09079311043024063, + "p05": 0.07304700464010239, + "p95": 0.11515989899635315 + }, + { + "label": "completed_group_4", + "mean": 0.1263950914144516, + "median": 0.12563186883926392, + "p05": 0.10441911220550537, + "p95": 0.1519687920808792 + }, + { + "label": "completed_group_5", + "mean": 0.14375528693199158, + "median": 0.1422564834356308, + "p05": 0.10313796997070312, + "p95": 0.1929876059293747 + }, + { + "label": "completed_group_6", + "mean": 0.28775152564048767, + "median": 0.28676605224609375, + "p05": 0.21690022945404053, + "p95": 0.3624127507209778 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7161309719085693, + "group": 7, + "latest_source_mass": 0.32536065578460693, + "layer": 25, + "max_source_mass": 0.32536065578460693, + "mixer_index": 49, + "normalized_entropy": 0.8252845475628168, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0837959572672844, + "median": 0.0849243700504303, + "p05": 0.0271806251257658, + "p95": 0.15180703997612 + }, + { + "label": "completed_group_1", + "mean": 0.04434927552938461, + "median": 0.04302988201379776, + "p05": 0.03217232972383499, + "p95": 0.06002792343497276 + }, + { + "label": "completed_group_2", + "mean": 0.056708935648202896, + "median": 0.05563671886920929, + "p05": 0.03992946073412895, + "p95": 0.07731831818819046 + }, + { + "label": "completed_group_3", + "mean": 0.06495873630046844, + "median": 0.06223570555448532, + "p05": 0.04220382124185562, + "p95": 0.0949486792087555 + }, + { + "label": "completed_group_4", + "mean": 0.04885463789105415, + "median": 0.046898528933525085, + "p05": 0.030923232436180115, + "p95": 0.0743112787604332 + }, + { + "label": "completed_group_5", + "mean": 0.09997115284204483, + "median": 0.08650931715965271, + "p05": 0.04111029952764511, + "p95": 0.20292627811431885 + }, + { + "label": "completed_group_6", + "mean": 0.2760006785392761, + "median": 0.27142390608787537, + "p05": 0.13761089742183685, + "p95": 0.42605364322662354 + }, + { + "label": "current_group_partial", + "mean": 0.32536065578460693, + "median": 0.3348004221916199, + "p05": 0.10654139518737793, + "p95": 0.5236068964004517 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9285151958465576, + "group": 7, + "latest_source_mass": 0.2804902195930481, + "layer": 26, + "max_source_mass": 0.2804902195930481, + "mixer_index": 50, + "normalized_entropy": 0.9274197697756124, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10564712435007095, + "median": 0.10497278720140457, + "p05": 0.04608207941055298, + "p95": 0.16350959241390228 + }, + { + "label": "completed_group_1", + "mean": 0.09173665940761566, + "median": 0.09041133522987366, + "p05": 0.07509034872055054, + "p95": 0.11238400638103485 + }, + { + "label": "completed_group_2", + "mean": 0.07135222852230072, + "median": 0.06981296837329865, + "p05": 0.0522611029446125, + "p95": 0.09555281698703766 + }, + { + "label": "completed_group_3", + "mean": 0.07045470923185349, + "median": 0.06927657127380371, + "p05": 0.055566973984241486, + "p95": 0.08908769488334656 + }, + { + "label": "completed_group_4", + "mean": 0.09785081446170807, + "median": 0.0971219539642334, + "p05": 0.07590530812740326, + "p95": 0.12285276502370834 + }, + { + "label": "completed_group_5", + "mean": 0.10425601899623871, + "median": 0.10151226818561554, + "p05": 0.06786561012268066, + "p95": 0.15156632661819458 + }, + { + "label": "completed_group_6", + "mean": 0.1782122254371643, + "median": 0.1762663722038269, + "p05": 0.11771149188280106, + "p95": 0.24932436645030975 + }, + { + "label": "current_group_partial", + "mean": 0.2804902195930481, + "median": 0.2783043086528778, + "p05": 0.14811429381370544, + "p95": 0.4168073534965515 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7219383716583252, + "group": 7, + "latest_source_mass": 0.3661728501319885, + "layer": 26, + "max_source_mass": 0.3661728501319885, + "mixer_index": 51, + "normalized_entropy": 0.8280773165026276, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0923302173614502, + "median": 0.08522969484329224, + "p05": 0.056062981486320496, + "p95": 0.1509525179862976 + }, + { + "label": "completed_group_1", + "mean": 0.04646994173526764, + "median": 0.04543843865394592, + "p05": 0.032575178891420364, + "p95": 0.06306323409080505 + }, + { + "label": "completed_group_2", + "mean": 0.05750556290149689, + "median": 0.05649096518754959, + "p05": 0.03895717114210129, + "p95": 0.07940060645341873 + }, + { + "label": "completed_group_3", + "mean": 0.06549686938524246, + "median": 0.062135741114616394, + "p05": 0.04169235751032829, + "p95": 0.09948549419641495 + }, + { + "label": "completed_group_4", + "mean": 0.054404959082603455, + "median": 0.05241036415100098, + "p05": 0.03378590941429138, + "p95": 0.08157843351364136 + }, + { + "label": "completed_group_5", + "mean": 0.08427833020687103, + "median": 0.07576382160186768, + "p05": 0.041103728115558624, + "p95": 0.15204203128814697 + }, + { + "label": "completed_group_6", + "mean": 0.2333412617444992, + "median": 0.22215056419372559, + "p05": 0.10060560703277588, + "p95": 0.3983197808265686 + }, + { + "label": "current_group_partial", + "mean": 0.3661728501319885, + "median": 0.3673480749130249, + "p05": 0.16410952806472778, + "p95": 0.5635213255882263 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9470824003219604, + "group": 7, + "latest_source_mass": 0.26050227880477905, + "layer": 27, + "max_source_mass": 0.26050227880477905, + "mixer_index": 52, + "normalized_entropy": 0.9363487077155573, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10804469138383865, + "median": 0.10379163920879364, + "p05": 0.05421917140483856, + "p95": 0.17028658092021942 + }, + { + "label": "completed_group_1", + "mean": 0.07850409299135208, + "median": 0.0776081532239914, + "p05": 0.063654825091362, + "p95": 0.09574449062347412 + }, + { + "label": "completed_group_2", + "mean": 0.07298290729522705, + "median": 0.07200999557971954, + "p05": 0.05533483251929283, + "p95": 0.09299935400485992 + }, + { + "label": "completed_group_3", + "mean": 0.08511961251497269, + "median": 0.08372659981250763, + "p05": 0.06449062377214432, + "p95": 0.11086944490671158 + }, + { + "label": "completed_group_4", + "mean": 0.09968686103820801, + "median": 0.09877584874629974, + "p05": 0.07606595754623413, + "p95": 0.1266063004732132 + }, + { + "label": "completed_group_5", + "mean": 0.11502514779567719, + "median": 0.11171717941761017, + "p05": 0.07302433997392654, + "p95": 0.1681303083896637 + }, + { + "label": "completed_group_6", + "mean": 0.18013443052768707, + "median": 0.1765173077583313, + "p05": 0.11514104157686234, + "p95": 0.2599799633026123 + }, + { + "label": "current_group_partial", + "mean": 0.26050227880477905, + "median": 0.2575443983078003, + "p05": 0.13687226176261902, + "p95": 0.39756521582603455 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7855088710784912, + "group": 7, + "latest_source_mass": 0.28804832696914673, + "layer": 27, + "max_source_mass": 0.28804832696914673, + "mixer_index": 53, + "normalized_entropy": 0.858648264589397, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08493785560131073, + "median": 0.05951457470655441, + "p05": 0.02441210113465786, + "p95": 0.22703105211257935 + }, + { + "label": "completed_group_1", + "mean": 0.05972691997885704, + "median": 0.058127306401729584, + "p05": 0.03591413050889969, + "p95": 0.08903861790895462 + }, + { + "label": "completed_group_2", + "mean": 0.07017223536968231, + "median": 0.06906847655773163, + "p05": 0.04151676222681999, + "p95": 0.10273467004299164 + }, + { + "label": "completed_group_3", + "mean": 0.07963316142559052, + "median": 0.07547932863235474, + "p05": 0.042773112654685974, + "p95": 0.12874338030815125 + }, + { + "label": "completed_group_4", + "mean": 0.0645163506269455, + "median": 0.061876002699136734, + "p05": 0.0366181954741478, + "p95": 0.10098200291395187 + }, + { + "label": "completed_group_5", + "mean": 0.11586236953735352, + "median": 0.10182450711727142, + "p05": 0.048887670040130615, + "p95": 0.2313569337129593 + }, + { + "label": "completed_group_6", + "mean": 0.23710274696350098, + "median": 0.22610372304916382, + "p05": 0.10011275857686996, + "p95": 0.4084784686565399 + }, + { + "label": "current_group_partial", + "mean": 0.28804832696914673, + "median": 0.27438053488731384, + "p05": 0.07688213139772415, + "p95": 0.5529112815856934 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.973182201385498, + "group": 7, + "latest_source_mass": 0.23531918227672577, + "layer": 28, + "max_source_mass": 0.23531918227672577, + "mixer_index": 54, + "normalized_entropy": 0.9489000589030754, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.104981929063797, + "median": 0.09386023879051208, + "p05": 0.04212673753499985, + "p95": 0.18614700436592102 + }, + { + "label": "completed_group_1", + "mean": 0.0840599462389946, + "median": 0.08250434696674347, + "p05": 0.05170809105038643, + "p95": 0.12236068397760391 + }, + { + "label": "completed_group_2", + "mean": 0.09310521930456161, + "median": 0.0931512713432312, + "p05": 0.06374019384384155, + "p95": 0.12329407781362534 + }, + { + "label": "completed_group_3", + "mean": 0.11566174775362015, + "median": 0.11439734697341919, + "p05": 0.07886623591184616, + "p95": 0.1571493148803711 + }, + { + "label": "completed_group_4", + "mean": 0.0963074117898941, + "median": 0.09477876126766205, + "p05": 0.06686149537563324, + "p95": 0.13022682070732117 + }, + { + "label": "completed_group_5", + "mean": 0.1096896305680275, + "median": 0.10702179372310638, + "p05": 0.06579374521970749, + "p95": 0.1643676459789276 + }, + { + "label": "completed_group_6", + "mean": 0.16087493300437927, + "median": 0.15455034375190735, + "p05": 0.09876372665166855, + "p95": 0.24324838817119598 + }, + { + "label": "current_group_partial", + "mean": 0.23531918227672577, + "median": 0.22382870316505432, + "p05": 0.11656329780817032, + "p95": 0.38286590576171875 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.802741289138794, + "group": 7, + "latest_source_mass": 0.2497689425945282, + "layer": 28, + "max_source_mass": 0.2913346588611603, + "mixer_index": 55, + "normalized_entropy": 0.8669353059487718, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08190082013607025, + "median": 0.06415966153144836, + "p05": 0.039168842136859894, + "p95": 0.23471535742282867 + }, + { + "label": "completed_group_1", + "mean": 0.05632811039686203, + "median": 0.055233441293239594, + "p05": 0.03936203569173813, + "p95": 0.076178178191185 + }, + { + "label": "completed_group_2", + "mean": 0.07102417945861816, + "median": 0.06932097673416138, + "p05": 0.04159427434206009, + "p95": 0.1046731173992157 + }, + { + "label": "completed_group_3", + "mean": 0.08421063423156738, + "median": 0.0812690332531929, + "p05": 0.0486646331846714, + "p95": 0.12889066338539124 + }, + { + "label": "completed_group_4", + "mean": 0.06341387331485748, + "median": 0.061601825058460236, + "p05": 0.03807957097887993, + "p95": 0.09577237069606781 + }, + { + "label": "completed_group_5", + "mean": 0.10201875865459442, + "median": 0.09330011904239655, + "p05": 0.05116701498627663, + "p95": 0.18108239769935608 + }, + { + "label": "completed_group_6", + "mean": 0.2913346588611603, + "median": 0.28823426365852356, + "p05": 0.13295966386795044, + "p95": 0.4586564898490906 + }, + { + "label": "current_group_partial", + "mean": 0.2497689425945282, + "median": 0.234481081366539, + "p05": 0.09318265318870544, + "p95": 0.4530518054962158 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.945815086364746, + "group": 8, + "latest_source_mass": 0.153213769197464, + "layer": 29, + "max_source_mass": 0.2451319396495819, + "mixer_index": 56, + "normalized_entropy": 0.9357392585284499, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1064736545085907, + "median": 0.0881282389163971, + "p05": 0.029344500973820686, + "p95": 0.23970629274845123 + }, + { + "label": "completed_group_1", + "mean": 0.07925166189670563, + "median": 0.07691921293735504, + "p05": 0.062442928552627563, + "p95": 0.1030145138502121 + }, + { + "label": "completed_group_2", + "mean": 0.07823626697063446, + "median": 0.07715678215026855, + "p05": 0.059018202126026154, + "p95": 0.10055352747440338 + }, + { + "label": "completed_group_3", + "mean": 0.09447507560253143, + "median": 0.09318602085113525, + "p05": 0.0684189647436142, + "p95": 0.12381350249052048 + }, + { + "label": "completed_group_4", + "mean": 0.09724453091621399, + "median": 0.09628183394670486, + "p05": 0.0699525699019432, + "p95": 0.12776269018650055 + }, + { + "label": "completed_group_5", + "mean": 0.1459731012582779, + "median": 0.14035329222679138, + "p05": 0.08667999505996704, + "p95": 0.2258891463279724 + }, + { + "label": "completed_group_6", + "mean": 0.2451319396495819, + "median": 0.24505369365215302, + "p05": 0.16207969188690186, + "p95": 0.3307104706764221 + }, + { + "label": "completed_group_7", + "mean": 0.153213769197464, + "median": 0.13417893648147583, + "p05": 0.05588865652680397, + "p95": 0.30438560247421265 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8717057704925537, + "group": 8, + "latest_source_mass": 0.3048541843891144, + "layer": 29, + "max_source_mass": 0.3048541843891144, + "mixer_index": 57, + "normalized_entropy": 0.8518500065030654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07336360216140747, + "median": 0.0611552819609642, + "p05": 0.024239415302872658, + "p95": 0.16916418075561523 + }, + { + "label": "completed_group_1", + "mean": 0.05720400810241699, + "median": 0.05402737483382225, + "p05": 0.0314149409532547, + "p95": 0.09488057345151901 + }, + { + "label": "completed_group_2", + "mean": 0.057025715708732605, + "median": 0.05547948181629181, + "p05": 0.03480810672044754, + "p95": 0.08415934443473816 + }, + { + "label": "completed_group_3", + "mean": 0.06655597686767578, + "median": 0.06265220046043396, + "p05": 0.035200610756874084, + "p95": 0.10949824750423431 + }, + { + "label": "completed_group_4", + "mean": 0.05032068490982056, + "median": 0.047904569655656815, + "p05": 0.02614513412117958, + "p95": 0.08244554698467255 + }, + { + "label": "completed_group_5", + "mean": 0.08329877257347107, + "median": 0.0731179416179657, + "p05": 0.02939312346279621, + "p95": 0.17248544096946716 + }, + { + "label": "completed_group_6", + "mean": 0.1901058554649353, + "median": 0.18234091997146606, + "p05": 0.08719424903392792, + "p95": 0.317789226770401 + }, + { + "label": "completed_group_7", + "mean": 0.11727118492126465, + "median": 0.09272899478673935, + "p05": 0.031939513981342316, + "p95": 0.2891285717487335 + }, + { + "label": "current_group_partial", + "mean": 0.3048541843891144, + "median": 0.29432767629623413, + "p05": 0.07913826406002045, + "p95": 0.5632759928703308 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.09096622467041, + "group": 8, + "latest_source_mass": 0.13795366883277893, + "layer": 30, + "max_source_mass": 0.17320571839809418, + "mixer_index": 58, + "normalized_entropy": 0.951639739623416, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10326676070690155, + "median": 0.09009090811014175, + "p05": 0.01757553033530712, + "p95": 0.2674753665924072 + }, + { + "label": "completed_group_1", + "mean": 0.08232800662517548, + "median": 0.08093428611755371, + "p05": 0.06430500745773315, + "p95": 0.10457758605480194 + }, + { + "label": "completed_group_2", + "mean": 0.07164021581411362, + "median": 0.0711187869310379, + "p05": 0.05190654471516609, + "p95": 0.09468771517276764 + }, + { + "label": "completed_group_3", + "mean": 0.08822821825742722, + "median": 0.08667590469121933, + "p05": 0.06148773431777954, + "p95": 0.12025904655456543 + }, + { + "label": "completed_group_4", + "mean": 0.10803373157978058, + "median": 0.10627314448356628, + "p05": 0.07042569667100906, + "p95": 0.1514367014169693 + }, + { + "label": "completed_group_5", + "mean": 0.13578416407108307, + "median": 0.12929600477218628, + "p05": 0.08227657526731491, + "p95": 0.21117588877677917 + }, + { + "label": "completed_group_6", + "mean": 0.17320571839809418, + "median": 0.1694163680076599, + "p05": 0.11322733014822006, + "p95": 0.24482715129852295 + }, + { + "label": "completed_group_7", + "mean": 0.09955951571464539, + "median": 0.08973086625337601, + "p05": 0.037010837346315384, + "p95": 0.1912725567817688 + }, + { + "label": "current_group_partial", + "mean": 0.13795366883277893, + "median": 0.1256815791130066, + "p05": 0.053015124052762985, + "p95": 0.2601332366466522 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.96881103515625, + "group": 8, + "latest_source_mass": 0.21570813655853271, + "layer": 30, + "max_source_mass": 0.21570813655853271, + "mixer_index": 59, + "normalized_entropy": 0.896044517007504, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08843819797039032, + "median": 0.08496083319187164, + "p05": 0.03690534085035324, + "p95": 0.1522325873374939 + }, + { + "label": "completed_group_1", + "mean": 0.06052382290363312, + "median": 0.058369796723127365, + "p05": 0.03549504280090332, + "p95": 0.09154202789068222 + }, + { + "label": "completed_group_2", + "mean": 0.0642135739326477, + "median": 0.06355445086956024, + "p05": 0.03948400169610977, + "p95": 0.09175601601600647 + }, + { + "label": "completed_group_3", + "mean": 0.07744920998811722, + "median": 0.07566337287425995, + "p05": 0.04564609378576279, + "p95": 0.11733324825763702 + }, + { + "label": "completed_group_4", + "mean": 0.051616813987493515, + "median": 0.049985647201538086, + "p05": 0.03238597884774208, + "p95": 0.07699409872293472 + }, + { + "label": "completed_group_5", + "mean": 0.06631231307983398, + "median": 0.061356961727142334, + "p05": 0.030705859884619713, + "p95": 0.1188126802444458 + }, + { + "label": "completed_group_6", + "mean": 0.16417551040649414, + "median": 0.15454982221126556, + "p05": 0.07227650284767151, + "p95": 0.28998225927352905 + }, + { + "label": "completed_group_7", + "mean": 0.2115624099969864, + "median": 0.19997704029083252, + "p05": 0.09131578356027603, + "p95": 0.3709705173969269 + }, + { + "label": "current_group_partial", + "mean": 0.21570813655853271, + "median": 0.2005728930234909, + "p05": 0.06839637458324432, + "p95": 0.4193759858608246 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.124063014984131, + "group": 8, + "latest_source_mass": 0.1258423924446106, + "layer": 31, + "max_source_mass": 0.17017140984535217, + "mixer_index": 60, + "normalized_entropy": 0.9667027380329118, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09220977127552032, + "median": 0.09152225404977798, + "p05": 0.03251190483570099, + "p95": 0.17071731388568878 + }, + { + "label": "completed_group_1", + "mean": 0.06839152425527573, + "median": 0.06702418625354767, + "p05": 0.04446800425648689, + "p95": 0.0977444052696228 + }, + { + "label": "completed_group_2", + "mean": 0.10039852559566498, + "median": 0.09736946225166321, + "p05": 0.06476961821317673, + "p95": 0.14545989036560059 + }, + { + "label": "completed_group_3", + "mean": 0.10902463644742966, + "median": 0.10749617218971252, + "p05": 0.07769118249416351, + "p95": 0.14509552717208862 + }, + { + "label": "completed_group_4", + "mean": 0.0849665105342865, + "median": 0.08355648815631866, + "p05": 0.059977494180202484, + "p95": 0.11480176448822021 + }, + { + "label": "completed_group_5", + "mean": 0.12137698382139206, + "median": 0.11852307617664337, + "p05": 0.07902064919471741, + "p95": 0.17207489907741547 + }, + { + "label": "completed_group_6", + "mean": 0.17017140984535217, + "median": 0.16817378997802734, + "p05": 0.11521546542644501, + "p95": 0.23236754536628723 + }, + { + "label": "completed_group_7", + "mean": 0.12761825323104858, + "median": 0.1237221211194992, + "p05": 0.062459878623485565, + "p95": 0.2092883139848709 + }, + { + "label": "current_group_partial", + "mean": 0.1258423924446106, + "median": 0.11723431199789047, + "p05": 0.06504593789577484, + "p95": 0.21421310305595398 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9633058309555054, + "group": 8, + "latest_source_mass": 0.15763342380523682, + "layer": 31, + "max_source_mass": 0.23511825501918793, + "mixer_index": 61, + "normalized_entropy": 0.8935389906004497, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0846441388130188, + "median": 0.06999644637107849, + "p05": 0.04030589759349823, + "p95": 0.20743247866630554 + }, + { + "label": "completed_group_1", + "mean": 0.05872901901602745, + "median": 0.05653517693281174, + "p05": 0.03710630163550377, + "p95": 0.08772030472755432 + }, + { + "label": "completed_group_2", + "mean": 0.07210008800029755, + "median": 0.07183568179607391, + "p05": 0.03840115666389465, + "p95": 0.10702595114707947 + }, + { + "label": "completed_group_3", + "mean": 0.07792744785547256, + "median": 0.0762966051697731, + "p05": 0.04050692170858383, + "p95": 0.122304767370224 + }, + { + "label": "completed_group_4", + "mean": 0.059186771512031555, + "median": 0.057271093130111694, + "p05": 0.03537197411060333, + "p95": 0.08958204090595245 + }, + { + "label": "completed_group_5", + "mean": 0.07445863634347916, + "median": 0.06928195059299469, + "p05": 0.03154807910323143, + "p95": 0.1351260244846344 + }, + { + "label": "completed_group_6", + "mean": 0.23511825501918793, + "median": 0.22703276574611664, + "p05": 0.0953572615981102, + "p95": 0.3982555568218231 + }, + { + "label": "completed_group_7", + "mean": 0.18020221590995789, + "median": 0.16846294701099396, + "p05": 0.07464539259672165, + "p95": 0.324230432510376 + }, + { + "label": "current_group_partial", + "mean": 0.15763342380523682, + "median": 0.12027859687805176, + "p05": 0.036883872002363205, + "p95": 0.4093724191188812 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1238560676574707, + "group": 8, + "latest_source_mass": 0.11516246944665909, + "layer": 32, + "max_source_mass": 0.1766587197780609, + "mixer_index": 62, + "normalized_entropy": 0.966608552245626, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10231245309114456, + "median": 0.10458895564079285, + "p05": 0.025718798860907555, + "p95": 0.1875312179327011 + }, + { + "label": "completed_group_1", + "mean": 0.07575537264347076, + "median": 0.07447351515293121, + "p05": 0.05878028646111488, + "p95": 0.09651422500610352 + }, + { + "label": "completed_group_2", + "mean": 0.08897508680820465, + "median": 0.08751188218593597, + "p05": 0.06392349302768707, + "p95": 0.11800213158130646 + }, + { + "label": "completed_group_3", + "mean": 0.0967075377702713, + "median": 0.09579810500144958, + "p05": 0.06915527582168579, + "p95": 0.12801720201969147 + }, + { + "label": "completed_group_4", + "mean": 0.09801244735717773, + "median": 0.09654392302036285, + "p05": 0.07431379705667496, + "p95": 0.12754815816879272 + }, + { + "label": "completed_group_5", + "mean": 0.12438583374023438, + "median": 0.12010829150676727, + "p05": 0.08193086087703705, + "p95": 0.1824759691953659 + }, + { + "label": "completed_group_6", + "mean": 0.1766587197780609, + "median": 0.17520052194595337, + "p05": 0.12216726690530777, + "p95": 0.2341049611568451 + }, + { + "label": "completed_group_7", + "mean": 0.12203006446361542, + "median": 0.1200856938958168, + "p05": 0.05817953124642372, + "p95": 0.19647708535194397 + }, + { + "label": "current_group_partial", + "mean": 0.11516246944665909, + "median": 0.10430245846509933, + "p05": 0.05224763602018356, + "p95": 0.2294922173023224 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0835556983947754, + "group": 8, + "latest_source_mass": 0.09660109132528305, + "layer": 32, + "max_source_mass": 0.22511571645736694, + "mixer_index": 63, + "normalized_entropy": 0.9482670637704002, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09133295714855194, + "median": 0.08019149303436279, + "p05": 0.04441600665450096, + "p95": 0.19931957125663757 + }, + { + "label": "completed_group_1", + "mean": 0.07054752856492996, + "median": 0.06906121969223022, + "p05": 0.05099350959062576, + "p95": 0.09504306316375732 + }, + { + "label": "completed_group_2", + "mean": 0.09000501036643982, + "median": 0.0901019275188446, + "p05": 0.05998067185282707, + "p95": 0.11849517375230789 + }, + { + "label": "completed_group_3", + "mean": 0.10956105589866638, + "median": 0.1064089685678482, + "p05": 0.0759081020951271, + "p95": 0.1509535312652588 + }, + { + "label": "completed_group_4", + "mean": 0.07114207744598389, + "median": 0.06995616853237152, + "p05": 0.04991999268531799, + "p95": 0.09545011073350906 + }, + { + "label": "completed_group_5", + "mean": 0.11565887182950974, + "median": 0.11069735884666443, + "p05": 0.06978912651538849, + "p95": 0.17647549510002136 + }, + { + "label": "completed_group_6", + "mean": 0.22511571645736694, + "median": 0.22372552752494812, + "p05": 0.11895382404327393, + "p95": 0.33412501215934753 + }, + { + "label": "completed_group_7", + "mean": 0.13003569841384888, + "median": 0.12226496636867523, + "p05": 0.06884703785181046, + "p95": 0.21555939316749573 + }, + { + "label": "current_group_partial", + "mean": 0.09660109132528305, + "median": 0.09000182151794434, + "p05": 0.051288168877363205, + "p95": 0.16580010950565338 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.679484486579895, + "group": null, + "latest_source_mass": 0.2827349305152893, + "layer": null, + "max_source_mass": 0.32717907428741455, + "mixer_index": null, + "normalized_entropy": 0.7643663300981273, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08993907272815704, + "median": 0.09290459752082825, + "p05": 0.022419869899749756, + "p95": 0.15114322304725647 + }, + { + "label": "completed_group_1", + "mean": 0.043493133038282394, + "median": 0.042318835854530334, + "p05": 0.026918282732367516, + "p95": 0.06356139481067657 + }, + { + "label": "completed_group_2", + "mean": 0.04922422394156456, + "median": 0.047891765832901, + "p05": 0.03095712885260582, + "p95": 0.07266643643379211 + }, + { + "label": "completed_group_3", + "mean": 0.037924449890851974, + "median": 0.03562554344534874, + "p05": 0.020909149199724197, + "p95": 0.06224166974425316 + }, + { + "label": "completed_group_4", + "mean": 0.03939433768391609, + "median": 0.036868683993816376, + "p05": 0.020651251077651978, + "p95": 0.06597590446472168 + }, + { + "label": "completed_group_5", + "mean": 0.03898489475250244, + "median": 0.03362073004245758, + "p05": 0.014276490546762943, + "p95": 0.08282073587179184 + }, + { + "label": "completed_group_6", + "mean": 0.09112587571144104, + "median": 0.07136045396327972, + "p05": 0.02230823040008545, + "p95": 0.22286513447761536 + }, + { + "label": "completed_group_7", + "mean": 0.32717907428741455, + "median": 0.3213679790496826, + "p05": 0.08271751552820206, + "p95": 0.6128557920455933 + }, + { + "label": "completed_group_8", + "mean": 0.2827349305152893, + "median": 0.2627606987953186, + "p05": 0.05262754485011101, + "p95": 0.571351170539856 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "optimizer_state_before": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.08626964688301086, + 0.16337208449840546, + 0.15584443509578705, + 0.04305505007505417, + 0.1360139101743698, + 0.03393339365720749, + 0.03730965405702591, + 0.037494905292987823, + 0.05234000086784363, + 0.04223497211933136, + 0.041619230061769485, + 0.05722156912088394, + 0.062187470495700836, + 0.0369485504925251, + 0.04293901473283768, + 0.045369699597358704, + 0.04149825870990753, + 0.02440357394516468, + 0.026009807363152504, + 0.03634416684508324, + 0.038355328142642975, + 0.02631118707358837, + 0.033921729773283005, + 0.02731483057141304, + 0.027557097375392914, + 0.024315863847732544, + 0.02051021158695221, + 0.018631720915436745, + 0.019497457891702652, + 0.021715763956308365, + 0.017114246264100075, + 0.020576251670718193 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0936665578119573e-05, + "first_to_last_ratio": 1.168787114339494, + "imbalance_abs_log_ratio": 0.1559665567011555, + "last_quartile_mean": 1.7913155716087203e-05, + "mean": 2.42622821247096e-05, + "normalized": [ + 0.7126129382274812, + 0.7437894782846245, + 0.7319401693963385, + 0.6938624658898646, + 1.0850314310673168, + 1.0981344950491732, + 0.9606310812212513, + 0.877442710157735, + 1.0089375010128527, + 0.8940805493086196, + 0.8303081011723588, + 0.7704924450335486, + 0.7457227037132441, + 0.6956545192296728, + 0.6629687198417477, + 0.6359294291467756, + 1.5989410181555679, + 1.4898579524033908, + 1.348196746912536, + 1.1824041858349024, + 2.2648751765532618, + 2.0078719511613548, + 1.669089976380829, + 1.384721107610681, + 1.3105185686643073, + 1.0955274221780484, + 0.8401582843314063, + 0.6990787108453305, + 0.7286997339090275, + 0.5032506445515866, + 0.392614310962981, + 0.33665547179218436 + ], + "peak_layer": 21, + "peak_normalized": 2.2648751765532618, + "population_cv": 0.43871254744666655, + "rest_mean": 2.099399386713264e-05, + "spike_contrast": 1.9963347127217863, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.191103871562518e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7289616152993403e-05, + 1.8046030163532123e-05, + 1.77585388883017e-05, + 1.6834686903166585e-05, + 2.6325338694732636e-05, + 2.6643248929758556e-05, + 2.330710231035482e-05, + 2.128876258211676e-05, + 2.4479126295773312e-05, + 2.169243452954106e-05, + 2.014516940107569e-05, + 1.869390507636126e-05, + 1.8092934624291956e-05, + 1.687816620687954e-05, + 1.608513412065804e-05, + 1.5429099221364595e-05, + 3.8793958083260804e-05, + 3.614735396695323e-05, + 3.271032983320765e-05, + 2.868782394216396e-05, + 5.49510405107867e-05, + 4.871555574936792e-05, + 4.0495931898476556e-05, + 3.35964941768907e-05, + 3.179617124260403e-05, + 2.6579995392239653e-05, + 2.0384157323860563e-05, + 1.696124490990769e-05, + 1.767991852830164e-05, + 1.2210009117552545e-05, + 9.52571917878231e-06, + 8.168030035449192e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.209008865378564e-05, + "first_to_last_ratio": 1.1473326374136463, + "imbalance_abs_log_ratio": 0.13743980256699795, + "last_quartile_mean": 7.154863897085306e-05, + "mean": 9.638685185109352e-05, + "normalized": [ + 0.7014672692081079, + 0.730745142012726, + 0.7197152784749434, + 0.6836365522681692, + 1.073826467208612, + 1.0803379033203697, + 0.9514146538604249, + 0.8722415119444402, + 1.0096074578076277, + 0.8908657485913869, + 0.8272757726491637, + 0.7688961382914924, + 0.7419179025068791, + 0.6953651992846211, + 0.6657440869339828, + 0.6349379036282153, + 1.6060213837194506, + 1.5018800747743897, + 1.362846846779478, + 1.1942969804897212, + 2.2656295070514414, + 2.0104706485090547, + 1.6756941049043201, + 1.3967091226783026, + 1.3149112210845808, + 1.1000992011684827, + 0.8430071951587476, + 0.7025118587919716, + 0.7359466516142374, + 0.5075845580254474, + 0.3944123577040649, + 0.33998329955514606 + ], + "peak_layer": 21, + "peak_normalized": 2.2656295070514414, + "population_cv": 0.4413317448728873, + "rest_mean": 8.330888886121102e-05, + "spike_contrast": 2.004682264754327, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016700785199645906, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761222175555304e-05, + 7.043422374408692e-05, + 6.93710899213329e-05, + 6.589357508346438e-05, + 0.00010350275260861963, + 0.00010413036943646148, + 9.17038632906042e-05, + 8.407261339016259e-05, + 9.731288446346298e-05, + 8.586774492869154e-05, + 7.973850733833387e-05, + 7.411147817038e-05, + 7.15111309546046e-05, + 6.70240624458529e-05, + 6.416897667804733e-05, + 6.119966565165669e-05, + 0.0001547993451822549, + 0.00014476149226538837, + 0.0001313605171162635, + 0.0001151145261246711, + 0.00021837689564563334, + 0.00019378293654881418, + 0.00016151487943716347, + 0.00013462439528666437, + 0.00012674015306401998, + 0.00010603509872453287, + 8.125480962917209e-05, + 6.771290645701811e-05, + 7.093558087944984e-05, + 4.8924477596301585e-05, + 3.801616549026221e-05, + 3.2769919926067814e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07279830891638994, + "first_to_last_ratio": 1.1473325978581586, + "imbalance_abs_log_ratio": 0.13743976809095573, + "last_quartile_mean": 0.06345004844479263, + "mean": 0.08547682355856523, + "normalized": [ + 0.701467243783159, + 0.7307451127750545, + 0.7197152636360212, + 0.6836365259909346, + 1.0738264172711294, + 1.080337898385664, + 0.951414616770227, + 0.8722414984608238, + 1.0096074000217996, + 0.8908656785314788, + 0.8272757305190417, + 0.7688961528411616, + 0.7419179123849904, + 0.6953651763038863, + 0.6657440954344426, + 0.6349378767127872, + 1.6060213451352348, + 1.5018801837842675, + 1.3628468172791657, + 1.19429705497508, + 2.2656295095184866, + 2.010470763800267, + 1.6756941843442625, + 1.3967091688883524, + 1.314911253304382, + 1.1000991448544435, + 0.8430072555120299, + 0.7025118281998033, + 0.7359466794525621, + 0.5075845660770223, + 0.3944123343969408, + 0.3399833106550978 + ], + "peak_layer": 21, + "peak_normalized": 2.2656295095184866, + "population_cv": 0.4413317644266601, + "rest_mean": 0.07387915439903736, + "spike_contrast": 2.0046823522109167, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1481042370200157, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05995919182896614, + 0.062461771070957184, + 0.06151897460222244, + 0.05843507871031761, + 0.09178727120161057, + 0.09234385192394257, + 0.08132389932870865, + 0.07455643266439438, + 0.08629803359508514, + 0.0761483684182167, + 0.07071290165185928, + 0.06572280079126358, + 0.06341678649187088, + 0.05943760648369789, + 0.05690569058060646, + 0.054272472858428955, + 0.13727760314941406, + 0.12837594747543335, + 0.11649181693792343, + 0.10208471864461899, + 0.19365881383419037, + 0.17184865474700928, + 0.1432330161333084, + 0.11938626319169998, + 0.11239443719387054, + 0.09403298050165176, + 0.0720575824379921, + 0.060048479586839676, + 0.06290638446807861, + 0.04338671639561653, + 0.03371311351656914, + 0.029060693457722664 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.120508836218505e-06, + "first_to_last_ratio": 0.6479745229308365, + "imbalance_abs_log_ratio": 0.43390389986753336, + "last_quartile_mean": 4.815789395706815e-06, + "mean": 5.18821034489747e-06, + "normalized": [ + 0.6392329797809361, + 0.5994300881226403, + 0.5850824179653978, + 0.6009191324861043, + 0.6062512409598932, + 0.6024392926484339, + 0.5897072336534216, + 0.5886297066862235, + 0.7270901470558813, + 0.6920316696984324, + 0.6762839200674405, + 0.6775578709931118, + 0.6128406906582687, + 0.6178016007991107, + 0.624771802695437, + 0.6621615984140956, + 1.5142793639941121, + 1.477704889855872, + 1.4442885378272925, + 1.3463135645628785, + 2.4904030104436794, + 2.3638034305258637, + 2.088000001391446, + 1.747233095035389, + 1.6663782972572876, + 1.4047981189097183, + 1.0886105900542489, + 0.8884508074141, + 0.8978880977348557, + 0.6001555244868991, + 0.4703385709663715, + 0.40912270685515684 + ], + "peak_layer": 21, + "peak_normalized": 2.4904030104436794, + "population_cv": 0.5663270603947061, + "rest_mean": 4.159058141373896e-06, + "spike_contrast": 2.5836696383318847, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0745632243924774e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.316475158499088e-06, + 3.109969384240685e-06, + 3.0355306535057025e-06, + 3.1176948596112197e-06, + 3.145358959955047e-06, + 3.125581770291319e-06, + 3.0595251701015513e-06, + 3.0539347335434286e-06, + 3.772296622628346e-06, + 3.5904058677260764e-06, + 3.508703230181709e-06, + 3.5153127555531682e-06, + 3.1795464110473404e-06, + 3.2052846563601634e-06, + 3.2414475299447076e-06, + 3.4354336548858555e-06, + 7.856399861339014e-06, + 7.666643796255812e-06, + 7.4932727329724e-06, + 6.984957963140914e-06, + 1.2920734661747701e-05, + 1.2263909411558416e-05, + 1.0832983207365032e-05, + 9.06501281860983e-06, + 8.645521120342892e-06, + 7.288388133019907e-06, + 5.647940724884393e-06, + 4.609469669958344e-06, + 4.658432317228289e-06, + 3.113733100690297e-06, + 2.4402154394920217e-06, + 2.12261466003838e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.9889651311095804e-05, + "first_to_last_ratio": 1.3326615656169287, + "imbalance_abs_log_ratio": 0.28717811970161894, + "last_quartile_mean": 4.493988035392249e-05, + "mean": 6.640704737037595e-05, + "normalized": [ + 0.7737542671823434, + 0.7960594489342157, + 0.7852142182091284, + 0.752976874377423, + 1.1002658507118863, + 1.0894970423039605, + 0.9958150050309817, + 0.921272183824516, + 1.0695745385738102, + 0.9470829860046872, + 0.8771170672423568, + 0.8087454440169668, + 0.8138319929245809, + 0.7582700648479997, + 0.7300932386851152, + 0.6932937290424763, + 1.5635426486844544, + 1.4635271461161115, + 1.3107699454980482, + 1.1481322979739153, + 2.32725688601457, + 1.991760393656834, + 1.584233429529925, + 1.2840444859649542, + 1.264803263909513, + 1.026616211913306, + 0.7846647444878888, + 0.6569396053772728, + 0.6050973350000657, + 0.42438660382108917, + 0.34340864002846966, + 0.3079524101111341 + ], + "peak_layer": 21, + "peak_normalized": 2.32725688601457, + "population_cv": 0.43279320435052254, + "rest_mean": 5.791654122596675e-05, + "spike_contrast": 1.9382335024498314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011225578055018559, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.1382736273808405e-05, + 5.286395753500983e-05, + 5.2143757784506306e-05, + 5.000297096557915e-05, + 7.306540646823123e-05, + 7.23502816981636e-05, + 6.612913421122357e-05, + 6.117896555224434e-05, + 7.102728704921901e-05, + 6.289298471529037e-05, + 5.8246754633728415e-05, + 5.3706397011410445e-05, + 5.4044179705670103e-05, + 5.035447611589916e-05, + 4.8483336286153644e-05, + 4.6039589506108314e-05, + 0.00010383025073679164, + 9.718851651996374e-05, + 8.704436186235398e-05, + 7.624407589901239e-05, + 0.00015454625827260315, + 0.00013226692681200802, + 0.00010520426440052688, + 8.526960300514475e-05, + 8.399185026064515e-05, + 6.817455141572282e-05, + 5.2107268857071176e-05, + 4.362541949376464e-05, + 4.018272738903761e-05, + 2.8182261303300038e-05, + 2.2804753825766966e-05, + 2.0450210286071524e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.414140539665823e-05, + "first_to_last_ratio": 1.6936904784541074, + "imbalance_abs_log_ratio": 0.5269098631562702, + "last_quartile_mean": 2.606226223633712e-05, + "mean": 4.478222555803768e-05, + "normalized": [ + 0.874026545726401, + 0.9009891441910284, + 0.8916727093699365, + 0.8527000158254343, + 1.172545810112114, + 1.156435027941858, + 1.0621486326987206, + 0.9750045233682423, + 1.1460449634090804, + 1.011287374798366, + 0.9322859052145132, + 0.8551582509249812, + 0.907118809465358, + 0.8439303067882139, + 0.8153240377760277, + 0.7685666352135386, + 1.5553576798550997, + 1.437596357968014, + 1.2796321689041579, + 1.1209246602381584, + 2.385838207411615, + 1.9252114173862964, + 1.4016273255130902, + 1.0727508885758477, + 1.1785651553842893, + 0.9257194279728913, + 0.7253410162251672, + 0.6191416448731234, + 0.43450056827805394, + 0.29243769148345633, + 0.24680184853049614, + 0.23331524857642952 + ], + "peak_layer": 21, + "peak_normalized": 2.385838207411615, + "population_cv": 0.43498704004207234, + "rest_mean": 3.986614397204162e-05, + "spike_contrast": 1.7892140803093455, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.13290661224164e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.914085391443223e-05, + 4.034829908050597e-05, + 3.9931088394951075e-05, + 3.81858044420369e-05, + 5.2509210945572704e-05, + 5.178773426450789e-05, + 4.756537964567542e-05, + 4.366287248558365e-05, + 5.1322444051038474e-05, + 4.528769932221621e-05, + 4.1749837691895664e-05, + 3.829588968073949e-05, + 4.0622799133416265e-05, + 3.779307735385373e-05, + 3.6512024962576106e-05, + 3.441812441451475e-05, + 6.965237844269723e-05, + 6.437876436393708e-05, + 5.7304776419186965e-05, + 5.019750096835196e-05, + 0.00010684314474929124, + 8.621525194030255e-05, + 6.276799103943631e-05, + 4.804017225978896e-05, + 5.277877062326297e-05, + 4.1455776226939633e-05, + 3.2482384995091707e-05, + 2.7726540793082677e-05, + 1.9457902453723364e-05, + 1.3096010661683977e-05, + 1.1052336049033329e-05, + 1.0448376087879296e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015771267680975143, + "first_to_last_ratio": 1.0243014562120425, + "imbalance_abs_log_ratio": 0.024010874124910462, + "last_quartile_mean": 0.0001539709583084914, + "mean": 0.00019892108525709773, + "normalized": [ + 0.6590708750981731, + 0.6828508221536124, + 0.6699110125304764, + 0.6473590320765046, + 0.9801195462127644, + 0.9691430493294225, + 0.887896737937672, + 0.8463722857737056, + 0.988608068348346, + 0.8930841762598842, + 0.8341812095096794, + 0.7796271143503263, + 0.7202755659725629, + 0.6755421311480478, + 0.6527815148816499, + 0.632514433083119, + 1.5594662296068276, + 1.4738278124849744, + 1.3649033855466377, + 1.1834547507508266, + 2.341778688768703, + 2.0989567426400018, + 1.7777537161382369, + 1.4882782565844777, + 1.4127403878123095, + 1.1574678874298643, + 0.8851612824909503, + 0.7258447229726291, + 0.7423942540292697, + 0.5132421599816662, + 0.4020767269150468, + 0.35331542118163173 + ], + "peak_layer": 21, + "peak_normalized": 2.341778688768703, + "population_cv": 0.465450630495743, + "rest_mean": 0.00016857082745344896, + "spike_contrast": 2.1522850832359604, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00036281247739680114, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0001311030937358737, + 0.000135833426611498, + 0.00013325942563824356, + 0.00012877336121164262, + 0.00019496644381433725, + 0.00019278298714198172, + 0.0001766213827067986, + 0.00016836129361763597, + 0.00019665498984977603, + 0.00017765327356755733, + 0.00016593623149674386, + 0.00015508427168242633, + 0.0001432779972674325, + 0.00013437957386486232, + 0.0001298520073760301, + 0.00012582045746967196, + 0.0003102107148151845, + 0.00029317542794160545, + 0.00027150806272402406, + 0.0002354141033720225, + 0.0004658291582018137, + 0.00041752675315365195, + 0.00035363269853405654, + 0.00029604992596432567, + 0.0002810238511301577, + 0.00023024476831778884, + 0.0001760772429406643, + 0.00014438582002185285, + 0.00014767787070013583, + 0.00010209468746325001, + 7.998153887456283e-05, + 7.028188701951876e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13547764718532562, + 0.039667773991823196, + 0.026703152805566788, + 0.05304057523608208, + 0.016798952594399452, + 0.00753376679494977, + 0.01294450368732214, + 0.03526800498366356, + 0.030294200405478477, + 0.034149833023548126, + 0.04367009550333023, + 0.05930330231785774, + 0.04527241364121437, + 0.04281846433877945, + 0.04812675341963768, + 0.06094835698604584, + 0.038031257688999176, + 0.040106989443302155, + 0.04961257055401802, + 0.07085621356964111, + 0.03753058984875679, + 0.04544757679104805, + 0.05368982255458832, + 0.06174425780773163, + 0.05051252990961075, + 0.054937634617090225, + 0.05777662619948387, + 0.07133159041404724, + 0.0636468306183815, + 0.09261580556631088, + 0.1020733118057251, + 0.12878532707691193 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0824043303946382e-05, + "first_to_last_ratio": 1.424267056786447, + "imbalance_abs_log_ratio": 0.3536573352845024, + "last_quartile_mean": 1.4620883916904859e-05, + "mean": 2.2054508917790372e-05, + "normalized": [ + 0.7842104740080174, + 0.8183716989463029, + 0.8099804080045728, + 0.7638102866038118, + 1.1922163664937413, + 1.1967612607580402, + 1.039219827104321, + 0.9490935140140151, + 1.019003480899918, + 0.9331398258308763, + 0.8658119659177055, + 0.813018282797004, + 0.7774015040255573, + 0.7266621576498908, + 0.7030285425215664, + 0.6829301796443584, + 1.632275124747471, + 1.4994012377073724, + 1.316962971421436, + 1.1572238560225663, + 2.2864000936278734, + 1.8918326711874776, + 1.5430845130531994, + 1.2946151724909234, + 1.217184871626016, + 0.9287724220385267, + 0.7771760119482759, + 0.6578637330922696, + 0.5567850124802112, + 0.4325353313877837, + 0.37087941374039357, + 0.3623477882085046 + ], + "peak_layer": 21, + "peak_normalized": 2.2864000936278734, + "population_cv": 0.42456281874262314, + "rest_mean": 1.9413589850753648e-05, + "spike_contrast": 1.870621155539602, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6315471879788676e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7295376892434433e-05, + 1.8048785932478495e-05, + 1.7863720131572336e-05, + 1.6845460777403787e-05, + 2.6293746486771852e-05, + 2.6393981897854246e-05, + 2.2919482944416814e-05, + 2.0931791368639097e-05, + 2.247362135676667e-05, + 2.0579940610332415e-05, + 1.909505772346165e-05, + 1.793071896827314e-05, + 1.71452084032353e-05, + 1.6026177036110312e-05, + 1.5504949260503054e-05, + 1.5061689737194683e-05, + 3.599902629503049e-05, + 3.3068557968363166e-05, + 2.9044971597613767e-05, + 2.552200385252945e-05, + 5.042543125455268e-05, + 4.1723440517671406e-05, + 3.4031971154036e-05, + 2.855210186680779e-05, + 2.68444146058755e-05, + 2.048361966444645e-05, + 1.7140235286206007e-05, + 1.4508861568174325e-05, + 1.2279620023036841e-05, + 9.53935432335129e-06, + 8.179563337762374e-06, + 7.991402526386082e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.156287549354602e-05, + "first_to_last_ratio": 1.3967979973430424, + "imbalance_abs_log_ratio": 0.3341824723557829, + "last_quartile_mean": 5.839274945174111e-05, + "mean": 8.75740092851629e-05, + "normalized": [ + 0.7720751985061499, + 0.8035956764433113, + 0.7960739585260428, + 0.7524919125585545, + 1.1804156983562801, + 1.1778760010857563, + 1.0298326204965722, + 0.9385140198493713, + 1.0207128740101707, + 0.929812017339553, + 0.8640701821512353, + 0.8130222503970004, + 0.7736044339144612, + 0.7280851293348032, + 0.7061941984091654, + 0.6826155087046274, + 1.642189790371069, + 1.511740343745529, + 1.3334033796706957, + 1.1637276240936476, + 2.2882576030527964, + 1.895449319429852, + 1.5552551751392194, + 1.3067312409844312, + 1.222859749246748, + 0.9319757616478574, + 0.7801162715542103, + 0.6613743325062905, + 0.561952429248885, + 0.43456344744462927, + 0.3748431739999157, + 0.36656867778116903 + ], + "peak_layer": 21, + "peak_normalized": 2.2882576030527964, + "population_cv": 0.4268460433011898, + "rest_mean": 7.697251674943362e-05, + "spike_contrast": 1.8814776376552176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014482206897810103, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761372060282156e-05, + 7.037409523036331e-05, + 6.971538823563606e-05, + 6.589873373741284e-05, + 0.00010337373532820493, + 0.00010315132385585457, + 9.018657146953046e-05, + 8.21894354885444e-05, + 8.9387918706052e-05, + 8.142736623995006e-05, + 7.567009015474468e-05, + 7.119961810531095e-05, + 6.774764187866822e-05, + 6.376133387675509e-05, + 6.184425728861243e-05, + 5.977937689749524e-05, + 0.0001438131439499557, + 0.0001323891628999263, + 0.00011677147995214909, + 0.00010191229375777766, + 0.00020039189257659018, + 0.00016599209629930556, + 0.00013619993114843965, + 0.00011443569383118302, + 0.00010709073103498667, + 8.161685400409624e-05, + 6.831790960859507e-05, + 5.79192019358743e-05, + 4.92124272568617e-05, + 3.8056463381508365e-05, + 3.282651960034855e-05, + 3.2101888791657984e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07233077567070723, + "first_to_last_ratio": 1.3967980239296907, + "imbalance_abs_log_ratio": 0.33418249138977923, + "last_quartile_mean": 0.051783274626359344, + "mean": 0.07766150805400684, + "normalized": [ + 0.7720752144189068, + 0.8035957101529103, + 0.7960739453756324, + 0.7524919172501093, + 1.18041575335872, + 1.177876023895764, + 1.029832619521225, + 0.9385140290506606, + 1.020712886859698, + 0.9298120043210532, + 0.8640702058481301, + 0.8130222480432568, + 0.7736043957649873, + 0.7280851679574961, + 0.7061942119475418, + 0.6826155393692982, + 1.6421898574274953, + 1.51174026509585, + 1.333403346061099, + 1.1637275827667504, + 2.288257583675922, + 1.8954492273341264, + 1.5552551855736059, + 1.3067312459654492, + 1.2228597336334397, + 0.9319757582706437, + 0.7801162839815503, + 0.6613743139447658, + 0.5619524390301038, + 0.4345634639005874, + 0.3748431737420666, + 0.366568666461154 + ], + "peak_layer": 21, + "peak_normalized": 2.288257583675922, + "population_cv": 0.4268460335100709, + "rest_mean": 0.06825999867309022, + "spike_contrast": 1.8814776033915557, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12842965871095657, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05996052548289299, + 0.0624084547162056, + 0.06182430312037468, + 0.05843965709209442, + 0.0916728675365448, + 0.09147562831640244, + 0.07997835427522659, + 0.07288641482591629, + 0.07927010208368301, + 0.07221060246229172, + 0.0671049952507019, + 0.06314053386449814, + 0.06007928401231766, + 0.056544192135334015, + 0.05484410747885704, + 0.05301295220851898, + 0.12753494083881378, + 0.1174040287733078, + 0.10355411469936371, + 0.0903768390417099, + 0.17770953476428986, + 0.1472034454345703, + 0.12078346312046051, + 0.10148271918296814, + 0.09496913105249405, + 0.07237864285707474, + 0.060585007071495056, + 0.051363326609134674, + 0.0436420738697052, + 0.03374885395169258, + 0.02911088615655899, + 0.02846827544271946 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.100046171766735e-06, + "first_to_last_ratio": 0.7824902199643116, + "imbalance_abs_log_ratio": 0.24527385509409916, + "last_quartile_mean": 3.961769863281006e-06, + "mean": 4.656072917441634e-06, + "normalized": [ + 0.7040746582663436, + 0.6663619356936908, + 0.6544453195323515, + 0.6681125781418429, + 0.6743036280537461, + 0.6669931612309391, + 0.6466275673249758, + 0.6455375483541481, + 0.766065733134754, + 0.7325311317531802, + 0.713184552491017, + 0.7279868059896252, + 0.6487420214900163, + 0.6567739586893749, + 0.6698952555545961, + 0.731415182041189, + 1.5336327145648883, + 1.46016794598778, + 1.3941212250673451, + 1.3066200632810194, + 2.5346433175887912, + 2.2579967860286922, + 1.9597071198477958, + 1.7730016814327627, + 1.6211494054956397, + 1.2094753028064131, + 1.0219643022327503, + 0.8604311185936954, + 0.6846742153365659, + 0.5134546057775913, + 0.45040862343156707, + 0.44550053478491114 + ], + "peak_layer": 21, + "peak_normalized": 2.5346433175887912, + "population_cv": 0.5361380970640629, + "rest_mean": 3.768573976601517e-06, + "spike_contrast": 2.50719960829824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.448567197978264e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.278222948210896e-06, + 3.1026297619973775e-06, + 3.0471451282210182e-06, + 3.110780880888342e-06, + 3.139606860713684e-06, + 3.1055687941261567e-06, + 3.0107451038929867e-06, + 3.0056698960834183e-06, + 3.566857913028798e-06, + 3.4107183637388516e-06, + 3.3206392799911555e-06, + 3.3895596516231308e-06, + 3.020590156666003e-06, + 3.0579874419345288e-06, + 3.119081156910397e-06, + 3.405522420507623e-06, + 7.140705747588072e-06, + 6.798648428230081e-06, + 6.491130079666618e-06, + 6.083718290028628e-06, + 1.1801484106399585e-05, + 1.0513397683098447e-05, + 9.124539246840868e-06, + 8.255225111497566e-06, + 7.548189842054853e-06, + 5.631405201711459e-06, + 4.7583403102180455e-06, + 4.006230028608115e-06, + 3.187893071299186e-06, + 2.3906820842967136e-06, + 2.097135393341887e-06, + 2.0742829747177893e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.958586689303047e-05, + "first_to_last_ratio": 1.5974368171613162, + "imbalance_abs_log_ratio": 0.46840035541466, + "last_quartile_mean": 3.730092248588335e-05, + "mean": 6.105391446453723e-05, + "normalized": [ + 0.841921083374255, + 0.8656847679595863, + 0.8583386074530126, + 0.8192653470430583, + 1.1963759198218682, + 1.175200963423096, + 1.067618556331515, + 0.9832339424318312, + 1.0816814073351606, + 0.984735398821402, + 0.9088666847085389, + 0.8483372799998844, + 0.8441732679586115, + 0.788240769992009, + 0.7677328825781261, + 0.7384848720107285, + 1.592019271873236, + 1.469773741071107, + 1.2839883253170383, + 1.1290190536252611, + 2.340759428283372, + 1.8604406810329672, + 1.4645591028269243, + 1.2019442625318217, + 1.1579689537744897, + 0.8703227892706042, + 0.7241054984123771, + 0.6218678661137057, + 0.4685539623066287, + 0.375211854314897, + 0.3379493629430677, + 0.3316240950598181 + ], + "peak_layer": 21, + "peak_normalized": 2.340759428283372, + "population_cv": 0.4213561912347865, + "rest_mean": 5.421209425549023e-05, + "spike_contrast": 1.8077099757766004, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.799974359339103e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.140257781022228e-05, + 5.2853443776257336e-05, + 5.240493192104623e-05, + 5.001935642212629e-05, + 7.30434330762364e-05, + 7.175061909947544e-05, + 6.518229201901704e-05, + 6.003028101986274e-05, + 6.604088412132114e-05, + 6.012195080984384e-05, + 5.5489868827862665e-05, + 5.179431173019111e-05, + 5.154008249519393e-05, + 4.812518454855308e-05, + 4.687309774453752e-05, + 4.508739220909774e-05, + 9.71990084508434e-05, + 8.973544026957825e-05, + 7.839251338737085e-05, + 6.893103272886947e-05, + 0.00014291252591647208, + 0.00011358718620613217, + 8.941706619225442e-05, + 7.338340219575912e-05, + 7.069853745633736e-05, + 5.3136613132664934e-05, + 4.420947516337037e-05, + 3.796746750595048e-05, + 2.860705353668891e-05, + 2.2908152459422126e-05, + 2.0633131498470902e-05, + 2.0246949134161696e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.3941625335719436e-05, + "first_to_last_ratio": 1.9900849942152503, + "imbalance_abs_log_ratio": 0.6881773484852566, + "last_quartile_mean": 2.2080275698499463e-05, + "mean": 4.1715027776945135e-05, + "normalized": [ + 0.9392326235815728, + 0.967237934106481, + 0.9603764045369728, + 0.9164866155582375, + 1.2577449343388738, + 1.232917566027963, + 1.1238782174568975, + 1.0291368181202014, + 1.1532132229581178, + 1.0411575328021039, + 0.9502116988067248, + 0.8875233874351255, + 0.938078657297939, + 0.8686978298587725, + 0.846151620562448, + 0.8110684117898503, + 1.5702254340289712, + 1.447397095355284, + 1.2505015110000162, + 1.0952804003622336, + 2.3931370003923895, + 1.7749904370362708, + 1.2920876407146809, + 1.0187689124624395, + 1.0659664997461884, + 0.8050885780431608, + 0.6754901061622123, + 0.6027145753846552, + 0.3199047099172539, + 0.266999929991275, + 0.2517914183349422, + 0.24654227582974492 + ], + "peak_layer": 21, + "peak_normalized": 2.3931370003923895, + "population_cv": 0.43040686028555775, + "rest_mean": 3.7783076651168005e-05, + "spike_contrast": 1.666025359377284, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.294756385614165e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.918011498171836e-05, + 4.034835728816688e-05, + 4.006212839158252e-05, + 3.8231264625210315e-05, + 5.2466864872258157e-05, + 5.143119051354006e-05, + 4.688261105911806e-05, + 4.293047095416114e-05, + 4.810632162843831e-05, + 4.343191540101543e-05, + 3.963810740970075e-05, + 3.70230627595447e-05, + 3.9131977246142924e-05, + 3.6237754102330655e-05, + 3.5297238355269656e-05, + 3.383374132681638e-05, + 6.550199759658426e-05, + 6.037821003701538e-05, + 5.216470526647754e-05, + 4.568965232465416e-05, + 9.98297764454037e-05, + 7.404377538478002e-05, + 5.3899471822660416e-05, + 4.2497973481658846e-05, + 4.446682214620523e-05, + 3.3584292395971715e-05, + 2.8178088541608304e-05, + 2.5142255253740586e-05, + 1.3344833860173821e-05, + 1.1137909496028442e-05, + 1.0503486009838525e-05, + 1.0284517884429079e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015718752911197953, + "first_to_last_ratio": 1.257989643214417, + "imbalance_abs_log_ratio": 0.22951492550542246, + "last_quartile_mean": 0.0001249513697985094, + "mean": 0.0001807773865039053, + "normalized": [ + 0.7253332250389443, + 0.7487212712295568, + 0.7410605164905312, + 0.7105636788603447, + 1.0746269026158506, + 1.051977329709352, + 0.9748117817952817, + 0.9289757254955786, + 1.003966745690328, + 0.9387623076773945, + 0.8790880476775305, + 0.8198592543393499, + 0.76275878383003, + 0.7097319103357375, + 0.6897181090899124, + 0.6787200572192026, + 1.6053206581539505, + 1.5070819773591158, + 1.3319581978459651, + 1.179141640369754, + 2.363943782302499, + 1.985198270384524, + 1.6420557999668541, + 1.4171107521036512, + 1.2788263571440766, + 0.9695939295226049, + 0.8072936975349638, + 0.6913356827376413, + 0.5679249559528513, + 0.4460061883008719, + 0.39061284360066933, + 0.37791961962508147 + ], + "peak_layer": 21, + "peak_normalized": 2.363943782302499, + "population_cv": 0.44512103777248124, + "rest_mean": 0.00015609032642613683, + "spike_contrast": 2.012216375705279, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000314087510923855, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00013112384476698935, + 0.00013535187463276088, + 0.00013396698341239244, + 0.00012845384480897337, + 0.0001942682429216802, + 0.00019017371232621372, + 0.00017622392624616623, + 0.00016793780378066003, + 0.00018149448442272842, + 0.00016970699653029442, + 0.00015891923976596445, + 0.00014821201330050826, + 0.0001378895394736901, + 0.00012830347986891866, + 0.0001246854371856898, + 0.0001226972381118685, + 0.00029020567308180034, + 0.0002724463411141187, + 0.0002407879219390452, + 0.00021316214406397194, + 0.00042734757880680263, + 0.00035887895501218736, + 0.0002968465560115874, + 0.0002561815781518817, + 0.00023118288663681597, + 0.00017528065654914826, + 0.00014594044478144497, + 0.00012497785792220384, + 0.00010266798926750198, + 8.062783308560029e-05, + 7.061396900098771e-05, + 6.831932114437222e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.08626964688301086, + 0.27676042914390564, + 0.41335058212280273, + 0.43042075634002686, + 0.1360139101743698, + 0.15445689857006073, + 0.1670466512441635, + 0.17834730446338654, + 0.05234000086784363, + 0.08269349485635757, + 0.11548293381929398, + 0.16410532593727112, + 0.062187470495700836, + 0.0988159254193306, + 0.13199056684970856, + 0.17199495434761047, + 0.04149825870990753, + 0.06434677541255951, + 0.08728200942277908, + 0.11647984385490417, + 0.038355328142642975, + 0.06353354454040527, + 0.09196189045906067, + 0.1169828325510025, + 0.027557097375392914, + 0.06579430401325226, + 0.09665275365114212, + 0.12799666821956635, + 0.019497457891702652, + 0.07176875323057175, + 0.14131556451320648, + 0.22250843048095703 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0937054614478257e-05, + "first_to_last_ratio": 1.1688073264187404, + "imbalance_abs_log_ratio": 0.15598384976011784, + "last_quartile_mean": 1.791317879451526e-05, + "mean": 2.4262353377935142e-05, + "normalized": [ + 0.7126199170124646, + 0.7437761981288968, + 0.7319360705844501, + 0.6938843441336064, + 1.08510119201235, + 1.0981398918115777, + 0.9606406303851301, + 0.877454527863037, + 1.0089327386637434, + 0.8940806225700821, + 0.830291493091461, + 0.7704822352676253, + 0.7457403811840673, + 0.695661023016061, + 0.6629855157677164, + 0.6359363332477325, + 1.598966161144301, + 1.4898256875513118, + 1.3481551517699815, + 1.1823992139387995, + 2.2648560798112616, + 2.007845512240135, + 1.6690690306890001, + 1.384726637362728, + 1.3105208676356837, + 1.0955359004357528, + 0.840135724559922, + 0.6990787570111011, + 0.728702616979868, + 0.5032438811110385, + 0.39261683155295724, + 0.3366588314661564 + ], + "peak_layer": 21, + "peak_normalized": 2.2648560798112616, + "population_cv": 0.43870656486100834, + "rest_mean": 2.0994108227670142e-05, + "spike_contrast": 1.9963161442660275, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1910877189366144e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7289836250711232e-05, + 1.8045760953100398e-05, + 1.7758491594577208e-05, + 1.6835267160786316e-05, + 2.6327108571422286e-05, + 2.6643458113539964e-05, + 2.3307402443606406e-05, + 2.128911182808224e-05, + 2.4479082640027627e-05, + 2.1692500013159588e-05, + 2.014482561207842e-05, + 1.8693712263484485e-05, + 1.8093416656483896e-05, + 1.6878373571671546e-05, + 1.6085588868008927e-05, + 1.542931204312481e-05, + 3.8794682041043416e-05, + 3.6146677302895114e-05, + 3.270941670052707e-05, + 2.8687787562375888e-05, + 5.495073855854571e-05, + 4.871505734627135e-05, + 4.04955426347442e-05, + 3.3596727007534355e-05, + 3.179632039973512e-05, + 2.6580279154586606e-05, + 2.0383669834700413e-05, + 1.696129584161099e-05, + 1.768004040059168e-05, + 1.2209880878799595e-05, + 9.525808309263084e-06, + 8.168135536834598e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.209185853047529e-05, + "first_to_last_ratio": 1.1473585117140146, + "imbalance_abs_log_ratio": 0.13746235401177884, + "last_quartile_mean": 7.154856803026632e-05, + "mean": 9.638741801154538e-05, + "normalized": [ + 0.701459601059617, + 0.7307291493400122, + 0.7197289413331719, + 0.6836538994288254, + 1.0739127818136336, + 1.080318121017968, + 0.9514366180464413, + 0.8722525427024255, + 1.0096073400424832, + 0.8908667057242938, + 0.8272733289673383, + 0.7688963021177191, + 0.7419276606217693, + 0.6953759857067083, + 0.6657591236204922, + 0.6349357593481342, + 1.6060262927227642, + 1.5018600810151583, + 1.362811062623623, + 1.194289437021534, + 2.265619973525555, + 2.010448271283051, + 1.6756913579512287, + 1.3967040891245648, + 1.314898968363092, + 1.100096740194715, + 0.8429885804297745, + 0.7025081852971004, + 0.7359520665807775, + 0.5075723672109625, + 0.3944131736981434, + 0.33998549206692247 + ], + "peak_layer": 21, + "peak_normalized": 2.265619973525555, + "population_cv": 0.44132695283622186, + "rest_mean": 8.330956363890975e-05, + "spike_contrast": 2.0046657830023333, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016700783162377776, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761187978554517e-05, + 7.043309597065672e-05, + 6.937281432328746e-05, + 6.58956341794692e-05, + 0.00010351168020861223, + 0.00010412907431600615, + 9.170651901513338e-05, + 8.407417044509202e-05, + 9.731344471219927e-05, + 8.58683415572159e-05, + 7.973874016897753e-05, + 7.411192927975208e-05, + 7.151249155867845e-05, + 6.70254958095029e-05, + 6.417080294340849e-05, + 6.119981844676659e-05, + 0.0001548007276142016, + 0.00014476041542366147, + 0.0001313578395638615, + 0.0001151144751929678, + 0.00021837725944351405, + 0.0001937819179147482, + 0.0001615155633771792, + 0.00013462470087688416, + 0.00012673971650656313, + 0.00010603548435028642, + 8.125349268084392e-05, + 6.771295011276379e-05, + 7.093651947798207e-05, + 4.8923589929472655e-05, + 3.80164674425032e-05, + 3.27703237417154e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07279988285154104, + "first_to_last_ratio": 1.147358527887077, + "imbalance_abs_log_ratio": 0.13746236810768883, + "last_quartile_mean": 0.06344998627901077, + "mean": 0.08547732653096318, + "normalized": [ + 0.7014596295781764, + 0.7307291764187523, + 0.7197289409168642, + 0.6836539457276586, + 1.0739127543434965, + 1.0803181180576646, + 0.9514366494990735, + 0.8722525785288534, + 1.0096072992260057, + 0.8908667122670901, + 0.8272733032016506, + 0.7688963353200844, + 0.7419276673674994, + 0.6953759896953364, + 0.6657591362695728, + 0.6349357531030216, + 1.6060263641500938, + 1.5018600149298467, + 1.3628109924626157, + 1.1942894172476932, + 2.2656200131573585, + 2.010448125243662, + 1.6756912972456086, + 1.3967041753456768, + 1.3148989834510798, + 1.10009676829185, + 0.8429886102104137, + 0.7025081738287764, + 0.7359520241866415, + 0.5075723834629702, + 0.3944131950658124, + 0.3399854721990999 + ], + "peak_layer": 21, + "peak_normalized": 2.2656200131573585, + "population_cv": 0.44132694138829454, + "rest_mean": 0.07387975465368342, + "spike_contrast": 2.0046657621228294, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14810421466827392, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.059958893805742264, + 0.062460776418447495, + 0.06152050569653511, + 0.058436911553144455, + 0.0917951911687851, + 0.09234270453453064, + 0.08132626116275787, + 0.0745578184723854, + 0.08629853278398514, + 0.07614890486001968, + 0.070713110268116, + 0.06572320312261581, + 0.06341799348592758, + 0.059438880532979965, + 0.05690731108188629, + 0.05427261069417, + 0.13727883994579315, + 0.12837497889995575, + 0.11648944020271301, + 0.10208466649055481, + 0.19365914165973663, + 0.17184773087501526, + 0.14323361217975616, + 0.11938653886318207, + 0.1123940497636795, + 0.09403333067893982, + 0.07205641269683838, + 0.06004852056503296, + 0.06290721148252487, + 0.043385930359363556, + 0.033713385462760925, + 0.029061049222946167 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.1205255481836502e-06, + "first_to_last_ratio": 0.6479717368101099, + "imbalance_abs_log_ratio": 0.4339081996148503, + "last_quartile_mean": 4.8158358936234436e-06, + "mean": 5.188215141060937e-06, + "normalized": [ + 0.6392337912530196, + 0.5994569684566948, + 0.58506750248592, + 0.6009069195183814, + 0.6062570789746512, + 0.60249307876806, + 0.5896500227462853, + 0.5886480511239991, + 0.7270810166784253, + 0.6920427312432549, + 0.6762719442051294, + 0.6775019374497203, + 0.6128892519841179, + 0.6178328466527419, + 0.6247970380766965, + 0.662130571719626, + 1.514294792953365, + 1.477854106612226, + 1.4442983342347777, + 1.3462892680196958, + 2.490378094515379, + 2.363646805945382, + 2.0879100705160045, + 1.7472602290519488, + 1.6663413461773389, + 1.404838629343315, + 1.0886161574596454, + 0.888478735318369, + 0.8979335469262955, + 0.6001271407896346, + 0.4703453234739583, + 0.40912666732594116 + ], + "peak_layer": 21, + "peak_normalized": 2.490378094515379, + "population_cv": 0.5663116287539589, + "rest_mean": 4.159116037448661e-06, + "spike_contrast": 2.583565883667621, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0745350300567224e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.3164824344567023e-06, + 3.1101117201615125e-06, + 3.035456074940157e-06, + 3.117634378213552e-06, + 3.1453921565116616e-06, + 3.1258637136488687e-06, + 3.0592311759392032e-06, + 3.0540327315975446e-06, + 3.7722527395089855e-06, + 3.5904665764974197e-06, + 3.508644340399769e-06, + 3.515025809974759e-06, + 3.1798012969375122e-06, + 3.205449729648535e-06, + 3.2415814530395437e-06, + 3.435275857555098e-06, + 7.856487172830384e-06, + 7.667425052204635e-06, + 7.493330485885963e-06, + 6.984838364587631e-06, + 1.2920617336931173e-05, + 1.2263108146726154e-05, + 1.0832526641024742e-05, + 9.06516197574092e-06, + 8.645337402413134e-06, + 7.28860504750628e-06, + 5.647974830935709e-06, + 4.6096188270894345e-06, + 4.658672423829557e-06, + 3.1135887184063904e-06, + 2.4402527287747944e-06, + 2.122637170032249e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.989002966089174e-05, + "first_to_last_ratio": 1.332672418740526, + "imbalance_abs_log_ratio": 0.2871862636142851, + "last_quartile_mean": 4.4939798272025655e-05, + "mean": 6.640715508865469e-05, + "normalized": [ + 0.7737597503853951, + 0.7960652246538951, + 0.7852109175536554, + 0.752979652136534, + 1.1002854313277015, + 1.0894969185314263, + 0.9958159097459445, + 0.9212749625084216, + 1.0695780627909957, + 0.9470852845577153, + 0.8771168497054519, + 0.8087458852142236, + 0.8138331380491499, + 0.758274367943245, + 0.7300990118419535, + 0.6932937549003495, + 1.563540331611732, + 1.463522142564802, + 1.3107588349120618, + 1.1481313121287318, + 2.3272535492615467, + 1.9917532184740583, + 1.5842339276075041, + 1.284041417038419, + 1.2647947479005226, + 1.026614217953409, + 0.784658048181791, + 0.6569384849801976, + 0.6050998048035263, + 0.4243821080145578, + 0.34340906908209917, + 0.3079536636389831 + ], + "peak_layer": 21, + "peak_normalized": 2.3272535492615467, + "population_cv": 0.432791558992841, + "rest_mean": 5.791668829462853e-05, + "spike_contrast": 1.938226771623043, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011225567577639594, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.138318374520168e-05, + 5.2864426834275946e-05, + 5.2143623179290444e-05, + 5.000323653803207e-05, + 7.3066825279966e-05, + 7.235039083752781e-05, + 6.61293015582487e-05, + 6.117924931459129e-05, + 7.102763629518449e-05, + 6.289323937380686e-05, + 5.824683466926217e-05, + 5.370651342673227e-05, + 5.404434341471642e-05, + 5.035484355175868e-05, + 4.8483798309462145e-05, + 4.603966590366326e-05, + 0.00010383026528870687, + 9.7188341896981e-05, + 8.704376523382962e-05, + 7.62441341066733e-05, + 0.0001545462873764336, + 0.0001322666648775339, + 0.00010520446812734008, + 8.526953752152622e-05, + 8.399142097914591e-05, + 6.817452958784997e-05, + 5.2106908697169274e-05, + 4.362541585578583e-05, + 4.0182956581702456e-05, + 2.8182008463772945e-05, + 2.2804819309385493e-05, + 2.045032670139335e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.413846727402415e-05, + "first_to_last_ratio": 1.6934944304798134, + "imbalance_abs_log_ratio": 0.5267941045072542, + "last_quartile_mean": 2.6063544396492944e-05, + "mean": 4.4781047250808115e-05, + "normalized": [ + 0.8740739154412697, + 0.9009770250766957, + 0.891219784649098, + 0.852663960345562, + 1.1726907227894316, + 1.1562245824076949, + 1.0625075493400142, + 0.9748474712247537, + 1.1459503353768645, + 1.011393273941681, + 0.9321659114804322, + 0.8550679923070413, + 0.9071127008648373, + 0.8438105066025905, + 0.8153663695855309, + 0.7686703721764182, + 1.5553617227891403, + 1.437584791446802, + 1.2800134621565402, + 1.1209839695222465, + 2.385210126491363, + 1.925331940523583, + 1.4016057137556681, + 1.0729916373601829, + 1.178493073924358, + 0.9257430549345239, + 0.725497883628933, + 0.6192210996384143, + 0.4345528238561406, + 0.29242012088135483, + 0.2468894193108289, + 0.23335668617000463 + ], + "peak_layer": 21, + "peak_normalized": 2.385210126491363, + "population_cv": 0.4349311871982457, + "rest_mean": 3.986569293003952e-05, + "spike_contrast": 1.7891062550480503, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.132396058295853e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.9141945308074355e-05, + 4.034669473185204e-05, + 3.990975528722629e-05, + 3.818318509729579e-05, + 5.251431866781786e-05, + 5.177694765734486e-05, + 4.758020077133551e-05, + 4.36546906712465e-05, + 5.131685611559078e-05, + 4.5291249989531934e-05, + 4.174336572759785e-05, + 3.829084016615525e-05, + 4.062145671923645e-05, + 3.7786718166898936e-05, + 3.651295992312953e-05, + 3.442186425672844e-05, + 6.965072680031881e-05, + 6.437655247282237e-05, + 5.732034333050251e-05, + 5.019883610657416e-05, + 0.00010681220737751573, + 8.621838060207665e-05, + 6.27653716946952e-05, + 4.804968921234831e-05, + 5.277415402815677e-05, + 4.145574348513037e-05, + 3.2488555007148534e-05, + 2.772936932160519e-05, + 1.9459730538073927e-05, + 1.3094879250274971e-05, + 1.1055966751882806e-05, + 1.0449956789670978e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015772300503158476, + "first_to_last_ratio": 1.024373509074145, + "imbalance_abs_log_ratio": 0.02408121506561912, + "last_quartile_mean": 0.00015397021070384653, + "mean": 0.0001989226409477851, + "normalized": [ + 0.6591656486528042, + 0.6829581384685631, + 0.6699127961863572, + 0.6472837418583299, + 0.9802229283334125, + 0.9691875554589603, + 0.8879032543387678, + 0.8464550603982911, + 0.9884366921662843, + 0.8930886037902819, + 0.8342806121981681, + 0.779517431659101, + 0.7202640807077498, + 0.675490688069157, + 0.6529307639307617, + 0.6324384542682998, + 1.559547816640635, + 1.474378252555848, + 1.3651032473841427, + 1.1832746816977402, + 2.3416687863410064, + 2.09906600553135, + 1.7771997929015884, + 1.4880606167336252, + 1.412667012460112, + 1.1576278202661294, + 0.8849667940826665, + 0.7258688199763643, + 0.7424188068351703, + 0.5131968143231243, + 0.4021086961884269, + 0.3533095855967808 + ], + "peak_layer": 21, + "peak_normalized": 2.3416687863410064, + "population_cv": 0.4654332615698335, + "rest_mean": 0.0001685783755857739, + "spike_contrast": 2.152005989155233, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003627816739026457, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00013112297165207565, + 0.00013585583656094968, + 0.00013326082262210548, + 0.0001287593913730234, + 0.00019498853362165391, + 0.00019279334810562432, + 0.00017662406025920063, + 0.00016837907605804503, + 0.00019662243721541017, + 0.00017765554366633296, + 0.00016595730266999453, + 0.00015506366617046297, + 0.00014327683311421424, + 0.00013437039160635322, + 0.00012988271191716194, + 0.00012580632755998522, + 0.00031022937037050724, + 0.00029328721575438976, + 0.0002715499431360513, + 0.00023538012464996427, + 0.0004658109392039478, + 0.0004175517533440143, + 0.00035352527629584074, + 0.0002960089477710426, + 0.00028101145289838314, + 0.00023027838324196637, + 0.00017603993183001876, + 0.0001443917426513508, + 0.0001476839097449556, + 0.00010208646563114598, + 7.998852379387245e-05, + 7.028127583907917e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16424456238746643, + 0.29588937759399414, + 0.42337876558303833, + 0.45154979825019836, + 0.13891471922397614, + 0.1555756777524948, + 0.16914354264736176, + 0.1874145120382309, + 0.06211879849433899, + 0.09842939674854279, + 0.137006014585495, + 0.20118924975395203, + 0.08399304747581482, + 0.12138351052999496, + 0.1587214469909668, + 0.2071460783481598, + 0.0581100657582283, + 0.08358953893184662, + 0.10947012156248093, + 0.1472771316766739, + 0.05584336072206497, + 0.08318652212619781, + 0.11238116025924683, + 0.1379220336675644, + 0.05995604768395424, + 0.09392217546701431, + 0.1259489357471466, + 0.17086055874824524, + 0.06700707972049713, + 0.13971327245235443, + 0.2211977243423462, + 0.32473331689834595 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0823702925554244e-05, + "first_to_last_ratio": 1.4242510302785252, + "imbalance_abs_log_ratio": 0.35364608276130804, + "last_quartile_mean": 1.4620809452026151e-05, + "mean": 2.2054440051988422e-05, + "normalized": [ + 0.7841928807641645, + 0.8183408510588258, + 0.809986731151413, + 0.7637938668182616, + 1.192276091281044, + 1.1966830153099992, + 1.039215154291241, + 0.9490753634193776, + 1.0189872806238243, + 0.9331490903420598, + 0.8658120301757168, + 0.81301306861575, + 0.7774052511262896, + 0.7266654164061733, + 0.7030327996851689, + 0.6829495498627286, + 1.6322510246428148, + 1.499422250137046, + 1.3169815172043178, + 1.1572342326306069, + 2.2864238933994527, + 1.8918479809103845, + 1.543069371898747, + 1.294657154506133, + 1.2171810844243176, + 0.9287685590307506, + 0.7771677166674748, + 0.6578712720326597, + 0.5567773486562734, + 0.43254113576640124, + 0.3708732313504376, + 0.3623537858101448 + ], + "peak_layer": 21, + "peak_normalized": 2.2864238933994527, + "population_cv": 0.4245664907667088, + "rest_mean": 1.9413478454495175e-05, + "spike_contrast": 1.8706401721656991, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6315632678451947e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.729493487800937e-05, + 1.804804924177006e-05, + 1.78638038050849e-05, + 1.684504604781978e-05, + 2.629498158057686e-05, + 2.639217382238712e-05, + 2.2919308321434073e-05, + 2.093132570735179e-05, + 2.2473193894256838e-05, + 2.0580080672516488e-05, + 1.9094999515800737e-05, + 1.7930547983269207e-05, + 1.7145237507065758e-05, + 1.6026198863983154e-05, + 1.5504994735238142e-05, + 1.5062069905980024e-05, + 3.5998382372781634e-05, + 3.306891812826507e-05, + 2.9045289920759387e-05, + 2.5522153009660542e-05, + 5.0425798690412194e-05, + 4.172364788246341e-05, + 3.403153095860034e-05, + 2.855293860193342e-05, + 2.6844247258850373e-05, + 2.048347050731536e-05, + 1.7139998817583546e-05, + 1.450898253096966e-05, + 1.2279412658244837e-05, + 9.539452548779082e-06, + 8.179401447705459e-06, + 7.991509846760891e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "rest_mean": 7.697225831819836e-05, + "spike_contrast": 1.8814916302334765, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001448226597858593, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07233011117205024, + "first_to_last_ratio": 1.3967870631582553, + "imbalance_abs_log_ratio": 0.3341746442893017, + "last_quartile_mean": 0.051783205242827535, + "mean": 0.07766139676095918, + "normalized": [ + 0.7720675426341356, + 0.8035859729306402, + 0.7960780122636479, + 0.7524732326455996, + 1.1804674279966365, + 1.1778079658527474, + 1.0298294903648106, + 0.9385077949910019, + 1.020698424099846, + 0.9298230264026498, + 0.8640760490713295, + 0.8130195756795249, + 0.773607039370292, + 0.7280910561484961, + 0.7061969508238138, + 0.6826252958058289, + 1.64216899408666, + 1.511744542110794, + 1.3334176327397504, + 1.1637251251728324, + 2.288267386572505, + 1.8954517517435265, + 1.5552435035136007, + 1.3067718770227776, + 1.2228641722871045, + 0.9319785328940787, + 0.7801153392913037, + 0.6613715681663117, + 0.5619466726726858, + 0.43456725256605405, + 0.37483694737332934, + 0.36657384470568466 + ], + "peak_layer": 21, + "peak_normalized": 2.288267386572505, + "population_cv": 0.4268470914878033, + "rest_mean": 0.0682597693607763, + "spike_contrast": 1.8814916300573643, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12843018472194673, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05995984375476837, + 0.062407609075307846, + 0.061824530363082886, + 0.058438122272491455, + 0.0916767492890358, + 0.09147021174430847, + 0.07997799664735794, + 0.07288582623004913, + 0.07926886528730392, + 0.072211354970932, + 0.06710535287857056, + 0.06314023584127426, + 0.06007940322160721, + 0.05654456838965416, + 0.054844241589307785, + 0.0530136339366436, + 0.12753313779830933, + 0.11740419268608093, + 0.10355507582426071, + 0.09037651866674423, + 0.17771004140377045, + 0.14720343053340912, + 0.12078238278627396, + 0.1014857292175293, + 0.09496933966875076, + 0.07237875461578369, + 0.06058484688401222, + 0.05136303976178169, + 0.04364156350493431, + 0.03374909982085228, + 0.029110360890626907, + 0.02846863679587841 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.1000530213987076e-06, + "first_to_last_ratio": 0.7825005490137864, + "imbalance_abs_log_ratio": 0.2452606549530124, + "last_quartile_mean": 3.961726321222159e-06, + "mean": 4.656059921614997e-06, + "normalized": [ + 0.7040313055728622, + 0.6663442620495864, + 0.6544029514954961, + 0.6681637652166335, + 0.6743674803950376, + 0.6670078174062475, + 0.6466284443239834, + 0.6455370061269436, + 0.7660325155889827, + 0.7325145706427032, + 0.7131598309497137, + 0.7280186266136806, + 0.64870437442493, + 0.656793469733594, + 0.6699169519151534, + 0.7314456448845376, + 1.53365232903732, + 1.4601427212016398, + 1.3940973786195396, + 1.306558663485338, + 2.534675981169938, + 2.2580403975857246, + 1.9597194264610087, + 1.773041790596292, + 1.6211455309545355, + 1.2094494759638503, + 1.0219317012739941, + 0.8604275624593173, + 0.6846808632651675, + 0.5134649266877133, + 0.4503974279453127, + 0.4455048059532228 + ], + "peak_layer": 21, + "peak_normalized": 2.534675981169938, + "population_cv": 0.5361438958058841, + "rest_mean": 3.768541933755774e-06, + "spike_contrast": 2.5072447705625382, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.4486570560548e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.2780119454400847e-06, + 3.1025388125272e-06, + 3.046939355044742e-06, + 3.1110105283005396e-06, + 3.1398953979078215e-06, + 3.105628366029123e-06, + 3.0107407837931532e-06, + 3.005658982146997e-06, + 3.566693294487777e-06, + 3.4106317343685078e-06, + 3.320514906590688e-06, + 3.389698349565151e-06, + 3.020406438736245e-06, + 3.058069751205039e-06, + 3.119173470622627e-06, + 3.405654751986731e-06, + 7.140777142922161e-06, + 6.798512004024815e-06, + 6.4910009314189665e-06, + 6.083415428292938e-06, + 1.1801603250205517e-05, + 1.0513571396586485e-05, + 9.12457107915543e-06, + 8.255388820543885e-06, + 7.548150733782677e-06, + 5.631269232253544e-06, + 4.758175236929674e-06, + 4.006202289019711e-06, + 3.187915126545704e-06, + 2.3907234663056443e-06, + 2.0970774130546488e-06, + 2.074297071885667e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.95857286498358e-05, + "first_to_last_ratio": 1.5974327409797537, + "imbalance_abs_log_ratio": 0.4683978037101298, + "last_quartile_mean": 3.7300931126083015e-05, + "mean": 6.105390025368251e-05, + "normalized": [ + 0.8419145460820371, + 0.8656854461464137, + 0.8583401181387939, + 0.8192663123570689, + 1.1963719080724353, + 1.1751919414919585, + 1.0676149913033914, + 0.9832376272965914, + 1.0816749854357006, + 0.984736879340849, + 0.9088680283961561, + 0.8483339022774173, + 0.8441772779737997, + 0.7882426814662962, + 0.7677338358974833, + 0.7384876656990601, + 1.5920129687595903, + 1.4697744406924866, + 1.283993033566668, + 1.1290203889690138, + 2.340760926498109, + 1.860435632123778, + 1.4645539617726704, + 1.2019638482713701, + 1.15797649283655, + 0.8703237664687024, + 0.7241030451553558, + 0.6218659849233957, + 0.4685510324632258, + 0.37521378882561096, + 0.3379462835275562, + 0.3316262577704645 + ], + "peak_layer": 21, + "peak_normalized": 2.340760926498109, + "population_cv": 0.4213559298637173, + "rest_mean": 5.4212039955288036e-05, + "spike_contrast": 1.8077155175462722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.79999458650127e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.140216671861708e-05, + 5.285347288008779e-05, + 5.240501195657998e-05, + 5.0019403715850785e-05, + 7.304317114176229e-05, + 7.175005157478154e-05, + 6.518205918837339e-05, + 6.003049202263355e-05, + 6.604047666769475e-05, + 6.012202720739879e-05, + 5.548993794946e-05, + 5.1794093451462686e-05, + 5.154031532583758e-05, + 4.8125290049938485e-05, + 4.687314503826201e-05, + 4.5087552280165255e-05, + 9.7198600997217e-05, + 8.973546209745109e-05, + 7.839278259780258e-05, + 6.8931098212488e-05, + 0.000142912584124133, + 0.00011358685151208192, + 8.941673149820417e-05, + 7.338458090089262e-05, + 7.069898128975183e-05, + 5.3136660426389426e-05, + 4.420931509230286e-05, + 3.796734381467104e-05, + 2.8606867999769747e-05, + 2.2908265236765146e-05, + 2.0632938685594127e-05, + 2.0247076463419944e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.39446339441929e-05, + "first_to_last_ratio": 1.9902838030955072, + "imbalance_abs_log_ratio": 0.6882772431887607, + "last_quartile_mean": 2.207958175404201e-05, + "mean": 4.171556903997953e-05, + "normalized": [ + 0.9393219483952423, + 0.9671725353750462, + 0.9606035942825071, + 0.9165296658237206, + 1.25754251069206, + 1.233040231347365, + 1.1241638961033311, + 1.029104366181827, + 1.1530467776467852, + 1.041181174787795, + 0.9503458810810488, + 0.8874956508445306, + 0.9380795670187313, + 0.868712023499227, + 0.8459898570744635, + 0.811097306638917, + 1.57041471098841, + 1.447423664007262, + 1.250399123009104, + 1.0954141829132344, + 2.392471764387743, + 1.7753028127291355, + 1.2921083756975622, + 1.0187283101804303, + 1.065894151409315, + 0.8051193818729687, + 0.6754751497722067, + 0.6026093861097862, + 0.3198852539229254, + 0.2669904264118248, + 0.2517857094641488, + 0.2465506103313451 + ], + "peak_layer": 21, + "peak_normalized": 2.392471764387743, + "population_cv": 0.4303681174959756, + "rest_mean": 3.778425454786177e-05, + "spike_contrast": 1.6658967617763285, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.294466729741543e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.9184349589049816e-05, + 4.034615267300978e-05, + 4.007212555734441e-05, + 3.823355655185878e-05, + 5.245910142548382e-05, + 5.1436974899843335e-05, + 4.689513662015088e-05, + 4.2929674236802384e-05, + 4.810000245925039e-05, + 4.343346517998725e-05, + 3.9644219214096665e-05, + 3.702238609548658e-05, + 3.913252294296399e-05, + 3.6238816392142326e-05, + 3.5290948289912194e-05, + 3.3835385693237185e-05, + 6.551074329763651e-05, + 6.0380101785995066e-05, + 5.2161110943416134e-05, + 4.5695825974689797e-05, + 9.980332106351852e-05, + 7.40577670512721e-05, + 5.3901036153547466e-05, + 4.249683115631342e-05, + 4.446438106242567e-05, + 3.358601315994747e-05, + 2.8177830245113e-05, + 2.513819345040247e-05, + 1.3344195394893177e-05, + 1.113765756599605e-05, + 1.0503384146431927e-05, + 1.0284999007126316e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015719397015345749, + "first_to_last_ratio": 1.2580343238755638, + "imbalance_abs_log_ratio": 0.22955044238581457, + "last_quartile_mean": 0.00012495205191953573, + "mean": 0.00018078252287523355, + "normalized": [ + 0.7253227592072276, + 0.7485993006094415, + 0.7410050906139998, + 0.7104508417729404, + 1.0745241672934553, + 1.0524252536641314, + 0.9748384190355198, + 0.9289919933888637, + 1.0038768041568893, + 0.9389629508381758, + 0.8789427325469985, + 0.8198897306139241, + 0.7627520038160613, + 0.7096499260969147, + 0.6897553416925117, + 0.6787601781063402, + 1.6052869610938005, + 1.5071148227224969, + 1.3317635520239959, + 1.1792135859285877, + 2.364258320942869, + 1.9856700690456726, + 1.6414783683894763, + 1.4170804705838413, + 1.2788477375082987, + 0.9694880608375548, + 0.8072605380605845, + 0.691415531221962, + 0.5679201295495192, + 0.44599504581333255, + 0.3904354851204105, + 0.3780238277042025 + ], + "peak_layer": 21, + "peak_normalized": 2.364258320942869, + "population_cv": 0.4451626458968529, + "rest_mean": 0.0001560934222113617, + "spike_contrast": 2.0122799667677387, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00031410366646014156, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00013112567830830812, + 0.0001353336701868102, + 0.00013396076974458992, + 0.00012843709555454552, + 0.00019425518985372037, + 0.0001902600924950093, + 0.00017623374878894538, + 0.00016794551629573107, + 0.00018148338131140918, + 0.00016974809113889933, + 0.00015889748465269804, + 0.0001482217339798808, + 0.00013789223157800734, + 0.00012829230399802327, + 0.00012469571083784103, + 0.00012270797742530704, + 0.00029020782676525414, + 0.00027246001991443336, + 0.00024075957480818033, + 0.0002131812070729211, + 0.00042741658398881555, + 0.0003589744446799159, + 0.00029675060068257153, + 0.00025618338258937, + 0.0002311933203600347, + 0.00017526649753563106, + 0.00014593859668821096, + 0.0001249958440894261, + 0.00010267003381159157, + 8.062810957198963e-05, + 7.058391202008352e-05, + 6.834010127931833e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.062413111329078674, + 0.07552354782819748, + 0.14653873443603516, + 0.2105521410703659, + 0.2186809629201889, + 0.14272794127464294, + 0.1287408471107483, + 0.13253600895404816, + 0.13371841609477997, + 0.09585829079151154, + 0.09883332997560501, + 0.10020580142736435, + 0.11237040907144547, + 0.08538161963224411, + 0.08663177490234375, + 0.10358677804470062, + 0.10874549299478531, + 0.06467291712760925, + 0.07278503477573395, + 0.08346924930810928, + 0.09857872873544693, + 0.06093711405992508, + 0.06834813207387924, + 0.0680762529373169, + 0.07369214296340942, + 0.05633483827114105, + 0.05850831791758537, + 0.06479237228631973, + 0.06705968081951141, + 0.061747051775455475, + 0.0654391348361969, + 0.07205834239721298 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0717411839777924e-05, + "first_to_last_ratio": 1.3606399895734356, + "imbalance_abs_log_ratio": 0.3079551696283513, + "last_quartile_mean": 7.876743239876305e-06, + "mean": 1.4356073158694471e-05, + "normalized": [ + 0.6895052247736269, + 0.5576007830925034, + 0.38530829240192566, + 0.5472592297661263, + 0.37729668968423546, + 0.9131958145391527, + 1.3435179101111983, + 1.1586522070597358, + 1.5436961923733754, + 1.144684350345656, + 0.8911764746394834, + 0.9261744836030649, + 0.5468539631209283, + 0.8612404659079811, + 0.9670035934822814, + 0.889769793290417, + 0.9785434614239343, + 1.8290819610081128, + 1.5223690430889059, + 1.8851307751720068, + 1.8951492301881587, + 2.5144698996677146, + 2.058593895387357, + 1.1843681717440393, + 1.0244158825153251, + 0.9136429572594602, + 0.5473929037729057, + 0.6024761608101054, + 0.3781148252878133, + 0.4421926007844957, + 0.24629038211477544, + 0.23483238158319855 + ], + "peak_layer": 22, + "peak_normalized": 2.5144698996677146, + "population_cv": 0.5646113035804416, + "rest_mean": 1.2400990229892799e-05, + "spike_contrast": 2.008994484502458, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4913520974223502e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.898587450152263e-06, + 8.004957635421306e-06, + 5.531514034373686e-06, + 7.856493539293297e-06, + 5.41649887964013e-06, + 1.3109905921737663e-05, + 1.9287641407572664e-05, + 1.6633695850032382e-05, + 2.216141547251027e-05, + 1.643317227717489e-05, + 1.279379466723185e-05, + 1.3296228644321673e-05, + 7.850675501686055e-06, + 1.2364031135803089e-05, + 1.3882374332752079e-05, + 1.2773600246873684e-05, + 1.4048041521164123e-05, + 2.6258434445480816e-05, + 2.185524135711603e-05, + 2.7063075322075747e-05, + 2.7206900995224714e-05, + 3.6097913834964857e-05, + 2.9553324566222727e-05, + 1.7002876120386645e-05, + 1.4706589354318567e-05, + 1.3116325135342777e-05, + 7.858412573114038e-06, + 8.649191840959247e-06, + 5.428244094218826e-06, + 6.348149327095598e-06, + 3.5357627439225325e-06, + 3.3712708500388544e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.013745751763054e-05, + "first_to_last_ratio": 1.3002217470857882, + "imbalance_abs_log_ratio": 0.26253482460265976, + "last_quartile_mean": 3.0869701731717214e-05, + "mean": 5.617674656832605e-05, + "normalized": [ + 0.473654064493233, + 0.5256902444154287, + 0.3705068590661575, + 0.5387329076786941, + 0.37665460925127686, + 0.9021763170314637, + 1.3591261829588805, + 1.169340915014312, + 1.4504557484696843, + 1.2215794360380687, + 0.9103877586499164, + 0.9321134746924985, + 0.5414129154069731, + 0.8525397685215896, + 0.9808579608967221, + 0.9142526712062526, + 1.0619023556310665, + 1.8034129735144708, + 1.5888135435441055, + 1.899281403844036, + 1.9024910148779852, + 2.5497535502321647, + 2.089500431013523, + 1.18928036938163, + 1.0430326636758747, + 0.9143825787913804, + 0.5551399248346642, + 0.5581364123079678, + 0.38887965127025986, + 0.4518317596354874, + 0.24874412658654727, + 0.2359354070676856 + ], + "peak_layer": 22, + "peak_normalized": 2.5497535502321647, + "population_cv": 0.5766137085977164, + "rest_mean": 4.832436503353098e-05, + "spike_contrast": 2.0399565889343343, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.857960685621947e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.6608344342093915e-05, + 2.9531567633966915e-05, + 2.0813869923586026e-05, + 3.0264262022683397e-05, + 2.1159230527700856e-05, + 5.0681330321822315e-05, + 7.635128713445738e-05, + 6.568976823473349e-05, + 8.148188499035314e-05, + 6.862435839138925e-05, + 5.114262239658274e-05, + 5.236310244072229e-05, + 3.041481613763608e-05, + 4.7892910515656695e-05, + 5.510140908882022e-05, + 5.135974060976878e-05, + 5.9654219512594864e-05, + 0.00010130987357115373, + 8.925437578000128e-05, + 0.00010669545008568093, + 0.00010687575559131801, + 0.00014323685900308192, + 0.00011738133616745472, + 6.680990190943703e-05, + 5.859418160980567e-05, + 5.1367038395255804e-05, + 3.1185954867396504e-05, + 3.1354287784779444e-05, + 2.18459936149884e-05, + 2.538243825256359e-05, + 1.397363575961208e-05, + 1.3254083569336217e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03559429990127683, + "first_to_last_ratio": 1.3002217714306916, + "imbalance_abs_log_ratio": 0.26253484332631455, + "last_quartile_mean": 0.027375560603104532, + "mean": 0.04981810107710771, + "normalized": [ + 0.4736540568825577, + 0.5256902401367205, + 0.3705068792758801, + 0.5387329172865728, + 0.3766546277313775, + 0.9021763186028472, + 1.3591262384871188, + 1.1693409622603343, + 1.4504557122308583, + 1.2215794195883674, + 0.9103877486678197, + 0.9321134797583928, + 0.5414129179041209, + 0.8525397564320455, + 0.9808579422244733, + 0.9142526416467389, + 1.0619023214513024, + 1.8034129787908137, + 1.5888135716490905, + 1.8992814677966294, + 1.9024909329539326, + 2.549753462436128, + 2.0895004555311574, + 1.1892804001617727, + 1.0430326554170677, + 0.9143826055433697, + 0.5551398871242083, + 0.5581364223645632, + 0.38887964652236373, + 0.4518317875925886, + 0.24874413075622867, + 0.23593541479255786 + ], + "peak_layer": 22, + "peak_normalized": 2.549753462436128, + "population_cv": 0.576613701889011, + "rest_mean": 0.04285453077129744, + "spike_contrast": 2.0399565496359413, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0874213807284832, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.023596545681357384, + 0.026188889518380165, + 0.01845794916152954, + 0.026838650926947594, + 0.01876421831548214, + 0.04494471102952957, + 0.06770908832550049, + 0.05825434625148773, + 0.07225894927978516, + 0.06085676699876785, + 0.04535378888249397, + 0.0464361235499382, + 0.02697216346859932, + 0.04247191175818443, + 0.04886448010802269, + 0.04554633051156998, + 0.05290195718407631, + 0.08984261006116867, + 0.07915167510509491, + 0.09461859613656998, + 0.09477848559617996, + 0.1270238757133484, + 0.10409494489431381, + 0.0592476911842823, + 0.051961906254291534, + 0.045552805066108704, + 0.027656015008687973, + 0.027805296704173088, + 0.019373245537281036, + 0.02250940166413784, + 0.012391960248351097, + 0.011753854341804981 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.9162920753833532e-06, + "first_to_last_ratio": 0.5172826991978068, + "imbalance_abs_log_ratio": 0.6591657469591501, + "last_quartile_mean": 5.637714309614239e-06, + "mean": 5.31696678152116e-06, + "normalized": [ + 2.0689578843648566, + 0.7607510573443436, + 0.20852787293451516, + 0.2204829621221139, + 0.15259464703358938, + 0.24520248807427852, + 0.36182592024390203, + 0.3695605650129213, + 0.49013872876325654, + 0.6336599249096766, + 0.533666626864933, + 0.7065919516088963, + 0.4409106734215795, + 0.6204696462782906, + 0.6094215771745697, + 0.6245105255232631, + 0.9007337205411854, + 1.4762581910365227, + 1.5659025652557783, + 1.5082894705927152, + 1.6990202709747757, + 2.997351654903827, + 2.2171258799622118, + 2.105442927603072, + 1.6398303896765056, + 1.6495827585670528, + 1.159798242314629, + 1.3298749851714036, + 0.9845526396022054, + 0.7951372535426381, + 0.492335162782499, + 0.4314908357979929 + ], + "peak_layer": 22, + "peak_normalized": 2.997351654903827, + "population_cv": 0.6973153224686441, + "rest_mean": 4.202615000563297e-06, + "spike_contrast": 2.697003269910379, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.133446639869362e-05, + "top_five_layers": [ + 22, + 23, + 24, + 1, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.1000580343534239e-05, + 4.044888100906974e-06, + 1.1087357734140824e-06, + 1.1723005854946678e-06, + 8.113406693155412e-07, + 1.3037334838372772e-06, + 1.9238163986301515e-06, + 1.9649412479338935e-06, + 2.6060513391712448e-06, + 3.3691487715259427e-06, + 2.8374877274472965e-06, + 3.756925934794708e-06, + 2.344307404200663e-06, + 3.299016498203855e-06, + 3.2402742817794206e-06, + 3.3205017189175123e-06, + 4.789171271113446e-06, + 7.84921576268971e-06, + 8.325851922563743e-06, + 8.019525012059603e-06, + 9.033634341903962e-06, + 1.593681918166112e-05, + 1.1788384654209949e-05, + 1.1194570106454194e-05, + 8.71892370923888e-06, + 8.77077673067106e-06, + 6.166608727653511e-06, + 7.070901119732298e-06, + 5.2348336794239e-06, + 4.227718363836175e-06, + 2.61772970588936e-06, + 2.2942224404687295e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6059751689899713e-05, + "first_to_last_ratio": 1.3136640344119184, + "imbalance_abs_log_ratio": 0.27282020579643, + "last_quartile_mean": 1.983745539746451e-05, + "mean": 3.78474055935385e-05, + "normalized": [ + 0.4589265841635195, + 0.5077348811615244, + 0.3979439201623806, + 0.5340807477186122, + 0.3758546364010748, + 0.8955899993665016, + 1.2433535731478862, + 1.0948988680288454, + 1.2525996711021592, + 1.2070776025128485, + 0.9384156458392604, + 0.9009485279466896, + 0.5787782313540217, + 0.9014501901107522, + 1.0394159362329198, + 0.9428128555264055, + 1.0906052781563889, + 1.901538177289176, + 1.645145075817088, + 1.8092152293363273, + 1.9503048533412066, + 2.7077203289997303, + 2.1756312348621445, + 1.2568133833623651, + 1.064835664650375, + 0.9479102198797764, + 0.5517473493893477, + 0.45064300663075274, + 0.3856334920760448, + 0.38842166290606656, + 0.2110006297428454, + 0.19295254278496235 + ], + "peak_layer": 22, + "peak_normalized": 2.7077203289997303, + "population_cv": 0.6018204393551218, + "rest_mean": 3.202268221179515e-05, + "spike_contrast": 2.164119526172185, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.930091185495258e-05, + "top_five_layers": [ + 22, + 23, + 21, + 18, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7369180568493903e-05, + 1.9216447981307283e-05, + 1.5061144949868321e-05, + 2.021357067860663e-05, + 1.4225122868083417e-05, + 3.389575795154087e-05, + 4.7057706979103386e-05, + 4.143908154219389e-05, + 4.7407647798536345e-05, + 4.5684755605179816e-05, + 3.5516597563400865e-05, + 3.4098564356099814e-05, + 2.1905254470766522e-05, + 3.4117550967494026e-05, + 3.933919651899487e-05, + 3.568302054191008e-05, + 4.127658030483872e-05, + 7.196828664746135e-05, + 6.226447294466197e-05, + 6.847410259069875e-05, + 7.381397881545126e-05, + 0.00010248018952552229, + 8.23419977677986e-05, + 4.7567125875502825e-05, + 4.0301267290487885e-05, + 3.5875942558050156e-05, + 2.0882205717498437e-05, + 1.7055668649845757e-05, + 1.4595227185054682e-05, + 1.4700752217322588e-05, + 7.985826414369512e-06, + 7.30275314708706e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7762501443030487e-05, + "first_to_last_ratio": 1.3504079548741488, + "imbalance_abs_log_ratio": 0.3004067355961661, + "last_quartile_mean": 1.315343365604349e-05, + "mean": 2.547085748005884e-05, + "normalized": [ + 0.4678386796616822, + 0.5235377308817779, + 0.4345577931245205, + 0.5165838835160134, + 0.4023318777827698, + 0.9134377309740589, + 1.2276035467300503, + 1.0930340705226127, + 1.0712949866382682, + 1.183459795509594, + 0.9400000096186449, + 0.8234489915172668, + 0.6264659982400982, + 0.9214742940814803, + 1.1003532746415148, + 0.9455412775127405, + 1.1042840733917474, + 2.0049913068036904, + 1.659392669807193, + 1.6624482120354451, + 1.950545580645667, + 2.815676235928119, + 2.2178876005698376, + 1.2625215400320522, + 1.0704850029998623, + 1.0091870523571758, + 0.5486605781901173, + 0.4093310376700093, + 0.39616480433203954, + 0.33219377199499983, + 0.19307701107731026, + 0.17218958121164055 + ], + "peak_layer": 22, + "peak_normalized": 2.815676235928119, + "population_cv": 0.6142210141608739, + "rest_mean": 2.1398240985743338e-05, + "spike_contrast": 2.218078886997531, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.746298654936254e-05, + "top_five_layers": [ + 22, + 23, + 18, + 21, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1916252333321609e-05, + 1.3334954928723164e-05, + 1.1068559615523554e-05, + 1.3157834473531693e-05, + 1.0247737918689381e-05, + 2.326604226254858e-05, + 3.126811498077586e-05, + 2.7840515031130053e-05, + 2.728680192376487e-05, + 3.014373578480445e-05, + 2.3942606276250444e-05, + 2.0973951905034482e-05, + 1.5956626157276332e-05, + 2.347074041608721e-05, + 2.8026941436110064e-05, + 2.4083747121039778e-05, + 2.8127062250860035e-05, + 5.1068847824353725e-05, + 4.226615419611335e-05, + 4.234398147673346e-05, + 4.96820684929844e-05, + 7.171768811531365e-05, + 5.6491498980904e-05, + 3.21575062116608e-05, + 2.7266170945949852e-05, + 2.5704859581310302e-05, + 1.3974855392007157e-05, + 1.0426012522657402e-05, + 1.0090657269756775e-05, + 8.461260222247802e-06, + 4.917837031825911e-06, + 4.385816282592714e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.240166451083496e-05, + "first_to_last_ratio": 1.2489384660588578, + "imbalance_abs_log_ratio": 0.22229396336363683, + "last_quartile_mean": 5.797056178380444e-05, + "mean": 0.0001109465127342446, + "normalized": [ + 0.42709475759614274, + 0.4935186328821003, + 0.35950105255417747, + 0.5439303076283766, + 0.3371404368406644, + 0.8629008054703304, + 1.152554863343196, + 1.0440127885102133, + 1.3928110595765304, + 1.2466636602290324, + 0.9528986062756032, + 0.9925221789536534, + 0.5398804000340625, + 0.8579108303439223, + 1.0148735477787485, + 0.9293336653216302, + 1.0803774560382864, + 1.8519808466056424, + 1.6712809075368884, + 1.9834945531761723, + 1.984928279946317, + 2.6376973498062735, + 2.185821980542675, + 1.2767982858711757, + 1.0594658474140766, + 0.9033078060139075, + 0.5489550405623127, + 0.4646003337611295, + 0.361861238846865, + 0.4230227492827023, + 0.22047516727493316, + 0.19838456398225895 + ], + "peak_layer": 22, + "peak_normalized": 2.6376973498062735, + "population_cv": 0.610312229060459, + "rest_mean": 9.391535294761761e-05, + "spike_contrast": 2.1606134589645687, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002029147755820304, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.738467396236956e-05, + 5.475417128764093e-05, + 3.988538810517639e-05, + 6.034717080183327e-05, + 3.7404555769171566e-05, + 9.573583520250395e-05, + 0.00012787194282282144, + 0.00011582957813516259, + 0.00015452752995770425, + 0.00013831298565492034, + 0.00010572077735560015, + 0.0001101168745663017, + 5.989784767734818e-05, + 9.518221486359835e-05, + 0.00011259668099228293, + 0.00010310632933396846, + 0.00011986411118414253, + 0.00020547081658151, + 0.00018542278849054128, + 0.00022006180370226502, + 0.0002202208706876263, + 0.000292643322609365, + 0.00024250932619906962, + 0.00014165631728246808, + 0.00011754404113162309, + 0.00010021885100286454, + 6.090464739827439e-05, + 5.154578684596345e-05, + 4.0147242543753237e-05, + 4.693289884016849e-05, + 2.4460950953653082e-05, + 2.2010075554135256e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05931045487523079, + 0.07934419065713882, + 0.214645653963089, + 0.09514089673757553, + 0.12821291387081146, + 0.1321839988231659, + 0.12568901479244232, + 0.1335759311914444, + 0.07788944244384766, + 0.08352623134851456, + 0.07934392243623734, + 0.09343641996383667, + 0.07063088566064835, + 0.06870647519826889, + 0.07622843235731125, + 0.08666495233774185, + 0.057347409427165985, + 0.05777324363589287, + 0.06351501494646072, + 0.07469023764133453, + 0.051319729536771774, + 0.05102822557091713, + 0.05902284011244774, + 0.06656526774168015, + 0.050236333161592484, + 0.05184732377529144, + 0.061248794198036194, + 0.06608261168003082, + 0.048912134021520615, + 0.062179163098335266, + 0.07131517678499222, + 0.06802479177713394 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 5.254158395473496e-06, + "first_to_last_ratio": 0.18864041654826996, + "imbalance_abs_log_ratio": 1.6679126340303851, + "last_quartile_mean": 2.7852771381731145e-05, + "mean": 2.0484676667820167e-05, + "normalized": [ + 0.5568020220803858, + 0.21809281295181746, + 0.09241273564310358, + 0.28328072581730895, + 0.22196567434120804, + 0.13250710048221157, + 0.17764328005295377, + 0.3692327060422689, + 0.6392853696302555, + 0.4685551353916866, + 0.4862788629304224, + 0.46993740268393913, + 0.5775584528492825, + 0.5275503726871256, + 0.4833126690762866, + 0.4805557266835708, + 1.3911160583811022, + 1.414259456489031, + 1.4619454359332327, + 1.4084306067413008, + 2.477707616941058, + 2.525352305517634, + 2.2589272888438794, + 1.9997852708388022, + 2.0849570534601547, + 1.8699232722164576, + 1.4787615635622833, + 1.3487564232655025, + 1.428990706094454, + 1.0620321056863733, + 0.8416254501901257, + 0.762458336494781 + ], + "peak_layer": 22, + "peak_normalized": 2.525352305517634, + "population_cv": 0.7251468208972824, + "rest_mean": 1.566946546652751e-05, + "spike_contrast": 2.966713654279006, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.648681715480052e-05, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1405909390305169e-05, + 4.467560756893363e-06, + 1.8930450096377172e-06, + 5.802914074592991e-06, + 4.546895070234314e-06, + 2.7143651095684618e-06, + 3.6389651540957857e-06, + 7.563612598460168e-06, + 1.3095554095343687e-05, + 9.598200449545402e-06, + 9.961265277524944e-06, + 9.626515748095699e-06, + 1.1831098163384013e-05, + 1.0806698810483795e-05, + 9.900503755488899e-06, + 9.844028681982309e-06, + 2.8496562663349323e-05, + 2.8970647690584883e-05, + 2.9947479561087675e-05, + 2.8851245588157326e-05, + 5.07550394104328e-05, + 5.1731025450862944e-05, + 4.6273395128082484e-05, + 4.096495467820205e-05, + 4.270967110642232e-05, + 3.830477362498641e-05, + 3.0291952498373576e-05, + 2.7628839234239422e-05, + 2.927241257566493e-05, + 2.1755384295829572e-05, + 1.7240425222553313e-05, + 1.561871249577962e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.0563243310789403e-05, + "first_to_last_ratio": 0.184584542859635, + "imbalance_abs_log_ratio": 1.6896476935739415, + "last_quartile_mean": 0.00011140284550492652, + "mean": 8.183052003118974e-05, + "normalized": [ + 0.5514618447323059, + 0.2169408826973503, + 0.08472485593965659, + 0.28295099514537675, + 0.21989392642956848, + 0.1303640614627248, + 0.17290402482659964, + 0.35108448581118906, + 0.6388397786640535, + 0.4693907558756991, + 0.4878178928069798, + 0.4684568375678178, + 0.5735758405807379, + 0.5222431877258841, + 0.48692296371012395, + 0.47894342377930604, + 1.3828038952387727, + 1.4137992065669855, + 1.4773604231197706, + 1.4101154596735193, + 2.482879371711871, + 2.526943137473127, + 2.259311590178407, + 2.019190865749238, + 2.085961378955964, + 1.8764944434843154, + 1.4765035496268348, + 1.350239590483524, + 1.4291199659133378, + 1.0658000032658883, + 0.838995628185017, + 0.7679657326180543 + ], + "peak_layer": 22, + "peak_normalized": 2.526943137473127, + "population_cv": 0.7282772544392188, + "rest_mean": 6.251158794367695e-05, + "spike_contrast": 2.977891930556664, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001861527533037588, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.51264095318038e-05, + 1.7752385247149505e-05, + 6.933079021109734e-06, + 2.3154027076088823e-05, + 1.7994034351431765e-05, + 1.0667758942872752e-05, + 1.4148826267046388e-05, + 2.8729426048812456e-05, + 5.2276591304689646e-05, + 3.841048965114169e-05, + 3.9918391848914325e-05, + 3.833406663034111e-05, + 4.6936009312048554e-05, + 4.2735431634355336e-05, + 3.984515933552757e-05, + 3.91921894333791e-05, + 0.00011315556184854358, + 0.00011569192429305986, + 0.00012089317169738933, + 0.00011539048136910424, + 0.00020317531016189605, + 0.00020678107102867216, + 0.00018488064233679324, + 0.00016523143858648837, + 0.00017069530440494418, + 0.0001535545161459595, + 0.00012082305329386145, + 0.00011049080785596743, + 0.00011694562999764457, + 8.721496851649135e-05, + 6.865544855827466e-05, + 6.2843035266269e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.018235690018627793, + "first_to_last_ratio": 0.1845845477639835, + "imbalance_abs_log_ratio": 1.6896476670042821, + "last_quartile_mean": 0.09879315597936511, + "mean": 0.07256812266132329, + "normalized": [ + 0.5514618483157399, + 0.21694088816023477, + 0.08472485792509094, + 0.28295099765712173, + 0.21989393857080072, + 0.13036405951384217, + 0.17290402763813106, + 0.3510845056878191, + 0.6388397993688198, + 0.4693907637084327, + 0.48781790094052196, + 0.46845687595408214, + 0.5735758195754074, + 0.5222432048140645, + 0.48692297651101485, + 0.47894340052024936, + 1.3828038812071648, + 1.413799283122749, + 1.4773604270157898, + 1.4101154778648226, + 2.482879404994159, + 2.526942972591032, + 2.259311440497952, + 2.0191909931789223, + 2.0859612981908873, + 1.8764944638324905, + 1.4765035418797068, + 1.350239576611916, + 1.4291200320637618, + 1.065800015673522, + 0.8389956311315239, + 0.7679656952822267 + ], + "peak_layer": 22, + "peak_normalized": 2.526942972591032, + "population_cv": 0.7282772403307106, + "rest_mean": 0.0554359013180214, + "spike_contrast": 2.977891835258891, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1650821179151535, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04001855105161667, + 0.015742992982268333, + 0.006148323882371187, + 0.02053322270512581, + 0.015957290306687355, + 0.009460275061428547, + 0.012547320686280727, + 0.025477543473243713, + 0.04635940492153168, + 0.03406280651688576, + 0.03540002927184105, + 0.03399503603577614, + 0.0416233204305172, + 0.03789820894598961, + 0.03533508628606796, + 0.034756023436784744, + 0.10034748166799545, + 0.10259675979614258, + 0.10720927268266678, + 0.10232943296432495, + 0.18017789721488953, + 0.1833755075931549, + 0.1639539897441864, + 0.14652889966964722, + 0.15137429535388947, + 0.13617368042469025, + 0.10714709013700485, + 0.09798435121774673, + 0.10370855778455734, + 0.07734310626983643, + 0.06088433787226677, + 0.05572982877492905 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0810431092522776e-06, + "first_to_last_ratio": 0.14665609099981078, + "imbalance_abs_log_ratio": 1.9196649501572336, + "last_quartile_mean": 7.371279991730262e-06, + "mean": 5.266313571894443e-06, + "normalized": [ + 0.5313084313996962, + 0.2061221485022935, + 0.06868365811035125, + 0.2784811311506817, + 0.146950226496972, + 0.10534296524196422, + 0.12013039774147215, + 0.1851819388065788, + 0.48851680128093, + 0.4019870994251106, + 0.45293299696953215, + 0.4442480210738259, + 0.5787785597712432, + 0.5090619802943529, + 0.49010374567770754, + 0.468585442827116, + 1.3525598419019165, + 1.4672711047348668, + 1.4843408168011953, + 1.4896119798089702, + 2.508208189766812, + 2.524495568813508, + 2.388845240425297, + 2.1106199865726754, + 2.1011836342122523, + 1.9283517633380012, + 1.5593850323316485, + 1.452268444304737, + 1.4451925616379866, + 1.080259681295224, + 0.833132510373883, + 0.7978580989111985 + ], + "peak_layer": 22, + "peak_normalized": 2.524495568813508, + "population_cv": 0.7672111824742712, + "rest_mean": 3.972487093051313e-06, + "spike_contrast": 3.084459752954336, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2252976557647343e-05, + "top_five_layers": [ + 22, + 21, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7980368031421676e-06, + 1.0855038681256701e-06, + 3.6170968087390065e-07, + 1.4665689604953513e-06, + 7.738859721939662e-07, + 5.547690875573608e-07, + 6.326443440229923e-07, + 9.752261576068122e-07, + 2.5726826606842224e-06, + 2.1169901174289407e-06, + 2.385287189099472e-06, + 2.3395493826683378e-06, + 3.048029384444817e-06, + 2.6808800157596124e-06, + 2.581040007498814e-06, + 2.4677178771526087e-06, + 7.1230042522074655e-06, + 7.727109732513782e-06, + 7.817004188837018e-06, + 7.844763786124531e-06, + 1.3209010830905754e-05, + 1.3294785276229959e-05, + 1.2580408110807184e-05, + 1.1115186680399347e-05, + 1.1065491889894474e-05, + 1.0155305062653497e-05, + 8.212210559577215e-06, + 7.648101018276066e-06, + 7.610837201355025e-06, + 5.688986220775405e-06, + 4.387537046568468e-06, + 4.201770934741944e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.3072703836769506e-05, + "first_to_last_ratio": 0.1965017087277629, + "imbalance_abs_log_ratio": 1.6270841518950423, + "last_quartile_mean": 6.65271763864439e-05, + "mean": 5.1720684069778144e-05, + "normalized": [ + 0.5903214985473635, + 0.21317222278028428, + 0.07861556264200854, + 0.2924267620633159, + 0.20614520303339837, + 0.10942131635502071, + 0.17754724840214117, + 0.3543967343826241, + 0.6616591248095567, + 0.5148181180855445, + 0.5447776590149115, + 0.5218723062533341, + 0.647120485093606, + 0.5989696729166056, + 0.5592579173992114, + 0.5462615999231498, + 1.3998993635309975, + 1.4086638785298187, + 1.4578445936683029, + 1.4102355321134408, + 2.6382627874666973, + 2.6317584037312556, + 2.2233659362975056, + 1.922962332424717, + 2.1034577149820564, + 1.8142894928770414, + 1.4101545016386676, + 1.2891070706275354, + 1.273679909247231, + 0.9355476159814242, + 0.756254888856092, + 0.7077325463251402 + ], + "peak_layer": 21, + "peak_normalized": 2.6382627874666973, + "population_cv": 0.7263037787473766, + "rest_mean": 3.923146602944514e-05, + "spike_contrast": 3.0374205592556516, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011916246148757637, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0531831725966185e-05, + 1.1025413186871447e-05, + 4.066050678375177e-06, + 1.5124512174224947e-05, + 1.0661970918590669e-05, + 5.659345333697274e-06, + 9.182865142065566e-06, + 1.8329641534364782e-05, + 3.4221462556160986e-05, + 2.6626745238900185e-05, + 2.8176273190183565e-05, + 2.6991592676495202e-05, + 3.346951416460797e-05, + 3.097912122029811e-05, + 2.8925202059326693e-05, + 2.825302362907678e-05, + 7.240375271067023e-05, + 7.285705942194909e-05, + 7.540071965195239e-05, + 7.293834642041475e-05, + 0.0001364527561236173, + 0.0001361163449473679, + 0.00011499400716274977, + 9.945692727342248e-05, + 0.00010879227193072438, + 9.383629367221147e-05, + 7.293415546882898e-05, + 6.667349953204393e-05, + 6.587559619219974e-05, + 4.838716267840937e-05, + 3.911402018275112e-05, + 3.66044114343822e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.603489931147124e-06, + "first_to_last_ratio": 0.24312184321824193, + "imbalance_abs_log_ratio": 1.4141925488860427, + "last_quartile_mean": 3.538756459420256e-05, + "mean": 3.111726851301455e-05, + "normalized": [ + 0.6644756490180851, + 0.23325159254807834, + 0.07539325824570257, + 0.31873771579525234, + 0.21217410245742602, + 0.11960013716276979, + 0.20155955466263742, + 0.38669620325415055, + 0.7284622302362078, + 0.5930713003867844, + 0.6319198952855785, + 0.6040604923468361, + 0.7580808064483824, + 0.7072262953008145, + 0.6749843915485383, + 0.6486223376959581, + 1.4735493318409316, + 1.4845067822216826, + 1.4896409680595757, + 1.476749210387741, + 2.817405493456774, + 2.7673279203027326, + 2.139795377674301, + 1.694850102870422, + 2.0495665188393017, + 1.6765832039465556, + 1.3012719394146397, + 1.1897383429944555, + 1.0088778638486162, + 0.720783924666597, + 0.5951684074906716, + 0.5558686495917995 + ], + "peak_layer": 21, + "peak_normalized": 2.817405493456774, + "population_cv": 0.7201026839240032, + "rest_mean": 2.366186438646562e-05, + "spike_contrast": 3.0165184632367974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.137645079637877e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.067666719085537e-05, + 7.258152436406817e-06, + 2.3460322609025752e-06, + 9.918247087625787e-06, + 6.602278517675586e-06, + 3.721629582287278e-06, + 6.271982783800922e-06, + 1.2032929589622654e-05, + 2.2667754819849506e-05, + 1.845475890149828e-05, + 1.9663621060317382e-05, + 1.879671253846027e-05, + 2.3589404008816928e-05, + 2.2006950530339964e-05, + 2.1003670553909615e-05, + 2.018335544562433e-05, + 4.5852830226067454e-05, + 4.619379615178332e-05, + 4.635355799109675e-05, + 4.5952401706017554e-05, + 8.76699632499367e-05, + 8.611168595962226e-05, + 6.658458732999861e-05, + 5.273910574032925e-05, + 6.377691170200706e-05, + 5.2170689741615206e-05, + 4.0492028347216547e-05, + 3.7021407479187474e-05, + 3.139352338621393e-05, + 2.242882692371495e-05, + 1.8520015146350488e-05, + 1.729711402731482e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.8654464106002706e-05, + "first_to_last_ratio": 0.16200207727357052, + "imbalance_abs_log_ratio": 1.8201461211556016, + "last_quartile_mean": 0.0002386047435720684, + "mean": 0.00017063225249103198, + "normalized": [ + 0.5177972227971404, + 0.18422958638068, + 0.07755682312933251, + 0.27342225532372755, + 0.19007362166945208, + 0.0968768622809422, + 0.1517726472113855, + 0.32056442710365024, + 0.6168627493486994, + 0.4509605691569947, + 0.4954745315627688, + 0.46627480155634954, + 0.5531570230685467, + 0.5170724935984963, + 0.45563757978860675, + 0.4789901403808068, + 1.3287627572643317, + 1.3383475524992587, + 1.407284240753668, + 1.3213535994248635, + 2.4744625690346806, + 2.5664095372695273, + 2.4114518937040392, + 2.1183513814824377, + 2.201741632911359, + 1.9351067846360057, + 1.5669730065860399, + 1.4334150862915997, + 1.3851211553907088, + 1.0454346613928627, + 0.8612433454555685, + 0.7578174615454693 + ], + "peak_layer": 22, + "peak_normalized": 2.5664095372695273, + "population_cv": 0.7569752509959051, + "rest_mean": 0.00012783252026969826, + "spike_contrast": 3.14279031375294, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00040175080648623406, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.83529064594768e-05, + 3.1435509299626574e-05, + 1.3233695426606573e-05, + 4.6654655307065696e-05, + 3.2432690204586834e-05, + 1.653031722526066e-05, + 2.5897308660205454e-05, + 5.469863026519306e-05, + 0.00010525668039917946, + 7.69484176998958e-05, + 8.454393537249416e-05, + 7.956151966936886e-05, + 9.438642882741988e-05, + 8.822924428386614e-05, + 7.774646655889228e-05, + 8.173116657417268e-05, + 0.00022672978229820728, + 0.0002283652574988082, + 0.00024012807989493012, + 0.00022546554100699723, + 0.00042222312185913324, + 0.0004379122401587665, + 0.00041147146839648485, + 0.0003614590677898377, + 0.000375688134226948, + 0.00033019162947312, + 0.00026737613370642066, + 0.00024458684492856264, + 0.00023634634271729738, + 0.00017838487110566348, + 0.00014695589197799563, + 0.00012930810044053942 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 1.8565786854442996, + "forward": { + "activation_sha256": { + "attention_branch_output": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", + "mlp_branch_output": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", + "post_attention_state": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", + "post_mlp_state": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", + "pre_attention_input": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", + "pre_mlp_input": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f" + }, + "logits_sha256": "a911233a29684ae3ced35122e9aee12a584fd2d6b7e0272c227ab94eb4e607ec", + "loss_nats": 1.2868822813034058, + "mixer_summary_sha256": "998a9ce6e3ee4cab0180aeb81cc12c18ccf6b1832b43c8f455133227a7e72a26" + }, + "loss_nats": 1.2868822813034058, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.21598131954669952, + "group": 1, + "latest_source_mass": 0.061274416744709015, + "layer": 1, + "max_source_mass": 0.9387255907058716, + "mixer_index": 1, + "normalized_entropy": 0.31159517863467795, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9387255907058716, + "median": 0.9528161883354187, + "p05": 0.8532108068466187, + "p95": 0.9815421104431152 + }, + { + "label": "current_group_partial", + "mean": 0.061274416744709015, + "median": 0.047183819115161896, + "p05": 0.018457919359207153, + "p95": 0.14678917825222015 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5377621054649353, + "group": 1, + "latest_source_mass": 0.2917928099632263, + "layer": 2, + "max_source_mass": 0.7082071900367737, + "mixer_index": 2, + "normalized_entropy": 0.7758267227322699, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7082071900367737, + "median": 0.7268136143684387, + "p05": 0.38468095660209656, + "p95": 0.923936128616333 + }, + { + "label": "current_group_partial", + "mean": 0.2917928099632263, + "median": 0.27318641543388367, + "p05": 0.07606390863656998, + "p95": 0.6153190732002258 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.43774232268333435, + "group": 1, + "latest_source_mass": 0.170285165309906, + "layer": 2, + "max_source_mass": 0.829714834690094, + "mixer_index": 3, + "normalized_entropy": 0.6315286781224629, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.829714834690094, + "median": 0.85542893409729, + "p05": 0.687734842300415, + "p95": 0.9260365962982178 + }, + { + "label": "current_group_partial", + "mean": 0.170285165309906, + "median": 0.14457105100154877, + "p05": 0.07396337389945984, + "p95": 0.31226515769958496 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6749181151390076, + "group": 1, + "latest_source_mass": 0.45927006006240845, + "layer": 3, + "max_source_mass": 0.5407299995422363, + "mixer_index": 4, + "normalized_entropy": 0.9737010177171727, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5407299995422363, + "median": 0.5310744643211365, + "p05": 0.4232047200202942, + "p95": 0.6916534900665283 + }, + { + "label": "current_group_partial", + "mean": 0.45927006006240845, + "median": 0.4689255356788635, + "p05": 0.3083464205265045, + "p95": 0.576795220375061 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6663151383399963, + "group": 1, + "latest_source_mass": 0.49885618686676025, + "layer": 3, + "max_source_mass": 0.5011438131332397, + "mixer_index": 5, + "normalized_entropy": 0.9612895457523564, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5011438131332397, + "median": 0.4804786741733551, + "p05": 0.3554112911224365, + "p95": 0.7207092642784119 + }, + { + "label": "current_group_partial", + "mean": 0.49885618686676025, + "median": 0.5195213556289673, + "p05": 0.27929070591926575, + "p95": 0.6445887684822083 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6865031719207764, + "group": 1, + "latest_source_mass": 0.4691343307495117, + "layer": 4, + "max_source_mass": 0.5308656692504883, + "mixer_index": 6, + "normalized_entropy": 0.9904147216846476, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5308656692504883, + "median": 0.5278033018112183, + "p05": 0.45786383748054504, + "p95": 0.6093498468399048 + }, + { + "label": "current_group_partial", + "mean": 0.4691343307495117, + "median": 0.47219666838645935, + "p05": 0.3906501829624176, + "p95": 0.5421361327171326 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4133813679218292, + "group": 1, + "latest_source_mass": 0.15208598971366882, + "layer": 4, + "max_source_mass": 0.8479140400886536, + "mixer_index": 7, + "normalized_entropy": 0.5963832494967191, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8479140400886536, + "median": 0.8514594435691833, + "p05": 0.7530128359794617, + "p95": 0.9402707815170288 + }, + { + "label": "current_group_partial", + "mean": 0.15208598971366882, + "median": 0.14854055643081665, + "p05": 0.05972917377948761, + "p95": 0.24698717892169952 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6850745677947998, + "group": 2, + "latest_source_mass": 0.4601486325263977, + "layer": 5, + "max_source_mass": 0.5398513674736023, + "mixer_index": 8, + "normalized_entropy": 0.9883536815967077, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5398513674736023, + "median": 0.5387102365493774, + "p05": 0.46027374267578125, + "p95": 0.623892605304718 + }, + { + "label": "completed_group_1", + "mean": 0.4601486325263977, + "median": 0.46128979325294495, + "p05": 0.376107394695282, + "p95": 0.5397262573242188 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074119567871094, + "group": 2, + "latest_source_mass": 0.4627366364002228, + "layer": 5, + "max_source_mass": 0.4627366364002228, + "mixer_index": 9, + "normalized_entropy": 0.9169858804405273, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32744842767715454, + "median": 0.30657994747161865, + "p05": 0.16577640175819397, + "p95": 0.5453717708587646 + }, + { + "label": "completed_group_1", + "mean": 0.20981493592262268, + "median": 0.2177819311618805, + "p05": 0.14342713356018066, + "p95": 0.2433384209871292 + }, + { + "label": "current_group_partial", + "mean": 0.4627366364002228, + "median": 0.4847465753555298, + "p05": 0.27109938859939575, + "p95": 0.6081984639167786 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0489801168441772, + "group": 2, + "latest_source_mass": 0.4763939380645752, + "layer": 6, + "max_source_mass": 0.4763939380645752, + "mixer_index": 10, + "normalized_entropy": 0.9548228503031734, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28095996379852295, + "median": 0.2765299677848816, + "p05": 0.23549969494342804, + "p95": 0.3424558639526367 + }, + { + "label": "completed_group_1", + "mean": 0.24264612793922424, + "median": 0.23937124013900757, + "p05": 0.21875721216201782, + "p95": 0.2814398407936096 + }, + { + "label": "current_group_partial", + "mean": 0.4763939380645752, + "median": 0.48882943391799927, + "p05": 0.38478636741638184, + "p95": 0.528400719165802 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.8527575731277466, + "group": 2, + "latest_source_mass": 0.6454645395278931, + "layer": 6, + "max_source_mass": 0.6454645395278931, + "mixer_index": 11, + "normalized_entropy": 0.7762133938639787, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20522089302539825, + "median": 0.1591135561466217, + "p05": 0.11052663624286652, + "p95": 0.47355735301971436 + }, + { + "label": "completed_group_1", + "mean": 0.1493145078420639, + "median": 0.15203402936458588, + "p05": 0.10004506260156631, + "p95": 0.18058395385742188 + }, + { + "label": "current_group_partial", + "mean": 0.6454645395278931, + "median": 0.68653404712677, + "p05": 0.4207671880722046, + "p95": 0.7333818674087524 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.049942135810852, + "group": 2, + "latest_source_mass": 0.4395729899406433, + "layer": 7, + "max_source_mass": 0.4395729899406433, + "mixer_index": 12, + "normalized_entropy": 0.9556985177033998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35788866877555847, + "median": 0.36233147978782654, + "p05": 0.30507102608680725, + "p95": 0.3985886573791504 + }, + { + "label": "completed_group_1", + "mean": 0.20253834128379822, + "median": 0.19928529858589172, + "p05": 0.180739626288414, + "p95": 0.23214691877365112 + }, + { + "label": "current_group_partial", + "mean": 0.4395729899406433, + "median": 0.43975552916526794, + "p05": 0.3959323763847351, + "p95": 0.4790603220462799 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9791408181190491, + "group": 2, + "latest_source_mass": 0.5214842557907104, + "layer": 7, + "max_source_mass": 0.5214842557907104, + "mixer_index": 13, + "normalized_entropy": 0.8912523810434521, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31465935707092285, + "median": 0.28992873430252075, + "p05": 0.20534247159957886, + "p95": 0.507034420967102 + }, + { + "label": "completed_group_1", + "mean": 0.1638563871383667, + "median": 0.1667816936969757, + "p05": 0.11925681680440903, + "p95": 0.19515612721443176 + }, + { + "label": "current_group_partial", + "mean": 0.5214842557907104, + "median": 0.5455081462860107, + "p05": 0.35950881242752075, + "p95": 0.6097541451454163 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0485806465148926, + "group": 2, + "latest_source_mass": 0.4457142949104309, + "layer": 8, + "max_source_mass": 0.4457142949104309, + "mixer_index": 14, + "normalized_entropy": 0.9544592367395849, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.346283882856369, + "median": 0.353412926197052, + "p05": 0.26436588168144226, + "p95": 0.4084087312221527 + }, + { + "label": "completed_group_1", + "mean": 0.20800183713436127, + "median": 0.20826736092567444, + "p05": 0.18089595437049866, + "p95": 0.23509056866168976 + }, + { + "label": "current_group_partial", + "mean": 0.4457142949104309, + "median": 0.4416901469230652, + "p05": 0.3883572816848755, + "p95": 0.5136083364486694 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9801348447799683, + "group": 2, + "latest_source_mass": 0.543371319770813, + "layer": 8, + "max_source_mass": 0.543371319770813, + "mixer_index": 15, + "normalized_entropy": 0.8921571831025336, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2748759388923645, + "median": 0.2577607035636902, + "p05": 0.17991375923156738, + "p95": 0.41774845123291016 + }, + { + "label": "completed_group_1", + "mean": 0.1817527413368225, + "median": 0.17968356609344482, + "p05": 0.130966454744339, + "p95": 0.23766928911209106 + }, + { + "label": "current_group_partial", + "mean": 0.543371319770813, + "median": 0.5535831451416016, + "p05": 0.4452384412288666, + "p95": 0.6142411828041077 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0688585042953491, + "group": 3, + "latest_source_mass": 0.36256325244903564, + "layer": 9, + "max_source_mass": 0.4051227569580078, + "mixer_index": 16, + "normalized_entropy": 0.9729169383233166, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4051227569580078, + "median": 0.4008670449256897, + "p05": 0.3286888897418976, + "p95": 0.48389261960983276 + }, + { + "label": "completed_group_1", + "mean": 0.23231396079063416, + "median": 0.23353952169418335, + "p05": 0.2008797824382782, + "p95": 0.2593720257282257 + }, + { + "label": "completed_group_2", + "mean": 0.36256325244903564, + "median": 0.3638863265514374, + "p05": 0.3081618845462799, + "p95": 0.4163197875022888 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2915732860565186, + "group": 3, + "latest_source_mass": 0.36661261320114136, + "layer": 9, + "max_source_mass": 0.36661261320114136, + "mixer_index": 17, + "normalized_entropy": 0.931673187369201, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.26621371507644653, + "median": 0.23494771122932434, + "p05": 0.16815711557865143, + "p95": 0.4694361090660095 + }, + { + "label": "completed_group_1", + "mean": 0.1108347550034523, + "median": 0.11330658197402954, + "p05": 0.06620892137289047, + "p95": 0.13892118632793427 + }, + { + "label": "completed_group_2", + "mean": 0.2563389539718628, + "median": 0.26728466153144836, + "p05": 0.16010139882564545, + "p95": 0.3091832995414734 + }, + { + "label": "current_group_partial", + "mean": 0.36661261320114136, + "median": 0.36950358748435974, + "p05": 0.29210716485977173, + "p95": 0.42882072925567627 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3568191528320312, + "group": 3, + "latest_source_mass": 0.30908840894699097, + "layer": 10, + "max_source_mass": 0.30908840894699097, + "mixer_index": 18, + "normalized_entropy": 0.978738131586968, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.294538676738739, + "median": 0.2961598038673401, + "p05": 0.2467859834432602, + "p95": 0.34424129128456116 + }, + { + "label": "completed_group_1", + "mean": 0.1710844486951828, + "median": 0.17187124490737915, + "p05": 0.1508811116218567, + "p95": 0.1898101568222046 + }, + { + "label": "completed_group_2", + "mean": 0.2252884805202484, + "median": 0.22875282168388367, + "p05": 0.18443678319454193, + "p95": 0.25181543827056885 + }, + { + "label": "current_group_partial", + "mean": 0.30908840894699097, + "median": 0.3080518841743469, + "p05": 0.2719307243824005, + "p95": 0.3494151830673218 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.320021152496338, + "group": 3, + "latest_source_mass": 0.3312946557998657, + "layer": 10, + "max_source_mass": 0.3312946557998657, + "mixer_index": 19, + "normalized_entropy": 0.9521939852875004, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30465462803840637, + "median": 0.311603307723999, + "p05": 0.21955373883247375, + "p95": 0.40013614296913147 + }, + { + "label": "completed_group_1", + "mean": 0.12911874055862427, + "median": 0.12981033325195312, + "p05": 0.09804825484752655, + "p95": 0.15542283654212952 + }, + { + "label": "completed_group_2", + "mean": 0.23493197560310364, + "median": 0.24103470146656036, + "p05": 0.16810482740402222, + "p95": 0.28502923250198364 + }, + { + "label": "current_group_partial", + "mean": 0.3312946557998657, + "median": 0.325168639421463, + "p05": 0.2647673785686493, + "p95": 0.41786038875579834 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3578683137893677, + "group": 3, + "latest_source_mass": 0.30723780393600464, + "layer": 11, + "max_source_mass": 0.30723780393600464, + "mixer_index": 20, + "normalized_entropy": 0.9794949412420898, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29558902978897095, + "median": 0.2938143014907837, + "p05": 0.24335844814777374, + "p95": 0.3542291522026062 + }, + { + "label": "completed_group_1", + "mean": 0.17812997102737427, + "median": 0.17726755142211914, + "p05": 0.15839283168315887, + "p95": 0.20142368972301483 + }, + { + "label": "completed_group_2", + "mean": 0.21904319524765015, + "median": 0.2218058705329895, + "p05": 0.18076662719249725, + "p95": 0.24698719382286072 + }, + { + "label": "current_group_partial", + "mean": 0.30723780393600464, + "median": 0.30536961555480957, + "p05": 0.26194074749946594, + "p95": 0.35952210426330566 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.258887767791748, + "group": 3, + "latest_source_mass": 0.34303945302963257, + "layer": 11, + "max_source_mass": 0.35744374990463257, + "mixer_index": 21, + "normalized_entropy": 0.9080955698144659, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35744374990463257, + "median": 0.359974205493927, + "p05": 0.17910680174827576, + "p95": 0.5207242965698242 + }, + { + "label": "completed_group_1", + "mean": 0.12067858874797821, + "median": 0.11769672483205795, + "p05": 0.08151324093341827, + "p95": 0.16476252675056458 + }, + { + "label": "completed_group_2", + "mean": 0.17883820831775665, + "median": 0.17648206651210785, + "p05": 0.0995911955833435, + "p95": 0.2739725708961487 + }, + { + "label": "current_group_partial", + "mean": 0.34303945302963257, + "median": 0.33267199993133545, + "p05": 0.22800606489181519, + "p95": 0.48691368103027344 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3539760112762451, + "group": 3, + "latest_source_mass": 0.3175881803035736, + "layer": 12, + "max_source_mass": 0.3175881803035736, + "mixer_index": 22, + "normalized_entropy": 0.976687238475429, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2736307978630066, + "median": 0.27999642491340637, + "p05": 0.18969643115997314, + "p95": 0.3464604914188385 + }, + { + "label": "completed_group_1", + "mean": 0.1722733974456787, + "median": 0.17083553969860077, + "p05": 0.1506597101688385, + "p95": 0.1988438367843628 + }, + { + "label": "completed_group_2", + "mean": 0.2365075945854187, + "median": 0.24049004912376404, + "p05": 0.19099700450897217, + "p95": 0.26942259073257446 + }, + { + "label": "current_group_partial", + "mean": 0.3175881803035736, + "median": 0.3092706799507141, + "p05": 0.268545538187027, + "p95": 0.3924510180950165 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3203014135360718, + "group": 3, + "latest_source_mass": 0.3395293354988098, + "layer": 12, + "max_source_mass": 0.3395293354988098, + "mixer_index": 23, + "normalized_entropy": 0.9523961508935896, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28207293152809143, + "median": 0.29242122173309326, + "p05": 0.1354818046092987, + "p95": 0.4195851981639862 + }, + { + "label": "completed_group_1", + "mean": 0.15420743823051453, + "median": 0.15396761894226074, + "p05": 0.11267110705375671, + "p95": 0.1951446533203125 + }, + { + "label": "completed_group_2", + "mean": 0.22419027984142303, + "median": 0.22542357444763184, + "p05": 0.1655150204896927, + "p95": 0.2747117578983307 + }, + { + "label": "current_group_partial", + "mean": 0.3395293354988098, + "median": 0.32681703567504883, + "p05": 0.2524653375148773, + "p95": 0.46652013063430786 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3734662532806396, + "group": 4, + "latest_source_mass": 0.283242404460907, + "layer": 13, + "max_source_mass": 0.283242404460907, + "mixer_index": 24, + "normalized_entropy": 0.9907464762181619, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2766023874282837, + "median": 0.282479852437973, + "p05": 0.22528076171875, + "p95": 0.3097468614578247 + }, + { + "label": "completed_group_1", + "mean": 0.21176087856292725, + "median": 0.2058793306350708, + "p05": 0.18811854720115662, + "p95": 0.2617422640323639 + }, + { + "label": "completed_group_2", + "mean": 0.22839435935020447, + "median": 0.22951346635818481, + "p05": 0.20004916191101074, + "p95": 0.2551718056201935 + }, + { + "label": "completed_group_3", + "mean": 0.283242404460907, + "median": 0.2849352955818176, + "p05": 0.2205706238746643, + "p95": 0.34444743394851685 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5325813293457031, + "group": 4, + "latest_source_mass": 0.27591708302497864, + "layer": 13, + "max_source_mass": 0.27591708302497864, + "mixer_index": 25, + "normalized_entropy": 0.9522463199762954, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17451074719429016, + "median": 0.16826045513153076, + "p05": 0.08149556815624237, + "p95": 0.3126368522644043 + }, + { + "label": "completed_group_1", + "mean": 0.10643941164016724, + "median": 0.10269765555858612, + "p05": 0.08178127557039261, + "p95": 0.13947664201259613 + }, + { + "label": "completed_group_2", + "mean": 0.1691833734512329, + "median": 0.17118802666664124, + "p05": 0.12382284551858902, + "p95": 0.2069271057844162 + }, + { + "label": "completed_group_3", + "mean": 0.27394935488700867, + "median": 0.27278125286102295, + "p05": 0.2122008204460144, + "p95": 0.33945947885513306 + }, + { + "label": "current_group_partial", + "mean": 0.27591708302497864, + "median": 0.27714359760284424, + "p05": 0.21523593366146088, + "p95": 0.33170604705810547 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.560360074043274, + "group": 4, + "latest_source_mass": 0.3255118429660797, + "layer": 14, + "max_source_mass": 0.3255118429660797, + "mixer_index": 26, + "normalized_entropy": 0.9695062244951087, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19377154111862183, + "median": 0.19472965598106384, + "p05": 0.17005598545074463, + "p95": 0.22297024726867676 + }, + { + "label": "completed_group_1", + "mean": 0.17211595177650452, + "median": 0.17136064171791077, + "p05": 0.1543424427509308, + "p95": 0.1928609162569046 + }, + { + "label": "completed_group_2", + "mean": 0.14605136215686798, + "median": 0.14629822969436646, + "p05": 0.13246256113052368, + "p95": 0.15914371609687805 + }, + { + "label": "completed_group_3", + "mean": 0.16254930198192596, + "median": 0.16272446513175964, + "p05": 0.14075018465518951, + "p95": 0.18463709950447083 + }, + { + "label": "current_group_partial", + "mean": 0.3255118429660797, + "median": 0.3261105418205261, + "p05": 0.3035835921764374, + "p95": 0.3457673192024231 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.540909767150879, + "group": 4, + "latest_source_mass": 0.2995486855506897, + "layer": 14, + "max_source_mass": 0.2995486855506897, + "mixer_index": 27, + "normalized_entropy": 0.9574210693349581, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21921303868293762, + "median": 0.21722173690795898, + "p05": 0.1413341462612152, + "p95": 0.2906968593597412 + }, + { + "label": "completed_group_1", + "mean": 0.10205383598804474, + "median": 0.09984464198350906, + "p05": 0.07455356419086456, + "p95": 0.13362759351730347 + }, + { + "label": "completed_group_2", + "mean": 0.16381604969501495, + "median": 0.16600549221038818, + "p05": 0.11865930259227753, + "p95": 0.2019941210746765 + }, + { + "label": "completed_group_3", + "mean": 0.215368390083313, + "median": 0.2136734575033188, + "p05": 0.1728287935256958, + "p95": 0.2628125846385956 + }, + { + "label": "current_group_partial", + "mean": 0.2995486855506897, + "median": 0.2982075810432434, + "p05": 0.23863884806632996, + "p95": 0.3662579655647278 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5142419338226318, + "group": 4, + "latest_source_mass": 0.369165301322937, + "layer": 15, + "max_source_mass": 0.369165301322937, + "mixer_index": 28, + "normalized_entropy": 0.940851412859105, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2000826597213745, + "median": 0.20369741320610046, + "p05": 0.15802279114723206, + "p95": 0.24072687327861786 + }, + { + "label": "completed_group_1", + "mean": 0.15480534732341766, + "median": 0.15228059887886047, + "p05": 0.13543802499771118, + "p95": 0.1815909892320633 + }, + { + "label": "completed_group_2", + "mean": 0.10827909409999847, + "median": 0.10761409997940063, + "p05": 0.09670238941907883, + "p95": 0.12229038774967194 + }, + { + "label": "completed_group_3", + "mean": 0.16766759753227234, + "median": 0.16637766361236572, + "p05": 0.1398160457611084, + "p95": 0.20042110979557037 + }, + { + "label": "current_group_partial", + "mean": 0.369165301322937, + "median": 0.3697334825992584, + "p05": 0.3279486298561096, + "p95": 0.40613043308258057 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5333527326583862, + "group": 4, + "latest_source_mass": 0.29974091053009033, + "layer": 15, + "max_source_mass": 0.29974091053009033, + "mixer_index": 29, + "normalized_entropy": 0.9527256198031004, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21770668029785156, + "median": 0.19699469208717346, + "p05": 0.1311802864074707, + "p95": 0.3720983564853668 + }, + { + "label": "completed_group_1", + "mean": 0.10525081306695938, + "median": 0.10493603348731995, + "p05": 0.08330865204334259, + "p95": 0.12839283049106598 + }, + { + "label": "completed_group_2", + "mean": 0.15970060229301453, + "median": 0.16236470639705658, + "p05": 0.11309029906988144, + "p95": 0.19861777126789093 + }, + { + "label": "completed_group_3", + "mean": 0.217601016163826, + "median": 0.21904586255550385, + "p05": 0.16268108785152435, + "p95": 0.2699928283691406 + }, + { + "label": "current_group_partial", + "mean": 0.29974091053009033, + "median": 0.30079299211502075, + "p05": 0.22156940400600433, + "p95": 0.37281978130340576 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.512608289718628, + "group": 4, + "latest_source_mass": 0.35851576924324036, + "layer": 16, + "max_source_mass": 0.35851576924324036, + "mixer_index": 30, + "normalized_entropy": 0.9398363727066501, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21392285823822021, + "median": 0.22146853804588318, + "p05": 0.1518818885087967, + "p95": 0.25874194502830505 + }, + { + "label": "completed_group_1", + "mean": 0.1860550194978714, + "median": 0.1838144212961197, + "p05": 0.1660098135471344, + "p95": 0.21311265230178833 + }, + { + "label": "completed_group_2", + "mean": 0.10291537642478943, + "median": 0.1024642288684845, + "p05": 0.09207107126712799, + "p95": 0.11504144221544266 + }, + { + "label": "completed_group_3", + "mean": 0.1385909616947174, + "median": 0.13646918535232544, + "p05": 0.11220520734786987, + "p95": 0.1728362739086151 + }, + { + "label": "current_group_partial", + "mean": 0.35851576924324036, + "median": 0.3558155298233032, + "p05": 0.31809645891189575, + "p95": 0.40622422099113464 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5262645483016968, + "group": 4, + "latest_source_mass": 0.3142552077770233, + "layer": 16, + "max_source_mass": 0.3142552077770233, + "mixer_index": 31, + "normalized_entropy": 0.9483214832396903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20246261358261108, + "median": 0.18459102511405945, + "p05": 0.12276223301887512, + "p95": 0.38892465829849243 + }, + { + "label": "completed_group_1", + "mean": 0.10530521720647812, + "median": 0.10532709956169128, + "p05": 0.0785030871629715, + "p95": 0.1342671513557434 + }, + { + "label": "completed_group_2", + "mean": 0.16650916635990143, + "median": 0.17111444473266602, + "p05": 0.1085265576839447, + "p95": 0.2094469517469406 + }, + { + "label": "completed_group_3", + "mean": 0.21146778762340546, + "median": 0.2099919468164444, + "p05": 0.15801574289798737, + "p95": 0.26882117986679077 + }, + { + "label": "current_group_partial", + "mean": 0.3142552077770233, + "median": 0.31609731912612915, + "p05": 0.22627028822898865, + "p95": 0.397154837846756 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5794764757156372, + "group": 5, + "latest_source_mass": 0.2760102152824402, + "layer": 17, + "max_source_mass": 0.2760102152824402, + "mixer_index": 32, + "normalized_entropy": 0.9813839126772218, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2210189402103424, + "median": 0.22435861825942993, + "p05": 0.17291823029518127, + "p95": 0.262204647064209 + }, + { + "label": "completed_group_1", + "mean": 0.18839938938617706, + "median": 0.18652752041816711, + "p05": 0.16997642815113068, + "p95": 0.2126670628786087 + }, + { + "label": "completed_group_2", + "mean": 0.14016608893871307, + "median": 0.14067986607551575, + "p05": 0.1265786588191986, + "p95": 0.15091627836227417 + }, + { + "label": "completed_group_3", + "mean": 0.17440536618232727, + "median": 0.17192018032073975, + "p05": 0.15062645077705383, + "p95": 0.20691406726837158 + }, + { + "label": "completed_group_4", + "mean": 0.2760102152824402, + "median": 0.27511972188949585, + "p05": 0.24412119388580322, + "p95": 0.30901142954826355 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6602650880813599, + "group": 5, + "latest_source_mass": 0.3575916588306427, + "layer": 17, + "max_source_mass": 0.3575916588306427, + "mixer_index": 33, + "normalized_entropy": 0.9266115885502495, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12695768475532532, + "median": 0.11564654856920242, + "p05": 0.07159632444381714, + "p95": 0.21949690580368042 + }, + { + "label": "completed_group_1", + "mean": 0.08036886155605316, + "median": 0.07964898645877838, + "p05": 0.06650256365537643, + "p95": 0.09638062119483948 + }, + { + "label": "completed_group_2", + "mean": 0.12672865390777588, + "median": 0.1287270039319992, + "p05": 0.09939455986022949, + "p95": 0.14712141454219818 + }, + { + "label": "completed_group_3", + "mean": 0.15391482412815094, + "median": 0.15372109413146973, + "p05": 0.13333335518836975, + "p95": 0.17581847310066223 + }, + { + "label": "completed_group_4", + "mean": 0.154438316822052, + "median": 0.15402798354625702, + "p05": 0.12481103092432022, + "p95": 0.18498417735099792 + }, + { + "label": "current_group_partial", + "mean": 0.3575916588306427, + "median": 0.36001941561698914, + "p05": 0.3110165596008301, + "p95": 0.39699214696884155 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5925545692443848, + "group": 5, + "latest_source_mass": 0.41908538341522217, + "layer": 18, + "max_source_mass": 0.41908538341522217, + "mixer_index": 34, + "normalized_entropy": 0.8888216284580353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14075040817260742, + "median": 0.1455841362476349, + "p05": 0.09744156897068024, + "p95": 0.1818566620349884 + }, + { + "label": "completed_group_1", + "mean": 0.11028306186199188, + "median": 0.10887216031551361, + "p05": 0.09629634767770767, + "p95": 0.12874113023281097 + }, + { + "label": "completed_group_2", + "mean": 0.07537610828876495, + "median": 0.07517752051353455, + "p05": 0.0666593387722969, + "p95": 0.08491044491529465 + }, + { + "label": "completed_group_3", + "mean": 0.10244421660900116, + "median": 0.10141433775424957, + "p05": 0.08653581142425537, + "p95": 0.12175601720809937 + }, + { + "label": "completed_group_4", + "mean": 0.1520608365535736, + "median": 0.1513972282409668, + "p05": 0.1301831305027008, + "p95": 0.17589205503463745 + }, + { + "label": "current_group_partial", + "mean": 0.41908538341522217, + "median": 0.4191858768463135, + "p05": 0.3649080991744995, + "p95": 0.4721684455871582 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5918848514556885, + "group": 5, + "latest_source_mass": 0.4151553511619568, + "layer": 18, + "max_source_mass": 0.4151553511619568, + "mixer_index": 35, + "normalized_entropy": 0.8884478518433735, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11773698031902313, + "median": 0.10857899487018585, + "p05": 0.06324320286512375, + "p95": 0.22339528799057007 + }, + { + "label": "completed_group_1", + "mean": 0.07344591617584229, + "median": 0.07228962332010269, + "p05": 0.05982384458184242, + "p95": 0.09060496836900711 + }, + { + "label": "completed_group_2", + "mean": 0.11976176500320435, + "median": 0.12151201069355011, + "p05": 0.08953659236431122, + "p95": 0.14441059529781342 + }, + { + "label": "completed_group_3", + "mean": 0.14735516905784607, + "median": 0.14620742201805115, + "p05": 0.11767779290676117, + "p95": 0.17902395129203796 + }, + { + "label": "completed_group_4", + "mean": 0.126544788479805, + "median": 0.12564796209335327, + "p05": 0.09858805686235428, + "p95": 0.1567486822605133 + }, + { + "label": "current_group_partial", + "mean": 0.4151553511619568, + "median": 0.4184337854385376, + "p05": 0.3375611901283264, + "p95": 0.4846380352973938 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6208878755569458, + "group": 5, + "latest_source_mass": 0.3991435170173645, + "layer": 19, + "max_source_mass": 0.3991435170173645, + "mixer_index": 36, + "normalized_entropy": 0.9046347477964072, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14785674214363098, + "median": 0.15170978009700775, + "p05": 0.09146977216005325, + "p95": 0.20172694325447083 + }, + { + "label": "completed_group_1", + "mean": 0.12492769956588745, + "median": 0.12345598638057709, + "p05": 0.10790181905031204, + "p95": 0.14623555541038513 + }, + { + "label": "completed_group_2", + "mean": 0.08781818300485611, + "median": 0.08839264512062073, + "p05": 0.07453933358192444, + "p95": 0.09894395619630814 + }, + { + "label": "completed_group_3", + "mean": 0.10019385814666748, + "median": 0.09901996701955795, + "p05": 0.08430047333240509, + "p95": 0.12013883143663406 + }, + { + "label": "completed_group_4", + "mean": 0.14006000757217407, + "median": 0.13918349146842957, + "p05": 0.11816112697124481, + "p95": 0.16475799679756165 + }, + { + "label": "current_group_partial", + "mean": 0.3991435170173645, + "median": 0.39726996421813965, + "p05": 0.34116071462631226, + "p95": 0.4634247422218323 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5470075607299805, + "group": 5, + "latest_source_mass": 0.44330859184265137, + "layer": 19, + "max_source_mass": 0.44330859184265137, + "mixer_index": 37, + "normalized_entropy": 0.8634013589985261, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12043030560016632, + "median": 0.1012125313282013, + "p05": 0.061379045248031616, + "p95": 0.25343188643455505 + }, + { + "label": "completed_group_1", + "mean": 0.06855648756027222, + "median": 0.06761305779218674, + "p05": 0.05337192490696907, + "p95": 0.08790726959705353 + }, + { + "label": "completed_group_2", + "mean": 0.11753211170434952, + "median": 0.1186976358294487, + "p05": 0.08412563800811768, + "p95": 0.14895018935203552 + }, + { + "label": "completed_group_3", + "mean": 0.1339646875858307, + "median": 0.13253431022167206, + "p05": 0.09857159107923508, + "p95": 0.1730121374130249 + }, + { + "label": "completed_group_4", + "mean": 0.1162077933549881, + "median": 0.11592038720846176, + "p05": 0.08609852194786072, + "p95": 0.14767511188983917 + }, + { + "label": "current_group_partial", + "mean": 0.44330859184265137, + "median": 0.4434061646461487, + "p05": 0.3389148414134979, + "p95": 0.5490255355834961 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.683492660522461, + "group": 5, + "latest_source_mass": 0.3365415334701538, + "layer": 20, + "max_source_mass": 0.3365415334701538, + "mixer_index": 38, + "normalized_entropy": 0.9395751435586169, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1658952683210373, + "median": 0.1726911962032318, + "p05": 0.10183650255203247, + "p95": 0.22522984445095062 + }, + { + "label": "completed_group_1", + "mean": 0.1435280442237854, + "median": 0.14210543036460876, + "p05": 0.12450659275054932, + "p95": 0.16782894730567932 + }, + { + "label": "completed_group_2", + "mean": 0.09461991488933563, + "median": 0.0948714017868042, + "p05": 0.08168491721153259, + "p95": 0.10723742842674255 + }, + { + "label": "completed_group_3", + "mean": 0.10784181207418442, + "median": 0.10693192481994629, + "p05": 0.08930590003728867, + "p95": 0.12958656251430511 + }, + { + "label": "completed_group_4", + "mean": 0.15157344937324524, + "median": 0.15053030848503113, + "p05": 0.128616601228714, + "p95": 0.17799487709999084 + }, + { + "label": "current_group_partial", + "mean": 0.3365415334701538, + "median": 0.33349889516830444, + "p05": 0.2778944969177246, + "p95": 0.40724658966064453 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5540567636489868, + "group": 5, + "latest_source_mass": 0.42266085743904114, + "layer": 20, + "max_source_mass": 0.42266085743904114, + "mixer_index": 39, + "normalized_entropy": 0.8673355940563396, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12987135350704193, + "median": 0.1083056628704071, + "p05": 0.03783264383673668, + "p95": 0.3301126956939697 + }, + { + "label": "completed_group_1", + "mean": 0.07178181409835815, + "median": 0.07081792503595352, + "p05": 0.05334944650530815, + "p95": 0.09311570972204208 + }, + { + "label": "completed_group_2", + "mean": 0.12132491171360016, + "median": 0.12282539904117584, + "p05": 0.07926258444786072, + "p95": 0.15763431787490845 + }, + { + "label": "completed_group_3", + "mean": 0.1441602110862732, + "median": 0.14273977279663086, + "p05": 0.09847753494977951, + "p95": 0.19304539263248444 + }, + { + "label": "completed_group_4", + "mean": 0.11020088195800781, + "median": 0.10934801399707794, + "p05": 0.07139573246240616, + "p95": 0.1517774760723114 + }, + { + "label": "current_group_partial", + "mean": 0.42266085743904114, + "median": 0.42266106605529785, + "p05": 0.28481048345565796, + "p95": 0.5570361018180847 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7208337783813477, + "group": 6, + "latest_source_mass": 0.27070939540863037, + "layer": 21, + "max_source_mass": 0.27070939540863037, + "mixer_index": 40, + "normalized_entropy": 0.9604156182429642, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18048501014709473, + "median": 0.18919861316680908, + "p05": 0.11199048161506653, + "p95": 0.23807701468467712 + }, + { + "label": "completed_group_1", + "mean": 0.1685236394405365, + "median": 0.1669977903366089, + "p05": 0.14678362011909485, + "p95": 0.19381441175937653 + }, + { + "label": "completed_group_2", + "mean": 0.08726631104946136, + "median": 0.08729851245880127, + "p05": 0.07629868388175964, + "p95": 0.0980045348405838 + }, + { + "label": "completed_group_3", + "mean": 0.11160479485988617, + "median": 0.1099197044968605, + "p05": 0.09190923720598221, + "p95": 0.13723208010196686 + }, + { + "label": "completed_group_4", + "mean": 0.18141084909439087, + "median": 0.18020854890346527, + "p05": 0.15656134486198425, + "p95": 0.2108723670244217 + }, + { + "label": "completed_group_5", + "mean": 0.27070939540863037, + "median": 0.2684924602508545, + "p05": 0.2159801721572876, + "p95": 0.33192309737205505 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6729483604431152, + "group": 6, + "latest_source_mass": 0.3803577423095703, + "layer": 21, + "max_source_mass": 0.3803577423095703, + "mixer_index": 41, + "normalized_entropy": 0.8597253893019091, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09229261428117752, + "median": 0.07798201590776443, + "p05": 0.04784023016691208, + "p95": 0.20285779237747192 + }, + { + "label": "completed_group_1", + "mean": 0.051420148462057114, + "median": 0.05041736364364624, + "p05": 0.040625106543302536, + "p95": 0.06547587364912033 + }, + { + "label": "completed_group_2", + "mean": 0.08079522103071213, + "median": 0.08136115223169327, + "p05": 0.05947277322411537, + "p95": 0.10017222166061401 + }, + { + "label": "completed_group_3", + "mean": 0.09550285339355469, + "median": 0.09445887804031372, + "p05": 0.07334083318710327, + "p95": 0.12144046276807785 + }, + { + "label": "completed_group_4", + "mean": 0.07590493559837341, + "median": 0.07492390275001526, + "p05": 0.055510398000478745, + "p95": 0.10084333270788193 + }, + { + "label": "completed_group_5", + "mean": 0.22372648119926453, + "median": 0.22069036960601807, + "p05": 0.1407623589038849, + "p95": 0.32290777564048767 + }, + { + "label": "current_group_partial", + "mean": 0.3803577423095703, + "median": 0.37903881072998047, + "p05": 0.2855539321899414, + "p95": 0.476589173078537 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.710235595703125, + "group": 6, + "latest_source_mass": 0.4011716842651367, + "layer": 22, + "max_source_mass": 0.4011716842651367, + "mixer_index": 42, + "normalized_entropy": 0.8788872376935791, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11657392978668213, + "median": 0.11877645552158356, + "p05": 0.06819256395101547, + "p95": 0.1684979349374771 + }, + { + "label": "completed_group_1", + "mean": 0.10617930442094803, + "median": 0.10509790480136871, + "p05": 0.090352863073349, + "p95": 0.12587188184261322 + }, + { + "label": "completed_group_2", + "mean": 0.061112720519304276, + "median": 0.061012350022792816, + "p05": 0.05164862796664238, + "p95": 0.07133162766695023 + }, + { + "label": "completed_group_3", + "mean": 0.06796726584434509, + "median": 0.06712733209133148, + "p05": 0.05444999784231186, + "p95": 0.08481802046298981 + }, + { + "label": "completed_group_4", + "mean": 0.103337861597538, + "median": 0.10192783921957016, + "p05": 0.0832459107041359, + "p95": 0.12805812060832977 + }, + { + "label": "completed_group_5", + "mean": 0.14365722239017487, + "median": 0.14159655570983887, + "p05": 0.10498540848493576, + "p95": 0.19020715355873108 + }, + { + "label": "current_group_partial", + "mean": 0.4011716842651367, + "median": 0.40084803104400635, + "p05": 0.32083970308303833, + "p95": 0.47994357347488403 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5640671253204346, + "group": 6, + "latest_source_mass": 0.4689176678657532, + "layer": 22, + "max_source_mass": 0.4689176678657532, + "mixer_index": 43, + "normalized_entropy": 0.8037715030571925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.090422622859478, + "median": 0.0774901732802391, + "p05": 0.04492149129509926, + "p95": 0.1988787055015564 + }, + { + "label": "completed_group_1", + "mean": 0.04561382159590721, + "median": 0.044634923338890076, + "p05": 0.03501313924789429, + "p95": 0.05900419130921364 + }, + { + "label": "completed_group_2", + "mean": 0.07565715163946152, + "median": 0.07550699263811111, + "p05": 0.053458306938409805, + "p95": 0.09821777045726776 + }, + { + "label": "completed_group_3", + "mean": 0.08878682553768158, + "median": 0.08692863583564758, + "p05": 0.06425619125366211, + "p95": 0.11807316541671753 + }, + { + "label": "completed_group_4", + "mean": 0.06555216014385223, + "median": 0.06414324045181274, + "p05": 0.04460844025015831, + "p95": 0.09090004861354828 + }, + { + "label": "completed_group_5", + "mean": 0.165049746632576, + "median": 0.15503892302513123, + "p05": 0.08368627727031708, + "p95": 0.2792147397994995 + }, + { + "label": "current_group_partial", + "mean": 0.4689176678657532, + "median": 0.46879667043685913, + "p05": 0.3151778280735016, + "p95": 0.6170428991317749 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7556228637695312, + "group": 6, + "latest_source_mass": 0.3706088662147522, + "layer": 23, + "max_source_mass": 0.3706088662147522, + "mixer_index": 44, + "normalized_entropy": 0.9022116795175967, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13281464576721191, + "median": 0.13658174872398376, + "p05": 0.08060654997825623, + "p95": 0.17402096092700958 + }, + { + "label": "completed_group_1", + "mean": 0.12175846099853516, + "median": 0.12066285312175751, + "p05": 0.10334118455648422, + "p95": 0.14384035766124725 + }, + { + "label": "completed_group_2", + "mean": 0.07235272228717804, + "median": 0.07218228280544281, + "p05": 0.059715576469898224, + "p95": 0.08584222197532654 + }, + { + "label": "completed_group_3", + "mean": 0.0707377940416336, + "median": 0.06942892074584961, + "p05": 0.05500924959778786, + "p95": 0.09052760899066925 + }, + { + "label": "completed_group_4", + "mean": 0.10551950335502625, + "median": 0.103996641933918, + "p05": 0.08408617228269577, + "p95": 0.13205930590629578 + }, + { + "label": "completed_group_5", + "mean": 0.12620802223682404, + "median": 0.12379512935876846, + "p05": 0.08937962353229523, + "p95": 0.17139491438865662 + }, + { + "label": "current_group_partial", + "mean": 0.3706088662147522, + "median": 0.3705657124519348, + "p05": 0.282781720161438, + "p95": 0.4561466872692108 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.594892978668213, + "group": 6, + "latest_source_mass": 0.4636557698249817, + "layer": 23, + "max_source_mass": 0.4636557698249817, + "mixer_index": 45, + "normalized_entropy": 0.8196128579947488, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10721131414175034, + "median": 0.10240983963012695, + "p05": 0.05563424155116081, + "p95": 0.18401460349559784 + }, + { + "label": "completed_group_1", + "mean": 0.053925663232803345, + "median": 0.0527028888463974, + "p05": 0.04038037359714508, + "p95": 0.07054222375154495 + }, + { + "label": "completed_group_2", + "mean": 0.07673212885856628, + "median": 0.07705660909414291, + "p05": 0.05211106687784195, + "p95": 0.10031480342149734 + }, + { + "label": "completed_group_3", + "mean": 0.08865120261907578, + "median": 0.08699032664299011, + "p05": 0.06242028623819351, + "p95": 0.11899784207344055 + }, + { + "label": "completed_group_4", + "mean": 0.06888366490602493, + "median": 0.06751501560211182, + "p05": 0.04784276708960533, + "p95": 0.09609999507665634 + }, + { + "label": "completed_group_5", + "mean": 0.14094024896621704, + "median": 0.13063763082027435, + "p05": 0.07297889143228531, + "p95": 0.24012833833694458 + }, + { + "label": "current_group_partial", + "mean": 0.4636557698249817, + "median": 0.46312135457992554, + "p05": 0.31259939074516296, + "p95": 0.6109440326690674 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7775511741638184, + "group": 6, + "latest_source_mass": 0.3569435477256775, + "layer": 24, + "max_source_mass": 0.3569435477256775, + "mixer_index": 46, + "normalized_entropy": 0.9134806018801903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13637229800224304, + "median": 0.13456633687019348, + "p05": 0.09586701542139053, + "p95": 0.18876218795776367 + }, + { + "label": "completed_group_1", + "mean": 0.10478455573320389, + "median": 0.10326151549816132, + "p05": 0.0880400538444519, + "p95": 0.12566551566123962 + }, + { + "label": "completed_group_2", + "mean": 0.07710614800453186, + "median": 0.07678486406803131, + "p05": 0.06292064487934113, + "p95": 0.09218747913837433 + }, + { + "label": "completed_group_3", + "mean": 0.08548048138618469, + "median": 0.08430111408233643, + "p05": 0.0665682703256607, + "p95": 0.10857507586479187 + }, + { + "label": "completed_group_4", + "mean": 0.11000140011310577, + "median": 0.10904969274997711, + "p05": 0.0881001353263855, + "p95": 0.13561299443244934 + }, + { + "label": "completed_group_5", + "mean": 0.12931162118911743, + "median": 0.12692990899085999, + "p05": 0.08648453652858734, + "p95": 0.1814313679933548 + }, + { + "label": "current_group_partial", + "mean": 0.3569435477256775, + "median": 0.3568228483200073, + "p05": 0.26841166615486145, + "p95": 0.4493965804576874 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.654036045074463, + "group": 6, + "latest_source_mass": 0.4265251159667969, + "layer": 24, + "max_source_mass": 0.4265251159667969, + "mixer_index": 47, + "normalized_entropy": 0.8500063817835848, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10232305526733398, + "median": 0.08623291552066803, + "p05": 0.04679499566555023, + "p95": 0.23077142238616943 + }, + { + "label": "completed_group_1", + "mean": 0.061968062072992325, + "median": 0.060845747590065, + "p05": 0.0479469932615757, + "p95": 0.07941868156194687 + }, + { + "label": "completed_group_2", + "mean": 0.08900615572929382, + "median": 0.08864577114582062, + "p05": 0.0641680359840393, + "p95": 0.11413820832967758 + }, + { + "label": "completed_group_3", + "mean": 0.1005433052778244, + "median": 0.09813467413187027, + "p05": 0.0731433779001236, + "p95": 0.1343638002872467 + }, + { + "label": "completed_group_4", + "mean": 0.07864196598529816, + "median": 0.07639941573143005, + "p05": 0.05487418174743652, + "p95": 0.10941869765520096 + }, + { + "label": "completed_group_5", + "mean": 0.14099235832691193, + "median": 0.12885531783103943, + "p05": 0.07315157353878021, + "p95": 0.24481485784053802 + }, + { + "label": "current_group_partial", + "mean": 0.4265251159667969, + "median": 0.42903727293014526, + "p05": 0.2803359627723694, + "p95": 0.5635722875595093 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.84956693649292, + "group": 7, + "latest_source_mass": 0.28775152564048767, + "layer": 25, + "max_source_mass": 0.28775152564048767, + "mixer_index": 48, + "normalized_entropy": 0.9504893827656096, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14851760864257812, + "median": 0.14813844859600067, + "p05": 0.10312555730342865, + "p95": 0.20065288245677948 + }, + { + "label": "completed_group_1", + "mean": 0.11630287766456604, + "median": 0.11500488221645355, + "p05": 0.09825178980827332, + "p95": 0.13750022649765015 + }, + { + "label": "completed_group_2", + "mean": 0.08512166142463684, + "median": 0.08458003401756287, + "p05": 0.0710662305355072, + "p95": 0.10092862695455551 + }, + { + "label": "completed_group_3", + "mean": 0.09215595573186874, + "median": 0.09079311043024063, + "p05": 0.07304700464010239, + "p95": 0.11515989899635315 + }, + { + "label": "completed_group_4", + "mean": 0.1263950914144516, + "median": 0.12563186883926392, + "p05": 0.10441911220550537, + "p95": 0.1519687920808792 + }, + { + "label": "completed_group_5", + "mean": 0.14375528693199158, + "median": 0.1422564834356308, + "p05": 0.10313796997070312, + "p95": 0.1929876059293747 + }, + { + "label": "completed_group_6", + "mean": 0.28775152564048767, + "median": 0.28676605224609375, + "p05": 0.21690022945404053, + "p95": 0.3624127507209778 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7161309719085693, + "group": 7, + "latest_source_mass": 0.32536065578460693, + "layer": 25, + "max_source_mass": 0.32536065578460693, + "mixer_index": 49, + "normalized_entropy": 0.8252845475628168, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0837959572672844, + "median": 0.0849243700504303, + "p05": 0.0271806251257658, + "p95": 0.15180703997612 + }, + { + "label": "completed_group_1", + "mean": 0.04434927552938461, + "median": 0.04302988201379776, + "p05": 0.03217232972383499, + "p95": 0.06002792343497276 + }, + { + "label": "completed_group_2", + "mean": 0.056708935648202896, + "median": 0.05563671886920929, + "p05": 0.03992946073412895, + "p95": 0.07731831818819046 + }, + { + "label": "completed_group_3", + "mean": 0.06495873630046844, + "median": 0.06223570555448532, + "p05": 0.04220382124185562, + "p95": 0.0949486792087555 + }, + { + "label": "completed_group_4", + "mean": 0.04885463789105415, + "median": 0.046898528933525085, + "p05": 0.030923232436180115, + "p95": 0.0743112787604332 + }, + { + "label": "completed_group_5", + "mean": 0.09997115284204483, + "median": 0.08650931715965271, + "p05": 0.04111029952764511, + "p95": 0.20292627811431885 + }, + { + "label": "completed_group_6", + "mean": 0.2760006785392761, + "median": 0.27142390608787537, + "p05": 0.13761089742183685, + "p95": 0.42605364322662354 + }, + { + "label": "current_group_partial", + "mean": 0.32536065578460693, + "median": 0.3348004221916199, + "p05": 0.10654139518737793, + "p95": 0.5236068964004517 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9285151958465576, + "group": 7, + "latest_source_mass": 0.2804902195930481, + "layer": 26, + "max_source_mass": 0.2804902195930481, + "mixer_index": 50, + "normalized_entropy": 0.9274197697756124, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10564712435007095, + "median": 0.10497278720140457, + "p05": 0.04608207941055298, + "p95": 0.16350959241390228 + }, + { + "label": "completed_group_1", + "mean": 0.09173665940761566, + "median": 0.09041133522987366, + "p05": 0.07509034872055054, + "p95": 0.11238400638103485 + }, + { + "label": "completed_group_2", + "mean": 0.07135222852230072, + "median": 0.06981296837329865, + "p05": 0.0522611029446125, + "p95": 0.09555281698703766 + }, + { + "label": "completed_group_3", + "mean": 0.07045470923185349, + "median": 0.06927657127380371, + "p05": 0.055566973984241486, + "p95": 0.08908769488334656 + }, + { + "label": "completed_group_4", + "mean": 0.09785081446170807, + "median": 0.0971219539642334, + "p05": 0.07590530812740326, + "p95": 0.12285276502370834 + }, + { + "label": "completed_group_5", + "mean": 0.10425601899623871, + "median": 0.10151226818561554, + "p05": 0.06786561012268066, + "p95": 0.15156632661819458 + }, + { + "label": "completed_group_6", + "mean": 0.1782122254371643, + "median": 0.1762663722038269, + "p05": 0.11771149188280106, + "p95": 0.24932436645030975 + }, + { + "label": "current_group_partial", + "mean": 0.2804902195930481, + "median": 0.2783043086528778, + "p05": 0.14811429381370544, + "p95": 0.4168073534965515 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7219383716583252, + "group": 7, + "latest_source_mass": 0.3661728501319885, + "layer": 26, + "max_source_mass": 0.3661728501319885, + "mixer_index": 51, + "normalized_entropy": 0.8280773165026276, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0923302173614502, + "median": 0.08522969484329224, + "p05": 0.056062981486320496, + "p95": 0.1509525179862976 + }, + { + "label": "completed_group_1", + "mean": 0.04646994173526764, + "median": 0.04543843865394592, + "p05": 0.032575178891420364, + "p95": 0.06306323409080505 + }, + { + "label": "completed_group_2", + "mean": 0.05750556290149689, + "median": 0.05649096518754959, + "p05": 0.03895717114210129, + "p95": 0.07940060645341873 + }, + { + "label": "completed_group_3", + "mean": 0.06549686938524246, + "median": 0.062135741114616394, + "p05": 0.04169235751032829, + "p95": 0.09948549419641495 + }, + { + "label": "completed_group_4", + "mean": 0.054404959082603455, + "median": 0.05241036415100098, + "p05": 0.03378590941429138, + "p95": 0.08157843351364136 + }, + { + "label": "completed_group_5", + "mean": 0.08427833020687103, + "median": 0.07576382160186768, + "p05": 0.041103728115558624, + "p95": 0.15204203128814697 + }, + { + "label": "completed_group_6", + "mean": 0.2333412617444992, + "median": 0.22215056419372559, + "p05": 0.10060560703277588, + "p95": 0.3983197808265686 + }, + { + "label": "current_group_partial", + "mean": 0.3661728501319885, + "median": 0.3673480749130249, + "p05": 0.16410952806472778, + "p95": 0.5635213255882263 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9470824003219604, + "group": 7, + "latest_source_mass": 0.26050227880477905, + "layer": 27, + "max_source_mass": 0.26050227880477905, + "mixer_index": 52, + "normalized_entropy": 0.9363487077155573, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10804469138383865, + "median": 0.10379163920879364, + "p05": 0.05421917140483856, + "p95": 0.17028658092021942 + }, + { + "label": "completed_group_1", + "mean": 0.07850409299135208, + "median": 0.0776081532239914, + "p05": 0.063654825091362, + "p95": 0.09574449062347412 + }, + { + "label": "completed_group_2", + "mean": 0.07298290729522705, + "median": 0.07200999557971954, + "p05": 0.05533483251929283, + "p95": 0.09299935400485992 + }, + { + "label": "completed_group_3", + "mean": 0.08511961251497269, + "median": 0.08372659981250763, + "p05": 0.06449062377214432, + "p95": 0.11086944490671158 + }, + { + "label": "completed_group_4", + "mean": 0.09968686103820801, + "median": 0.09877584874629974, + "p05": 0.07606595754623413, + "p95": 0.1266063004732132 + }, + { + "label": "completed_group_5", + "mean": 0.11502514779567719, + "median": 0.11171717941761017, + "p05": 0.07302433997392654, + "p95": 0.1681303083896637 + }, + { + "label": "completed_group_6", + "mean": 0.18013443052768707, + "median": 0.1765173077583313, + "p05": 0.11514104157686234, + "p95": 0.2599799633026123 + }, + { + "label": "current_group_partial", + "mean": 0.26050227880477905, + "median": 0.2575443983078003, + "p05": 0.13687226176261902, + "p95": 0.39756521582603455 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7855088710784912, + "group": 7, + "latest_source_mass": 0.28804832696914673, + "layer": 27, + "max_source_mass": 0.28804832696914673, + "mixer_index": 53, + "normalized_entropy": 0.858648264589397, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08493785560131073, + "median": 0.05951457470655441, + "p05": 0.02441210113465786, + "p95": 0.22703105211257935 + }, + { + "label": "completed_group_1", + "mean": 0.05972691997885704, + "median": 0.058127306401729584, + "p05": 0.03591413050889969, + "p95": 0.08903861790895462 + }, + { + "label": "completed_group_2", + "mean": 0.07017223536968231, + "median": 0.06906847655773163, + "p05": 0.04151676222681999, + "p95": 0.10273467004299164 + }, + { + "label": "completed_group_3", + "mean": 0.07963316142559052, + "median": 0.07547932863235474, + "p05": 0.042773112654685974, + "p95": 0.12874338030815125 + }, + { + "label": "completed_group_4", + "mean": 0.0645163506269455, + "median": 0.061876002699136734, + "p05": 0.0366181954741478, + "p95": 0.10098200291395187 + }, + { + "label": "completed_group_5", + "mean": 0.11586236953735352, + "median": 0.10182450711727142, + "p05": 0.048887670040130615, + "p95": 0.2313569337129593 + }, + { + "label": "completed_group_6", + "mean": 0.23710274696350098, + "median": 0.22610372304916382, + "p05": 0.10011275857686996, + "p95": 0.4084784686565399 + }, + { + "label": "current_group_partial", + "mean": 0.28804832696914673, + "median": 0.27438053488731384, + "p05": 0.07688213139772415, + "p95": 0.5529112815856934 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.973182201385498, + "group": 7, + "latest_source_mass": 0.23531918227672577, + "layer": 28, + "max_source_mass": 0.23531918227672577, + "mixer_index": 54, + "normalized_entropy": 0.9489000589030754, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.104981929063797, + "median": 0.09386023879051208, + "p05": 0.04212673753499985, + "p95": 0.18614700436592102 + }, + { + "label": "completed_group_1", + "mean": 0.0840599462389946, + "median": 0.08250434696674347, + "p05": 0.05170809105038643, + "p95": 0.12236068397760391 + }, + { + "label": "completed_group_2", + "mean": 0.09310521930456161, + "median": 0.0931512713432312, + "p05": 0.06374019384384155, + "p95": 0.12329407781362534 + }, + { + "label": "completed_group_3", + "mean": 0.11566174775362015, + "median": 0.11439734697341919, + "p05": 0.07886623591184616, + "p95": 0.1571493148803711 + }, + { + "label": "completed_group_4", + "mean": 0.0963074117898941, + "median": 0.09477876126766205, + "p05": 0.06686149537563324, + "p95": 0.13022682070732117 + }, + { + "label": "completed_group_5", + "mean": 0.1096896305680275, + "median": 0.10702179372310638, + "p05": 0.06579374521970749, + "p95": 0.1643676459789276 + }, + { + "label": "completed_group_6", + "mean": 0.16087493300437927, + "median": 0.15455034375190735, + "p05": 0.09876372665166855, + "p95": 0.24324838817119598 + }, + { + "label": "current_group_partial", + "mean": 0.23531918227672577, + "median": 0.22382870316505432, + "p05": 0.11656329780817032, + "p95": 0.38286590576171875 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.802741289138794, + "group": 7, + "latest_source_mass": 0.2497689425945282, + "layer": 28, + "max_source_mass": 0.2913346588611603, + "mixer_index": 55, + "normalized_entropy": 0.8669353059487718, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08190082013607025, + "median": 0.06415966153144836, + "p05": 0.039168842136859894, + "p95": 0.23471535742282867 + }, + { + "label": "completed_group_1", + "mean": 0.05632811039686203, + "median": 0.055233441293239594, + "p05": 0.03936203569173813, + "p95": 0.076178178191185 + }, + { + "label": "completed_group_2", + "mean": 0.07102417945861816, + "median": 0.06932097673416138, + "p05": 0.04159427434206009, + "p95": 0.1046731173992157 + }, + { + "label": "completed_group_3", + "mean": 0.08421063423156738, + "median": 0.0812690332531929, + "p05": 0.0486646331846714, + "p95": 0.12889066338539124 + }, + { + "label": "completed_group_4", + "mean": 0.06341387331485748, + "median": 0.061601825058460236, + "p05": 0.03807957097887993, + "p95": 0.09577237069606781 + }, + { + "label": "completed_group_5", + "mean": 0.10201875865459442, + "median": 0.09330011904239655, + "p05": 0.05116701498627663, + "p95": 0.18108239769935608 + }, + { + "label": "completed_group_6", + "mean": 0.2913346588611603, + "median": 0.28823426365852356, + "p05": 0.13295966386795044, + "p95": 0.4586564898490906 + }, + { + "label": "current_group_partial", + "mean": 0.2497689425945282, + "median": 0.234481081366539, + "p05": 0.09318265318870544, + "p95": 0.4530518054962158 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.945815086364746, + "group": 8, + "latest_source_mass": 0.153213769197464, + "layer": 29, + "max_source_mass": 0.2451319396495819, + "mixer_index": 56, + "normalized_entropy": 0.9357392585284499, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1064736545085907, + "median": 0.0881282389163971, + "p05": 0.029344500973820686, + "p95": 0.23970629274845123 + }, + { + "label": "completed_group_1", + "mean": 0.07925166189670563, + "median": 0.07691921293735504, + "p05": 0.062442928552627563, + "p95": 0.1030145138502121 + }, + { + "label": "completed_group_2", + "mean": 0.07823626697063446, + "median": 0.07715678215026855, + "p05": 0.059018202126026154, + "p95": 0.10055352747440338 + }, + { + "label": "completed_group_3", + "mean": 0.09447507560253143, + "median": 0.09318602085113525, + "p05": 0.0684189647436142, + "p95": 0.12381350249052048 + }, + { + "label": "completed_group_4", + "mean": 0.09724453091621399, + "median": 0.09628183394670486, + "p05": 0.0699525699019432, + "p95": 0.12776269018650055 + }, + { + "label": "completed_group_5", + "mean": 0.1459731012582779, + "median": 0.14035329222679138, + "p05": 0.08667999505996704, + "p95": 0.2258891463279724 + }, + { + "label": "completed_group_6", + "mean": 0.2451319396495819, + "median": 0.24505369365215302, + "p05": 0.16207969188690186, + "p95": 0.3307104706764221 + }, + { + "label": "completed_group_7", + "mean": 0.153213769197464, + "median": 0.13417893648147583, + "p05": 0.05588865652680397, + "p95": 0.30438560247421265 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8717057704925537, + "group": 8, + "latest_source_mass": 0.3048541843891144, + "layer": 29, + "max_source_mass": 0.3048541843891144, + "mixer_index": 57, + "normalized_entropy": 0.8518500065030654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07336360216140747, + "median": 0.0611552819609642, + "p05": 0.024239415302872658, + "p95": 0.16916418075561523 + }, + { + "label": "completed_group_1", + "mean": 0.05720400810241699, + "median": 0.05402737483382225, + "p05": 0.0314149409532547, + "p95": 0.09488057345151901 + }, + { + "label": "completed_group_2", + "mean": 0.057025715708732605, + "median": 0.05547948181629181, + "p05": 0.03480810672044754, + "p95": 0.08415934443473816 + }, + { + "label": "completed_group_3", + "mean": 0.06655597686767578, + "median": 0.06265220046043396, + "p05": 0.035200610756874084, + "p95": 0.10949824750423431 + }, + { + "label": "completed_group_4", + "mean": 0.05032068490982056, + "median": 0.047904569655656815, + "p05": 0.02614513412117958, + "p95": 0.08244554698467255 + }, + { + "label": "completed_group_5", + "mean": 0.08329877257347107, + "median": 0.0731179416179657, + "p05": 0.02939312346279621, + "p95": 0.17248544096946716 + }, + { + "label": "completed_group_6", + "mean": 0.1901058554649353, + "median": 0.18234091997146606, + "p05": 0.08719424903392792, + "p95": 0.317789226770401 + }, + { + "label": "completed_group_7", + "mean": 0.11727118492126465, + "median": 0.09272899478673935, + "p05": 0.031939513981342316, + "p95": 0.2891285717487335 + }, + { + "label": "current_group_partial", + "mean": 0.3048541843891144, + "median": 0.29432767629623413, + "p05": 0.07913826406002045, + "p95": 0.5632759928703308 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.09096622467041, + "group": 8, + "latest_source_mass": 0.13795366883277893, + "layer": 30, + "max_source_mass": 0.17320571839809418, + "mixer_index": 58, + "normalized_entropy": 0.951639739623416, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10326676070690155, + "median": 0.09009090811014175, + "p05": 0.01757553033530712, + "p95": 0.2674753665924072 + }, + { + "label": "completed_group_1", + "mean": 0.08232800662517548, + "median": 0.08093428611755371, + "p05": 0.06430500745773315, + "p95": 0.10457758605480194 + }, + { + "label": "completed_group_2", + "mean": 0.07164021581411362, + "median": 0.0711187869310379, + "p05": 0.05190654471516609, + "p95": 0.09468771517276764 + }, + { + "label": "completed_group_3", + "mean": 0.08822821825742722, + "median": 0.08667590469121933, + "p05": 0.06148773431777954, + "p95": 0.12025904655456543 + }, + { + "label": "completed_group_4", + "mean": 0.10803373157978058, + "median": 0.10627314448356628, + "p05": 0.07042569667100906, + "p95": 0.1514367014169693 + }, + { + "label": "completed_group_5", + "mean": 0.13578416407108307, + "median": 0.12929600477218628, + "p05": 0.08227657526731491, + "p95": 0.21117588877677917 + }, + { + "label": "completed_group_6", + "mean": 0.17320571839809418, + "median": 0.1694163680076599, + "p05": 0.11322733014822006, + "p95": 0.24482715129852295 + }, + { + "label": "completed_group_7", + "mean": 0.09955951571464539, + "median": 0.08973086625337601, + "p05": 0.037010837346315384, + "p95": 0.1912725567817688 + }, + { + "label": "current_group_partial", + "mean": 0.13795366883277893, + "median": 0.1256815791130066, + "p05": 0.053015124052762985, + "p95": 0.2601332366466522 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.96881103515625, + "group": 8, + "latest_source_mass": 0.21570813655853271, + "layer": 30, + "max_source_mass": 0.21570813655853271, + "mixer_index": 59, + "normalized_entropy": 0.896044517007504, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08843819797039032, + "median": 0.08496083319187164, + "p05": 0.03690534085035324, + "p95": 0.1522325873374939 + }, + { + "label": "completed_group_1", + "mean": 0.06052382290363312, + "median": 0.058369796723127365, + "p05": 0.03549504280090332, + "p95": 0.09154202789068222 + }, + { + "label": "completed_group_2", + "mean": 0.0642135739326477, + "median": 0.06355445086956024, + "p05": 0.03948400169610977, + "p95": 0.09175601601600647 + }, + { + "label": "completed_group_3", + "mean": 0.07744920998811722, + "median": 0.07566337287425995, + "p05": 0.04564609378576279, + "p95": 0.11733324825763702 + }, + { + "label": "completed_group_4", + "mean": 0.051616813987493515, + "median": 0.049985647201538086, + "p05": 0.03238597884774208, + "p95": 0.07699409872293472 + }, + { + "label": "completed_group_5", + "mean": 0.06631231307983398, + "median": 0.061356961727142334, + "p05": 0.030705859884619713, + "p95": 0.1188126802444458 + }, + { + "label": "completed_group_6", + "mean": 0.16417551040649414, + "median": 0.15454982221126556, + "p05": 0.07227650284767151, + "p95": 0.28998225927352905 + }, + { + "label": "completed_group_7", + "mean": 0.2115624099969864, + "median": 0.19997704029083252, + "p05": 0.09131578356027603, + "p95": 0.3709705173969269 + }, + { + "label": "current_group_partial", + "mean": 0.21570813655853271, + "median": 0.2005728930234909, + "p05": 0.06839637458324432, + "p95": 0.4193759858608246 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.124063014984131, + "group": 8, + "latest_source_mass": 0.1258423924446106, + "layer": 31, + "max_source_mass": 0.17017140984535217, + "mixer_index": 60, + "normalized_entropy": 0.9667027380329118, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09220977127552032, + "median": 0.09152225404977798, + "p05": 0.03251190483570099, + "p95": 0.17071731388568878 + }, + { + "label": "completed_group_1", + "mean": 0.06839152425527573, + "median": 0.06702418625354767, + "p05": 0.04446800425648689, + "p95": 0.0977444052696228 + }, + { + "label": "completed_group_2", + "mean": 0.10039852559566498, + "median": 0.09736946225166321, + "p05": 0.06476961821317673, + "p95": 0.14545989036560059 + }, + { + "label": "completed_group_3", + "mean": 0.10902463644742966, + "median": 0.10749617218971252, + "p05": 0.07769118249416351, + "p95": 0.14509552717208862 + }, + { + "label": "completed_group_4", + "mean": 0.0849665105342865, + "median": 0.08355648815631866, + "p05": 0.059977494180202484, + "p95": 0.11480176448822021 + }, + { + "label": "completed_group_5", + "mean": 0.12137698382139206, + "median": 0.11852307617664337, + "p05": 0.07902064919471741, + "p95": 0.17207489907741547 + }, + { + "label": "completed_group_6", + "mean": 0.17017140984535217, + "median": 0.16817378997802734, + "p05": 0.11521546542644501, + "p95": 0.23236754536628723 + }, + { + "label": "completed_group_7", + "mean": 0.12761825323104858, + "median": 0.1237221211194992, + "p05": 0.062459878623485565, + "p95": 0.2092883139848709 + }, + { + "label": "current_group_partial", + "mean": 0.1258423924446106, + "median": 0.11723431199789047, + "p05": 0.06504593789577484, + "p95": 0.21421310305595398 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9633058309555054, + "group": 8, + "latest_source_mass": 0.15763342380523682, + "layer": 31, + "max_source_mass": 0.23511825501918793, + "mixer_index": 61, + "normalized_entropy": 0.8935389906004497, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0846441388130188, + "median": 0.06999644637107849, + "p05": 0.04030589759349823, + "p95": 0.20743247866630554 + }, + { + "label": "completed_group_1", + "mean": 0.05872901901602745, + "median": 0.05653517693281174, + "p05": 0.03710630163550377, + "p95": 0.08772030472755432 + }, + { + "label": "completed_group_2", + "mean": 0.07210008800029755, + "median": 0.07183568179607391, + "p05": 0.03840115666389465, + "p95": 0.10702595114707947 + }, + { + "label": "completed_group_3", + "mean": 0.07792744785547256, + "median": 0.0762966051697731, + "p05": 0.04050692170858383, + "p95": 0.122304767370224 + }, + { + "label": "completed_group_4", + "mean": 0.059186771512031555, + "median": 0.057271093130111694, + "p05": 0.03537197411060333, + "p95": 0.08958204090595245 + }, + { + "label": "completed_group_5", + "mean": 0.07445863634347916, + "median": 0.06928195059299469, + "p05": 0.03154807910323143, + "p95": 0.1351260244846344 + }, + { + "label": "completed_group_6", + "mean": 0.23511825501918793, + "median": 0.22703276574611664, + "p05": 0.0953572615981102, + "p95": 0.3982555568218231 + }, + { + "label": "completed_group_7", + "mean": 0.18020221590995789, + "median": 0.16846294701099396, + "p05": 0.07464539259672165, + "p95": 0.324230432510376 + }, + { + "label": "current_group_partial", + "mean": 0.15763342380523682, + "median": 0.12027859687805176, + "p05": 0.036883872002363205, + "p95": 0.4093724191188812 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1238560676574707, + "group": 8, + "latest_source_mass": 0.11516246944665909, + "layer": 32, + "max_source_mass": 0.1766587197780609, + "mixer_index": 62, + "normalized_entropy": 0.966608552245626, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10231245309114456, + "median": 0.10458895564079285, + "p05": 0.025718798860907555, + "p95": 0.1875312179327011 + }, + { + "label": "completed_group_1", + "mean": 0.07575537264347076, + "median": 0.07447351515293121, + "p05": 0.05878028646111488, + "p95": 0.09651422500610352 + }, + { + "label": "completed_group_2", + "mean": 0.08897508680820465, + "median": 0.08751188218593597, + "p05": 0.06392349302768707, + "p95": 0.11800213158130646 + }, + { + "label": "completed_group_3", + "mean": 0.0967075377702713, + "median": 0.09579810500144958, + "p05": 0.06915527582168579, + "p95": 0.12801720201969147 + }, + { + "label": "completed_group_4", + "mean": 0.09801244735717773, + "median": 0.09654392302036285, + "p05": 0.07431379705667496, + "p95": 0.12754815816879272 + }, + { + "label": "completed_group_5", + "mean": 0.12438583374023438, + "median": 0.12010829150676727, + "p05": 0.08193086087703705, + "p95": 0.1824759691953659 + }, + { + "label": "completed_group_6", + "mean": 0.1766587197780609, + "median": 0.17520052194595337, + "p05": 0.12216726690530777, + "p95": 0.2341049611568451 + }, + { + "label": "completed_group_7", + "mean": 0.12203006446361542, + "median": 0.1200856938958168, + "p05": 0.05817953124642372, + "p95": 0.19647708535194397 + }, + { + "label": "current_group_partial", + "mean": 0.11516246944665909, + "median": 0.10430245846509933, + "p05": 0.05224763602018356, + "p95": 0.2294922173023224 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0835556983947754, + "group": 8, + "latest_source_mass": 0.09660109132528305, + "layer": 32, + "max_source_mass": 0.22511571645736694, + "mixer_index": 63, + "normalized_entropy": 0.9482670637704002, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09133295714855194, + "median": 0.08019149303436279, + "p05": 0.04441600665450096, + "p95": 0.19931957125663757 + }, + { + "label": "completed_group_1", + "mean": 0.07054752856492996, + "median": 0.06906121969223022, + "p05": 0.05099350959062576, + "p95": 0.09504306316375732 + }, + { + "label": "completed_group_2", + "mean": 0.09000501036643982, + "median": 0.0901019275188446, + "p05": 0.05998067185282707, + "p95": 0.11849517375230789 + }, + { + "label": "completed_group_3", + "mean": 0.10956105589866638, + "median": 0.1064089685678482, + "p05": 0.0759081020951271, + "p95": 0.1509535312652588 + }, + { + "label": "completed_group_4", + "mean": 0.07114207744598389, + "median": 0.06995616853237152, + "p05": 0.04991999268531799, + "p95": 0.09545011073350906 + }, + { + "label": "completed_group_5", + "mean": 0.11565887182950974, + "median": 0.11069735884666443, + "p05": 0.06978912651538849, + "p95": 0.17647549510002136 + }, + { + "label": "completed_group_6", + "mean": 0.22511571645736694, + "median": 0.22372552752494812, + "p05": 0.11895382404327393, + "p95": 0.33412501215934753 + }, + { + "label": "completed_group_7", + "mean": 0.13003569841384888, + "median": 0.12226496636867523, + "p05": 0.06884703785181046, + "p95": 0.21555939316749573 + }, + { + "label": "current_group_partial", + "mean": 0.09660109132528305, + "median": 0.09000182151794434, + "p05": 0.051288168877363205, + "p95": 0.16580010950565338 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.679484486579895, + "group": null, + "latest_source_mass": 0.2827349305152893, + "layer": null, + "max_source_mass": 0.32717907428741455, + "mixer_index": null, + "normalized_entropy": 0.7643663300981273, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08993907272815704, + "median": 0.09290459752082825, + "p05": 0.022419869899749756, + "p95": 0.15114322304725647 + }, + { + "label": "completed_group_1", + "mean": 0.043493133038282394, + "median": 0.042318835854530334, + "p05": 0.026918282732367516, + "p95": 0.06356139481067657 + }, + { + "label": "completed_group_2", + "mean": 0.04922422394156456, + "median": 0.047891765832901, + "p05": 0.03095712885260582, + "p95": 0.07266643643379211 + }, + { + "label": "completed_group_3", + "mean": 0.037924449890851974, + "median": 0.03562554344534874, + "p05": 0.020909149199724197, + "p95": 0.06224166974425316 + }, + { + "label": "completed_group_4", + "mean": 0.03939433768391609, + "median": 0.036868683993816376, + "p05": 0.020651251077651978, + "p95": 0.06597590446472168 + }, + { + "label": "completed_group_5", + "mean": 0.03898489475250244, + "median": 0.03362073004245758, + "p05": 0.014276490546762943, + "p95": 0.08282073587179184 + }, + { + "label": "completed_group_6", + "mean": 0.09112587571144104, + "median": 0.07136045396327972, + "p05": 0.02230823040008545, + "p95": 0.22286513447761536 + }, + { + "label": "completed_group_7", + "mean": 0.32717907428741455, + "median": 0.3213679790496826, + "p05": 0.08271751552820206, + "p95": 0.6128557920455933 + }, + { + "label": "completed_group_8", + "mean": 0.2827349305152893, + "median": 0.2627606987953186, + "p05": 0.05262754485011101, + "p95": 0.571351170539856 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "optimizer_state_before": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c901fb19f26257116155a8180e3a604d86c7950d0f8a51691ca86859e9cf8f6b", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.08626964688301086, + 0.16337208449840546, + 0.15584443509578705, + 0.04305505007505417, + 0.1360139101743698, + 0.03393339365720749, + 0.03730965405702591, + 0.037494905292987823, + 0.05234000086784363, + 0.04223497211933136, + 0.041619230061769485, + 0.05722156912088394, + 0.062187470495700836, + 0.0369485504925251, + 0.04293901473283768, + 0.045369699597358704, + 0.04149825870990753, + 0.02440357394516468, + 0.026009807363152504, + 0.03634416684508324, + 0.038355328142642975, + 0.02631118707358837, + 0.033921729773283005, + 0.02731483057141304, + 0.027557097375392914, + 0.024315863847732544, + 0.02051021158695221, + 0.018631720915436745, + 0.019497457891702652, + 0.021715763956308365, + 0.017114246264100075, + 0.020576251670718193 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.943075613387919e-06, + "first_to_last_ratio": 2.307872264355047, + "imbalance_abs_log_ratio": 0.8363260023807616, + "last_quartile_mean": 3.875030586186767e-06, + "mean": 6.099237623402587e-06, + "normalized": [ + 1.6993132354180294, + 1.6239172875947692, + 1.5554831244671488, + 1.442125192626584, + 1.4104481551972334, + 1.403523053069228, + 1.3221099712249675, + 1.2731697595883604, + 1.2000943569582707, + 1.1235452195545337, + 1.0767048612407788, + 1.0261714183282156, + 1.0093778108098996, + 0.9743388011452827, + 0.9573184625534406, + 0.928757504863306, + 0.9116899710167641, + 0.9026661339368709, + 0.886533931006931, + 0.8702403098677887, + 0.8533876523625498, + 0.8403774938169696, + 0.8216913043945464, + 0.8043724353132603, + 0.7863149194889869, + 0.7481207616594623, + 0.7006856536668127, + 0.6618189820973319, + 0.6190766705086127, + 0.569713863763572, + 0.5180828888655294, + 0.4788288135939631 + ], + "peak_layer": 1, + "peak_normalized": 1.6993132354180294, + "population_cv": 0.3198013047414009, + "rest_mean": 6.301157672741856e-06, + "spike_contrast": 0.7949125568843288, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.008869356970535e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.0364515219407622e-05, + 9.904657417791896e-06, + 9.487261195317842e-06, + 8.795864232524764e-06, + 8.602658454037737e-06, + 8.5604206105927e-06, + 8.063862878771033e-06, + 7.765364898659755e-06, + 7.319660653593019e-06, + 6.8527692747011315e-06, + 6.567078798980219e-06, + 6.258863322727848e-06, + 6.156435119919479e-06, + 5.94272387388628e-06, + 5.838912784383865e-06, + 5.664712716679787e-06, + 5.5606137721042614e-06, + 5.505575245479122e-06, + 5.407181106420467e-06, + 5.307802439347142e-06, + 5.205014076636871e-06, + 5.125662028149236e-06, + 5.011690518585965e-06, + 4.906058620690601e-06, + 4.795921540790005e-06, + 4.562966296361992e-06, + 4.273648301023059e-06, + 4.03659123549005e-06, + 3.775895720536937e-06, + 3.474820232440834e-06, + 3.1599106478097383e-06, + 2.920490715041524e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.6375963190948823e-05, + "first_to_last_ratio": 2.3430311680370033, + "imbalance_abs_log_ratio": 0.8514454620564724, + "last_quartile_mean": 1.5525172557318e-05, + "mean": 2.4540472736589436e-05, + "normalized": [ + 1.7240884069821072, + 1.6433567785450998, + 1.5754725715380682, + 1.4596583982024929, + 1.430627916212154, + 1.420003858884998, + 1.3309607801353691, + 1.274107560343628, + 1.196376835996058, + 1.1165969342919553, + 1.0683735188874326, + 1.0196491868738, + 1.0014614962907418, + 0.9675840271884015, + 0.9499774532397648, + 0.9220954178226054, + 0.9056958468427084, + 0.8955638114822986, + 0.879813549394944, + 0.8616003608371102, + 0.8461107858930287, + 0.8325957474316453, + 0.8164485619288154, + 0.8006967432790473, + 0.7812168094045981, + 0.7432274953816075, + 0.6968240720487157, + 0.659313141540847, + 0.6166001053246104, + 0.5675655726585476, + 0.5174401121065899, + 0.47889614301020944 + ], + "peak_layer": 1, + "peak_normalized": 1.7240884069821072, + "population_cv": 0.3275253935067736, + "rest_mean": 2.5379330947072487e-05, + "spike_contrast": 0.7884620142947147, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0010638399980964e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.230994454701431e-05, + 4.032875222037546e-05, + 3.866284168907441e-05, + 3.582070712582208e-05, + 3.510828537400812e-05, + 3.4847565984819084e-05, + 3.266240673838183e-05, + 3.126720184809528e-05, + 2.9359653126448393e-05, + 2.7401816623751074e-05, + 2.6218391212751158e-05, + 2.5022673071362078e-05, + 2.4576338546467014e-05, + 2.374496943957638e-05, + 2.3312895791605115e-05, + 2.262865746160969e-05, + 2.2226204237085767e-05, + 2.197755929955747e-05, + 2.1591040422208607e-05, + 2.1144080164958723e-05, + 2.076395867334213e-05, + 2.0432293240446597e-05, + 2.0036033674841747e-05, + 1.964947659871541e-05, + 1.9171429812558927e-05, + 1.823915408749599e-05, + 1.710039214231074e-05, + 1.6179856174858287e-05, + 1.5131658074096777e-05, + 1.3928327462053858e-05, + 1.2698224963969551e-05, + 1.175233774119988e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03225856809876859, + "first_to_last_ratio": 2.343031178392381, + "imbalance_abs_log_ratio": 0.8514454664761225, + "last_quartile_mean": 0.013767878292128444, + "mean": 0.02176273672375828, + "normalized": [ + 1.724088391019206, + 1.6433568148707862, + 1.575472621674219, + 1.4596583934186327, + 1.4306278920801987, + 1.4200037621599195, + 1.3309608349416662, + 1.2741075749322037, + 1.1963768254907436, + 1.1165969105087108, + 1.0683735628480904, + 1.0196491787212385, + 1.001461486526046, + 0.9675840094241758, + 0.9499774647530201, + 0.9220954786760075, + 0.9056959063736247, + 0.8955638261021951, + 0.8798135299584785, + 0.8616003323196685, + 0.8461108250575042, + 0.8325956787797999, + 0.8164485932514832, + 0.8006966709217642, + 0.7812167599757752, + 0.7432275143610545, + 0.6968240828090287, + 0.6593131498508736, + 0.6166000906652189, + 0.5675655757499506, + 0.5174401039988743, + 0.478896157779841 + ], + "peak_layer": 1, + "peak_normalized": 1.724088391019206, + "population_cv": 0.3275253950778613, + "rest_mean": 0.02250664467336955, + "spike_contrast": 0.788461987710435, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01774563379585743, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.037520881742239, + 0.0357639417052269, + 0.03428659588098526, + 0.03176616132259369, + 0.031134378165006638, + 0.0309031680226326, + 0.02896535024046898, + 0.027728067710995674, + 0.02603643387556076, + 0.02430020458996296, + 0.023250732570886612, + 0.022190356627106667, + 0.02179454267024994, + 0.02105727605521679, + 0.02067410945892334, + 0.02006732113659382, + 0.019710421562194824, + 0.019489919766783714, + 0.019147150218486786, + 0.018750781193375587, + 0.018413687124848366, + 0.018119560554623604, + 0.01776815578341484, + 0.017425350844860077, + 0.01700141467154026, + 0.016174664720892906, + 0.015164799056947231, + 0.014348458498716354, + 0.013418905436992645, + 0.012351780198514462, + 0.011260912753641605, + 0.010422090999782085 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4063504494288281e-06, + "first_to_last_ratio": 1.3888652705487472, + "imbalance_abs_log_ratio": 0.32848706162254393, + "last_quartile_mean": 1.0125895428814147e-06, + "mean": 1.1880723089774392e-06, + "normalized": [ + 2.22105949166683, + 1.3045309379325452, + 1.1542973756533603, + 1.0756338384718074, + 0.9439046493177333, + 0.9303001868869332, + 0.9181101208153997, + 0.9219604060672644, + 0.9152949161005884, + 0.9117060563851229, + 0.9184975703018127, + 0.9267480721956175, + 0.9157813092197382, + 0.9328035373473063, + 0.9602268081831139, + 0.9736851517288183, + 0.975330448461193, + 0.9909657433074398, + 1.017059016511325, + 1.0393447779680196, + 1.0410638518191953, + 1.0608138238488845, + 1.067044114935786, + 1.0654680021941936, + 1.0596368347940859, + 1.0074277055068903, + 0.9486012184070485, + 0.8859287521128595, + 0.8284090099343453, + 0.7515767911699255, + 0.6962507460711886, + 0.6405387346836281 + ], + "peak_layer": 1, + "peak_normalized": 2.22105949166683, + "population_cv": 0.2514651034600585, + "rest_mean": 1.1751343499276522e-06, + "spike_contrast": 1.0704625287514868, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2579372878462892e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.638779278640868e-06, + 1.5498770835620235e-06, + 1.3713887483390863e-06, + 1.277930778087466e-06, + 1.1214269761694595e-06, + 1.105263891076902e-06, + 1.0907812111327075e-06, + 1.0953556284221122e-06, + 1.0874365443669376e-06, + 1.0831727195181884e-06, + 1.0912415291386424e-06, + 1.1010437219738378e-06, + 1.0880144145630766e-06, + 1.1082380524385371e-06, + 1.1408188811401487e-06, + 1.1568083664315054e-06, + 1.1587630979192909e-06, + 1.1773389587688143e-06, + 1.2083396541129332e-06, + 1.234816750184109e-06, + 1.2368591342237778e-06, + 1.2603235290953307e-06, + 1.2677255654125474e-06, + 1.2658530295084347e-06, + 1.2589251809913549e-06, + 1.1968969602094148e-06, + 1.1270068398516742e-06, + 1.0525474181122263e-06, + 9.84209805210412e-07, + 8.92927573659108e-07, + 8.271962315120618e-07, + 7.610063335050654e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6793285996973282e-05, + "first_to_last_ratio": 2.59826300786115, + "imbalance_abs_log_ratio": 0.954843147867069, + "last_quartile_mean": 1.031199917633785e-05, + "mean": 1.738467395284715e-05, + "normalized": [ + 1.78773045454333, + 1.7041491227711507, + 1.6364518370910566, + 1.5208900106186054, + 1.4934955268940733, + 1.479012186458667, + 1.3884863387277648, + 1.3193967179334984, + 1.2374664756707232, + 1.1441290673436744, + 1.0866714663117618, + 1.0281062332426787, + 1.0083205726565403, + 0.9674201139660152, + 0.9490038741339376, + 0.9147691923366695, + 0.8949298556467083, + 0.8863389586087415, + 0.8627560014145367, + 0.8374599603804893, + 0.8137503987308741, + 0.7895173642792594, + 0.7624546156174374, + 0.7419648877747225, + 0.7224315990451545, + 0.6853503573208848, + 0.6466114342849871, + 0.616257335993725, + 0.5815823113447253, + 0.5363768303272249, + 0.49518141464512766, + 0.46153748388525473 + ], + "peak_layer": 1, + "peak_normalized": 1.78773045454333, + "population_cv": 0.365262728647077, + "rest_mean": 1.813793328943171e-05, + "spike_contrast": 0.7342111872828369, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3317073535290546e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.107911106781103e-05, + 2.9626076866406947e-05, + 2.844918162736576e-05, + 2.6440176952746697e-05, + 2.5963932785089128e-05, + 2.5712144633871503e-05, + 2.413838228676468e-05, + 2.293728175573051e-05, + 2.1512951207114384e-05, + 1.989031079574488e-05, + 1.8891429135692306e-05, + 1.7873291653813794e-05, + 1.752932439558208e-05, + 1.6818283256725408e-05, + 1.6498122931807302e-05, + 1.5902964150882326e-05, + 1.5558063751086593e-05, + 1.5408713807119057e-05, + 1.4998731785453856e-05, + 1.4558968359779101e-05, + 1.414678536093561e-05, + 1.3725501958106179e-05, + 1.3255024896352552e-05, + 1.2898817658424377e-05, + 1.2559237802634016e-05, + 1.1914592505490873e-05, + 1.1241128959227353e-05, + 1.0713432857301086e-05, + 1.0110618859471288e-05, + 9.324736311100423e-06, + 8.608567441115156e-06, + 8.0236786743626e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.938115951816144e-05, + "first_to_last_ratio": 2.5763378474631047, + "imbalance_abs_log_ratio": 0.946368951607931, + "last_quartile_mean": 7.522755424815841e-06, + "mean": 1.236969426088308e-05, + "normalized": [ + 1.8099938946844014, + 1.728964663406708, + 1.6762891339633759, + 1.547604731622059, + 1.523697296804946, + 1.516312046701088, + 1.4092172441217883, + 1.322529595442431, + 1.2410675898615309, + 1.1253325211094913, + 1.0636520363082502, + 1.00702477602901, + 0.9880485860245845, + 0.9448364468322119, + 0.9350182198861665, + 0.8997048366987048, + 0.8789956378042715, + 0.8856125407337307, + 0.8462418433091708, + 0.8155022213016762, + 0.7864781089853847, + 0.750887679658867, + 0.7217551899516604, + 0.7099517597798954, + 0.7012053955236477, + 0.6724904602282508, + 0.6442506502382093, + 0.6188711504662027, + 0.5967590010995487, + 0.5664323381260263, + 0.5432338046756088, + 0.522038598621101 + ], + "peak_layer": 1, + "peak_normalized": 1.8099938946844014, + "population_cv": 0.37345755309966855, + "rest_mean": 1.2978888814056214e-05, + "spike_contrast": 0.699601006205894, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.08004367374815e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.2389071091311052e-05, + 2.13867642742116e-05, + 2.073518407996744e-05, + 1.914339736686088e-05, + 1.8847669707611203e-05, + 1.8756316421786323e-05, + 1.7431586456950754e-05, + 1.635928674659226e-05, + 1.5351626643678173e-05, + 1.3920019227953162e-05, + 1.3157050489098765e-05, + 1.2456588592613116e-05, + 1.2221858924021944e-05, + 1.1687337973853573e-05, + 1.1565889508347027e-05, + 1.1129073755000718e-05, + 1.0872907296288759e-05, + 1.0954756362480111e-05, + 1.0467752872500569e-05, + 1.0087513146572746e-05, + 9.72849375102669e-06, + 9.2882510216441e-06, + 8.927891030907631e-06, + 8.781886208453216e-06, + 8.673696356709115e-06, + 8.318501386384014e-06, + 7.96918357082177e-06, + 7.655246918147895e-06, + 7.381726391031407e-06, + 7.006594842096092e-06, + 6.719636076013558e-06, + 6.457457857322879e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.503894994442817e-05, + "first_to_last_ratio": 2.2837759148871593, + "imbalance_abs_log_ratio": 0.8258301761605996, + "last_quartile_mean": 3.285740490355238e-05, + "mean": 5.1171222082757595e-05, + "normalized": [ + 1.7250540829663195, + 1.6170039295082501, + 1.558641962352184, + 1.4475704461622705, + 1.405871545434971, + 1.379249740977325, + 1.3192988177043423, + 1.2787392731618172, + 1.2121135966900498, + 1.1204005426005206, + 1.0730027281223469, + 1.0188897789208062, + 0.9948840289702825, + 0.9615972823180635, + 0.9476620305313541, + 0.9234741371548107, + 0.9102494722311251, + 0.908601578994704, + 0.8796202280241118, + 0.8553643685604198, + 0.8418081912263009, + 0.8360712420605473, + 0.8303072059918611, + 0.8176670067611138, + 0.7905207401435352, + 0.7558412574586363, + 0.7089931435932589, + 0.6694146283776838, + 0.6276662460639247, + 0.5712927135223572, + 0.5255955797489182, + 0.48753247366578784 + ], + "peak_layer": 1, + "peak_normalized": 1.7250540829663195, + "population_cv": 0.3189635247096411, + "rest_mean": 5.2845896250262204e-05, + "spike_contrast": 0.7971854877572201, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.212798157823272e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.827312558423728e-05, + 8.274406718555838e-05, + 7.975761400302872e-05, + 7.407394878100604e-05, + 7.194016507128254e-05, + 7.057789480313659e-05, + 6.751013279426843e-05, + 6.543465133290738e-05, + 6.202533404575661e-05, + 5.733226498705335e-05, + 5.4906860896153376e-05, + 5.213783515500836e-05, + 5.0909431593026966e-05, + 4.920610808767378e-05, + 4.849302422371693e-05, + 4.7255300160031766e-05, + 4.6578577894251794e-05, + 4.6494253183482215e-05, + 4.50112420367077e-05, + 4.3770040065282956e-05, + 4.3076353904325515e-05, + 4.278278720448725e-05, + 4.248783443472348e-05, + 4.184101999271661e-05, + 4.0451912354910746e-05, + 3.867732084472664e-05, + 3.6280045605963096e-05, + 3.4254764614161104e-05, + 3.2118448871187866e-05, + 2.9233746317913756e-05, + 2.6895368137047626e-05, + 2.4947632482508197e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ae15e4a0d4226f581b200c0985062e16db0c3d230d4858d667731f6d827f77ba", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13547764718532562, + 0.039667773991823196, + 0.026703152805566788, + 0.05304057523608208, + 0.016798952594399452, + 0.00753376679494977, + 0.01294450368732214, + 0.03526800498366356, + 0.030294200405478477, + 0.034149833023548126, + 0.04367009550333023, + 0.05930330231785774, + 0.04527241364121437, + 0.04281846433877945, + 0.04812675341963768, + 0.06094835698604584, + 0.038031257688999176, + 0.040106989443302155, + 0.04961257055401802, + 0.07085621356964111, + 0.03753058984875679, + 0.04544757679104805, + 0.05368982255458832, + 0.06174425780773163, + 0.05051252990961075, + 0.054937634617090225, + 0.05777662619948387, + 0.07133159041404724, + 0.0636468306183815, + 0.09261580556631088, + 0.1020733118057251, + 0.12878532707691193 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.744960837248072e-06, + "first_to_last_ratio": 2.4076765961345377, + "imbalance_abs_log_ratio": 0.8786622144961542, + "last_quartile_mean": 3.6321160621355375e-06, + "mean": 5.9219457781978235e-06, + "normalized": [ + 1.5966609800930127, + 1.6382818782556905, + 1.6078801806645238, + 1.4395757389908588, + 1.4506416663399584, + 1.4388820164000453, + 1.348677960564097, + 1.29303133390772, + 1.1804814091096145, + 1.1203943128344216, + 1.0731688825386334, + 1.0349744395148408, + 1.011916879057038, + 0.9796669183445198, + 0.9673471607471578, + 0.9423088740875128, + 0.9226627178910438, + 0.9142650193290811, + 0.8983732123895815, + 0.882245122030563, + 0.8656871601790304, + 0.8478113250594501, + 0.8280201889812614, + 0.8103923861805131, + 0.7714582232411454, + 0.7214453089906826, + 0.6827625571385767, + 0.6375172771181937, + 0.58753815055203, + 0.5337410156600697, + 0.4936733910977551, + 0.4785163127113774 + ], + "peak_layer": 2, + "peak_normalized": 1.6382818782556905, + "population_cv": 0.32643117815601047, + "rest_mean": 6.114218354856281e-06, + "spike_contrast": 0.798740506276366, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.883673864242155e-06, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.455339750275016e-06, + 9.701816452434286e-06, + 9.52177924773423e-06, + 8.525089469912928e-06, + 8.590621291659772e-06, + 8.52098128234502e-06, + 7.986797754711006e-06, + 7.657261448912323e-06, + 6.990746896917699e-06, + 6.634914370806655e-06, + 6.355247933242936e-06, + 6.12906251262757e-06, + 5.992516889818944e-06, + 5.801534371130401e-06, + 5.728577434638282e-06, + 5.580302058660891e-06, + 5.463958586915396e-06, + 5.414227871369803e-06, + 5.320117452356499e-06, + 5.224607775744516e-06, + 5.126552423462272e-06, + 5.0206926971441135e-06, + 4.903490662400145e-06, + 4.79909977002535e-06, + 4.568533768178895e-06, + 4.272360001777997e-06, + 4.0432828427583445e-06, + 3.775342747758259e-06, + 3.4793690701917512e-06, + 3.1607853543391684e-06, + 2.9235070542199537e-06, + 2.8337476578599308e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.557536911102943e-05, + "first_to_last_ratio": 2.4432305702295483, + "imbalance_abs_log_ratio": 0.8933211677768842, + "last_quartile_mean": 1.4560790759787778e-05, + "mean": 2.3831157136555703e-05, + "normalized": [ + 1.614913653123857, + 1.6568181419735588, + 1.628341463280827, + 1.4605653558450646, + 1.471807278919832, + 1.4559143731134858, + 1.3583919651208736, + 1.295720862036415, + 1.177691966325428, + 1.114101795346675, + 1.0640412563479125, + 1.026954832666515, + 1.0041441491510337, + 0.9733761741275299, + 0.9597031214537947, + 0.934968890890384, + 0.9158576063976391, + 0.906677613403137, + 0.8903545981719634, + 0.8734940797341108, + 0.8579208364426144, + 0.8414682922506719, + 0.8239516565784237, + 0.8048355634088795, + 0.765892761498337, + 0.717371762357387, + 0.6794952654450417, + 0.6347194240846251, + 0.5854549882448604, + 0.5328880600388105, + 0.4936169323623591, + 0.4785452798579532 + ], + "peak_layer": 2, + "peak_normalized": 1.6568181419735588, + "population_cv": 0.333834364988221, + "rest_mean": 2.463076385471073e-05, + "spike_contrast": 0.7922320628633921, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9513280858518555e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.8485261029563844e-05, + 3.9483893488068134e-05, + 3.880526128341444e-05, + 3.4806962503353134e-05, + 3.507487053866498e-05, + 3.469612420303747e-05, + 3.237205237383023e-05, + 3.087852746830322e-05, + 2.806576230796054e-05, + 2.6550334951025434e-05, + 2.5357334379805252e-05, + 2.4473521989420988e-05, + 2.392991700617131e-05, + 2.319668055861257e-05, + 2.2870835891808383e-05, + 2.2281390556599945e-05, + 2.1825946532771923e-05, + 2.1607176677207462e-05, + 2.121818033629097e-05, + 2.081637467199471e-05, + 2.0445246263989247e-05, + 2.0053163098054938e-05, + 1.9635721400845796e-05, + 1.918016278068535e-05, + 1.8252110749017447e-05, + 1.7095799194066785e-05, + 1.6193158444366418e-05, + 1.5126098332984839e-05, + 1.395206982124364e-05, + 1.2699339094979223e-05, + 1.1763462680391967e-05, + 1.1404287761251908e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03154859389178455, + "first_to_last_ratio": 2.443230668285183, + "imbalance_abs_log_ratio": 0.8933212079104819, + "last_quartile_mean": 0.012912654667161405, + "mean": 0.021133708796696737, + "normalized": [ + 1.6149137386398011, + 1.656818103118424, + 1.6283414676061427, + 1.4605652763644328, + 1.471807315279864, + 1.4559144163177982, + 1.3583919563693225, + 1.2957209363886424, + 1.1776919990513512, + 1.1141018064612538, + 1.0640412283046075, + 1.0269548301404738, + 1.0041441194885832, + 0.9733762067371586, + 0.9597031065420684, + 0.9349689110855055, + 0.9158576300153286, + 0.9066776260380441, + 0.8903546223359304, + 0.8734940758449745, + 0.857920846942776, + 0.8414682830013448, + 0.8239516505093816, + 0.8048355219472574, + 0.7658927106397581, + 0.7173717399936698, + 0.6794952447102416, + 0.634719402274497, + 0.5854550026361336, + 0.5328880508022114, + 0.4936169320855568, + 0.47854524232746515 + ], + "peak_layer": 2, + "peak_normalized": 1.656818103118424, + "population_cv": 0.3338343750142194, + "rest_mean": 0.0218428081177451, + "spike_contrast": 0.7922320413087064, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017304572463035583, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03412911668419838, + 0.03501471132040024, + 0.034412894397974014, + 0.030867161229252815, + 0.03110474720597267, + 0.030768871307373047, + 0.02870786003768444, + 0.027383388951420784, + 0.02488899976015091, + 0.023545103147625923, + 0.022487137466669083, + 0.021703364327549934, + 0.021221289411187172, + 0.02057104930281639, + 0.020282085984945297, + 0.01975936070084572, + 0.019355468451976776, + 0.019161460921168327, + 0.018816495314240456, + 0.018460169434547424, + 0.01813104934990406, + 0.01778334565460682, + 0.017413154244422913, + 0.017009159550070763, + 0.016186153516173363, + 0.015160725452005863, + 0.014360254630446434, + 0.013413975015282631, + 0.012372835539281368, + 0.011261900886893272, + 0.010431956499814987, + 0.010113435797393322 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.344927397894935e-06, + "first_to_last_ratio": 1.4244585458149095, + "imbalance_abs_log_ratio": 0.3537917736653531, + "last_quartile_mean": 9.441674535537459e-07, + "mean": 1.1411028104646448e-06, + "normalized": [ + 2.099538361004741, + 1.2695654171364545, + 1.1994886284285255, + 1.0098034387773935, + 0.9820379553982943, + 0.96728171717488, + 0.9479405561697765, + 0.9533095585319065, + 0.9297712304918835, + 0.9212055346840066, + 0.9197232556499785, + 0.928745849673345, + 0.9294609860429008, + 0.9490990412220468, + 0.9764042390024628, + 0.987029961936741, + 0.9894569223661618, + 1.0074939408416628, + 1.0338473835735458, + 1.0587716494844093, + 1.0614411066989373, + 1.080242378600304, + 1.0792969998024888, + 1.0997107657330087, + 1.050181644220671, + 0.9776416301573645, + 0.927097788120657, + 0.865770311315491, + 0.7904398407173965, + 0.71069112651653, + 0.6594538672142737, + 0.6380569133117612 + ], + "peak_layer": 1, + "peak_normalized": 2.099538361004741, + "population_cv": 0.23657254437119296, + "rest_mean": 1.1254285844323156e-06, + "spike_contrast": 1.0891349731067184, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2257436310392222e-06, + "top_five_layers": [ + 1, + 2, + 3, + 24, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.395789124420844e-06, + 1.4487046655631275e-06, + 1.3687398450201727e-06, + 1.1522895420057466e-06, + 1.1206062708879472e-06, + 1.1037678859793232e-06, + 1.0816976327987504e-06, + 1.0878242164835683e-06, + 1.0609645642034593e-06, + 1.0511902246435056e-06, + 1.0494987918718834e-06, + 1.0597944992696284e-06, + 1.0606105433907942e-06, + 1.0830195833477774e-06, + 1.114177621275303e-06, + 1.1263026635788265e-06, + 1.129072074945725e-06, + 1.1496541674205218e-06, + 1.1797261549872928e-06, + 1.2081673048669472e-06, + 1.2112134299968602e-06, + 1.2326676142038195e-06, + 1.2315888398006791e-06, + 1.2548830454761628e-06, + 1.1983652257185895e-06, + 1.1155896117998054e-06, + 1.0579138916000375e-06, + 9.879329354589572e-07, + 9.019731237458473e-07, + 8.109716418402968e-07, + 7.525046612499864e-07, + 7.280885370164469e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.630348421917006e-05, + "first_to_last_ratio": 2.69255008587913, + "imbalance_abs_log_ratio": 0.9904887318732114, + "last_quartile_mean": 9.768986046765349e-06, + "mean": 1.6957483779833638e-05, + "normalized": [ + 1.6937424339749063, + 1.716030827709739, + 1.683456753729431, + 1.520726904446963, + 1.5301933816635431, + 1.5109070875973878, + 1.4130243932548368, + 1.3410631406221931, + 1.2180743459632961, + 1.139878912155672, + 1.078689369482256, + 1.0334527787088414, + 1.0088730407419506, + 0.9710503607378324, + 0.9566665199949184, + 0.9261587427208572, + 0.9027566962629673, + 0.8950285985522498, + 0.8707223394543085, + 0.846760516774566, + 0.8234152149025455, + 0.7960865825504274, + 0.7684196280360551, + 0.7461253882143986, + 0.7082562076848088, + 0.6648380180827745, + 0.633671939252038, + 0.597754010876058, + 0.5540441605387132, + 0.5092663534964215, + 0.477652324967025, + 0.4632130268500192 + ], + "peak_layer": 2, + "peak_normalized": 1.716030827709739, + "population_cv": 0.37037219142929884, + "rest_mean": 1.7684581103417764e-05, + "spike_contrast": 0.7368655302760281, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3031158232479356e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.872160985134542e-05, + 2.909956492658239e-05, + 2.8547190595418215e-05, + 2.578770181571599e-05, + 2.5948229449568316e-05, + 2.5621182430768386e-05, + 2.396133822912816e-05, + 2.2741056454833597e-05, + 2.065547596430406e-05, + 1.932947816385422e-05, + 1.8291857486474328e-05, + 1.752475873217918e-05, + 1.7107948224293068e-05, + 1.6466570741613396e-05, + 1.622265699552372e-05, + 1.570532185724005e-05, + 1.530848203401547e-05, + 1.5177432942437008e-05, + 1.4765259948035236e-05, + 1.435892772860825e-05, + 1.3963050150778145e-05, + 1.3499625310942065e-05, + 1.3030463378527202e-05, + 1.265240916836774e-05, + 1.201024315378163e-05, + 1.1273979907855392e-05, + 1.074548163160216e-05, + 1.0136403943761252e-05, + 9.395194865646772e-06, + 8.63587592903059e-06, + 8.099781553028151e-06, + 7.854927389416844e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.907835189740581e-05, + "first_to_last_ratio": 2.6287461590420254, + "imbalance_abs_log_ratio": 0.9665069868989548, + "last_quartile_mean": 7.257586219111545e-06, + "mean": 1.212781258175255e-05, + "normalized": [ + 1.7354359984845942, + 1.7341266298424, + 1.7101923309667906, + 1.5428934146424107, + 1.5519006110270992, + 1.5422286813294432, + 1.4249678951055762, + 1.34311375683423, + 1.2181905393168442, + 1.117207245110816, + 1.0512122908225914, + 1.0107192781417305, + 0.9868283999100976, + 0.9491456552989723, + 0.9450899871696192, + 0.910761731031231, + 0.8806493269523064, + 0.8929657163538394, + 0.8505844681035736, + 0.8211947665220615, + 0.7893804081857143, + 0.7583735688378634, + 0.7282248684002433, + 0.7172124481825861, + 0.6937135556263685, + 0.6567877100820053, + 0.6342047756355106, + 0.6086855107660424, + 0.5784802660010331, + 0.5544185549076066, + 0.5338091599726986, + 0.5273004504361006 + ], + "peak_layer": 1, + "peak_normalized": 1.7354359984845942, + "population_cv": 0.3752406439924748, + "rest_mean": 1.2717626355475255e-05, + "spike_contrast": 0.7031829646260871, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.942818203649949e-06, + "top_five_layers": [ + 1, + 2, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1047042537247762e-05, + 2.1031162759754807e-05, + 2.0740892068715766e-05, + 1.8711922166403383e-05, + 1.8821159756043926e-05, + 1.8703860405366868e-05, + 1.7281743566854857e-05, + 1.628903191885911e-05, + 1.4773986549698748e-05, + 1.354928008368006e-05, + 1.2748905646731146e-05, + 1.2257813978067134e-05, + 1.196806988446042e-05, + 1.1511060620250646e-05, + 1.1461874237284064e-05, + 1.1045547580579296e-05, + 1.0680349987524096e-05, + 1.0829720849869773e-05, + 1.031572901410982e-05, + 9.959296221495606e-06, + 9.57345764618367e-06, + 9.197412509820424e-06, + 8.831774721329566e-06, + 8.698218152858317e-06, + 8.41322798805777e-06, + 7.96539825387299e-06, + 7.691516657359898e-06, + 7.3820237957988866e-06, + 7.015700248302892e-06, + 6.723884325765539e-06, + 6.473937446571654e-06, + 6.395001037162729e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.32765902284882e-05, + "first_to_last_ratio": 2.3797059401792184, + "imbalance_abs_log_ratio": 0.8669769255033447, + "last_quartile_mean": 3.079228781643906e-05, + "mean": 4.9631882802714244e-05, + "normalized": [ + 1.6036295653160026, + 1.6264177096181267, + 1.605845107850632, + 1.4451626399742143, + 1.462521070493661, + 1.4193768503503077, + 1.3493265351473542, + 1.2989331677271607, + 1.1827368823021753, + 1.1159788020275245, + 1.069537069411288, + 1.0194230453577149, + 0.999117032890771, + 0.9700459717654388, + 0.9607374093144786, + 0.9410261468020766, + 0.9223083820569733, + 0.9136749788161789, + 0.8935785288486232, + 0.8650748037657058, + 0.859486397149459, + 0.8548828389913368, + 0.8395424095808984, + 0.8183290259422422, + 0.7781609011469676, + 0.7313438031724356, + 0.6926703962811562, + 0.6467311298885312, + 0.5885390175628117, + 0.5417837841386413, + 0.5025152241470534, + 0.48156337216205825 + ], + "peak_layer": 2, + "peak_normalized": 1.6264177096181267, + "population_cv": 0.32403192338043996, + "rest_mean": 5.119362982373719e-05, + "spike_contrast": 0.80475733076633, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.119844888919033e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.959115464473143e-05, + 8.072217315202579e-05, + 7.970111619215459e-05, + 7.172614277806133e-05, + 7.258767436724156e-05, + 7.044634548947215e-05, + 6.696961645502597e-05, + 6.44684987491928e-05, + 5.8701458328869194e-05, + 5.538812911254354e-05, + 5.308313848217949e-05, + 5.059588511358015e-05, + 4.958805948263034e-05, + 4.814520798390731e-05, + 4.768320650327951e-05, + 4.670489943237044e-05, + 4.577590152621269e-05, + 4.534740946837701e-05, + 4.4349984818836674e-05, + 4.2935291276080534e-05, + 4.2657928133849055e-05, + 4.242944487486966e-05, + 4.166807048022747e-05, + 4.061521030962467e-05, + 3.86215906473808e-05, + 3.629796992754564e-05, + 3.437853592913598e-05, + 3.2098483643494546e-05, + 2.921029954450205e-05, + 2.6889749278780073e-05, + 2.4940776711446233e-05, + 2.3900896849227138e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "325eabd236a76c354560999f425d509e669c9276a3ef345243b3badd5d5fb399", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.08626964688301086, + 0.27676042914390564, + 0.41335058212280273, + 0.43042075634002686, + 0.1360139101743698, + 0.15445689857006073, + 0.1670466512441635, + 0.17834730446338654, + 0.05234000086784363, + 0.08269349485635757, + 0.11548293381929398, + 0.16410532593727112, + 0.062187470495700836, + 0.0988159254193306, + 0.13199056684970856, + 0.17199495434761047, + 0.04149825870990753, + 0.06434677541255951, + 0.08728200942277908, + 0.11647984385490417, + 0.038355328142642975, + 0.06353354454040527, + 0.09196189045906067, + 0.1169828325510025, + 0.027557097375392914, + 0.06579430401325226, + 0.09665275365114212, + 0.12799666821956635, + 0.019497457891702652, + 0.07176875323057175, + 0.14131556451320648, + 0.22250843048095703 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.943195439314877e-06, + "first_to_last_ratio": 2.307887867651198, + "imbalance_abs_log_ratio": 0.8363327632590157, + "last_quartile_mean": 3.875056307833802e-06, + "mean": 6.099261987912996e-06, + "normalized": [ + 1.6993364194524787, + 1.6239631401407362, + 1.5555352150060116, + 1.4421180897792774, + 1.410443415622084, + 1.403515806193484, + 1.322136600562147, + 1.2731514024188308, + 1.2000780810869136, + 1.123528951240296, + 1.07670935798164, + 1.026140851109733, + 1.0093877955434476, + 0.9743169405661355, + 0.9573130726734715, + 0.928737466637045, + 0.9116867764670428, + 0.9026819876648067, + 0.886529122118542, + 0.8702279611747841, + 0.8533837960178821, + 0.8403751805988604, + 0.8216892894948097, + 0.8043572928719456, + 0.7863148352924635, + 0.748123886904242, + 0.7006953058091231, + 0.6618269255548263, + 0.619067487305518, + 0.5697120352939504, + 0.5180849945326964, + 0.47883051688477524 + ], + "peak_layer": 1, + "peak_normalized": 1.6993364194524787, + "population_cv": 0.31980786012147844, + "rest_mean": 6.301188132393118e-06, + "spike_contrast": 0.7949073575459225, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.008860807720339e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.0364698027842678e-05, + 9.904976650432218e-06, + 9.487616807746235e-06, + 8.795856047072448e-06, + 8.602663911005948e-06, + 8.560410606150981e-06, + 8.064057510637213e-06, + 7.765283953631297e-06, + 7.319590622500982e-06, + 6.8526974246196914e-06, + 6.567132459167624e-06, + 6.258701887418283e-06, + 6.1565206124214455e-06, + 5.942614279774716e-06, + 5.8389032346894965e-06, + 5.664613127009943e-06, + 5.560616500588367e-06, + 5.505693934537703e-06, + 5.407173375715502e-06, + 5.307748324412387e-06, + 5.205011348152766e-06, + 5.125668394612148e-06, + 5.01169824929093e-06, + 4.905985861114459e-06, + 4.795940185431391e-06, + 4.563003585644765e-06, + 4.273724243830657e-06, + 4.036655809613876e-06, + 3.7758547932753572e-06, + 3.4748229609249393e-06, + 3.159936113661388e-06, + 2.920512770288042e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.637614054241567e-05, + "first_to_last_ratio": 2.3430282822498043, + "imbalance_abs_log_ratio": 0.8514442304087609, + "last_quartile_mean": 1.552526737214066e-05, + "mean": 2.454050053302126e-05, + "normalized": [ + 1.7240897155140114, + 1.6433731511507879, + 1.5755074032746206, + 1.4596353977652112, + 1.4306148809963868, + 1.42000847672355, + 1.3309924792154737, + 1.2740991497308294, + 1.1963637696230847, + 1.1165890726973144, + 1.0683694180110759, + 1.019625350629474, + 1.001471406127935, + 0.96757892864483, + 0.9499723005167983, + 0.9220974865087133, + 0.905693931519684, + 0.8955733965356255, + 0.8798087726326785, + 0.8615996072890131, + 0.8461088639386269, + 0.8325951008583086, + 0.8164437828166039, + 0.8006795295233357, + 0.7812208907088943, + 0.743232064447096, + 0.6968341045755492, + 0.659316323215381, + 0.6166001110749636, + 0.567565559827195, + 0.5174384883075808, + 0.4789010855993677 + ], + "peak_layer": 1, + "peak_normalized": 1.7240897155140114, + "population_cv": 0.32752712068205064, + "rest_mean": 2.5379378308166957e-05, + "spike_contrast": 0.7884574753667308, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.001056054723449e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.231002458254807e-05, + 4.0329199691768736e-05, + 3.8663740269839764e-05, + 3.582018325687386e-05, + 3.510800524963997e-05, + 3.484771877992898e-05, + 3.266322164563462e-05, + 3.126703086309135e-05, + 2.9359365726122633e-05, + 2.740165473369416e-05, + 2.621832027216442e-05, + 2.5022116460604593e-05, + 2.4576609575888142e-05, + 2.374487121414859e-05, + 2.331279574718792e-05, + 2.262873385916464e-05, + 2.2226182409212925e-05, + 2.1977819415042177e-05, + 2.1590947653749026e-05, + 2.1144085621926934e-05, + 2.0763935026479885e-05, + 2.043230051640421e-05, + 2.0035939087392762e-05, + 1.964907642104663e-05, + 1.9171551684848964e-05, + 1.823928687372245e-05, + 1.7100657714763656e-05, + 1.6179952581296675e-05, + 1.513167535449611e-05, + 1.3928342923463788e-05, + 1.2698199498117901e-05, + 1.1752472346415743e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03225872525945306, + "first_to_last_ratio": 2.3430283290295004, + "imbalance_abs_log_ratio": 0.8514442503742458, + "last_quartile_mean": 0.013767962111160159, + "mean": 0.021762761170975864, + "normalized": [ + 1.7240897066312928, + 1.6433731135962664, + 1.5755074838038876, + 1.4596353565498406, + 1.430614987282464, + 1.4200084149681802, + 1.33099254819824, + 1.2740991253937513, + 1.1963637558959541, + 1.116589065853211, + 1.0683694526758873, + 1.019625352307586, + 1.0014714024680393, + 0.9675788998216788, + 0.9499723749288511, + 0.9220975240302813, + 0.9056938618956135, + 0.8955734330612176, + 0.8798086901303112, + 0.8615996212060729, + 0.8461089331030892, + 0.8325950858372491, + 0.8164437390157344, + 0.8006795096166728, + 0.7812208465363841, + 0.7432320715401213, + 0.6968340841998046, + 0.6593163034940038, + 0.6166001255103924, + 0.5675655372951299, + 0.5174384956683723, + 0.4789010974844188 + ], + "peak_layer": 1, + "peak_normalized": 1.7240897066312928, + "population_cv": 0.327527126007556, + "rest_mean": 0.022506686410418263, + "spike_contrast": 0.788457463457284, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017745564877986907, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03752095252275467, + 0.035764336585998535, + 0.03428739309310913, + 0.03176569566130638, + 0.03113413229584694, + 0.030903303995728493, + 0.02896607294678688, + 0.027727914974093437, + 0.026036178693175316, + 0.02430006116628647, + 0.023250669240951538, + 0.02218986302614212, + 0.02179478295147419, + 0.021057188510894775, + 0.020674021914601326, + 0.02006738819181919, + 0.019710399210453033, + 0.01949015073478222, + 0.01914706639945507, + 0.018750786781311035, + 0.018413666635751724, + 0.0181195680052042, + 0.017768070101737976, + 0.017424996942281723, + 0.017001522704958916, + 0.016174782067537308, + 0.015165033750236034, + 0.014348543249070644, + 0.013418921269476414, + 0.012351793237030506, + 0.011260890401899815, + 0.010422210209071636 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4063588764656743e-06, + "first_to_last_ratio": 1.3888550856996456, + "imbalance_abs_log_ratio": 0.3284797283795997, + "last_quartile_mean": 1.0126030360879668e-06, + "mean": 1.1880729502422582e-06, + "normalized": [ + 2.221048723833859, + 1.3045184639250926, + 1.1544035427877724, + 1.075575269686293, + 0.9438353386502788, + 0.9303903032947847, + 0.9181045536874566, + 0.9219724438423569, + 0.9151919379546727, + 0.911597530149608, + 0.918441095823346, + 0.9266988657126964, + 0.9158950689226896, + 0.9328019812734085, + 0.9602261942086033, + 0.9737021374713742, + 0.9753680066905094, + 0.9909324824121717, + 1.0171062169192169, + 1.0393085245664904, + 1.041121852253469, + 1.0608184185386729, + 1.0669551213298931, + 1.0655289558498546, + 1.0596357844023474, + 1.0074707964383398, + 0.9486502738773706, + 0.8859198532008371, + 0.8284323896376146, + 0.751570117802247, + 0.6962432413548182, + 0.6405345135018543 + ], + "peak_layer": 1, + "peak_normalized": 2.221048723833859, + "population_cv": 0.25146614998929095, + "rest_mean": 1.17513350990824e-06, + "spike_contrast": 1.0704706464750395, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.257945928045956e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.638767909957096e-06, + 1.5498631000809837e-06, + 1.3715156228499836e-06, + 1.2778618838638067e-06, + 1.1213452353331377e-06, + 1.1053715525122243e-06, + 1.0907751857303083e-06, + 1.0953705213978537e-06, + 1.0873147857637377e-06, + 1.0830443670784007e-06, + 1.0911750223385752e-06, + 1.1009858553734375e-06, + 1.0881501566473162e-06, + 1.1082368018833222e-06, + 1.140818767453311e-06, + 1.1568291711228085e-06, + 1.158808345280704e-06, + 1.1773000778703135e-06, + 1.2083963838449563e-06, + 1.2347743449936388e-06, + 1.2369287105684634e-06, + 1.2603296681845677e-06, + 1.2676205187744927e-06, + 1.2659261301450897e-06, + 1.2589246125571663e-06, + 1.1969488014074159e-06, + 1.127065729633614e-06, + 1.0525374136705068e-06, + 9.842381132330047e-07, + 8.929201271712373e-07, + 8.271877618426515e-07, + 7.610017291881377e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6793339202413335e-05, + "first_to_last_ratio": 2.5982590296524397, + "imbalance_abs_log_ratio": 0.9548416167627336, + "last_quartile_mean": 1.0312035442439083e-05, + "mean": 1.738467847189895e-05, + "normalized": [ + 1.7877226656098162, + 1.7041567364320032, + 1.6364649092006618, + 1.5208911847479025, + 1.4934925228742508, + 1.4790155687407116, + 1.3884983243453457, + 1.3193915619028695, + 1.2374632243089856, + 1.1441214457108106, + 1.0866704514151964, + 1.028096130607215, + 1.0083233448700089, + 0.9674170374331021, + 0.9490011162834874, + 0.9147690591791905, + 0.8949296230146019, + 0.8863450061150394, + 0.8627491853453367, + 0.8374612075319247, + 0.813747832986454, + 0.7895165835739041, + 0.7624542081580346, + 0.7419568475234603, + 0.7224344978897704, + 0.6853530565410577, + 0.6466163408419674, + 0.6162592684360938, + 0.5815819509020856, + 0.5363784696388306, + 0.49517950718561515, + 0.4615411306542665 + ], + "peak_layer": 1, + "peak_normalized": 1.7877226656098162, + "population_cv": 0.3652632831630456, + "rest_mean": 1.8137943731778284e-05, + "spike_contrast": 0.7342092502590932, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3317046068550553e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.107898373855278e-05, + 2.962621692859102e-05, + 2.8449416276998818e-05, + 2.644020423758775e-05, + 2.596388731035404e-05, + 2.571221011749003e-05, + 2.41385969275143e-05, + 2.2937198082217947e-05, + 2.1512900275411084e-05, + 1.9890183466486633e-05, + 1.889141640276648e-05, + 1.787312066880986e-05, + 1.7529377146274783e-05, + 1.6818234144011512e-05, + 1.6498079276061617e-05, + 1.590296596987173e-05, + 1.5558063751086593e-05, + 1.540882294648327e-05, + 1.4998617189121433e-05, + 1.455899382563075e-05, + 1.414674443367403e-05, + 1.372549195366446e-05, + 1.3255021258373745e-05, + 1.2898681234219112e-05, + 1.255929146282142e-05, + 1.191464252769947e-05, + 1.1241217180213425e-05, + 1.0713469237089157e-05, + 1.011061522149248e-05, + 9.324767233920284e-06, + 8.608536518295296e-06, + 8.023744157981128e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.938144146151899e-05, + "first_to_last_ratio": 2.5765408114136847, + "imbalance_abs_log_ratio": 0.9464477285262795, + "last_quartile_mean": 7.522272255755524e-06, + "mean": 1.2369578129778347e-05, + "normalized": [ + 1.8100777970421271, + 1.7287935495739235, + 1.6764760419525127, + 1.547606614625523, + 1.5236827794729195, + 1.516550097896885, + 1.4092148868125007, + 1.322506866159222, + 1.2411937962106272, + 1.1252563246839273, + 1.0634665882834902, + 1.0069909967052106, + 0.9883696891505255, + 0.9448880364107244, + 0.9350719230888654, + 0.899703210365348, + 0.8790680055200518, + 0.8856083557133494, + 0.8461178812403954, + 0.8154512032643977, + 0.7864210833714382, + 0.7508010562653247, + 0.7218124054512957, + 0.7098562229603778, + 0.7011466870106873, + 0.6724037625473418, + 0.6441132480887107, + 0.6188962247087517, + 0.5967934262190382, + 0.5664043851970144, + 0.5432476544778881, + 0.5220091995296049 + ], + "peak_layer": 1, + "peak_normalized": 1.8100777970421271, + "population_cv": 0.37348547055792297, + "rest_mean": 1.2978877226420014e-05, + "spike_contrast": 0.6995491866916842, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.079363007913343e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.2389898731489666e-05, + 2.1384446881711483e-05, + 2.0737301383633167e-05, + 1.9143240933772177e-05, + 1.8847313185688108e-05, + 1.875908492365852e-05, + 1.7431393644073978e-05, + 1.6358852008124813e-05, + 1.5353043636423536e-05, + 1.391894602420507e-05, + 1.3154633052181453e-05, + 1.2456053809728473e-05, + 1.2225716091052163e-05, + 1.1687866390275303e-05, + 1.156644520960981e-05, + 1.1128949154226575e-05, + 1.0873700375668705e-05, + 1.095460174838081e-05, + 1.0466121239005588e-05, + 1.0086787369800732e-05, + 9.727697033667937e-06, + 9.287092325394042e-06, + 8.928514944273047e-06, + 8.780622010817751e-06, + 8.67288872541394e-06, + 8.317350875586271e-06, + 7.96740914665861e-06, + 7.65548520575976e-06, + 7.382082912954502e-06, + 7.00618329574354e-06, + 6.719744305883069e-06, + 6.457033578044502e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.505454232159536e-05, + "first_to_last_ratio": 2.2842016822178994, + "imbalance_abs_log_ratio": 0.826016590092333, + "last_quartile_mean": 3.2858106578714796e-05, + "mean": 5.1174989891933365e-05, + "normalized": [ + 1.7246853715589219, + 1.6173583287304225, + 1.5597620217115067, + 1.4476516844938752, + 1.4063499712999827, + 1.3795411723395463, + 1.3191459492505877, + 1.2785090603756792, + 1.2120772438445848, + 1.1201281732875392, + 1.0727436588021377, + 1.0187485782963843, + 0.994689288551661, + 0.9615888999095473, + 0.9476802661591673, + 0.9234563342926768, + 0.910357048796321, + 0.9086057002160716, + 0.879517219234551, + 0.8554380245700975, + 0.8417689607464686, + 0.835900421698054, + 0.830223112057046, + 0.8174852430317888, + 0.7904688641941676, + 0.7556805383589447, + 0.7090539036922652, + 0.6694271894936573, + 0.6276488245612287, + 0.5712677128918332, + 0.5256154951349393, + 0.4874257384183444 + ], + "peak_layer": 1, + "peak_normalized": 1.7246853715589219, + "population_cv": 0.31905266441255564, + "rest_mean": 5.285078771487827e-05, + "spike_contrast": 0.7970681889415223, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2125681648030874e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.826075645629317e-05, + 8.276829612441361e-05, + 7.98208056949079e-05, + 7.408356032101437e-05, + 7.196994556579739e-05, + 7.059800554998219e-05, + 6.750728061888367e-05, + 6.54276882414706e-05, + 6.202804070198908e-05, + 5.732254794565961e-05, + 5.489764589583501e-05, + 5.213444819673896e-05, + 5.0903214287245646e-05, + 4.920930223306641e-05, + 4.849752804148011e-05, + 4.725786857306957e-05, + 4.658751277020201e-05, + 4.64978875243105e-05, + 4.5009284804109484e-05, + 4.3777032260550186e-05, + 4.3077518057543784e-05, + 4.277719563106075e-05, + 4.248665936756879e-05, + 4.1834799048956484e-05, + 4.045223613502458e-05, + 3.8671943912049755e-05, + 3.6285826354287565e-05, + 3.4257929655723274e-05, + 3.211992225260474e-05, + 2.9234619432827458e-05, + 2.6898367650574073e-05, + 2.494400723662693e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "0ad1f569f422f17bfa7a4e03139f3040905ebe4b0be1770b20df5a8dc8fd9ee5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16424456238746643, + 0.29588937759399414, + 0.42337876558303833, + 0.45154979825019836, + 0.13891471922397614, + 0.1555756777524948, + 0.16914354264736176, + 0.1874145120382309, + 0.06211879849433899, + 0.09842939674854279, + 0.137006014585495, + 0.20118924975395203, + 0.08399304747581482, + 0.12138351052999496, + 0.1587214469909668, + 0.2071460783481598, + 0.0581100657582283, + 0.08358953893184662, + 0.10947012156248093, + 0.1472771316766739, + 0.05584336072206497, + 0.08318652212619781, + 0.11238116025924683, + 0.1379220336675644, + 0.05995604768395424, + 0.09392217546701431, + 0.1259489357471466, + 0.17086055874824524, + 0.06700707972049713, + 0.13971327245235443, + 0.2211977243423462, + 0.32473331689834595 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.744833621676662e-06, + "first_to_last_ratio": 2.407641608620192, + "imbalance_abs_log_ratio": 0.8786476827402625, + "last_quartile_mean": 3.6321160052921186e-06, + "mean": 5.921910066319924e-06, + "normalized": [ + 1.5966143979437075, + 1.6382704100570205, + 1.6078846551669277, + 1.4395458713949063, + 1.4506247663066902, + 1.438881478650706, + 1.3486619814552512, + 1.2930475784613584, + 1.1804812328505396, + 1.1204054464016595, + 1.0731833404752842, + 1.034984213263227, + 1.0119477847689309, + 0.9796848055347882, + 0.9673465438864248, + 0.9423091045127517, + 0.9226633673753865, + 0.9142662324966246, + 0.8983748672567047, + 0.882237157596605, + 0.8656713400367232, + 0.8478121374802432, + 0.8280229553999554, + 0.8103965821084333, + 0.7714594199143477, + 0.7214453593654997, + 0.6827608384252767, + 0.6375225038763053, + 0.5875414633089651, + 0.533745309436336, + 0.49368765640464074, + 0.47851919838777923 + ], + "peak_layer": 2, + "peak_normalized": 1.6382704100570205, + "population_cv": 0.326425065508555, + "rest_mean": 6.114182985617879e-06, + "spike_contrast": 0.798738983376607, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.883636302110972e-06, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.455006875214167e-06, + 9.70169003267074e-06, + 9.52174832491437e-06, + 8.524861186742783e-06, + 8.590469406044576e-06, + 8.520926712662913e-06, + 7.986654964042827e-06, + 7.65731147112092e-06, + 6.990703695919365e-06, + 6.6349402914056554e-06, + 6.355295226967428e-06, + 6.129083431005711e-06, + 5.99266377321328e-06, + 5.8016053117171396e-06, + 5.728539235860808e-06, + 5.580269771598978e-06, + 5.463929483084939e-06, + 5.414202405518154e-06, + 5.320095169736305e-06, + 5.224529104452813e-06, + 5.126427822688129e-06, + 5.020667231292464e-06, + 4.903477474726969e-06, + 4.799095677299192e-06, + 4.568513304548105e-06, + 4.272334535926348e-06, + 4.043248281959677e-06, + 3.775350933210575e-06, + 3.4793677059496986e-06, + 3.160791720802081e-06, + 2.923573902080534e-06, + 2.8337476578599308e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.5575103993323864e-05, + "first_to_last_ratio": 2.443213468991071, + "imbalance_abs_log_ratio": 0.893314168315378, + "last_quartile_mean": 1.456078416595119e-05, + "mean": 2.3831100747884193e-05, + "normalized": [ + 1.614905261758675, + 1.6568148874352646, + 1.6283515751571318, + 1.4605379751384047, + 1.4717695441533916, + 1.4559185813526447, + 1.358376097224691, + 1.295738430337987, + 1.1776907075521434, + 1.1141057290944703, + 1.0640572841848701, + 1.0269673379730528, + 1.0041631647243947, + 0.9733861101490422, + 0.9597079874490758, + 0.9349675920893245, + 0.9158525986134215, + 0.9066797587654788, + 0.890357239209801, + 0.8734858422464332, + 0.8579185157197757, + 0.8414715045701692, + 0.8239471946120328, + 0.8048352542714806, + 0.7658980848440458, + 0.7173626211575344, + 0.6794957283273223, + 0.6347225670052176, + 0.5854538165358278, + 0.5328910383364602, + 0.49362355782826073, + 0.47854641218217414 + ], + "peak_layer": 2, + "peak_normalized": 1.6568148874352646, + "population_cv": 0.33383138748240165, + "rest_mean": 2.4630704299650257e-05, + "spike_contrast": 0.7922323832463255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.951324156834744e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.848496999125928e-05, + 3.94837225030642e-05, + 3.880541044054553e-05, + 3.48062276316341e-05, + 3.507388828438707e-05, + 3.4696142392931506e-05, + 3.237159762647934e-05, + 3.087887307628989e-05, + 2.8065665901522152e-05, + 2.6550365873845294e-05, + 2.535765634092968e-05, + 2.4473762096022256e-05, + 2.393031354586128e-05, + 2.3196862457552925e-05, + 2.2870897737448104e-05, + 2.2281306883087382e-05, + 2.182577554776799e-05, + 2.1607176677207462e-05, + 2.1218193069216795e-05, + 2.081612910842523e-05, + 2.0445142581593245e-05, + 2.0053192201885395e-05, + 1.9635568605735898e-05, + 1.9180110029992647e-05, + 1.825219442253001e-05, + 1.7095540897571482e-05, + 1.6193131159525365e-05, + 1.5126137441257015e-05, + 1.3952008885098621e-05, + 1.2699380022240803e-05, + 1.176359273813432e-05, + 1.1404287761251908e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03154835826717317, + "first_to_last_ratio": 2.4432135220636924, + "imbalance_abs_log_ratio": 0.8933141900378437, + "last_quartile_mean": 0.012912648846395314, + "mean": 0.021133658214239404, + "normalized": [ + 1.6149054410358779, + 1.6568148414506838, + 1.628351534517943, + 1.460537924411657, + 1.4717696782723049, + 1.4559186942111129, + 1.3583760820142625, + 1.2957384038760298, + 1.1776906753918865, + 1.1141057069236509, + 1.0640572599127858, + 1.0269673356633326, + 1.0041631806471485, + 0.9733861162020112, + 0.9597079595045668, + 0.934967535297851, + 0.9158525948921301, + 0.9066797961300305, + 0.890357194041685, + 0.8734858545515346, + 0.8579184935162031, + 0.8414714427912773, + 0.8239471886412217, + 0.8048352448740587, + 0.7658981573629571, + 0.7173626162098946, + 0.679495681209669, + 0.6347225519706674, + 0.5854538479398482, + 0.5328910449060651, + 0.49362353392574315, + 0.4785463877039095 + ], + "peak_layer": 2, + "peak_normalized": 1.6568148414506838, + "population_cv": 0.33383140220751223, + "rest_mean": 0.021842754652930632, + "spike_contrast": 0.7922323772924418, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01730453744530678, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.034128859639167786, + 0.035014558583498, + 0.03441302478313446, + 0.030866509303450584, + 0.03110387735068798, + 0.03076888807117939, + 0.028707455843687057, + 0.02738369256258011, + 0.024888912215828896, + 0.023545129224658012, + 0.022487422451376915, + 0.021703576669096947, + 0.021221641451120377, + 0.020571209490299225, + 0.020282140001654625, + 0.0197592843323946, + 0.01935531571507454, + 0.019161460921168327, + 0.018816504627466202, + 0.018459951505064964, + 0.0181309562176466, + 0.01778336986899376, + 0.01741301827132702, + 0.017009112983942032, + 0.01618622988462448, + 0.015160496346652508, + 0.01436022948473692, + 0.013414009474217892, + 0.01237278152257204, + 0.011261937208473682, + 0.010432071052491665, + 0.010113435797393322 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3449252946884371e-06, + "first_to_last_ratio": 1.4244745637229155, + "imbalance_abs_log_ratio": 0.35380301851203094, + "last_quartile_mean": 9.441553601163832e-07, + "mean": 1.1411003768557748e-06, + "normalized": [ + 2.099491828546412, + 1.269444086454149, + 1.1995747754012265, + 1.0098229278397233, + 0.9820952443149423, + 0.9671699039875943, + 0.9480579483618607, + 0.9533142816284239, + 0.9297807852159039, + 0.9211908612615112, + 0.919724320468778, + 0.9287522140783918, + 0.9295613022426437, + 0.9490597192634187, + 0.9764897109479356, + 0.9869635221203849, + 0.989406229128281, + 1.0074504593684357, + 1.0338404225654974, + 1.058785663749283, + 1.0614877053824159, + 1.080292006254891, + 1.0793236111118636, + 1.0996580161619744, + 1.0501518033485862, + 0.9775929043037735, + 0.9270853191053471, + 0.8657999542590211, + 0.7904345026237674, + 0.7106884577741329, + 0.6594512386425827, + 0.6380582740868475 + ], + "peak_layer": 1, + "peak_normalized": 2.099491828546412, + "population_cv": 0.2365628847316153, + "rest_mean": 1.1254244832849104e-06, + "spike_contrast": 1.0891447807859125, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2257502021384426e-06, + "top_five_layers": [ + 1, + 2, + 3, + 24, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.3957309167599306e-06, + 1.448563125450164e-06, + 1.368835228277021e-06, + 1.1523093235155102e-06, + 1.120669253396045e-06, + 1.1036379419238074e-06, + 1.081829282156832e-06, + 1.0878272860281868e-06, + 1.0609732044031261e-06, + 1.0511712389416061e-06, + 1.049497768690344e-06, + 1.0597995014904882e-06, + 1.0607227522996254e-06, + 1.082972403310123e-06, + 1.114272777158476e-06, + 1.126224447034474e-06, + 1.1290118209217326e-06, + 1.1496020988488453e-06, + 1.1797156957982224e-06, + 1.2081807199137984e-06, + 1.2112640206396463e-06, + 1.2327216154517373e-06, + 1.2316165793890832e-06, + 1.2548201766549028e-06, + 1.1983286185568431e-06, + 1.1155316315125674e-06, + 1.0578974070085678e-06, + 9.879646540866815e-07, + 9.019651088237879e-07, + 8.109668669931125e-07, + 7.525000569330587e-07, + 7.280885370164469e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6303443064534804e-05, + "first_to_last_ratio": 2.692545967098127, + "imbalance_abs_log_ratio": 0.9904872021768188, + "last_quartile_mean": 9.768985705704836e-06, + "mean": 1.6957480056589702e-05, + "normalized": [ + 1.6937361552633041, + 1.7160304536137982, + 1.6834660265718648, + 1.5207187641970439, + 1.5301774129520833, + 1.5109128899887163, + 1.4130194473875266, + 1.3410670821715687, + 1.2180739698019514, + 1.1398798060375233, + 1.078696685989418, + 1.0334569745212194, + 1.008882916344115, + 0.9710505739450633, + 0.9566696262711185, + 0.9261570152534361, + 0.9027531401068917, + 0.895031047688981, + 0.8707202243782249, + 0.8467588255080271, + 0.8234129285380357, + 0.7960884736247433, + 0.7684181877378946, + 0.7461244793595948, + 0.7082594203206751, + 0.6648334442793923, + 0.6336722392847842, + 0.5977542493887017, + 0.5540428877068647, + 0.5092666262142399, + 0.47765489699849767, + 0.4632131285546998 + ], + "peak_layer": 2, + "peak_normalized": 1.7160304536137982, + "population_cv": 0.370371578681264, + "rest_mean": 1.7684576926479133e-05, + "spike_contrast": 0.7368656323168925, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3031156959186773e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.87214970740024e-05, + 2.9099552193656564e-05, + 2.854734157153871e-05, + 2.578755811555311e-05, + 2.5947952963178977e-05, + 2.5621275199227966e-05, + 2.3961249098647386e-05, + 2.2741118300473318e-05, + 2.065546505036764e-05, + 1.932948907779064e-05, + 1.8291977539774962e-05, + 1.752482603478711e-05, + 1.7108111933339387e-05, + 1.6466570741613396e-05, + 1.6222706108237617e-05, + 1.5705289115430787e-05, + 1.5308418369386345e-05, + 1.5177471141214482e-05, + 1.476522083976306e-05, + 1.4358895896293689e-05, + 1.3963008314021863e-05, + 1.3499654414772522e-05, + 1.3030436093686149e-05, + 1.2652390978473704e-05, + 1.2010294994979631e-05, + 1.1273899872321635e-05, + 1.0745484360086266e-05, + 1.0136405762750655e-05, + 9.395171218784526e-06, + 8.635878657514695e-06, + 8.099823389784433e-06, + 7.854927389416844e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9079276171396486e-05, + "first_to_last_ratio": 2.6285962135440917, + "imbalance_abs_log_ratio": 0.9664499445808603, + "last_quartile_mean": 7.258351843120181e-06, + "mean": 1.2128285078460976e-05, + "normalized": [ + 1.7356747962648031, + 1.7341306113468407, + 1.7105708428728739, + 1.5429534394890942, + 1.5518170549190342, + 1.5419721261714732, + 1.4249221294985488, + 1.3429376987128008, + 1.218113684845076, + 1.1171416737474187, + 1.0512005083729412, + 1.010591189648245, + 0.9869796783812811, + 0.9490114917860836, + 0.945008399357438, + 0.9108007889697958, + 0.8807169292049036, + 0.8929396267645643, + 0.8504587187244783, + 0.8212221659423008, + 0.7893862502127975, + 0.7583794939936823, + 0.7281575784579098, + 0.7171946304338384, + 0.6937537204046544, + 0.6568753570132616, + 0.6340749327739144, + 0.6088859412265378, + 0.578600809495843, + 0.5544537228265827, + 0.5337941004053197, + 0.5272799077356636 + ], + "peak_layer": 1, + "peak_normalized": 1.7356747962648031, + "population_cv": 0.37524903134285426, + "rest_mean": 1.27181367335854e-05, + "spike_contrast": 0.7031758132598661, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.943086140789091e-06, + "top_five_layers": [ + 1, + 2, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1050758732599206e-05, + 2.1032030417700298e-05, + 2.074629082926549e-05, + 1.8713379176915623e-05, + 1.882087963167578e-05, + 1.8701477529248223e-05, + 1.7281861801166087e-05, + 1.6287531252601184e-05, + 1.4773630027775653e-05, + 1.3549012692237739e-05, + 1.2749259440170135e-05, + 1.2256738045834936e-05, + 1.1970370906055905e-05, + 1.1509881915117148e-05, + 1.1461331268947106e-05, + 1.1046451618312858e-05, + 1.0681585990823805e-05, + 1.0829826351255178e-05, + 1.031460578815313e-05, + 9.96001654129941e-06, + 9.573901479598135e-06, + 9.197842700814363e-06, + 8.831302693579346e-06, + 8.698340934643056e-06, + 8.414042895310558e-06, + 7.966771590872668e-06, + 7.690241545788012e-06, + 7.384742275462486e-06, + 7.017435564193875e-06, + 6.724572813254781e-06, + 6.474007022916339e-06, + 6.395001037162729e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.327860384975793e-05, + "first_to_last_ratio": 2.380013015374498, + "imbalance_abs_log_ratio": 0.8671059563131774, + "last_quartile_mean": 3.078916097365436e-05, + "mean": 4.963039305039274e-05, + "normalized": [ + 1.603330252581107, + 1.6263391318078697, + 1.606460956677952, + 1.4448408316016137, + 1.4626423772053405, + 1.419568110995152, + 1.3492771702415232, + 1.299432930512805, + 1.1824076365210394, + 1.115525432190159, + 1.069721713913707, + 1.0194221990001253, + 0.9991025293542795, + 0.9701679624855374, + 0.9607540796701428, + 0.941138763470014, + 0.9221596303852333, + 0.9137187507583548, + 0.8936702231039447, + 0.8650472606162771, + 0.8595412236639356, + 0.8548091765162906, + 0.8394520870170261, + 0.8185169785606647, + 0.7782339575226548, + 0.7312396774119493, + 0.6924473875661084, + 0.6467258401932515, + 0.5884903459061062, + 0.5417511914242714, + 0.5024863639066834, + 0.48157782721888 + ], + "peak_layer": 2, + "peak_normalized": 1.6263391318078697, + "population_cv": 0.3240577046299135, + "rest_mean": 5.1191814068092584e-05, + "spike_contrast": 0.8047911625091714, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.119871955481358e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.957391062518582e-05, + 8.071585034485906e-05, + 7.97292887000367e-05, + 7.17080183676444e-05, + 7.259151607286185e-05, + 7.045372331049293e-05, + 6.696515629300848e-05, + 6.449136708397418e-05, + 5.8683355746325105e-05, + 5.5363965657306835e-05, + 5.309070911607705e-05, + 5.059432442067191e-05, + 4.958585122949444e-05, + 4.81498173030559e-05, + 4.768260259879753e-05, + 4.67090867459774e-05, + 4.576714491122402e-05, + 4.534822073765099e-05, + 4.435320443008095e-05, + 4.293263555155136e-05, + 4.265936877345666e-05, + 4.242451541358605e-05, + 4.1662337025627494e-05, + 4.062331936438568e-05, + 3.862405719701201e-05, + 3.629171260399744e-05, + 3.436643601162359e-05, + 3.2097257644636557e-05, + 2.920700717368163e-05, + 2.6887324565905146e-05, + 2.493859574315138e-05, + 2.3900896849227138e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b2099d933b249b5476dd300b5c609d1c183350a0e7084ee2e3694f173783432f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.062413111329078674, + 0.07552354782819748, + 0.14653873443603516, + 0.2105521410703659, + 0.2186809629201889, + 0.14272794127464294, + 0.1287408471107483, + 0.13253600895404816, + 0.13371841609477997, + 0.09585829079151154, + 0.09883332997560501, + 0.10020580142736435, + 0.11237040907144547, + 0.08538161963224411, + 0.08663177490234375, + 0.10358677804470062, + 0.10874549299478531, + 0.06467291712760925, + 0.07278503477573395, + 0.08346924930810928, + 0.09857872873544693, + 0.06093711405992508, + 0.06834813207387924, + 0.0680762529373169, + 0.07369214296340942, + 0.05633483827114105, + 0.05850831791758537, + 0.06479237228631973, + 0.06705968081951141, + 0.061747051775455475, + 0.0654391348361969, + 0.07205834239721298 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.896821650390848e-06, + "first_to_last_ratio": 3.086921357209413, + "imbalance_abs_log_ratio": 1.1271742697207559, + "last_quartile_mean": 1.5863124076531676e-06, + "mean": 3.5260151562965802e-06, + "normalized": [ + 2.4218303084255877, + 1.2262927966730428, + 0.9438390302215882, + 1.3886420618812072, + 0.4539768042487178, + 1.1428851913549358, + 1.7589416507010018, + 1.7737466699170399, + 1.825038365263652, + 1.4555799135606915, + 1.1601907943732277, + 1.2524020951634824, + 0.6438615738360847, + 1.151558369997547, + 1.2084021893674217, + 1.345408620035675, + 0.6220571382439607, + 0.9702308357427584, + 1.0337141829385006, + 1.2671021334514148, + 0.7373548929504563, + 0.9774444054740276, + 0.9713309430114792, + 0.6690638964335255, + 0.5006396015841585, + 0.6207090232052809, + 0.3879420916285575, + 0.589315985595003, + 0.2987117204227787, + 0.45844781171783394, + 0.36830553154718904, + 0.37503337103217216 + ], + "peak_layer": 1, + "peak_normalized": 2.4218303084255877, + "population_cv": 0.49741945720200614, + "rest_mean": 3.6754354368895292e-06, + "spike_contrast": 0.7398159178102259, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.7191456410946557e-06, + "top_five_layers": [ + 1, + 9, + 8, + 7, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.539410373487044e-06, + 4.32392698712647e-06, + 3.327990725665586e-06, + 4.89637295686407e-06, + 1.6007290923880646e-06, + 4.029830506624421e-06, + 6.202054919413058e-06, + 6.25425764155807e-06, + 6.435112936742371e-06, + 5.1323968364158645e-06, + 4.09085032515577e-06, + 4.415988769324031e-06, + 2.2702656679030042e-06, + 4.060412265971536e-06, + 4.260844434611499e-06, + 4.743931185657857e-06, + 2.1933828975306824e-06, + 3.421048631935264e-06, + 3.644891876319889e-06, + 4.467821327125421e-06, + 2.5999245281127514e-06, + 3.446483788138721e-06, + 3.4249276268383255e-06, + 2.3591294393554563e-06, + 1.7652628230280243e-06, + 2.188629423471866e-06, + 1.3678896948476904e-06, + 2.0779370970558375e-06, + 1.0532620535741444e-06, + 1.6164939324880834e-06, + 1.2986508863832569e-06, + 1.3223733503764379e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.0633759504562477e-05, + "first_to_last_ratio": 3.384548225953592, + "imbalance_abs_log_ratio": 1.2192204339279695, + "last_quartile_mean": 6.096459003401833e-06, + "mean": 1.4159107664113435e-05, + "normalized": [ + 3.1246660167166214, + 1.2194113419753856, + 0.9163650527941182, + 1.3371893358177132, + 0.4570845763492036, + 1.1361310632504436, + 1.7660054721762457, + 1.7013730653458237, + 1.7160657990991948, + 1.4561034988782278, + 1.164959879782242, + 1.19728584675021, + 0.6527944132565722, + 1.141018877615477, + 1.2123794000378127, + 1.3469149905738007, + 0.5843570471425387, + 0.938924512245779, + 1.003661170695441, + 1.2206463029313366, + 0.695795797421837, + 0.9665811243416496, + 0.9496890218619827, + 0.6500522038437716, + 0.48232230036649704, + 0.5819961943923555, + 0.38373225704750635, + 0.5379015492539144, + 0.28024614530736863, + 0.446258402189272, + 0.3666855435712217, + 0.36540179696843594 + ], + "peak_layer": 1, + "peak_normalized": 3.1246660167166214, + "population_cv": 0.5666838267588304, + "rest_mean": 1.4817537400328243e-05, + "spike_contrast": 0.7156106174780821, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0603587088553468e-05, + "top_five_layers": [ + 1, + 7, + 9, + 8, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.4242482545087114e-05, + 1.726577647787053e-05, + 1.2974911442142911e-05, + 1.8933407773147337e-05, + 6.471909728134051e-06, + 1.60866020451067e-05, + 2.5005061615956947e-05, + 2.408992440905422e-05, + 2.4297960408148356e-05, + 2.0617126210709102e-05, + 1.649479236220941e-05, + 1.6952499208855443e-05, + 9.242986379831564e-06, + 1.615580913494341e-05, + 1.7166210454888642e-05, + 1.9071114365942776e-05, + 8.273974344774615e-06, + 1.3294333257363178e-05, + 1.4210946574166883e-05, + 1.7283262423006818e-05, + 9.85184760793345e-06, + 1.3685926205653232e-05, + 1.344674910797039e-05, + 9.204159141518176e-06, + 6.829253379692091e-06, + 8.240546776505653e-06, + 5.433306341728894e-06, + 7.616205948579591e-06, + 3.96803534385981e-06, + 6.318620762613136e-06, + 5.1919400902988855e-06, + 5.173763383936603e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.01829822437139228, + "first_to_last_ratio": 3.3845482144866907, + "imbalance_abs_log_ratio": 1.219220430539954, + "last_quartile_mean": 0.005406400858191773, + "mean": 0.012556438210594933, + "normalized": [ + 3.1246659927740246, + 1.2194113871462446, + 0.9163651029165816, + 1.337189393706543, + 0.45708456224334865, + 1.136131091380754, + 1.7660054527899205, + 1.701373061401254, + 1.7160657272955568, + 1.4561034762492286, + 1.1649599187562338, + 1.1972858308410934, + 0.6527944366098802, + 1.1410188066701286, + 1.2123793905582598, + 1.3469149814270192, + 0.5843570399372514, + 0.9389244849330981, + 1.0036612348062732, + 1.2206462587941367, + 0.6957957692173607, + 0.9665811150881268, + 0.9496889847306975, + 0.6500522635247569, + 0.48232233035374666, + 0.5819961795520987, + 0.38373222955368735, + 0.5379015675830991, + 0.28024616194939816, + 0.44625838932572864, + 0.3666855640843364, + 0.3654018138001321 + ], + "peak_layer": 1, + "peak_normalized": 3.1246659927740246, + "population_cv": 0.566683818054122, + "rest_mean": 0.013140340275303633, + "spike_contrast": 0.715610620741758, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.009403367061167955, + "top_five_layers": [ + 1, + 7, + 9, + 8, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03923467546701431, + 0.015311463735997677, + 0.011506281793117523, + 0.01679033599793911, + 0.005739354062825441, + 0.014265759848058224, + 0.022174738347530365, + 0.021363185718655586, + 0.021547673270106316, + 0.018283473327755928, + 0.014627747237682343, + 0.015033645555377007, + 0.008196773007512093, + 0.014327132143080235, + 0.015223166905343533, + 0.01691245473921299, + 0.0073374430648982525, + 0.01178954727947712, + 0.012602410279214382, + 0.01532696932554245, + 0.008736716583371162, + 0.012136816047132015, + 0.011924711056053638, + 0.008162341080605984, + 0.006056250538676977, + 0.007307799067348242, + 0.004818310029804707, + 0.00675412779673934, + 0.003518893616273999, + 0.00560341589152813, + 0.0046042646281421185, + 0.004588145297020674 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.353687545308958e-06, + "first_to_last_ratio": 2.0714788069877605, + "imbalance_abs_log_ratio": 0.7282627517264126, + "last_quartile_mean": 1.1362353973254358e-06, + "mean": 1.408354910914511e-06, + "normalized": [ + 8.562446711412006, + 2.3789661370966213, + 0.4619949054197906, + 0.5268622892149433, + 0.20998667107627778, + 0.31572440518666567, + 0.4964571094095077, + 0.41741635581071684, + 0.503305459225638, + 0.6375462734944312, + 0.6385460296455763, + 0.6762080183809729, + 0.5854533273996188, + 0.7765606105827553, + 0.8348084946779258, + 0.7212436224371831, + 0.4309937062730567, + 0.8125935658899578, + 0.874841120374869, + 0.8623676992823301, + 0.6400613645240014, + 1.1225318571258882, + 0.9563878029262699, + 1.1024404313909701, + 0.7907538382800698, + 1.0058032833852923, + 0.7964771092624024, + 1.1595820855067065, + 0.6910708445662769, + 0.7332770216661861, + 0.6924978683240438, + 0.5847939807510477 + ], + "peak_layer": 1, + "peak_normalized": 8.562446711412006, + "population_cv": 1.4067449803535663, + "rest_mean": 1.4285843490226938e-06, + "spike_contrast": 0.9093729369351278, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2991159451303246e-06, + "top_five_layers": [ + 1, + 2, + 28, + 22, + 24 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 1.2058963875460904e-05, + 3.3504286420793505e-06, + 6.506527938654472e-07, + 7.420090923915268e-07, + 2.957357594368659e-07, + 4.446520165402035e-07, + 6.991878080953029e-07, + 5.878703746020619e-07, + 7.088327151905105e-07, + 8.978914252111281e-07, + 8.992994366963103e-07, + 9.523408834866132e-07, + 8.245260687544942e-07, + 1.0936729495369946e-06, + 1.1757066431528074e-06, + 1.0157669976251782e-06, + 6.069921028029057e-07, + 1.1444201390986564e-06, + 1.2320867881498998e-06, + 1.214519784298318e-06, + 9.014335660140205e-07, + 1.580923253641231e-06, + 1.3469334589899518e-06, + 1.5526273955401848e-06, + 1.1136620514662354e-06, + 1.4165279935696162e-06, + 1.121722448260698e-06, + 1.6331031247318606e-06, + 9.732730177347548e-07, + 1.0327142945243395e-06, + 9.752827736519976e-07, + 8.23597474663984e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.2558051935229742e-05, + "first_to_last_ratio": 3.235991308724262, + "imbalance_abs_log_ratio": 1.174335313384314, + "last_quartile_mean": 3.880743406625697e-06, + "mean": 9.272452984987467e-06, + "normalized": [ + 1.947517132469556, + 1.2659703720728832, + 1.00333727887001, + 1.3279268679179317, + 0.5164022830958469, + 1.2531052602985686, + 1.8259856606852272, + 1.694473384142606, + 1.5673596610865692, + 1.56833061106792, + 1.2405222428920049, + 1.2135192597759437, + 0.7282554475336783, + 1.2012094109417673, + 1.3067699865079492, + 1.2637529494818742, + 0.6470120244072932, + 1.0867341236920782, + 1.1179134962672703, + 1.2614932519699678, + 0.7458582439755571, + 1.0936916340494032, + 1.0525562723887423, + 0.7221117539250419, + 0.5393185760918967, + 0.6195458807889652, + 0.4064975180219839, + 0.4502100069225183, + 0.3003691735935886, + 0.42059397269238735, + 0.3160004972452703, + 0.29565576512769975 + ], + "peak_layer": 1, + "peak_normalized": 1.947517132469556, + "population_cv": 0.45871035705065244, + "rest_mean": 9.563149029013908e-06, + "spike_contrast": 0.8054558518198625, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.702694347244687e-06, + "top_five_layers": [ + 1, + 7, + 8, + 10, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8058261048281565e-05, + 1.17386507554329e-05, + 9.303397746407427e-06, + 1.2313139450270683e-05, + 4.7883158913464285e-06, + 1.161935961135896e-05, + 1.6931366189965047e-05, + 1.5711924788774922e-05, + 1.4533268767991103e-05, + 1.4542271856043953e-05, + 1.1502684174047317e-05, + 1.125230028264923e-05, + 6.752714398317039e-06, + 1.1138157788082026e-05, + 1.2116963262087665e-05, + 1.171808980870992e-05, + 5.999388577038189e-06, + 1.007669106911635e-05, + 1.0365800335421227e-05, + 1.1697136869770475e-05, + 6.915935500728665e-06, + 1.014120425679721e-05, + 9.759778549778275e-06, + 6.69574728817679e-06, + 5.0008061407424975e-06, + 5.7447100516583305e-06, + 3.7692291243729414e-06, + 4.174551122559933e-06, + 2.7851590402860893e-06, + 3.899937837559264e-06, + 2.9300997539394302e-06, + 2.741454181887093e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 7.895624435150239e-06, + "first_to_last_ratio": 3.058028124449071, + "imbalance_abs_log_ratio": 1.1177703044721228, + "last_quartile_mean": 2.581933230771938e-06, + "mean": 6.122625247684255e-06, + "normalized": [ + 1.1319461307239276, + 1.3085216761923217, + 1.0639933002407227, + 1.2290715675273767, + 0.5847062304385443, + 1.3366939702161282, + 1.897298452770328, + 1.7644209127072272, + 1.3831256030397758, + 1.6077197781270347, + 1.29630788384933, + 1.1373671133182328, + 0.7956691079721259, + 1.243813918811557, + 1.3698450981038646, + 1.232490514336823, + 0.692747757146363, + 1.2511518200321439, + 1.1980660456404937, + 1.2681707953518462, + 0.7704716066493089, + 1.1938208830614028, + 1.1123242489274907, + 0.7566266249200412, + 0.584885451820435, + 0.6871683496836065, + 0.42379305915423937, + 0.4006394094629471, + 0.32523481684807415, + 0.389650977401113, + 0.2995172170061151, + 0.2627396785190583 + ], + "peak_layer": 7, + "peak_normalized": 1.897298452770328, + "population_cv": 0.4334998696783958, + "rest_mean": 6.25457262567916e-06, + "spike_contrast": 0.8649846648673781, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.410109406511765e-06, + "top_five_layers": [ + 7, + 8, + 10, + 9, + 15 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.930481958988821e-06, + 8.01158785179723e-06, + 6.5144322434207425e-06, + 7.52514461055398e-06, + 3.579937128961319e-06, + 8.184076250472572e-06, + 1.1616447409323882e-05, + 1.0802888027683366e-05, + 8.468359737889841e-06, + 9.843465704761911e-06, + 7.936807378428057e-06, + 6.9636726038879715e-06, + 4.871583769272547e-06, + 7.615406502736732e-06, + 8.387048183067236e-06, + 7.546077540609986e-06, + 4.241434908180963e-06, + 7.660333722014911e-06, + 7.335309419431724e-06, + 7.764534529997036e-06, + 4.717308911494911e-06, + 7.309317879844457e-06, + 6.810344530094881e-06, + 4.6325412768055685e-06, + 3.581034434319008e-06, + 4.207274287182372e-06, + 2.594726083771093e-06, + 2.45296496359515e-06, + 1.9912909010599833e-06, + 2.3856869120209012e-06, + 1.8338316749577643e-06, + 1.6086565892692306e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.8174724750206224e-05, + "first_to_last_ratio": 3.4131964974538564, + "imbalance_abs_log_ratio": 1.2276492415031157, + "last_quartile_mean": 1.1184449761003634e-05, + "mean": 2.727851558859129e-05, + "normalized": [ + 2.560639234860951, + 1.2774901614756382, + 0.9638213100637651, + 1.437728242920233, + 0.4488834649809974, + 1.1881397139629066, + 1.757440362013926, + 1.5614006473172555, + 1.6987735612467263, + 1.5843521015029483, + 1.2353981365841489, + 1.257208792964521, + 0.6596706652646716, + 1.1343619132828286, + 1.279586514141272, + 1.2620503153781926, + 0.6179136513723597, + 0.948834235599364, + 1.071059237777408, + 1.2838045586802798, + 0.7435873864059896, + 1.0227779780310893, + 1.0169155522739846, + 0.7080864895405166, + 0.5037032405562084, + 0.5772180835558656, + 0.3939239236704607, + 0.465560531237428, + 0.2747838533410833, + 0.43521029512005854, + 0.3246502639074058, + 0.3050255809695173 + ], + "peak_layer": 1, + "peak_normalized": 2.560639234860951, + "population_cv": 0.5127605859467733, + "rest_mean": 2.829381118216487e-05, + "spike_contrast": 0.7703422929828946, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.179591938329395e-05, + "top_five_layers": [ + 1, + 7, + 9, + 10, + 8 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.985043728491291e-05, + 3.48480352840852e-05, + 2.6291614631190896e-05, + 3.921909228665754e-05, + 1.224487459694501e-05, + 3.241068770876154e-05, + 4.79403643112164e-05, + 4.2592691897880286e-05, + 4.634002107195556e-05, + 4.321877349866554e-05, + 3.3699827326927334e-05, + 3.429478965699673e-05, + 1.799483652575873e-05, + 3.094370913458988e-05, + 3.4905220672953874e-05, + 3.442685920163058e-05, + 1.6855767171364278e-05, + 2.588278948678635e-05, + 2.9216906114015728e-05, + 3.502028266666457e-05, + 2.0283960111555643e-05, + 2.7899865017388947e-05, + 2.773994674498681e-05, + 1.9315548343001865e-05, + 1.374027669953648e-05, + 1.574565249029547e-05, + 1.0745659892563708e-05, + 1.2699800208793022e-05, + 7.495695626857923e-06, + 1.1871890819747932e-05, + 8.855977284838445e-06, + 8.320645065396093e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "59db4720b6344fca1116e578ec4d190ae2504f1b9d034222d6eb02932b7f4d0f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05931045487523079, + 0.07934419065713882, + 0.214645653963089, + 0.09514089673757553, + 0.12821291387081146, + 0.1321839988231659, + 0.12568901479244232, + 0.1335759311914444, + 0.07788944244384766, + 0.08352623134851456, + 0.07934392243623734, + 0.09343641996383667, + 0.07063088566064835, + 0.06870647519826889, + 0.07622843235731125, + 0.08666495233774185, + 0.057347409427165985, + 0.05777324363589287, + 0.06351501494646072, + 0.07469023764133453, + 0.051319729536771774, + 0.05102822557091713, + 0.05902284011244774, + 0.06656526774168015, + 0.050236333161592484, + 0.05184732377529144, + 0.061248794198036194, + 0.06608261168003082, + 0.048912134021520615, + 0.062179163098335266, + 0.07131517678499222, + 0.06802479177713394 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.536313694179171e-06, + "first_to_last_ratio": 0.32103533438342263, + "imbalance_abs_log_ratio": 1.136204085949217, + "last_quartile_mean": 7.90041912068773e-06, + "mean": 5.024183989732478e-06, + "normalized": [ + 1.4240775191442525, + 0.5380590571289816, + 0.1905335652531641, + 0.6822754406556624, + 0.2910317874157833, + 0.1585066509591937, + 0.2416971045584352, + 0.5123870473279437, + 0.7644347964709719, + 0.6121884879234236, + 0.6815584070993507, + 0.7497887841806089, + 0.7911538454261898, + 0.7760261752123336, + 0.7322264401935603, + 0.7150908135381492, + 0.8515627911494139, + 0.9152360407363572, + 1.0880793237379123, + 1.1939585333655534, + 1.1057515489086724, + 1.3427709644104338, + 1.488668647713089, + 1.573111703813592, + 1.6787238134633642, + 1.809834874575413, + 1.572907328432356, + 1.5205734730600422, + 1.9211083090017804, + 1.5699327524673685, + 1.322832236174336, + 1.1839117365023109 + ], + "peak_layer": 29, + "peak_normalized": 1.9211083090017804, + "population_cv": 0.48692469008524586, + "rest_mean": 4.616847960463659e-06, + "spike_contrast": 1.5646602638087805, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.223798547784099e-06, + "top_five_layers": [ + 29, + 26, + 25, + 24, + 27 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.1548274718225e-06, + 2.703307700357982e-06, + 9.572756880515954e-07, + 3.4278773455298506e-06, + 1.4621972468376043e-06, + 7.963665780152951e-07, + 1.214330723087187e-06, + 2.5743267997313524e-06, + 3.840661065623863e-06, + 3.0757475997233996e-06, + 3.424274837016128e-06, + 3.767076805161196e-06, + 3.974902483605547e-06, + 3.898898285115138e-06, + 3.6788403576792916e-06, + 3.5927478165831417e-06, + 4.278408141544787e-06, + 4.598314262693748e-06, + 5.466710717882961e-06, + 5.9986673477396835e-06, + 5.555499228648841e-06, + 6.74632838126854e-06, + 7.4793451858568005e-06, + 7.903602636361029e-06, + 8.434217306785285e-06, + 9.092943400901277e-06, + 7.902575816842727e-06, + 7.639640898560174e-06, + 9.65200160862878e-06, + 7.887630999903195e-06, + 6.646152542089112e-06, + 5.948190391791286e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.0282310029197106e-05, + "first_to_last_ratio": 0.32628005375076213, + "imbalance_abs_log_ratio": 1.1199992058116255, + "last_quartile_mean": 3.151375608467788e-05, + "mean": 2.007631842104729e-05, + "normalized": [ + 1.459931390393604, + 0.545480683473369, + 0.19734981318978928, + 0.6879078045288014, + 0.29465579930095187, + 0.15846117316623343, + 0.24241735602570375, + 0.5110850599423152, + 0.7578637625848365, + 0.6102749921347288, + 0.6808791746019699, + 0.7378534750263886, + 0.7874390409852465, + 0.7700487415879345, + 0.7271376355617218, + 0.7086964687616041, + 0.8475748227839637, + 0.9199112127298047, + 1.0869552322670417, + 1.1894614876142768, + 1.1112082208533107, + 1.3353228072817924, + 1.4957799353947956, + 1.5787202238925846, + 1.6775886487367329, + 1.8106175804235458, + 1.5746959684661916, + 1.5206109758006878, + 1.9184410970777135, + 1.5638430908546315, + 1.3084920518741907, + 1.1832942726835374 + ], + "peak_layer": 29, + "peak_normalized": 1.9184410970777135, + "population_cv": 0.4874445604692638, + "rest_mean": 1.8441496498174576e-05, + "spike_contrast": 1.5673541899064982, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.8904356804559937e-05, + "top_five_layers": [ + 29, + 26, + 25, + 24, + 27 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.9310047466424294e-05, + 1.0951243893941864e-05, + 3.962057689932408e-06, + 1.3810656128043775e-05, + 5.915603651374113e-06, + 3.181316969858017e-06, + 4.866848030360416e-06, + 1.0260706403641962e-05, + 1.5215114217426162e-05, + 1.2252075066498946e-05, + 1.3669547115569003e-05, + 1.4813381312706042e-05, + 1.5808876923983917e-05, + 1.5459743735846132e-05, + 1.4598246707464568e-05, + 1.4228015970729757e-05, + 1.7016182027873583e-05, + 1.846843042585533e-05, + 2.1822059352416545e-05, + 2.3880007574916817e-05, + 2.2308970073936507e-05, + 2.6808365873876028e-05, + 3.0029754270799458e-05, + 3.16948899126146e-05, + 3.36798038915731e-05, + 3.6350535083329305e-05, + 3.1614097679266706e-05, + 3.052827014471404e-05, + 3.851523433695547e-05, + 3.139621185255237e-05, + 2.6269703084835783e-05, + 2.3756192604196258e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.009118455491261557, + "first_to_last_ratio": 0.3262800607417022, + "imbalance_abs_log_ratio": 1.1199991843854307, + "last_quartile_mean": 0.027946713846176863, + "mean": 0.017803879898565356, + "normalized": [ + 1.4599314269226145, + 0.5454806821468461, + 0.19734980888248807, + 0.6879078250232088, + 0.29465581055680906, + 0.15846116652987666, + 0.2424173583964748, + 0.5110850711933042, + 0.7578637834127073, + 0.6102749946338123, + 0.6808791796688727, + 0.7378534997474308, + 0.7874390718063808, + 0.7700486867302203, + 0.7271376730207798, + 0.7086964305299408, + 0.847574856844415, + 0.919911238227211, + 1.0869552482975884, + 1.1894614745664287, + 1.1112081941897007, + 1.3353228272158537, + 1.4957798460287703, + 1.5787202151640234, + 1.6775885884762012, + 1.810617543503823, + 1.5746959998764927, + 1.5206111322850961, + 1.918440985885323, + 1.5638431207118677, + 1.3084920400476525, + 1.1832942194777856 + ], + "peak_layer": 29, + "peak_normalized": 1.918440985885323, + "population_cv": 0.4874445523784538, + "rest_mean": 0.01635410358129, + "spike_contrast": 1.5673541435299125, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.025632672011852264, + "top_five_layers": [ + 29, + 26, + 25, + 24, + 27 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.025992443785071373, + 0.00971167255192995, + 0.0035135922953486443, + 0.012247428297996521, + 0.005246016662567854, + 0.0028212235774844885, + 0.004315969534218311, + 0.009099297225475311, + 0.013492915779352188, + 0.01086526270955801, + 0.012122291140258312, + 0.01313665509223938, + 0.014019470661878586, + 0.01370985433459282, + 0.01294587180018425, + 0.012617546133697033, + 0.01509012095630169, + 0.016377989202737808, + 0.01935202069580555, + 0.021177029237151146, + 0.01978381723165512, + 0.023773927241563797, + 0.026630684733390808, + 0.02810734510421753, + 0.029867585748434067, + 0.032236017286777496, + 0.02803569845855236, + 0.027072777971625328, + 0.03415569290518761, + 0.027842475101351738, + 0.023296235129237175, + 0.02106722816824913 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.048277736425689e-07, + "first_to_last_ratio": 0.2857864675257622, + "imbalance_abs_log_ratio": 1.2525103640623867, + "last_quartile_mean": 2.116362537663008e-06, + "mean": 1.2933788453572959e-06, + "normalized": [ + 1.5722659861144699, + 0.5494596534444791, + 0.14380470914509727, + 0.6710949567625404, + 0.22255230509223908, + 0.12256240809816367, + 0.18626377655013246, + 0.27306728115170176, + 0.6044679656212456, + 0.5171048294185702, + 0.6237768050810616, + 0.6102501003317593, + 0.7436430071434332, + 0.7420143247015445, + 0.7014560094306781, + 0.6999767554604397, + 0.8688625721017827, + 0.9411031514274796, + 1.1366340862434177, + 1.2603718481576007, + 1.1663554971537906, + 1.3398176046889787, + 1.583025715064311, + 1.629627005627822, + 1.7375189329126879, + 1.8474466033760384, + 1.712075834943869, + 1.6006522975801964, + 1.9409014549228467, + 1.6098356440532264, + 1.3816138029988174, + 1.2603970751995792 + ], + "peak_layer": 29, + "peak_normalized": 1.9409014549228467, + "population_cv": 0.5453879627305627, + "rest_mean": 1.1757127585572864e-06, + "spike_contrast": 1.6405161039879685, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9287757140773467e-06, + "top_five_layers": [ + 29, + 26, + 25, + 27, + 24 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.033535565715283e-06, + 7.106594921424403e-07, + 1.859939686710277e-07, + 8.679800203026389e-07, + 2.878444433918048e-07, + 1.5851962587021262e-07, + 2.409096282462997e-07, + 3.5317944480084407e-07, + 7.818060794306803e-07, + 6.688124472020718e-07, + 8.067797239164065e-07, + 7.892845701462647e-07, + 9.61812133937201e-07, + 9.597056305210572e-07, + 9.072483635463868e-07, + 9.053351277543698e-07, + 1.123768470279174e-06, + 1.2172029073553858e-06, + 1.4700984820592566e-06, + 1.6301382856909186e-06, + 1.5085395261849044e-06, + 1.732891746542009e-06, + 2.047451971520786e-06, + 2.10772509490198e-06, + 2.247270231237053e-06, + 2.3894483547337586e-06, + 2.2143626665638294e-06, + 2.0702498204627773e-06, + 2.510320882720407e-06, + 2.082127366520581e-06, + 1.786950065252313e-06, + 1.6301709138133447e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 6.5630383971893025e-06, + "first_to_last_ratio": 0.33029070877691014, + "imbalance_abs_log_ratio": 1.1077820766301152, + "last_quartile_mean": 1.9870490518769657e-05, + "mean": 1.2924320181184612e-05, + "normalized": [ + 1.4791032175094332, + 0.509681391761457, + 0.16554536291443128, + 0.6646319035237539, + 0.2907163787243681, + 0.15580076690764444, + 0.2616909410389207, + 0.5352725058862023, + 0.8032176162539202, + 0.6632295542966498, + 0.7433292538374848, + 0.8060642557026766, + 0.8297942030089529, + 0.8051034128510668, + 0.768957453371839, + 0.7423222477435537, + 0.8644340163366496, + 0.9439214195795605, + 1.097648605420341, + 1.189066040117497, + 1.0986102223507093, + 1.338134608877611, + 1.4379233534932891, + 1.5062047000334828, + 1.6649733200863754, + 1.7681836335559407, + 1.5403186324010156, + 1.4902361589548763, + 1.9021040439797992, + 1.5128593847489753, + 1.2614380410420527, + 1.1594833536894698 + ], + "peak_layer": 29, + "peak_normalized": 1.9021040439797992, + "population_cv": 0.47010582540802626, + "rest_mean": 1.1945017537005629e-05, + "spike_contrast": 1.5246988464712319, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.8212554459751117e-05, + "top_five_layers": [ + 29, + 26, + 25, + 27, + 30 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.911640356411226e-05, + 6.587285497516859e-06, + 2.139561274816515e-06, + 8.589915523771197e-06, + 3.75731156054826e-06, + 2.0136189959885087e-06, + 3.3821775105025154e-06, + 6.918033250258304e-06, + 1.0381041647633538e-05, + 8.571791113354266e-06, + 9.607025276636705e-06, + 1.0417832527309656e-05, + 1.072452596417861e-05, + 1.0405414286651649e-05, + 9.938252333085984e-06, + 9.594010407454334e-06, + 1.117222200264223e-05, + 1.2199542652524542e-05, + 1.4186362022883259e-05, + 1.5367870219051838e-05, + 1.4198790267982986e-05, + 1.7294480130658485e-05, + 1.8584181816549972e-05, + 1.9466671801637858e-05, + 2.151864828192629e-05, + 2.285257141920738e-05, + 1.9907571186195128e-05, + 1.9260289263911545e-05, + 2.4583401682320982e-05, + 1.9552679077605717e-05, + 1.6303229131153785e-05, + 1.4985534107836429e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.125818321654151e-06, + "first_to_last_ratio": 0.323071664836056, + "imbalance_abs_log_ratio": 1.1298811078121074, + "last_quartile_mean": 1.2770597891176294e-05, + "mean": 8.254944056318436e-06, + "normalized": [ + 1.4528370129337378, + 0.4887545271444068, + 0.1464999865512414, + 0.6321858434529055, + 0.27861137744593806, + 0.1604870466652074, + 0.2845859883398777, + 0.5544355837055989, + 0.8055064100672278, + 0.6948443934131001, + 0.7505943725961226, + 0.8501485246227231, + 0.8625059475864164, + 0.7928262823143416, + 0.7999235841614649, + 0.7664878864471988, + 0.8659165482877662, + 0.9625161170669155, + 1.085645016298278, + 1.1880281512105406, + 1.0543357106739477, + 1.3441904964711768, + 1.3657148728576123, + 1.4362250429780776, + 1.6635267721014393, + 1.753447817081851, + 1.5686511233757214, + 1.557344226245061, + 1.8968012391308895, + 1.5319357137581802, + 1.2230350653804842, + 1.1814513196345497 + ], + "peak_layer": 29, + "peak_normalized": 1.8968012391308895, + "population_cv": 0.4662704135356736, + "rest_mean": 7.685049350011806e-06, + "spike_contrast": 1.4746002210586762, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1332375470374245e-05, + "top_five_layers": [ + 29, + 26, + 25, + 27, + 28 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.1993088264716789e-05, + 4.034641278849449e-06, + 1.2093491932319012e-06, + 5.21865877090022e-06, + 2.2999213342700386e-06, + 1.3248115919850534e-06, + 2.349241412957781e-06, + 4.576834726321977e-06, + 6.649410352110863e-06, + 5.73590159547166e-06, + 6.196114554768428e-06, + 7.0179285103222355e-06, + 7.119938345567789e-06, + 6.544736606883816e-06, + 6.603324436582625e-06, + 6.327314622467384e-06, + 7.148092663555872e-06, + 7.945516699692234e-06, + 8.961938874563202e-06, + 9.807105925574433e-06, + 8.703482308192179e-06, + 1.1096217349404469e-05, + 1.1273899872321635e-05, + 1.1855957382067572e-05, + 1.373232043988537e-05, + 1.4474613635684364e-05, + 1.2949127267347649e-05, + 1.28557894640835e-05, + 1.565798811498098e-05, + 1.2646043614950031e-05, + 1.0096086043631658e-05, + 9.7528145488468e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.0554489708501933e-05, + "first_to_last_ratio": 0.304193026420869, + "imbalance_abs_log_ratio": 1.1900928237429556, + "last_quartile_mean": 6.75705487083178e-05, + "mean": 4.220494211892856e-05, + "normalized": [ + 1.4416703556828947, + 0.48115035856614824, + 0.16644163758868316, + 0.6569569491237993, + 0.2862080852971654, + 0.14160291640607714, + 0.23550628823607553, + 0.48659285300774935, + 0.769034088597553, + 0.5943108767709541, + 0.6762795951535582, + 0.7435203621700468, + 0.778579992579065, + 0.7456281602454271, + 0.7046027708376329, + 0.7063643980108446, + 0.8365760311947629, + 0.8942705494380042, + 1.096453061256065, + 1.1557136235607102, + 1.1214101214270462, + 1.3312370970047724, + 1.5301222792609177, + 1.6116848329794238, + 1.7202394076902243, + 1.8048142344035467, + 1.596568646119442, + 1.523076115359806, + 2.001887880882806, + 1.5733919180733085, + 1.3426993534506797, + 1.2454051596248104 + ], + "peak_layer": 29, + "peak_normalized": 2.001887880882806, + "population_cv": 0.5063397050013957, + "rest_mean": 3.858673785038668e-05, + "spike_contrast": 1.6001157031841702, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.174324516905472e-05, + "top_five_layers": [ + 29, + 26, + 25, + 24, + 27 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 6.0845613916171715e-05, + 2.0306923033786006e-05, + 7.024659680610057e-06, + 2.772683001239784e-05, + 1.2079395673936233e-05, + 5.976342890789965e-06, + 9.939529263647273e-06, + 2.0536623196676373e-05, + 3.24570391967427e-05, + 2.5082856154767796e-05, + 2.854234116966836e-05, + 3.138023384963162e-05, + 3.285992352175526e-05, + 3.1469193345401436e-05, + 2.9737719160038978e-05, + 2.981206853291951e-05, + 3.530764297465794e-05, + 3.7742636777693406e-05, + 4.627573798643425e-05, + 4.877682658843696e-05, + 4.732904926640913e-05, + 5.61847846256569e-05, + 6.457872223109007e-05, + 6.802106508985162e-05, + 7.260260463226587e-05, + 7.617208029842004e-05, + 6.738308729836717e-05, + 6.428133929148316e-05, + 8.448956214124337e-05, + 6.640491483267397e-05, + 5.666854849550873e-05, + 5.25622526765801e-05 + ] + } + } + } + } + } + }, + "step": 8000 + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 8.061194200882085, + "cross_entropy_nats": 5.587594032287598, + "step": 0 + }, + { + "bits_per_byte": 5.191908918496252, + "cross_entropy_nats": 3.598757028579712, + "step": 100 + }, + { + "bits_per_byte": 3.0881952483500545, + "cross_entropy_nats": 2.1405738294124603, + "step": 500 + }, + { + "bits_per_byte": 1.948972795980338, + "cross_entropy_nats": 1.3509249985218048, + "step": 2000 + }, + { + "bits_per_byte": 1.7872753095784035, + "cross_entropy_nats": 1.2388448417186737, + "step": 4000 + }, + { + "bits_per_byte": 1.7030966812870931, + "cross_entropy_nats": 1.1804966628551483, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "94814c85fe1adce99ef9f8a23b6b40ed1a12e214f38d376a95be44bd8a14d14a", + "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76", + "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "final_public_parameters": "73e15de1ee917017dc95607378bdbf84b7b54f65be48f3497fdb5654f3f0479e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "3f93c11463780b386fff00c53b0424f687e1136b1b80f7b626f058adef96c764" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "3eff5c07fa0cfbd5ceed5922cbd09525ad514f801cc0a440c03fd2fa15df4c84", + "1": "e83e4001c5811b19c69a54c7ea7a802af996a34aa57072cde43a76bc8369e2d0", + "7999": "d04c4ca879f0209183a48ac54a0c4ffe3a0f62a78888479d4641f6c304426442" + }, + "parent_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "parent_path": "experiments/k3/attnres_gradient/manifest.json", + "path": "experiments/k3/attnres_spike/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "aggregation_groups": 8, + "blocks_per_group": 4, + "d_ff": 768, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + } + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", + "round05_equivalence": { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true + }, + "run_kind": "formal", + "schema_version": 1, + "seed": 2026073003, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 224.65581085475935, + "measured_steps": 7980, + "median_ms": 227.5267465010984, + "p95_ms": 230.52696228027344, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 8.082576459898554, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.602415084838867, + "step": 1, + "unclipped_grad_norm": 19.083253860473633 + }, + { + "bits_per_byte": 7.337116283872504, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.0857014656066895, + "step": 10, + "unclipped_grad_norm": 11.73559856414795 + }, + { + "bits_per_byte": 6.794594947933964, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.709654331207275, + "step": 20, + "unclipped_grad_norm": 3.3705036640167236 + }, + { + "bits_per_byte": 6.669727976170927, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.623103141784668, + "step": 30, + "unclipped_grad_norm": 3.037728786468506 + }, + { + "bits_per_byte": 6.488515360401613, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.497496128082275, + "step": 40, + "unclipped_grad_norm": 3.071707010269165 + }, + { + "bits_per_byte": 6.170372909737745, + "learning_rate": 3.75e-05, + "loss_nats": 4.276976585388184, + "step": 50, + "unclipped_grad_norm": 2.8251144886016846 + }, + { + "bits_per_byte": 5.9458358004242795, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.121339321136475, + "step": 60, + "unclipped_grad_norm": 2.6051926612854004 + }, + { + "bits_per_byte": 5.763860269615738, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9952034950256348, + "step": 70, + "unclipped_grad_norm": 2.513765573501587 + }, + { + "bits_per_byte": 5.595572150058764, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8785550594329834, + "step": 80, + "unclipped_grad_norm": 2.595860719680786 + }, + { + "bits_per_byte": 5.4051244099034195, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.746546745300293, + "step": 90, + "unclipped_grad_norm": 2.317351818084717 + }, + { + "bits_per_byte": 5.1779332643218385, + "learning_rate": 7.5e-05, + "loss_nats": 3.5890698432922363, + "step": 100, + "unclipped_grad_norm": 2.214414358139038 + }, + { + "bits_per_byte": 4.995566300999934, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.462662696838379, + "step": 110, + "unclipped_grad_norm": 2.0333311557769775 + }, + { + "bits_per_byte": 4.799066491360726, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3264594078063965, + "step": 120, + "unclipped_grad_norm": 1.8019887208938599 + }, + { + "bits_per_byte": 4.633962458601826, + "learning_rate": 9.75e-05, + "loss_nats": 3.2120180130004883, + "step": 130, + "unclipped_grad_norm": 1.6018544435501099 + }, + { + "bits_per_byte": 4.476944362155892, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.1031813621520996, + "step": 140, + "unclipped_grad_norm": 1.3790134191513062 + }, + { + "bits_per_byte": 4.275779695094682, + "learning_rate": 0.0001125, + "loss_nats": 2.963744640350342, + "step": 150, + "unclipped_grad_norm": 1.2061023712158203 + }, + { + "bits_per_byte": 4.128819144071206, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.861879348754883, + "step": 160, + "unclipped_grad_norm": 1.0463906526565552 + }, + { + "bits_per_byte": 4.021535678532382, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7875161170959473, + "step": 170, + "unclipped_grad_norm": 0.9023508429527283 + }, + { + "bits_per_byte": 3.919193618518149, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.7165780067443848, + "step": 180, + "unclipped_grad_norm": 0.9033810496330261 + }, + { + "bits_per_byte": 3.8348498868820102, + "learning_rate": 0.0001425, + "loss_nats": 2.6581153869628906, + "step": 190, + "unclipped_grad_norm": 0.8499347567558289 + }, + { + "bits_per_byte": 3.7665400977858114, + "learning_rate": 0.00015, + "loss_nats": 2.610766649246216, + "step": 200, + "unclipped_grad_norm": 0.8223105072975159 + }, + { + "bits_per_byte": 3.6822345462981643, + "learning_rate": 0.0001575, + "loss_nats": 2.552330493927002, + "step": 210, + "unclipped_grad_norm": 0.8248683214187622 + }, + { + "bits_per_byte": 3.6564082555833983, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5344290733337402, + "step": 220, + "unclipped_grad_norm": 1.0457000732421875 + }, + { + "bits_per_byte": 3.6121584953776043, + "learning_rate": 0.0001725, + "loss_nats": 2.5037574768066406, + "step": 230, + "unclipped_grad_norm": 1.2628414630889893 + }, + { + "bits_per_byte": 3.5855720413460244, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4853291511535645, + "step": 240, + "unclipped_grad_norm": 0.9464337229728699 + }, + { + "bits_per_byte": 3.5024318443006237, + "learning_rate": 0.0001875, + "loss_nats": 2.4277007579803467, + "step": 250, + "unclipped_grad_norm": 0.8258991837501526 + }, + { + "bits_per_byte": 3.520142273721174, + "learning_rate": 0.000195, + "loss_nats": 2.439976692199707, + "step": 260, + "unclipped_grad_norm": 1.1166419982910156 + }, + { + "bits_per_byte": 3.497664141253405, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.42439603805542, + "step": 270, + "unclipped_grad_norm": 0.9347699880599976 + }, + { + "bits_per_byte": 3.454170760746106, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.3942487239837646, + "step": 280, + "unclipped_grad_norm": 1.2552868127822876 + }, + { + "bits_per_byte": 3.4220065653817433, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.3719542026519775, + "step": 290, + "unclipped_grad_norm": 0.8032089471817017 + }, + { + "bits_per_byte": 3.435558454304431, + "learning_rate": 0.000225, + "loss_nats": 2.38134765625, + "step": 300, + "unclipped_grad_norm": 1.544488787651062 + }, + { + "bits_per_byte": 3.330305071976392, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.308391571044922, + "step": 310, + "unclipped_grad_norm": 0.8757423162460327 + }, + { + "bits_per_byte": 3.390824390953808, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3503403663635254, + "step": 320, + "unclipped_grad_norm": 0.8823365569114685 + }, + { + "bits_per_byte": 3.3979087003081485, + "learning_rate": 0.0002475, + "loss_nats": 2.355250835418701, + "step": 330, + "unclipped_grad_norm": 0.898759663105011 + }, + { + "bits_per_byte": 3.325636774901372, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3051557540893555, + "step": 340, + "unclipped_grad_norm": 0.9536095261573792 + }, + { + "bits_per_byte": 3.3170438337336625, + "learning_rate": 0.0002625, + "loss_nats": 2.2991995811462402, + "step": 350, + "unclipped_grad_norm": 1.2654696702957153 + }, + { + "bits_per_byte": 3.313786482326321, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2969417572021484, + "step": 360, + "unclipped_grad_norm": 1.3773932456970215 + }, + { + "bits_per_byte": 3.3236555347634327, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.3037824630737305, + "step": 370, + "unclipped_grad_norm": 1.2408338785171509 + }, + { + "bits_per_byte": 3.208911117864879, + "learning_rate": 0.000285, + "loss_nats": 2.224247694015503, + "step": 380, + "unclipped_grad_norm": 1.0960922241210938 + }, + { + "bits_per_byte": 3.2804789462990227, + "learning_rate": 0.0002925, + "loss_nats": 2.2738547325134277, + "step": 390, + "unclipped_grad_norm": 0.9866538643836975 + }, + { + "bits_per_byte": 3.27038631641579, + "learning_rate": 0.0003, + "loss_nats": 2.2668590545654297, + "step": 400, + "unclipped_grad_norm": 1.2531954050064087 + }, + { + "bits_per_byte": 3.2889521875417316, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.2797279357910156, + "step": 410, + "unclipped_grad_norm": 1.448500633239746 + }, + { + "bits_per_byte": 3.205672684549132, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.2220029830932617, + "step": 420, + "unclipped_grad_norm": 0.9654834866523743 + }, + { + "bits_per_byte": 3.218386673996816, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2308156490325928, + "step": 430, + "unclipped_grad_norm": 0.974204421043396 + }, + { + "bits_per_byte": 3.1822768526563587, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2057862281799316, + "step": 440, + "unclipped_grad_norm": 1.4095590114593506 + }, + { + "bits_per_byte": 3.237966210866943, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.244387149810791, + "step": 450, + "unclipped_grad_norm": 0.9198341369628906 + }, + { + "bits_per_byte": 3.1857660366770637, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.208204746246338, + "step": 460, + "unclipped_grad_norm": 1.4268778562545776 + }, + { + "bits_per_byte": 3.1576792059993566, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.1887364387512207, + "step": 470, + "unclipped_grad_norm": 1.2876354455947876 + }, + { + "bits_per_byte": 3.131057323541699, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.170283555984497, + "step": 480, + "unclipped_grad_norm": 1.0718865394592285 + }, + { + "bits_per_byte": 3.1147361699748184, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.158970594406128, + "step": 490, + "unclipped_grad_norm": 0.8680974841117859 + }, + { + "bits_per_byte": 3.059730958749366, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1208438873291016, + "step": 500, + "unclipped_grad_norm": 1.296195387840271 + }, + { + "bits_per_byte": 3.12068642572937, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.163094997406006, + "step": 510, + "unclipped_grad_norm": 1.0469720363616943 + }, + { + "bits_per_byte": 3.1397613655018657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.176316738128662, + "step": 520, + "unclipped_grad_norm": 1.2089059352874756 + }, + { + "bits_per_byte": 2.9844827015867583, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.06868577003479, + "step": 530, + "unclipped_grad_norm": 1.1353814601898193 + }, + { + "bits_per_byte": 3.0015299659055814, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.0805020332336426, + "step": 540, + "unclipped_grad_norm": 0.919367253780365 + }, + { + "bits_per_byte": 2.987599371185694, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.0708460807800293, + "step": 550, + "unclipped_grad_norm": 0.8667266368865967 + }, + { + "bits_per_byte": 2.995183462123442, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.0761029720306396, + "step": 560, + "unclipped_grad_norm": 0.9804732203483582 + }, + { + "bits_per_byte": 2.962294187903042, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.0533058643341064, + "step": 570, + "unclipped_grad_norm": 0.9414609670639038 + }, + { + "bits_per_byte": 2.956716790535561, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0494399070739746, + "step": 580, + "unclipped_grad_norm": 0.8727462291717529 + }, + { + "bits_per_byte": 2.9113505189603663, + "learning_rate": 0.00029958384005397226, + "loss_nats": 2.0179944038391113, + "step": 590, + "unclipped_grad_norm": 1.1642946004867554 + }, + { + "bits_per_byte": 2.8665478345320485, + "learning_rate": 0.0002995389065559004, + "loss_nats": 1.986939549446106, + "step": 600, + "unclipped_grad_norm": 0.9213871359825134 + }, + { + "bits_per_byte": 2.875505550902238, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.9931485652923584, + "step": 610, + "unclipped_grad_norm": 0.7122263312339783 + }, + { + "bits_per_byte": 2.878308696128651, + "learning_rate": 0.00029944214367070535, + "loss_nats": 1.9950915575027466, + "step": 620, + "unclipped_grad_norm": 1.4622896909713745 + }, + { + "bits_per_byte": 2.790202156074285, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.934020757675171, + "step": 630, + "unclipped_grad_norm": 1.1472660303115845 + }, + { + "bits_per_byte": 2.7890029210498186, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9331895112991333, + "step": 640, + "unclipped_grad_norm": 1.1472764015197754 + }, + { + "bits_per_byte": 2.7734051265124666, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.9223779439926147, + "step": 650, + "unclipped_grad_norm": 0.9182148575782776 + }, + { + "bits_per_byte": 2.795219749913207, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.937498688697815, + "step": 660, + "unclipped_grad_norm": 1.083983302116394 + }, + { + "bits_per_byte": 2.7702638633944576, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.9202005863189697, + "step": 670, + "unclipped_grad_norm": 1.1721765995025635 + }, + { + "bits_per_byte": 2.755497089026075, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.9099650382995605, + "step": 680, + "unclipped_grad_norm": 1.280714750289917 + }, + { + "bits_per_byte": 2.7128121989604823, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8803781270980835, + "step": 690, + "unclipped_grad_norm": 1.0380759239196777 + }, + { + "bits_per_byte": 2.7091302223881604, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8778259754180908, + "step": 700, + "unclipped_grad_norm": 1.1401211023330688 + }, + { + "bits_per_byte": 2.6870259802033667, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.862504482269287, + "step": 710, + "unclipped_grad_norm": 1.6450653076171875 + }, + { + "bits_per_byte": 2.688485424978589, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.863516092300415, + "step": 720, + "unclipped_grad_norm": 1.0721391439437866 + }, + { + "bits_per_byte": 2.65192277331841, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8381727933883667, + "step": 730, + "unclipped_grad_norm": 1.0645538568496704 + }, + { + "bits_per_byte": 2.6143492076364594, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.8121287822723389, + "step": 740, + "unclipped_grad_norm": 1.1877458095550537 + }, + { + "bits_per_byte": 2.6277774410331767, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.8214365243911743, + "step": 750, + "unclipped_grad_norm": 1.1091431379318237 + }, + { + "bits_per_byte": 2.540277655771055, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.7607862949371338, + "step": 760, + "unclipped_grad_norm": 1.222191333770752 + }, + { + "bits_per_byte": 2.5790566478529025, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.787665843963623, + "step": 770, + "unclipped_grad_norm": 1.254854679107666 + }, + { + "bits_per_byte": 2.5138998166949715, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7425025701522827, + "step": 780, + "unclipped_grad_norm": 0.9573470950126648 + }, + { + "bits_per_byte": 2.5258618979930834, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7507940530776978, + "step": 790, + "unclipped_grad_norm": 1.108724594116211 + }, + { + "bits_per_byte": 2.430956195819507, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.6850104331970215, + "step": 800, + "unclipped_grad_norm": 0.9832104444503784 + }, + { + "bits_per_byte": 2.468076415233783, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7107402086257935, + "step": 810, + "unclipped_grad_norm": 0.9492170214653015 + }, + { + "bits_per_byte": 2.4602348662676934, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.7053048610687256, + "step": 820, + "unclipped_grad_norm": 1.192505121231079 + }, + { + "bits_per_byte": 2.4583616312344954, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.7040064334869385, + "step": 830, + "unclipped_grad_norm": 0.9113606214523315 + }, + { + "bits_per_byte": 2.4751598646748696, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7156500816345215, + "step": 840, + "unclipped_grad_norm": 1.3671480417251587 + }, + { + "bits_per_byte": 2.386899400147981, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6544725894927979, + "step": 850, + "unclipped_grad_norm": 1.0324512720108032 + }, + { + "bits_per_byte": 2.472668523994471, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.7139232158660889, + "step": 860, + "unclipped_grad_norm": 1.503003478050232 + }, + { + "bits_per_byte": 2.4348980381772827, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6877427101135254, + "step": 870, + "unclipped_grad_norm": 1.1144746541976929 + }, + { + "bits_per_byte": 2.3709108610278693, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.64339017868042, + "step": 880, + "unclipped_grad_norm": 1.0134795904159546 + }, + { + "bits_per_byte": 2.3377405671559597, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.6203982830047607, + "step": 890, + "unclipped_grad_norm": 1.0625330209732056 + }, + { + "bits_per_byte": 2.389408283058768, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6562116146087646, + "step": 900, + "unclipped_grad_norm": 0.9719302654266357 + }, + { + "bits_per_byte": 2.3782030974071033, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6484447717666626, + "step": 910, + "unclipped_grad_norm": 1.0672563314437866 + }, + { + "bits_per_byte": 2.3530695528100125, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.6310235261917114, + "step": 920, + "unclipped_grad_norm": 1.1197935342788696 + }, + { + "bits_per_byte": 2.297453115243394, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.5924731492996216, + "step": 930, + "unclipped_grad_norm": 1.0027272701263428 + }, + { + "bits_per_byte": 2.344061961471073, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6247799396514893, + "step": 940, + "unclipped_grad_norm": 1.0651428699493408 + }, + { + "bits_per_byte": 2.367935217202641, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.6413276195526123, + "step": 950, + "unclipped_grad_norm": 0.9822106957435608 + }, + { + "bits_per_byte": 2.309206925551311, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6006202697753906, + "step": 960, + "unclipped_grad_norm": 0.8860561847686768 + }, + { + "bits_per_byte": 2.2419830349126597, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5540242195129395, + "step": 970, + "unclipped_grad_norm": 1.1816905736923218 + }, + { + "bits_per_byte": 2.3851068249780383, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.65323007106781, + "step": 980, + "unclipped_grad_norm": 0.9257189035415649 + }, + { + "bits_per_byte": 2.232579711494085, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.547506332397461, + "step": 990, + "unclipped_grad_norm": 0.9634203910827637 + }, + { + "bits_per_byte": 2.2247954327507324, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5421106815338135, + "step": 1000, + "unclipped_grad_norm": 0.809658944606781 + }, + { + "bits_per_byte": 2.2354428786656486, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.5494909286499023, + "step": 1010, + "unclipped_grad_norm": 1.3530596494674683 + }, + { + "bits_per_byte": 2.2067197121797495, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5295815467834473, + "step": 1020, + "unclipped_grad_norm": 1.0303069353103638 + }, + { + "bits_per_byte": 2.1954008459958647, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5217359066009521, + "step": 1030, + "unclipped_grad_norm": 0.9071640968322754 + }, + { + "bits_per_byte": 2.291673982226451, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5884673595428467, + "step": 1040, + "unclipped_grad_norm": 1.3239798545837402 + }, + { + "bits_per_byte": 2.196653911590051, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5226044654846191, + "step": 1050, + "unclipped_grad_norm": 0.9085557460784912 + }, + { + "bits_per_byte": 2.225028985190604, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.5422725677490234, + "step": 1060, + "unclipped_grad_norm": 0.9488101005554199 + }, + { + "bits_per_byte": 2.275690258620563, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5773882865905762, + "step": 1070, + "unclipped_grad_norm": 1.047879695892334 + }, + { + "bits_per_byte": 2.235951603346901, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5498435497283936, + "step": 1080, + "unclipped_grad_norm": 0.8919407725334167 + }, + { + "bits_per_byte": 2.1733449309359636, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5064479112625122, + "step": 1090, + "unclipped_grad_norm": 1.0036922693252563 + }, + { + "bits_per_byte": 2.169043472855234, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5034663677215576, + "step": 1100, + "unclipped_grad_norm": 0.9904995560646057 + }, + { + "bits_per_byte": 2.124962083664635, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4729114770889282, + "step": 1110, + "unclipped_grad_norm": 0.9176430702209473 + }, + { + "bits_per_byte": 2.176794214981668, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.5088387727737427, + "step": 1120, + "unclipped_grad_norm": 0.9229135513305664 + }, + { + "bits_per_byte": 2.210271497885368, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.53204345703125, + "step": 1130, + "unclipped_grad_norm": 0.871993899345398 + }, + { + "bits_per_byte": 2.123456375556331, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.4718677997589111, + "step": 1140, + "unclipped_grad_norm": 0.8673478960990906 + }, + { + "bits_per_byte": 2.1911007637763418, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.518755316734314, + "step": 1150, + "unclipped_grad_norm": 0.8959671258926392 + }, + { + "bits_per_byte": 2.0953000638876755, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4523513317108154, + "step": 1160, + "unclipped_grad_norm": 0.9113589525222778 + }, + { + "bits_per_byte": 2.1570418355474237, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.495147466659546, + "step": 1170, + "unclipped_grad_norm": 0.8980047702789307 + }, + { + "bits_per_byte": 2.1555593450969863, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4941198825836182, + "step": 1180, + "unclipped_grad_norm": 0.948503315448761 + }, + { + "bits_per_byte": 2.1334348089593913, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4787843227386475, + "step": 1190, + "unclipped_grad_norm": 1.1094794273376465 + }, + { + "bits_per_byte": 2.1317098229712372, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4775886535644531, + "step": 1200, + "unclipped_grad_norm": 0.8304017782211304 + }, + { + "bits_per_byte": 2.1385674591917407, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.482342004776001, + "step": 1210, + "unclipped_grad_norm": 0.8620252013206482 + }, + { + "bits_per_byte": 2.0980762078379027, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.4542756080627441, + "step": 1220, + "unclipped_grad_norm": 0.88591068983078 + }, + { + "bits_per_byte": 2.108927281211441, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4617969989776611, + "step": 1230, + "unclipped_grad_norm": 1.0479243993759155 + }, + { + "bits_per_byte": 2.081558994049039, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.442826747894287, + "step": 1240, + "unclipped_grad_norm": 0.9365729689598083 + }, + { + "bits_per_byte": 2.0172010223043655, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.3982172012329102, + "step": 1250, + "unclipped_grad_norm": 0.8966301083564758 + }, + { + "bits_per_byte": 2.0690183631134245, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.4341342449188232, + "step": 1260, + "unclipped_grad_norm": 0.8974369168281555 + }, + { + "bits_per_byte": 2.101840220134686, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4568846225738525, + "step": 1270, + "unclipped_grad_norm": 0.9220007061958313 + }, + { + "bits_per_byte": 2.1256708241688407, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.473402738571167, + "step": 1280, + "unclipped_grad_norm": 0.9313441514968872 + }, + { + "bits_per_byte": 2.076867651298803, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.439574956893921, + "step": 1290, + "unclipped_grad_norm": 0.8451681137084961 + }, + { + "bits_per_byte": 2.032476865319302, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4088056087493896, + "step": 1300, + "unclipped_grad_norm": 0.9314314126968384 + }, + { + "bits_per_byte": 2.058903719450881, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4271233081817627, + "step": 1310, + "unclipped_grad_norm": 0.8796194195747375 + }, + { + "bits_per_byte": 2.0275063947267156, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4053603410720825, + "step": 1320, + "unclipped_grad_norm": 0.9140937924385071 + }, + { + "bits_per_byte": 2.037489299678698, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4122799634933472, + "step": 1330, + "unclipped_grad_norm": 0.821657657623291 + }, + { + "bits_per_byte": 2.0199990080512524, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4001566171646118, + "step": 1340, + "unclipped_grad_norm": 0.8321346044540405 + }, + { + "bits_per_byte": 2.048903616233812, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.420191764831543, + "step": 1350, + "unclipped_grad_norm": 0.9421875476837158 + }, + { + "bits_per_byte": 2.00944598061166, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.3928418159484863, + "step": 1360, + "unclipped_grad_norm": 0.8682821393013 + }, + { + "bits_per_byte": 2.0435301902902543, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4164671897888184, + "step": 1370, + "unclipped_grad_norm": 0.8776147961616516 + }, + { + "bits_per_byte": 2.0135448431296727, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.39568293094635, + "step": 1380, + "unclipped_grad_norm": 0.87440025806427 + }, + { + "bits_per_byte": 2.0533221944997795, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4232544898986816, + "step": 1390, + "unclipped_grad_norm": 0.7801422476768494 + }, + { + "bits_per_byte": 2.1564656936670334, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.4947481155395508, + "step": 1400, + "unclipped_grad_norm": 0.8561526536941528 + }, + { + "bits_per_byte": 2.0373921294809607, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.412212610244751, + "step": 1410, + "unclipped_grad_norm": 0.8376067876815796 + }, + { + "bits_per_byte": 2.0325960492963495, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.4088882207870483, + "step": 1420, + "unclipped_grad_norm": 0.751315712928772 + }, + { + "bits_per_byte": 1.9860154082234123, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.376600980758667, + "step": 1430, + "unclipped_grad_norm": 0.9057046175003052 + }, + { + "bits_per_byte": 2.015529006972677, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.3970582485198975, + "step": 1440, + "unclipped_grad_norm": 0.8342951536178589 + }, + { + "bits_per_byte": 1.9818307263626185, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3737003803253174, + "step": 1450, + "unclipped_grad_norm": 0.8142686486244202 + }, + { + "bits_per_byte": 2.1060192265680016, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.459781289100647, + "step": 1460, + "unclipped_grad_norm": 0.8357279896736145 + }, + { + "bits_per_byte": 2.079546797033944, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.441431999206543, + "step": 1470, + "unclipped_grad_norm": 0.7720605731010437 + }, + { + "bits_per_byte": 2.025708316111945, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.404114007949829, + "step": 1480, + "unclipped_grad_norm": 0.8328851461410522 + }, + { + "bits_per_byte": 2.0422327531721454, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.4155678749084473, + "step": 1490, + "unclipped_grad_norm": 0.9698525071144104 + }, + { + "bits_per_byte": 1.9217496191796013, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.3320553302764893, + "step": 1500, + "unclipped_grad_norm": 0.7396577000617981 + }, + { + "bits_per_byte": 1.9436662282923034, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.347246766090393, + "step": 1510, + "unclipped_grad_norm": 0.809975266456604 + }, + { + "bits_per_byte": 1.9993110429963143, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3858168125152588, + "step": 1520, + "unclipped_grad_norm": 0.8165730834007263 + }, + { + "bits_per_byte": 2.03089187320895, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4077069759368896, + "step": 1530, + "unclipped_grad_norm": 0.7620540261268616 + }, + { + "bits_per_byte": 2.002577509484152, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3880809545516968, + "step": 1540, + "unclipped_grad_norm": 0.8928612470626831 + }, + { + "bits_per_byte": 2.0629018001354886, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4298945665359497, + "step": 1550, + "unclipped_grad_norm": 0.7963082194328308 + }, + { + "bits_per_byte": 1.9429003895480113, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3467159271240234, + "step": 1560, + "unclipped_grad_norm": 0.7710158228874207 + }, + { + "bits_per_byte": 1.9502758655302628, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.3518282175064087, + "step": 1570, + "unclipped_grad_norm": 0.8057935833930969 + }, + { + "bits_per_byte": 1.9693647358974786, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.3650596141815186, + "step": 1580, + "unclipped_grad_norm": 0.8588574528694153 + }, + { + "bits_per_byte": 1.972771368245769, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3674209117889404, + "step": 1590, + "unclipped_grad_norm": 0.8608299493789673 + }, + { + "bits_per_byte": 1.9610373359427014, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3592875003814697, + "step": 1600, + "unclipped_grad_norm": 0.8587453365325928 + }, + { + "bits_per_byte": 1.9257736692444893, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3348445892333984, + "step": 1610, + "unclipped_grad_norm": 0.8028180003166199 + }, + { + "bits_per_byte": 1.929578785394829, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3374820947647095, + "step": 1620, + "unclipped_grad_norm": 0.7991352677345276 + }, + { + "bits_per_byte": 1.9716548568763677, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3666470050811768, + "step": 1630, + "unclipped_grad_norm": 0.7573540806770325 + }, + { + "bits_per_byte": 1.9236465878186182, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3333702087402344, + "step": 1640, + "unclipped_grad_norm": 0.8524133563041687 + }, + { + "bits_per_byte": 1.9335809936630575, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.3402562141418457, + "step": 1650, + "unclipped_grad_norm": 0.7908568978309631 + }, + { + "bits_per_byte": 1.9524230689262851, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3533165454864502, + "step": 1660, + "unclipped_grad_norm": 0.8385109901428223 + }, + { + "bits_per_byte": 1.8864807930018634, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3076088428497314, + "step": 1670, + "unclipped_grad_norm": 0.8104477524757385 + }, + { + "bits_per_byte": 1.918672677573015, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3299225568771362, + "step": 1680, + "unclipped_grad_norm": 0.7970330119132996 + }, + { + "bits_per_byte": 1.9517956762159374, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.352881669998169, + "step": 1690, + "unclipped_grad_norm": 0.7619603276252747 + }, + { + "bits_per_byte": 1.8990867773448057, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3163466453552246, + "step": 1700, + "unclipped_grad_norm": 0.8769286274909973 + }, + { + "bits_per_byte": 1.938959579086141, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3439843654632568, + "step": 1710, + "unclipped_grad_norm": 0.7246651649475098 + }, + { + "bits_per_byte": 1.847509524316351, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.2805960178375244, + "step": 1720, + "unclipped_grad_norm": 0.7415109276771545 + }, + { + "bits_per_byte": 1.944612132872049, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3479024171829224, + "step": 1730, + "unclipped_grad_norm": 0.8057509660720825 + }, + { + "bits_per_byte": 1.956270664791388, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3559834957122803, + "step": 1740, + "unclipped_grad_norm": 0.8188740611076355 + }, + { + "bits_per_byte": 1.908994697861012, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3232142925262451, + "step": 1750, + "unclipped_grad_norm": 0.7595267295837402 + }, + { + "bits_per_byte": 1.9438418225788343, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3473684787750244, + "step": 1760, + "unclipped_grad_norm": 0.7832275629043579 + }, + { + "bits_per_byte": 1.9277638524802738, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.33622407913208, + "step": 1770, + "unclipped_grad_norm": 0.7276267409324646 + }, + { + "bits_per_byte": 1.987644083927083, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.377729892730713, + "step": 1780, + "unclipped_grad_norm": 0.7699368000030518 + }, + { + "bits_per_byte": 1.8704368754681737, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.2964880466461182, + "step": 1790, + "unclipped_grad_norm": 0.7781511545181274 + }, + { + "bits_per_byte": 2.006532250540742, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.390822172164917, + "step": 1800, + "unclipped_grad_norm": 0.8113785982131958 + }, + { + "bits_per_byte": 1.8832052114335305, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.3053383827209473, + "step": 1810, + "unclipped_grad_norm": 0.8088275790214539 + }, + { + "bits_per_byte": 1.9263209180395349, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.335223913192749, + "step": 1820, + "unclipped_grad_norm": 0.7881855964660645 + }, + { + "bits_per_byte": 1.968589438107389, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3645222187042236, + "step": 1830, + "unclipped_grad_norm": 0.7948311567306519 + }, + { + "bits_per_byte": 1.9398172565659937, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3445788621902466, + "step": 1840, + "unclipped_grad_norm": 0.8343047499656677 + }, + { + "bits_per_byte": 1.8747907882574166, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.2995059490203857, + "step": 1850, + "unclipped_grad_norm": 0.7942258715629578 + }, + { + "bits_per_byte": 1.9135037390013332, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3263397216796875, + "step": 1860, + "unclipped_grad_norm": 0.769598126411438 + }, + { + "bits_per_byte": 1.987024774401326, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3773006200790405, + "step": 1870, + "unclipped_grad_norm": 0.7647593021392822 + }, + { + "bits_per_byte": 1.8769319722606586, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.300990104675293, + "step": 1880, + "unclipped_grad_norm": 0.8446611166000366 + }, + { + "bits_per_byte": 1.8953207012562119, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.3137362003326416, + "step": 1890, + "unclipped_grad_norm": 0.7048609852790833 + }, + { + "bits_per_byte": 1.915848206497895, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3279647827148438, + "step": 1900, + "unclipped_grad_norm": 0.7416327595710754 + }, + { + "bits_per_byte": 1.9362319342434569, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.3420937061309814, + "step": 1910, + "unclipped_grad_norm": 0.8084686398506165 + }, + { + "bits_per_byte": 1.8860243510464736, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.3072924613952637, + "step": 1920, + "unclipped_grad_norm": 0.8338462710380554 + }, + { + "bits_per_byte": 1.930916638435891, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.338409423828125, + "step": 1930, + "unclipped_grad_norm": 0.6878981590270996 + }, + { + "bits_per_byte": 1.9257588787365152, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.334834337234497, + "step": 1940, + "unclipped_grad_norm": 0.6981199383735657 + }, + { + "bits_per_byte": 1.8699291826828268, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.2961361408233643, + "step": 1950, + "unclipped_grad_norm": 0.76193767786026 + }, + { + "bits_per_byte": 1.859464038377825, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.2888822555541992, + "step": 1960, + "unclipped_grad_norm": 0.7367894053459167 + }, + { + "bits_per_byte": 1.8691335909399354, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.2955846786499023, + "step": 1970, + "unclipped_grad_norm": 0.8052698373794556 + }, + { + "bits_per_byte": 1.8850715671606937, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3066320419311523, + "step": 1980, + "unclipped_grad_norm": 0.7624203562736511 + }, + { + "bits_per_byte": 1.8715028239382214, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.297226905822754, + "step": 1990, + "unclipped_grad_norm": 0.779162585735321 + }, + { + "bits_per_byte": 1.9325518494802945, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3395428657531738, + "step": 2000, + "unclipped_grad_norm": 0.7498615384101868 + }, + { + "bits_per_byte": 1.9335421255839624, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.3402292728424072, + "step": 2010, + "unclipped_grad_norm": 0.7343811392784119 + }, + { + "bits_per_byte": 1.838237939608337, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.2741694450378418, + "step": 2020, + "unclipped_grad_norm": 0.7684847116470337 + }, + { + "bits_per_byte": 1.8631659649376469, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.2914482355117798, + "step": 2030, + "unclipped_grad_norm": 0.6878870129585266 + }, + { + "bits_per_byte": 1.8841837927169398, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.3060166835784912, + "step": 2040, + "unclipped_grad_norm": 0.853155255317688 + }, + { + "bits_per_byte": 1.8339112000779305, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.2711703777313232, + "step": 2050, + "unclipped_grad_norm": 0.8042898774147034 + }, + { + "bits_per_byte": 1.8136427007084468, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.2571213245391846, + "step": 2060, + "unclipped_grad_norm": 0.7331950664520264 + }, + { + "bits_per_byte": 1.9306452498128293, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.3382213115692139, + "step": 2070, + "unclipped_grad_norm": 0.7544640898704529 + }, + { + "bits_per_byte": 1.9159075405124426, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.3280059099197388, + "step": 2080, + "unclipped_grad_norm": 0.7521310448646545 + }, + { + "bits_per_byte": 1.8394872215842044, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2750353813171387, + "step": 2090, + "unclipped_grad_norm": 0.7430872321128845 + }, + { + "bits_per_byte": 1.7890550290451945, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.2400784492492676, + "step": 2100, + "unclipped_grad_norm": 0.7162277698516846 + }, + { + "bits_per_byte": 1.8155914361253733, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.2584720849990845, + "step": 2110, + "unclipped_grad_norm": 0.7056337594985962 + }, + { + "bits_per_byte": 1.834290249840432, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2714331150054932, + "step": 2120, + "unclipped_grad_norm": 0.8666519522666931 + }, + { + "bits_per_byte": 1.8865812308699674, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.307678461074829, + "step": 2130, + "unclipped_grad_norm": 0.762921154499054 + }, + { + "bits_per_byte": 1.9030090136803828, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3190653324127197, + "step": 2140, + "unclipped_grad_norm": 0.7508745789527893 + }, + { + "bits_per_byte": 1.8090877402003454, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2539640665054321, + "step": 2150, + "unclipped_grad_norm": 0.7472740411758423 + }, + { + "bits_per_byte": 1.898260228724759, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.3157737255096436, + "step": 2160, + "unclipped_grad_norm": 0.7288866639137268 + }, + { + "bits_per_byte": 1.8098697453138184, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.2545061111450195, + "step": 2170, + "unclipped_grad_norm": 0.7270416617393494 + }, + { + "bits_per_byte": 1.896411071262682, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.3144919872283936, + "step": 2180, + "unclipped_grad_norm": 0.7397056221961975 + }, + { + "bits_per_byte": 1.8517294626205718, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.283521056175232, + "step": 2190, + "unclipped_grad_norm": 0.7186560034751892 + }, + { + "bits_per_byte": 1.8299633383273748, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.268433928489685, + "step": 2200, + "unclipped_grad_norm": 0.6916610598564148 + }, + { + "bits_per_byte": 1.8274437924922347, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2666875123977661, + "step": 2210, + "unclipped_grad_norm": 0.7494430541992188 + }, + { + "bits_per_byte": 1.8262679471082857, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2658724784851074, + "step": 2220, + "unclipped_grad_norm": 0.782909095287323 + }, + { + "bits_per_byte": 1.782249847533204, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.2353614568710327, + "step": 2230, + "unclipped_grad_norm": 0.7370295524597168 + }, + { + "bits_per_byte": 1.8783687153259665, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.3019859790802002, + "step": 2240, + "unclipped_grad_norm": 0.7077252268791199 + }, + { + "bits_per_byte": 1.844123529886164, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2782490253448486, + "step": 2250, + "unclipped_grad_norm": 0.7420696020126343 + }, + { + "bits_per_byte": 1.897104505310961, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.3149726390838623, + "step": 2260, + "unclipped_grad_norm": 0.7444695234298706 + }, + { + "bits_per_byte": 1.7929542197055561, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.242781162261963, + "step": 2270, + "unclipped_grad_norm": 0.7164454460144043 + }, + { + "bits_per_byte": 1.8643507534194408, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2922694683074951, + "step": 2280, + "unclipped_grad_norm": 0.7152382135391235 + }, + { + "bits_per_byte": 1.8035947863144381, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2501566410064697, + "step": 2290, + "unclipped_grad_norm": 0.7178454399108887 + }, + { + "bits_per_byte": 1.7935200426268947, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.243173360824585, + "step": 2300, + "unclipped_grad_norm": 0.7068437337875366 + }, + { + "bits_per_byte": 1.8020768674379228, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2491044998168945, + "step": 2310, + "unclipped_grad_norm": 0.7017168998718262 + }, + { + "bits_per_byte": 1.8564474626816911, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.2867913246154785, + "step": 2320, + "unclipped_grad_norm": 0.6947462558746338 + }, + { + "bits_per_byte": 1.7936961528613782, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.243295431137085, + "step": 2330, + "unclipped_grad_norm": 0.6949398517608643 + }, + { + "bits_per_byte": 1.8129625093242843, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.2566498517990112, + "step": 2340, + "unclipped_grad_norm": 0.7047421336174011 + }, + { + "bits_per_byte": 1.871945851246844, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2975339889526367, + "step": 2350, + "unclipped_grad_norm": 0.7291849851608276 + }, + { + "bits_per_byte": 1.7987307729827362, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2467851638793945, + "step": 2360, + "unclipped_grad_norm": 0.7353813052177429 + }, + { + "bits_per_byte": 1.7311453748119714, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.1999385356903076, + "step": 2370, + "unclipped_grad_norm": 0.6746760010719299 + }, + { + "bits_per_byte": 1.834468595849377, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2715567350387573, + "step": 2380, + "unclipped_grad_norm": 0.7166123390197754 + }, + { + "bits_per_byte": 1.8260691351638882, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.2657346725463867, + "step": 2390, + "unclipped_grad_norm": 0.6613916158676147 + }, + { + "bits_per_byte": 1.8641103216735346, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2921028137207031, + "step": 2400, + "unclipped_grad_norm": 0.824955403804779 + }, + { + "bits_per_byte": 1.793840446305452, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.243395447731018, + "step": 2410, + "unclipped_grad_norm": 0.7929511070251465 + }, + { + "bits_per_byte": 1.8233349549804712, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2638394832611084, + "step": 2420, + "unclipped_grad_norm": 0.725745677947998 + }, + { + "bits_per_byte": 1.825773496987055, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.265529751777649, + "step": 2430, + "unclipped_grad_norm": 0.7284960746765137 + }, + { + "bits_per_byte": 1.7725216488871491, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.2286183834075928, + "step": 2440, + "unclipped_grad_norm": 0.7458904981613159 + }, + { + "bits_per_byte": 1.9001444706476118, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.317079782485962, + "step": 2450, + "unclipped_grad_norm": 0.8321223258972168 + }, + { + "bits_per_byte": 1.7654523020234334, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.223718285560608, + "step": 2460, + "unclipped_grad_norm": 0.7338651418685913 + }, + { + "bits_per_byte": 1.7862824967306352, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2381566762924194, + "step": 2470, + "unclipped_grad_norm": 0.7279908061027527 + }, + { + "bits_per_byte": 1.8666427662074894, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2938581705093384, + "step": 2480, + "unclipped_grad_norm": 0.7701565027236938 + }, + { + "bits_per_byte": 1.850673661126925, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.2827892303466797, + "step": 2490, + "unclipped_grad_norm": 0.6776601672172546 + }, + { + "bits_per_byte": 1.8000481600883451, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2476983070373535, + "step": 2500, + "unclipped_grad_norm": 0.7444605231285095 + }, + { + "bits_per_byte": 1.7603095048146828, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.220153570175171, + "step": 2510, + "unclipped_grad_norm": 0.7512273788452148 + }, + { + "bits_per_byte": 1.7805241736144466, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.2341653108596802, + "step": 2520, + "unclipped_grad_norm": 0.6968159079551697 + }, + { + "bits_per_byte": 1.803689032807111, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2502219676971436, + "step": 2530, + "unclipped_grad_norm": 0.7150991559028625 + }, + { + "bits_per_byte": 1.8049077018711248, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.2510666847229004, + "step": 2540, + "unclipped_grad_norm": 0.7343016862869263 + }, + { + "bits_per_byte": 1.742795479581418, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2080137729644775, + "step": 2550, + "unclipped_grad_norm": 0.7049802541732788 + }, + { + "bits_per_byte": 1.7179068382791558, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.1907622814178467, + "step": 2560, + "unclipped_grad_norm": 0.7055768370628357 + }, + { + "bits_per_byte": 1.7731971967397384, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2290866374969482, + "step": 2570, + "unclipped_grad_norm": 0.6996883749961853 + }, + { + "bits_per_byte": 1.7458676056737803, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2101432085037231, + "step": 2580, + "unclipped_grad_norm": 0.6933735609054565 + }, + { + "bits_per_byte": 1.7458146350173145, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2101064920425415, + "step": 2590, + "unclipped_grad_norm": 0.7094681859016418 + }, + { + "bits_per_byte": 1.7637799427264436, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2225590944290161, + "step": 2600, + "unclipped_grad_norm": 0.7199745178222656 + }, + { + "bits_per_byte": 1.786311561798631, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2381768226623535, + "step": 2610, + "unclipped_grad_norm": 0.7421150207519531 + }, + { + "bits_per_byte": 1.7252271078304804, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.1958363056182861, + "step": 2620, + "unclipped_grad_norm": 0.7062536478042603 + }, + { + "bits_per_byte": 1.7243496523457782, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.195228099822998, + "step": 2630, + "unclipped_grad_norm": 0.6937817931175232 + }, + { + "bits_per_byte": 1.7124126805146924, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.1869540214538574, + "step": 2640, + "unclipped_grad_norm": 0.7252171039581299 + }, + { + "bits_per_byte": 1.781446344588373, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2348045110702515, + "step": 2650, + "unclipped_grad_norm": 0.6726356744766235 + }, + { + "bits_per_byte": 1.6915262474806587, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.1724766492843628, + "step": 2660, + "unclipped_grad_norm": 0.6618354320526123 + }, + { + "bits_per_byte": 1.7563157236963467, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2173852920532227, + "step": 2670, + "unclipped_grad_norm": 0.7113842368125916 + }, + { + "bits_per_byte": 1.858153874543552, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.2879741191864014, + "step": 2680, + "unclipped_grad_norm": 0.7770810127258301 + }, + { + "bits_per_byte": 1.784997614346039, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.2372660636901855, + "step": 2690, + "unclipped_grad_norm": 0.7249491810798645 + }, + { + "bits_per_byte": 1.7001362149308035, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.1784446239471436, + "step": 2700, + "unclipped_grad_norm": 0.6978740096092224 + }, + { + "bits_per_byte": 1.7671420315681605, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2248895168304443, + "step": 2710, + "unclipped_grad_norm": 0.6759880185127258 + }, + { + "bits_per_byte": 1.8404900524213854, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.2757304906845093, + "step": 2720, + "unclipped_grad_norm": 0.7600791454315186 + }, + { + "bits_per_byte": 1.708491820040322, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.1842362880706787, + "step": 2730, + "unclipped_grad_norm": 0.6715761423110962 + }, + { + "bits_per_byte": 1.7849604660934528, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2372403144836426, + "step": 2740, + "unclipped_grad_norm": 0.7194302082061768 + }, + { + "bits_per_byte": 1.7485555225241152, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.2120063304901123, + "step": 2750, + "unclipped_grad_norm": 0.6991336941719055 + }, + { + "bits_per_byte": 1.8629487508496072, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2912976741790771, + "step": 2760, + "unclipped_grad_norm": 0.8377771973609924 + }, + { + "bits_per_byte": 1.7856594035865592, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.237724781036377, + "step": 2770, + "unclipped_grad_norm": 0.7150954604148865 + }, + { + "bits_per_byte": 1.7901013714930438, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.2408037185668945, + "step": 2780, + "unclipped_grad_norm": 0.6968044638633728 + }, + { + "bits_per_byte": 1.752669863480706, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2148581743240356, + "step": 2790, + "unclipped_grad_norm": 0.7223197817802429 + }, + { + "bits_per_byte": 1.7567675221201633, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2176984548568726, + "step": 2800, + "unclipped_grad_norm": 0.7307369709014893 + }, + { + "bits_per_byte": 1.7124591158304254, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.1869862079620361, + "step": 2810, + "unclipped_grad_norm": 0.7025272250175476 + }, + { + "bits_per_byte": 1.7803284573577647, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2340296506881714, + "step": 2820, + "unclipped_grad_norm": 0.7275472283363342 + }, + { + "bits_per_byte": 1.7225665362216327, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.1939921379089355, + "step": 2830, + "unclipped_grad_norm": 0.6614095568656921 + }, + { + "bits_per_byte": 1.7602923065495966, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2201416492462158, + "step": 2840, + "unclipped_grad_norm": 0.7144313454627991 + }, + { + "bits_per_byte": 1.7021394688480533, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.179833173751831, + "step": 2850, + "unclipped_grad_norm": 0.7220690846443176 + }, + { + "bits_per_byte": 1.744293620453084, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2090522050857544, + "step": 2860, + "unclipped_grad_norm": 0.7174125909805298 + }, + { + "bits_per_byte": 1.7433115995166573, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.2083715200424194, + "step": 2870, + "unclipped_grad_norm": 0.730305552482605 + }, + { + "bits_per_byte": 1.7421880368585705, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.2075927257537842, + "step": 2880, + "unclipped_grad_norm": 0.7075303792953491 + }, + { + "bits_per_byte": 1.7085926018737276, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.1843061447143555, + "step": 2890, + "unclipped_grad_norm": 0.6744402050971985 + }, + { + "bits_per_byte": 1.7378320602775172, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.204573392868042, + "step": 2900, + "unclipped_grad_norm": 0.6889722347259521 + }, + { + "bits_per_byte": 1.7031551983840492, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.180537223815918, + "step": 2910, + "unclipped_grad_norm": 0.7552688717842102 + }, + { + "bits_per_byte": 1.7508614659068837, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.2136046886444092, + "step": 2920, + "unclipped_grad_norm": 0.6863148808479309 + }, + { + "bits_per_byte": 1.701510700276499, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.1793973445892334, + "step": 2930, + "unclipped_grad_norm": 0.6827563047409058 + }, + { + "bits_per_byte": 1.7106488264474435, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.1857314109802246, + "step": 2940, + "unclipped_grad_norm": 0.6807736158370972 + }, + { + "bits_per_byte": 1.7703209588867088, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.227092981338501, + "step": 2950, + "unclipped_grad_norm": 0.6866815686225891 + }, + { + "bits_per_byte": 1.7651767858167513, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2235273122787476, + "step": 2960, + "unclipped_grad_norm": 0.741482138633728 + }, + { + "bits_per_byte": 1.7465264712092357, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.2105998992919922, + "step": 2970, + "unclipped_grad_norm": 0.6754798293113708 + }, + { + "bits_per_byte": 1.7663952828981142, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2243719100952148, + "step": 2980, + "unclipped_grad_norm": 0.688591480255127 + }, + { + "bits_per_byte": 1.7069632382394535, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.1831767559051514, + "step": 2990, + "unclipped_grad_norm": 0.7298949956893921 + }, + { + "bits_per_byte": 1.79113911480835, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.241523027420044, + "step": 3000, + "unclipped_grad_norm": 0.713433027267456 + }, + { + "bits_per_byte": 1.6766162115641075, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.1621417999267578, + "step": 3010, + "unclipped_grad_norm": 0.7228698134422302 + }, + { + "bits_per_byte": 1.7365501015979858, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.2036848068237305, + "step": 3020, + "unclipped_grad_norm": 0.6969108581542969 + }, + { + "bits_per_byte": 1.6833617150962485, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.1668174266815186, + "step": 3030, + "unclipped_grad_norm": 0.669261634349823 + }, + { + "bits_per_byte": 1.6671776317021056, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.1555994749069214, + "step": 3040, + "unclipped_grad_norm": 0.6839757561683655 + }, + { + "bits_per_byte": 1.8241205717296125, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2643840312957764, + "step": 3050, + "unclipped_grad_norm": 0.7963703870773315 + }, + { + "bits_per_byte": 1.7604708245411922, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2202653884887695, + "step": 3060, + "unclipped_grad_norm": 0.7280165553092957 + }, + { + "bits_per_byte": 1.6594571585222233, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.1502480506896973, + "step": 3070, + "unclipped_grad_norm": 0.7014188766479492 + }, + { + "bits_per_byte": 1.7706030104341237, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2272884845733643, + "step": 3080, + "unclipped_grad_norm": 0.7385008335113525 + }, + { + "bits_per_byte": 1.7493225651469633, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.2125380039215088, + "step": 3090, + "unclipped_grad_norm": 0.6976860165596008 + }, + { + "bits_per_byte": 1.706454857523503, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.1828243732452393, + "step": 3100, + "unclipped_grad_norm": 0.7350029349327087 + }, + { + "bits_per_byte": 1.8454660264587992, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.279179573059082, + "step": 3110, + "unclipped_grad_norm": 0.8262335658073425 + }, + { + "bits_per_byte": 1.7237298268720687, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.194798469543457, + "step": 3120, + "unclipped_grad_norm": 0.7770306468009949 + }, + { + "bits_per_byte": 1.6658043502349662, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.1546475887298584, + "step": 3130, + "unclipped_grad_norm": 0.6939164996147156 + }, + { + "bits_per_byte": 1.7147224075157799, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.1885550022125244, + "step": 3140, + "unclipped_grad_norm": 0.744745135307312 + }, + { + "bits_per_byte": 1.7012960659282221, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.179248571395874, + "step": 3150, + "unclipped_grad_norm": 0.6927894353866577 + }, + { + "bits_per_byte": 1.700822425707746, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.1789202690124512, + "step": 3160, + "unclipped_grad_norm": 0.6962236166000366 + }, + { + "bits_per_byte": 1.7730090477196945, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.2289562225341797, + "step": 3170, + "unclipped_grad_norm": 0.7581587433815002 + }, + { + "bits_per_byte": 1.7445914944043783, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2092586755752563, + "step": 3180, + "unclipped_grad_norm": 0.7753618359565735 + }, + { + "bits_per_byte": 1.7320080397886992, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.2005364894866943, + "step": 3190, + "unclipped_grad_norm": 0.7117310166358948 + }, + { + "bits_per_byte": 1.7215322325593436, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.1932752132415771, + "step": 3200, + "unclipped_grad_norm": 0.712298572063446 + }, + { + "bits_per_byte": 1.6960516269728116, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1756134033203125, + "step": 3210, + "unclipped_grad_norm": 0.7590813636779785 + }, + { + "bits_per_byte": 1.6593746068498092, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.150190830230713, + "step": 3220, + "unclipped_grad_norm": 0.6987801194190979 + }, + { + "bits_per_byte": 1.7324070395387008, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.2008130550384521, + "step": 3230, + "unclipped_grad_norm": 0.6938228607177734 + }, + { + "bits_per_byte": 1.7474627447605329, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.2112488746643066, + "step": 3240, + "unclipped_grad_norm": 0.78226238489151 + }, + { + "bits_per_byte": 1.7110863503112386, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.1860346794128418, + "step": 3250, + "unclipped_grad_norm": 0.6999595165252686 + }, + { + "bits_per_byte": 1.7142417160066183, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.18822181224823, + "step": 3260, + "unclipped_grad_norm": 0.700433075428009 + }, + { + "bits_per_byte": 1.748484321706658, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.2119569778442383, + "step": 3270, + "unclipped_grad_norm": 0.7274234294891357 + }, + { + "bits_per_byte": 1.738793787261142, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.20524001121521, + "step": 3280, + "unclipped_grad_norm": 0.7183425426483154 + }, + { + "bits_per_byte": 1.6962500949519075, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.175750970840454, + "step": 3290, + "unclipped_grad_norm": 0.7457373738288879 + }, + { + "bits_per_byte": 1.7210296992535226, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1929268836975098, + "step": 3300, + "unclipped_grad_norm": 0.7834483981132507 + }, + { + "bits_per_byte": 1.6835460804979734, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.166945219039917, + "step": 3310, + "unclipped_grad_norm": 0.6523665189743042 + }, + { + "bits_per_byte": 1.6871995079502522, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1694775819778442, + "step": 3320, + "unclipped_grad_norm": 0.7348048686981201 + }, + { + "bits_per_byte": 1.7008401399207849, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.178932547569275, + "step": 3330, + "unclipped_grad_norm": 0.7228167653083801 + }, + { + "bits_per_byte": 1.643402750046831, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1391199827194214, + "step": 3340, + "unclipped_grad_norm": 0.7094708681106567 + }, + { + "bits_per_byte": 1.686364876145615, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1688990592956543, + "step": 3350, + "unclipped_grad_norm": 0.7186631560325623 + }, + { + "bits_per_byte": 1.7065993229502279, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.182924509048462, + "step": 3360, + "unclipped_grad_norm": 0.6950210332870483 + }, + { + "bits_per_byte": 1.6548356407282379, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1470446586608887, + "step": 3370, + "unclipped_grad_norm": 0.6926314830780029 + }, + { + "bits_per_byte": 1.6926787032240904, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1732754707336426, + "step": 3380, + "unclipped_grad_norm": 0.7107762694358826 + }, + { + "bits_per_byte": 1.7003921251152874, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.1786220073699951, + "step": 3390, + "unclipped_grad_norm": 0.700124979019165 + }, + { + "bits_per_byte": 1.6889117672222422, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.1706644296646118, + "step": 3400, + "unclipped_grad_norm": 0.7049819827079773 + }, + { + "bits_per_byte": 1.6802605239358903, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.1646678447723389, + "step": 3410, + "unclipped_grad_norm": 0.6922183036804199 + }, + { + "bits_per_byte": 1.664921907253389, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1540359258651733, + "step": 3420, + "unclipped_grad_norm": 0.7144856452941895 + }, + { + "bits_per_byte": 1.698013089105902, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1769729852676392, + "step": 3430, + "unclipped_grad_norm": 0.6953245401382446 + }, + { + "bits_per_byte": 1.7072490734051875, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1833748817443848, + "step": 3440, + "unclipped_grad_norm": 0.727461040019989 + }, + { + "bits_per_byte": 1.7747906159999822, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2301911115646362, + "step": 3450, + "unclipped_grad_norm": 0.7409155368804932 + }, + { + "bits_per_byte": 1.7588486841782542, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.2191410064697266, + "step": 3460, + "unclipped_grad_norm": 0.748551070690155 + }, + { + "bits_per_byte": 1.7339454243506687, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.2018793821334839, + "step": 3470, + "unclipped_grad_norm": 0.7151429653167725 + }, + { + "bits_per_byte": 1.7067943512763062, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1830596923828125, + "step": 3480, + "unclipped_grad_norm": 0.7189637422561646 + }, + { + "bits_per_byte": 1.6200879219826143, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.1229593753814697, + "step": 3490, + "unclipped_grad_norm": 0.6848708391189575 + }, + { + "bits_per_byte": 1.6373195517031627, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.1349034309387207, + "step": 3500, + "unclipped_grad_norm": 0.6704034209251404 + }, + { + "bits_per_byte": 1.6849599498707166, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.167925238609314, + "step": 3510, + "unclipped_grad_norm": 0.7942876219749451 + }, + { + "bits_per_byte": 1.6514716600773613, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.1447129249572754, + "step": 3520, + "unclipped_grad_norm": 0.7149106860160828 + }, + { + "bits_per_byte": 1.6705435041621417, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.1579325199127197, + "step": 3530, + "unclipped_grad_norm": 0.7720932364463806 + }, + { + "bits_per_byte": 1.6828739722984016, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1664793491363525, + "step": 3540, + "unclipped_grad_norm": 0.6750161647796631 + }, + { + "bits_per_byte": 1.7101762181228726, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.185403823852539, + "step": 3550, + "unclipped_grad_norm": 0.6993052363395691 + }, + { + "bits_per_byte": 1.6591575647444203, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.1500403881072998, + "step": 3560, + "unclipped_grad_norm": 0.6886851787567139 + }, + { + "bits_per_byte": 1.7182958630354075, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1910319328308105, + "step": 3570, + "unclipped_grad_norm": 0.749891459941864 + }, + { + "bits_per_byte": 1.7607525321233055, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.2204606533050537, + "step": 3580, + "unclipped_grad_norm": 0.8161401152610779 + }, + { + "bits_per_byte": 1.690903842267186, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1720452308654785, + "step": 3590, + "unclipped_grad_norm": 0.6925192475318909 + }, + { + "bits_per_byte": 1.6772946831217612, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.1626120805740356, + "step": 3600, + "unclipped_grad_norm": 0.7061766982078552 + }, + { + "bits_per_byte": 1.6673109182565242, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1556918621063232, + "step": 3610, + "unclipped_grad_norm": 0.7162109017372131 + }, + { + "bits_per_byte": 1.6598764522250269, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.150538682937622, + "step": 3620, + "unclipped_grad_norm": 0.6953511238098145 + }, + { + "bits_per_byte": 1.6688818077895051, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.15678071975708, + "step": 3630, + "unclipped_grad_norm": 0.7265485525131226 + }, + { + "bits_per_byte": 1.6378186453559664, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.135249376296997, + "step": 3640, + "unclipped_grad_norm": 0.68751460313797 + }, + { + "bits_per_byte": 1.6996694540163617, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.1781210899353027, + "step": 3650, + "unclipped_grad_norm": 0.7145417928695679 + }, + { + "bits_per_byte": 1.6073453834148872, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1141269207000732, + "step": 3660, + "unclipped_grad_norm": 0.71860671043396 + }, + { + "bits_per_byte": 1.62879970316207, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1289979219436646, + "step": 3670, + "unclipped_grad_norm": 0.6769548654556274 + }, + { + "bits_per_byte": 1.702227523965295, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.179894208908081, + "step": 3680, + "unclipped_grad_norm": 0.7466121315956116 + }, + { + "bits_per_byte": 1.653425382991163, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1460671424865723, + "step": 3690, + "unclipped_grad_norm": 0.6637522578239441 + }, + { + "bits_per_byte": 1.5989233930021354, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1082892417907715, + "step": 3700, + "unclipped_grad_norm": 0.7335078120231628 + }, + { + "bits_per_byte": 1.6701862961962997, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1576849222183228, + "step": 3710, + "unclipped_grad_norm": 0.7247957587242126 + }, + { + "bits_per_byte": 1.6538059805975223, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1463309526443481, + "step": 3720, + "unclipped_grad_norm": 0.7534144520759583 + }, + { + "bits_per_byte": 1.7119505631318241, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1866337060928345, + "step": 3730, + "unclipped_grad_norm": 0.6951875686645508 + }, + { + "bits_per_byte": 1.6437530787066386, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.1393628120422363, + "step": 3740, + "unclipped_grad_norm": 0.7089563608169556 + }, + { + "bits_per_byte": 1.6267918057132467, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1276061534881592, + "step": 3750, + "unclipped_grad_norm": 0.6922760009765625 + }, + { + "bits_per_byte": 1.6201740852906965, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.1230190992355347, + "step": 3760, + "unclipped_grad_norm": 0.6971311569213867 + }, + { + "bits_per_byte": 1.6730486034546093, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1596689224243164, + "step": 3770, + "unclipped_grad_norm": 0.6925700902938843 + }, + { + "bits_per_byte": 1.6258194158052683, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.126932144165039, + "step": 3780, + "unclipped_grad_norm": 0.71840900182724 + }, + { + "bits_per_byte": 1.6611288298886098, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1514067649841309, + "step": 3790, + "unclipped_grad_norm": 0.6731990575790405 + }, + { + "bits_per_byte": 1.622545554063444, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1246628761291504, + "step": 3800, + "unclipped_grad_norm": 0.7068047523498535 + }, + { + "bits_per_byte": 1.613386446009094, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.118314266204834, + "step": 3810, + "unclipped_grad_norm": 0.679490327835083 + }, + { + "bits_per_byte": 1.723293162921528, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.1944957971572876, + "step": 3820, + "unclipped_grad_norm": 0.7167435884475708 + }, + { + "bits_per_byte": 1.6410942269242994, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1375198364257812, + "step": 3830, + "unclipped_grad_norm": 0.749890148639679 + }, + { + "bits_per_byte": 1.6271811744348, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1278760433197021, + "step": 3840, + "unclipped_grad_norm": 0.6869876980781555 + }, + { + "bits_per_byte": 1.6520392028252087, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.145106315612793, + "step": 3850, + "unclipped_grad_norm": 0.6890377998352051 + }, + { + "bits_per_byte": 1.6079827511189846, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1145687103271484, + "step": 3860, + "unclipped_grad_norm": 0.717820942401886 + }, + { + "bits_per_byte": 1.6751354409401786, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.1611154079437256, + "step": 3870, + "unclipped_grad_norm": 0.7368612289428711 + }, + { + "bits_per_byte": 1.6338718155013159, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1325136423110962, + "step": 3880, + "unclipped_grad_norm": 0.7661339640617371 + }, + { + "bits_per_byte": 1.656585564200767, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1482576131820679, + "step": 3890, + "unclipped_grad_norm": 0.7343658804893494 + }, + { + "bits_per_byte": 1.6647341021986468, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1539057493209839, + "step": 3900, + "unclipped_grad_norm": 0.7533658146858215 + }, + { + "bits_per_byte": 1.6762937440937398, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.16191828250885, + "step": 3910, + "unclipped_grad_norm": 0.7147406935691833 + }, + { + "bits_per_byte": 1.6322013480134854, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1313557624816895, + "step": 3920, + "unclipped_grad_norm": 0.6887187361717224 + }, + { + "bits_per_byte": 1.6221845624792828, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.1244126558303833, + "step": 3930, + "unclipped_grad_norm": 0.7910217642784119 + }, + { + "bits_per_byte": 1.63472175376188, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1331027746200562, + "step": 3940, + "unclipped_grad_norm": 0.730076789855957 + }, + { + "bits_per_byte": 1.6972579132859635, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.1764495372772217, + "step": 3950, + "unclipped_grad_norm": 0.7732967138290405 + }, + { + "bits_per_byte": 1.7135035664691152, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.187710165977478, + "step": 3960, + "unclipped_grad_norm": 0.7405394911766052 + }, + { + "bits_per_byte": 1.685746770498414, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1684706211090088, + "step": 3970, + "unclipped_grad_norm": 0.7160924673080444 + }, + { + "bits_per_byte": 1.6665311489175123, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1551513671875, + "step": 3980, + "unclipped_grad_norm": 0.7374541759490967 + }, + { + "bits_per_byte": 1.7040839046987084, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.1811809539794922, + "step": 3990, + "unclipped_grad_norm": 0.7841057181358337 + }, + { + "bits_per_byte": 1.693502844087025, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.17384672164917, + "step": 4000, + "unclipped_grad_norm": 0.7609017491340637 + }, + { + "bits_per_byte": 1.699465998540391, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.1779800653457642, + "step": 4010, + "unclipped_grad_norm": 0.7629905343055725 + }, + { + "bits_per_byte": 1.5838037102341824, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.097809076309204, + "step": 4020, + "unclipped_grad_norm": 0.6868616342544556 + }, + { + "bits_per_byte": 1.617556165379263, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1212044954299927, + "step": 4030, + "unclipped_grad_norm": 0.749467670917511 + }, + { + "bits_per_byte": 1.6189757101794846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1221884489059448, + "step": 4040, + "unclipped_grad_norm": 0.7323976755142212 + }, + { + "bits_per_byte": 1.6111234982890412, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1167457103729248, + "step": 4050, + "unclipped_grad_norm": 0.7304336428642273 + }, + { + "bits_per_byte": 1.6428754512392856, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1387544870376587, + "step": 4060, + "unclipped_grad_norm": 0.745829164981842 + }, + { + "bits_per_byte": 1.6097484969953932, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1157926321029663, + "step": 4070, + "unclipped_grad_norm": 0.7523763179779053 + }, + { + "bits_per_byte": 1.6445053308215125, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.1398842334747314, + "step": 4080, + "unclipped_grad_norm": 0.7511109113693237 + }, + { + "bits_per_byte": 1.6034616711931031, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1114349365234375, + "step": 4090, + "unclipped_grad_norm": 0.7438745498657227 + }, + { + "bits_per_byte": 1.561935084281071, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.082650899887085, + "step": 4100, + "unclipped_grad_norm": 0.6872318387031555 + }, + { + "bits_per_byte": 1.6907129415247284, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1719129085540771, + "step": 4110, + "unclipped_grad_norm": 0.7602059841156006 + }, + { + "bits_per_byte": 1.6125045189754696, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.1177029609680176, + "step": 4120, + "unclipped_grad_norm": 0.7136654257774353 + }, + { + "bits_per_byte": 1.5891117827704122, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1014883518218994, + "step": 4130, + "unclipped_grad_norm": 0.7062129378318787 + }, + { + "bits_per_byte": 1.6146775197691212, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1192091703414917, + "step": 4140, + "unclipped_grad_norm": 0.7150430679321289 + }, + { + "bits_per_byte": 1.6618133208390438, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.151881217956543, + "step": 4150, + "unclipped_grad_norm": 0.7448095679283142 + }, + { + "bits_per_byte": 1.6228898633304711, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.124901533126831, + "step": 4160, + "unclipped_grad_norm": 0.7362097501754761 + }, + { + "bits_per_byte": 1.5905947891688021, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1025162935256958, + "step": 4170, + "unclipped_grad_norm": 0.7651577591896057 + }, + { + "bits_per_byte": 1.5608787668394717, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.081918716430664, + "step": 4180, + "unclipped_grad_norm": 0.7135135531425476 + }, + { + "bits_per_byte": 1.713072749928704, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.1874115467071533, + "step": 4190, + "unclipped_grad_norm": 0.7463747262954712 + }, + { + "bits_per_byte": 1.5972510337051453, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.1071300506591797, + "step": 4200, + "unclipped_grad_norm": 0.7313898801803589 + }, + { + "bits_per_byte": 1.6169509584308768, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1207849979400635, + "step": 4210, + "unclipped_grad_norm": 0.7539675831794739 + }, + { + "bits_per_byte": 1.6536405332873922, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1462162733078003, + "step": 4220, + "unclipped_grad_norm": 0.7351350784301758 + }, + { + "bits_per_byte": 1.6357479742395782, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.1338140964508057, + "step": 4230, + "unclipped_grad_norm": 0.7141795754432678 + }, + { + "bits_per_byte": 1.6650933739562992, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1541547775268555, + "step": 4240, + "unclipped_grad_norm": 0.7625938653945923 + }, + { + "bits_per_byte": 1.6824343846427963, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1661746501922607, + "step": 4250, + "unclipped_grad_norm": 0.702212393283844 + }, + { + "bits_per_byte": 1.6672134040934852, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.155624270439148, + "step": 4260, + "unclipped_grad_norm": 0.775193989276886 + }, + { + "bits_per_byte": 1.643573356836487, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1392382383346558, + "step": 4270, + "unclipped_grad_norm": 0.7224892973899841 + }, + { + "bits_per_byte": 1.6848497089915135, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.167848825454712, + "step": 4280, + "unclipped_grad_norm": 0.792568564414978 + }, + { + "bits_per_byte": 1.6287462165576516, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.1289608478546143, + "step": 4290, + "unclipped_grad_norm": 0.7352984547615051 + }, + { + "bits_per_byte": 1.5679441581022175, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.0868160724639893, + "step": 4300, + "unclipped_grad_norm": 0.7459802031517029 + }, + { + "bits_per_byte": 1.7134996108681453, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1877074241638184, + "step": 4310, + "unclipped_grad_norm": 0.8436940312385559 + }, + { + "bits_per_byte": 1.585466610485375, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.0989617109298706, + "step": 4320, + "unclipped_grad_norm": 0.7326236367225647 + }, + { + "bits_per_byte": 1.5870087789156617, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1000306606292725, + "step": 4330, + "unclipped_grad_norm": 0.7154185175895691 + }, + { + "bits_per_byte": 1.5845594020020737, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.0983328819274902, + "step": 4340, + "unclipped_grad_norm": 0.7299889326095581 + }, + { + "bits_per_byte": 1.5087386145947432, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.0457779169082642, + "step": 4350, + "unclipped_grad_norm": 0.7116599082946777 + }, + { + "bits_per_byte": 1.6642632137005844, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1535793542861938, + "step": 4360, + "unclipped_grad_norm": 0.7841640114784241 + }, + { + "bits_per_byte": 1.6118272512963718, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1172335147857666, + "step": 4370, + "unclipped_grad_norm": 0.7682546973228455 + }, + { + "bits_per_byte": 1.5975416843851036, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1073315143585205, + "step": 4380, + "unclipped_grad_norm": 0.7542603611946106 + }, + { + "bits_per_byte": 1.6190350441940322, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1222295761108398, + "step": 4390, + "unclipped_grad_norm": 0.7354364991188049 + }, + { + "bits_per_byte": 1.5791293937663822, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.0945690870285034, + "step": 4400, + "unclipped_grad_norm": 0.7447035908699036 + }, + { + "bits_per_byte": 1.5777399459300616, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.0936059951782227, + "step": 4410, + "unclipped_grad_norm": 0.7433028221130371 + }, + { + "bits_per_byte": 1.5774630538621723, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.093414068222046, + "step": 4420, + "unclipped_grad_norm": 0.7514091730117798 + }, + { + "bits_per_byte": 1.5840658117940973, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.0979907512664795, + "step": 4430, + "unclipped_grad_norm": 0.748140811920166 + }, + { + "bits_per_byte": 1.5992336497042918, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.108504295349121, + "step": 4440, + "unclipped_grad_norm": 0.7693572044372559 + }, + { + "bits_per_byte": 1.5825853851354703, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.0969645977020264, + "step": 4450, + "unclipped_grad_norm": 0.7571823596954346 + }, + { + "bits_per_byte": 1.5385392523882981, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0664341449737549, + "step": 4460, + "unclipped_grad_norm": 0.7259396314620972 + }, + { + "bits_per_byte": 1.6428487939284018, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1387360095977783, + "step": 4470, + "unclipped_grad_norm": 0.7504101991653442 + }, + { + "bits_per_byte": 1.5687149843433847, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.0873503684997559, + "step": 4480, + "unclipped_grad_norm": 0.7544133067131042 + }, + { + "bits_per_byte": 1.5307804270772734, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.061056137084961, + "step": 4490, + "unclipped_grad_norm": 0.7395938634872437 + }, + { + "bits_per_byte": 1.6123227333135077, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1175769567489624, + "step": 4500, + "unclipped_grad_norm": 0.7624281644821167 + }, + { + "bits_per_byte": 1.5895855949735391, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1018167734146118, + "step": 4510, + "unclipped_grad_norm": 0.7442159652709961 + }, + { + "bits_per_byte": 1.5720557473363947, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.0896660089492798, + "step": 4520, + "unclipped_grad_norm": 0.7299280166625977 + }, + { + "bits_per_byte": 1.5930625682260326, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.10422682762146, + "step": 4530, + "unclipped_grad_norm": 0.7508688569068909 + }, + { + "bits_per_byte": 1.551802554422838, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.0756275653839111, + "step": 4540, + "unclipped_grad_norm": 0.7326505780220032 + }, + { + "bits_per_byte": 1.6245931795046165, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.126082181930542, + "step": 4550, + "unclipped_grad_norm": 0.7591264247894287 + }, + { + "bits_per_byte": 1.6222865481912445, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.124483346939087, + "step": 4560, + "unclipped_grad_norm": 0.7490679621696472 + }, + { + "bits_per_byte": 1.5636793323261218, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.083859920501709, + "step": 4570, + "unclipped_grad_norm": 0.7205959558486938 + }, + { + "bits_per_byte": 1.5242402708302625, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0565228462219238, + "step": 4580, + "unclipped_grad_norm": 0.7278034687042236 + }, + { + "bits_per_byte": 1.649223158899981, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1431543827056885, + "step": 4590, + "unclipped_grad_norm": 0.80039381980896 + }, + { + "bits_per_byte": 1.621592598195013, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1240023374557495, + "step": 4600, + "unclipped_grad_norm": 0.7594562768936157 + }, + { + "bits_per_byte": 1.591440943811047, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.1031028032302856, + "step": 4610, + "unclipped_grad_norm": 0.7573962211608887 + }, + { + "bits_per_byte": 1.5204752266375738, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.0539131164550781, + "step": 4620, + "unclipped_grad_norm": 0.7093440890312195 + }, + { + "bits_per_byte": 1.6007937043302685, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1095856428146362, + "step": 4630, + "unclipped_grad_norm": 0.7716385126113892 + }, + { + "bits_per_byte": 1.566827990698118, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.0860424041748047, + "step": 4640, + "unclipped_grad_norm": 0.7477832436561584 + }, + { + "bits_per_byte": 1.5540792607549598, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.0772056579589844, + "step": 4650, + "unclipped_grad_norm": 0.7563350200653076 + }, + { + "bits_per_byte": 1.5711925664117143, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0890676975250244, + "step": 4660, + "unclipped_grad_norm": 0.7477205991744995 + }, + { + "bits_per_byte": 1.6065126434194095, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.1135497093200684, + "step": 4670, + "unclipped_grad_norm": 0.7434800267219543 + }, + { + "bits_per_byte": 1.5613855997115644, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.0822700262069702, + "step": 4680, + "unclipped_grad_norm": 0.7484313249588013 + }, + { + "bits_per_byte": 1.533432227570927, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.0628942251205444, + "step": 4690, + "unclipped_grad_norm": 0.7767173647880554 + }, + { + "bits_per_byte": 1.6430371149310967, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.1388665437698364, + "step": 4700, + "unclipped_grad_norm": 0.7674224972724915 + }, + { + "bits_per_byte": 1.552380416129737, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.0760281085968018, + "step": 4710, + "unclipped_grad_norm": 0.7836162447929382 + }, + { + "bits_per_byte": 1.5203768525612804, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.053844928741455, + "step": 4720, + "unclipped_grad_norm": 0.7613595128059387 + }, + { + "bits_per_byte": 1.6524877335786587, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.1454172134399414, + "step": 4730, + "unclipped_grad_norm": 0.7924392223358154 + }, + { + "bits_per_byte": 1.587415861850254, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.1003128290176392, + "step": 4740, + "unclipped_grad_norm": 0.7647171020507812 + }, + { + "bits_per_byte": 1.6396726183322676, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1365344524383545, + "step": 4750, + "unclipped_grad_norm": 0.8181672096252441 + }, + { + "bits_per_byte": 1.5091890371573529, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.0460901260375977, + "step": 4760, + "unclipped_grad_norm": 0.7407304048538208 + }, + { + "bits_per_byte": 1.6098291568586478, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1158485412597656, + "step": 4770, + "unclipped_grad_norm": 0.7601819038391113 + }, + { + "bits_per_byte": 1.5582154435082103, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.0800726413726807, + "step": 4780, + "unclipped_grad_norm": 0.7307054996490479 + }, + { + "bits_per_byte": 1.5944504682184955, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.1051888465881348, + "step": 4790, + "unclipped_grad_norm": 0.7617009282112122 + }, + { + "bits_per_byte": 1.5823225956449518, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0967824459075928, + "step": 4800, + "unclipped_grad_norm": 0.7428550124168396 + }, + { + "bits_per_byte": 1.542078311377753, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.0688872337341309, + "step": 4810, + "unclipped_grad_norm": 0.773262083530426 + }, + { + "bits_per_byte": 1.6108488419956133, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.1165553331375122, + "step": 4820, + "unclipped_grad_norm": 0.8227523565292358 + }, + { + "bits_per_byte": 1.5982464692883394, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1078200340270996, + "step": 4830, + "unclipped_grad_norm": 0.783271312713623 + }, + { + "bits_per_byte": 1.6134836162068316, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1183816194534302, + "step": 4840, + "unclipped_grad_norm": 0.7595719695091248 + }, + { + "bits_per_byte": 1.5430993723759254, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.0695949792861938, + "step": 4850, + "unclipped_grad_norm": 0.8344528079032898 + }, + { + "bits_per_byte": 1.551674427347945, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0755387544631958, + "step": 4860, + "unclipped_grad_norm": 0.7924710512161255 + }, + { + "bits_per_byte": 1.5551434893984988, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.0779433250427246, + "step": 4870, + "unclipped_grad_norm": 0.7731344103813171 + }, + { + "bits_per_byte": 1.7162552888829201, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.1896175146102905, + "step": 4880, + "unclipped_grad_norm": 0.8454892039299011 + }, + { + "bits_per_byte": 1.5015098398136777, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0407673120498657, + "step": 4890, + "unclipped_grad_norm": 0.7279435396194458 + }, + { + "bits_per_byte": 1.6017640304464364, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1102582216262817, + "step": 4900, + "unclipped_grad_norm": 0.7495299577713013 + }, + { + "bits_per_byte": 1.5795956387328711, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.0948922634124756, + "step": 4910, + "unclipped_grad_norm": 0.8354763388633728 + }, + { + "bits_per_byte": 1.5130457481029513, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.048763394355774, + "step": 4920, + "unclipped_grad_norm": 0.7128972411155701 + }, + { + "bits_per_byte": 1.5263470583033318, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.057983160018921, + "step": 4930, + "unclipped_grad_norm": 0.7301722168922424 + }, + { + "bits_per_byte": 1.56106777577277, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0820497274398804, + "step": 4940, + "unclipped_grad_norm": 0.7844029664993286 + }, + { + "bits_per_byte": 1.5809642766684375, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0958409309387207, + "step": 4950, + "unclipped_grad_norm": 0.7815173864364624 + }, + { + "bits_per_byte": 1.6004612618661507, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.1093552112579346, + "step": 4960, + "unclipped_grad_norm": 0.8055645227432251 + }, + { + "bits_per_byte": 1.5618184800437862, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.0825700759887695, + "step": 4970, + "unclipped_grad_norm": 0.7773133516311646 + }, + { + "bits_per_byte": 1.606830467358204, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1137700080871582, + "step": 4980, + "unclipped_grad_norm": 0.7920810580253601 + }, + { + "bits_per_byte": 1.567882244347907, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.086773157119751, + "step": 4990, + "unclipped_grad_norm": 0.7929123640060425 + }, + { + "bits_per_byte": 1.5708692390280923, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.088843584060669, + "step": 5000, + "unclipped_grad_norm": 0.762309730052948 + }, + { + "bits_per_byte": 1.6157285057485438, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.1199376583099365, + "step": 5010, + "unclipped_grad_norm": 0.7734562158584595 + }, + { + "bits_per_byte": 1.5685849654593325, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.0872602462768555, + "step": 5020, + "unclipped_grad_norm": 0.8221466541290283 + }, + { + "bits_per_byte": 1.5125002191344146, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.0483852624893188, + "step": 5030, + "unclipped_grad_norm": 0.7515868544578552 + }, + { + "bits_per_byte": 1.5645020973278494, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.08443021774292, + "step": 5040, + "unclipped_grad_norm": 0.7403401732444763 + }, + { + "bits_per_byte": 1.5465288784167808, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.071972131729126, + "step": 5050, + "unclipped_grad_norm": 0.8082988262176514 + }, + { + "bits_per_byte": 1.6139199361920706, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.1186840534210205, + "step": 5060, + "unclipped_grad_norm": 0.8066977262496948 + }, + { + "bits_per_byte": 1.5533101543403012, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.0766725540161133, + "step": 5070, + "unclipped_grad_norm": 0.7855141758918762 + }, + { + "bits_per_byte": 1.6163216739113697, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.1203488111495972, + "step": 5080, + "unclipped_grad_norm": 0.776370644569397 + }, + { + "bits_per_byte": 1.5614495772576855, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.082314372062683, + "step": 5090, + "unclipped_grad_norm": 0.776827871799469 + }, + { + "bits_per_byte": 1.5419546558517827, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0688015222549438, + "step": 5100, + "unclipped_grad_norm": 0.7785685062408447 + }, + { + "bits_per_byte": 1.5800214677764075, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0951874256134033, + "step": 5110, + "unclipped_grad_norm": 0.8104537725448608 + }, + { + "bits_per_byte": 1.487382496941203, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0309749841690063, + "step": 5120, + "unclipped_grad_norm": 0.7389267086982727 + }, + { + "bits_per_byte": 1.6243671943013827, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1259255409240723, + "step": 5130, + "unclipped_grad_norm": 0.8053697943687439 + }, + { + "bits_per_byte": 1.5876562935961602, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.1004794836044312, + "step": 5140, + "unclipped_grad_norm": 0.8099423050880432 + }, + { + "bits_per_byte": 1.5188835272038386, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.0528098344802856, + "step": 5150, + "unclipped_grad_norm": 0.7463088035583496 + }, + { + "bits_per_byte": 1.5574305146896725, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.079528570175171, + "step": 5160, + "unclipped_grad_norm": 0.7795574069023132 + }, + { + "bits_per_byte": 1.604379370618107, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1120710372924805, + "step": 5170, + "unclipped_grad_norm": 0.8112488985061646 + }, + { + "bits_per_byte": 1.570943535533265, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0888950824737549, + "step": 5180, + "unclipped_grad_norm": 0.7960402965545654 + }, + { + "bits_per_byte": 1.4686855750353036, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0180152654647827, + "step": 5190, + "unclipped_grad_norm": 0.7735135555267334 + }, + { + "bits_per_byte": 1.5401951013508053, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0675818920135498, + "step": 5200, + "unclipped_grad_norm": 0.7935121655464172 + }, + { + "bits_per_byte": 1.522642552003747, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.055415391921997, + "step": 5210, + "unclipped_grad_norm": 0.7616920471191406 + }, + { + "bits_per_byte": 1.5296819738862126, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.0602947473526, + "step": 5220, + "unclipped_grad_norm": 0.799717128276825 + }, + { + "bits_per_byte": 1.60920967535024, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.1154191493988037, + "step": 5230, + "unclipped_grad_norm": 0.8263183832168579 + }, + { + "bits_per_byte": 1.5759482306733728, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.0923640727996826, + "step": 5240, + "unclipped_grad_norm": 0.9089767336845398 + }, + { + "bits_per_byte": 1.5751560785834986, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0918149948120117, + "step": 5250, + "unclipped_grad_norm": 0.8223497271537781 + }, + { + "bits_per_byte": 1.518346597367845, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.052437663078308, + "step": 5260, + "unclipped_grad_norm": 0.7989961504936218 + }, + { + "bits_per_byte": 1.5652323356634128, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0849363803863525, + "step": 5270, + "unclipped_grad_norm": 0.7858845591545105 + }, + { + "bits_per_byte": 1.4824085866955998, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0275273323059082, + "step": 5280, + "unclipped_grad_norm": 0.7963522672653198 + }, + { + "bits_per_byte": 1.5109090356486319, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.047282338142395, + "step": 5290, + "unclipped_grad_norm": 0.7786324620246887 + }, + { + "bits_per_byte": 1.4973765807654917, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.0379023551940918, + "step": 5300, + "unclipped_grad_norm": 0.8200122714042664 + }, + { + "bits_per_byte": 1.4979238295605373, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0382816791534424, + "step": 5310, + "unclipped_grad_norm": 0.798004686832428 + }, + { + "bits_per_byte": 1.4963071926424267, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.037161111831665, + "step": 5320, + "unclipped_grad_norm": 0.7881578207015991 + }, + { + "bits_per_byte": 1.5549873291515153, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0778350830078125, + "step": 5330, + "unclipped_grad_norm": 0.8092938661575317 + }, + { + "bits_per_byte": 1.6144941862633015, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.1190820932388306, + "step": 5340, + "unclipped_grad_norm": 0.8349001407623291 + }, + { + "bits_per_byte": 1.4860422361430292, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.030045986175537, + "step": 5350, + "unclipped_grad_norm": 0.7843849062919617 + }, + { + "bits_per_byte": 1.5082066722556244, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.0454092025756836, + "step": 5360, + "unclipped_grad_norm": 0.7918040752410889 + }, + { + "bits_per_byte": 1.577472340925319, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0934205055236816, + "step": 5370, + "unclipped_grad_norm": 0.8359649181365967 + }, + { + "bits_per_byte": 1.5013394050066726, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.040649175643921, + "step": 5380, + "unclipped_grad_norm": 0.7734270095825195 + }, + { + "bits_per_byte": 1.5016145772480531, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0408399105072021, + "step": 5390, + "unclipped_grad_norm": 0.7885358333587646 + }, + { + "bits_per_byte": 1.5491196250693782, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073767900466919, + "step": 5400, + "unclipped_grad_norm": 0.8202832341194153 + }, + { + "bits_per_byte": 1.4666546319112648, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.0166075229644775, + "step": 5410, + "unclipped_grad_norm": 0.8083077073097229 + }, + { + "bits_per_byte": 1.6009971598062391, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.1097266674041748, + "step": 5420, + "unclipped_grad_norm": 0.847724974155426 + }, + { + "bits_per_byte": 1.5289176829857782, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0597649812698364, + "step": 5430, + "unclipped_grad_norm": 0.8225540518760681 + }, + { + "bits_per_byte": 1.530357349756151, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.060762882232666, + "step": 5440, + "unclipped_grad_norm": 0.8045809864997864 + }, + { + "bits_per_byte": 1.5424515137301253, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.069145917892456, + "step": 5450, + "unclipped_grad_norm": 0.8051854968070984 + }, + { + "bits_per_byte": 1.5648894022575925, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0846986770629883, + "step": 5460, + "unclipped_grad_norm": 0.7835660576820374 + }, + { + "bits_per_byte": 1.5450825043230245, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.070969581604004, + "step": 5470, + "unclipped_grad_norm": 0.8599129915237427 + }, + { + "bits_per_byte": 1.4780804713039863, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.0245273113250732, + "step": 5480, + "unclipped_grad_norm": 0.8000594973564148 + }, + { + "bits_per_byte": 1.5424891779306642, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.0691720247268677, + "step": 5490, + "unclipped_grad_norm": 0.8055532574653625 + }, + { + "bits_per_byte": 1.5569413960306187, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0791895389556885, + "step": 5500, + "unclipped_grad_norm": 0.8300883769989014 + }, + { + "bits_per_byte": 1.5474785666148452, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.0726304054260254, + "step": 5510, + "unclipped_grad_norm": 0.826352059841156 + }, + { + "bits_per_byte": 1.5350409532870977, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.0640093088150024, + "step": 5520, + "unclipped_grad_norm": 0.853265643119812 + }, + { + "bits_per_byte": 1.5635150888945477, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.083746075630188, + "step": 5530, + "unclipped_grad_norm": 0.8060055375099182 + }, + { + "bits_per_byte": 1.5843347926600468, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.098177194595337, + "step": 5540, + "unclipped_grad_norm": 0.8706573843955994 + }, + { + "bits_per_byte": 1.5165710484803372, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0512069463729858, + "step": 5550, + "unclipped_grad_norm": 0.7865509986877441 + }, + { + "bits_per_byte": 1.5106121935932426, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0470765829086304, + "step": 5560, + "unclipped_grad_norm": 0.8064424991607666 + }, + { + "bits_per_byte": 1.5654490338035, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0850865840911865, + "step": 5570, + "unclipped_grad_norm": 0.8376733064651489 + }, + { + "bits_per_byte": 1.4567845475782482, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.0097661018371582, + "step": 5580, + "unclipped_grad_norm": 0.7972759008407593 + }, + { + "bits_per_byte": 1.5257076268074239, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.057539939880371, + "step": 5590, + "unclipped_grad_norm": 0.8159511089324951 + }, + { + "bits_per_byte": 1.5137768463517691, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0492701530456543, + "step": 5600, + "unclipped_grad_norm": 0.7912799715995789 + }, + { + "bits_per_byte": 1.5312812405565859, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0614032745361328, + "step": 5610, + "unclipped_grad_norm": 0.7786760330200195 + }, + { + "bits_per_byte": 1.4626550033827992, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0138351917266846, + "step": 5620, + "unclipped_grad_norm": 0.8055174350738525 + }, + { + "bits_per_byte": 1.4690646247978054, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0182780027389526, + "step": 5630, + "unclipped_grad_norm": 0.7851108312606812 + }, + { + "bits_per_byte": 1.4663743001903584, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.0164132118225098, + "step": 5640, + "unclipped_grad_norm": 0.8267287015914917 + }, + { + "bits_per_byte": 1.5738650048234715, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.090920090675354, + "step": 5650, + "unclipped_grad_norm": 0.8359448313713074 + }, + { + "bits_per_byte": 1.5300369460775933, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.060540795326233, + "step": 5660, + "unclipped_grad_norm": 0.8635727167129517 + }, + { + "bits_per_byte": 1.5230305448640935, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.0556843280792236, + "step": 5670, + "unclipped_grad_norm": 0.8095638751983643 + }, + { + "bits_per_byte": 1.524155483383387, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.0564640760421753, + "step": 5680, + "unclipped_grad_norm": 0.8353701233863831 + }, + { + "bits_per_byte": 1.566590998605229, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.0858781337738037, + "step": 5690, + "unclipped_grad_norm": 0.8297926187515259 + }, + { + "bits_per_byte": 1.499949785187701, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.0396859645843506, + "step": 5700, + "unclipped_grad_norm": 0.8335801959037781 + }, + { + "bits_per_byte": 1.5306388853556132, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.0609580278396606, + "step": 5710, + "unclipped_grad_norm": 0.8218200206756592 + }, + { + "bits_per_byte": 1.5663932185567369, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0857410430908203, + "step": 5720, + "unclipped_grad_norm": 0.8909743428230286 + }, + { + "bits_per_byte": 1.4614645394735268, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.013010025024414, + "step": 5730, + "unclipped_grad_norm": 0.7809724807739258 + }, + { + "bits_per_byte": 1.5314872757723195, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0615460872650146, + "step": 5740, + "unclipped_grad_norm": 0.8163876533508301 + }, + { + "bits_per_byte": 1.499649159513993, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0394775867462158, + "step": 5750, + "unclipped_grad_norm": 0.8667106628417969 + }, + { + "bits_per_byte": 1.4999243317553734, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.039668321609497, + "step": 5760, + "unclipped_grad_norm": 0.8516601324081421 + }, + { + "bits_per_byte": 1.450708744488567, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0055546760559082, + "step": 5770, + "unclipped_grad_norm": 0.7943668365478516 + }, + { + "bits_per_byte": 1.5466515020468459, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.0720571279525757, + "step": 5780, + "unclipped_grad_norm": 0.8325088024139404 + }, + { + "bits_per_byte": 1.494320105094358, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.0357837677001953, + "step": 5790, + "unclipped_grad_norm": 0.8366720080375671 + }, + { + "bits_per_byte": 1.4906410522271007, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033233642578125, + "step": 5800, + "unclipped_grad_norm": 0.8333538770675659 + }, + { + "bits_per_byte": 1.5321304908865463, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0619919300079346, + "step": 5810, + "unclipped_grad_norm": 0.8446563482284546 + }, + { + "bits_per_byte": 1.51830377368778, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.05240797996521, + "step": 5820, + "unclipped_grad_norm": 0.8496891856193542 + }, + { + "bits_per_byte": 1.4932847695361637, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0350661277770996, + "step": 5830, + "unclipped_grad_norm": 0.8205961585044861 + }, + { + "bits_per_byte": 1.456290785387621, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.009423851966858, + "step": 5840, + "unclipped_grad_norm": 0.8371089100837708 + }, + { + "bits_per_byte": 1.454681199758196, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0083081722259521, + "step": 5850, + "unclipped_grad_norm": 0.8174164295196533 + }, + { + "bits_per_byte": 1.4966561454410274, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.0374029874801636, + "step": 5860, + "unclipped_grad_norm": 0.9058852791786194 + }, + { + "bits_per_byte": 1.5418609253070623, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0687365531921387, + "step": 5870, + "unclipped_grad_norm": 0.8718268275260925 + }, + { + "bits_per_byte": 1.5420466665699941, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.0688652992248535, + "step": 5880, + "unclipped_grad_norm": 0.8315351009368896 + }, + { + "bits_per_byte": 1.525186175410008, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.0571784973144531, + "step": 5890, + "unclipped_grad_norm": 0.8407886028289795 + }, + { + "bits_per_byte": 1.4989159974733648, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.0389693975448608, + "step": 5900, + "unclipped_grad_norm": 0.8770707249641418 + }, + { + "bits_per_byte": 1.4937374278732347, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0353798866271973, + "step": 5910, + "unclipped_grad_norm": 0.8901946544647217 + }, + { + "bits_per_byte": 1.4996551789067731, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.03948175907135, + "step": 5920, + "unclipped_grad_norm": 0.85027015209198 + }, + { + "bits_per_byte": 1.5573538104273876, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.0794754028320312, + "step": 5930, + "unclipped_grad_norm": 0.8851437568664551 + }, + { + "bits_per_byte": 1.5404470559343193, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.0677565336227417, + "step": 5940, + "unclipped_grad_norm": 0.8759228587150574 + }, + { + "bits_per_byte": 1.5066732749405318, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0443463325500488, + "step": 5950, + "unclipped_grad_norm": 0.8818607926368713 + }, + { + "bits_per_byte": 1.5044743047665998, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0428221225738525, + "step": 5960, + "unclipped_grad_norm": 0.816109299659729 + }, + { + "bits_per_byte": 1.5173669841885302, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.0517586469650269, + "step": 5970, + "unclipped_grad_norm": 0.8590478301048279 + }, + { + "bits_per_byte": 1.4670959393933787, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0169134140014648, + "step": 5980, + "unclipped_grad_norm": 0.8544116616249084 + }, + { + "bits_per_byte": 1.4408131207318973, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 0.9986955523490906, + "step": 5990, + "unclipped_grad_norm": 0.8525132536888123 + }, + { + "bits_per_byte": 1.5172739415744134, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.0516941547393799, + "step": 6000, + "unclipped_grad_norm": 0.860197126865387 + }, + { + "bits_per_byte": 1.499894750739425, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0396478176116943, + "step": 6010, + "unclipped_grad_norm": 0.8407278060913086 + }, + { + "bits_per_byte": 1.4776142263374972, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.024204134941101, + "step": 6020, + "unclipped_grad_norm": 0.8550556302070618 + }, + { + "bits_per_byte": 1.5367169242197558, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.0651710033416748, + "step": 6030, + "unclipped_grad_norm": 0.8657232522964478 + }, + { + "bits_per_byte": 1.4931389282482321, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.0349650382995605, + "step": 6040, + "unclipped_grad_norm": 0.8972119092941284 + }, + { + "bits_per_byte": 1.498401597364634, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.0386128425598145, + "step": 6050, + "unclipped_grad_norm": 0.8974336981773376 + }, + { + "bits_per_byte": 1.5272425719663745, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0586038827896118, + "step": 6060, + "unclipped_grad_norm": 0.8948551416397095 + }, + { + "bits_per_byte": 1.5059438965182226, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.043840765953064, + "step": 6070, + "unclipped_grad_norm": 0.843971312046051 + }, + { + "bits_per_byte": 1.4791374766761891, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0252599716186523, + "step": 6080, + "unclipped_grad_norm": 0.8345775008201599 + }, + { + "bits_per_byte": 1.4884991802932552, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0317490100860596, + "step": 6090, + "unclipped_grad_norm": 0.8372970223426819 + }, + { + "bits_per_byte": 1.486896301987214, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0306379795074463, + "step": 6100, + "unclipped_grad_norm": 0.827723503112793 + }, + { + "bits_per_byte": 1.4668594632484424, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.0167495012283325, + "step": 6110, + "unclipped_grad_norm": 0.871588945388794 + }, + { + "bits_per_byte": 1.4677823221529722, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.017389178276062, + "step": 6120, + "unclipped_grad_norm": 0.8457695245742798 + }, + { + "bits_per_byte": 1.5061057321926845, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0439529418945312, + "step": 6130, + "unclipped_grad_norm": 0.9012824296951294 + }, + { + "bits_per_byte": 1.5178127632195666, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0520676374435425, + "step": 6140, + "unclipped_grad_norm": 0.8450613617897034 + }, + { + "bits_per_byte": 1.4675592606548031, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0172345638275146, + "step": 6150, + "unclipped_grad_norm": 0.885280191898346 + }, + { + "bits_per_byte": 1.5554258849112155, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 1.078139066696167, + "step": 6160, + "unclipped_grad_norm": 0.9099593758583069 + }, + { + "bits_per_byte": 1.421598961003528, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 0.985377311706543, + "step": 6170, + "unclipped_grad_norm": 0.7732160091400146 + }, + { + "bits_per_byte": 1.5028951600663776, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0417275428771973, + "step": 6180, + "unclipped_grad_norm": 0.869156002998352 + }, + { + "bits_per_byte": 1.4956368042693635, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.036696434020996, + "step": 6190, + "unclipped_grad_norm": 0.8908746838569641 + }, + { + "bits_per_byte": 1.4533810109176732, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0074069499969482, + "step": 6200, + "unclipped_grad_norm": 0.8600793480873108 + }, + { + "bits_per_byte": 1.475978499345141, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.0230703353881836, + "step": 6210, + "unclipped_grad_norm": 0.8739057779312134 + }, + { + "bits_per_byte": 1.4582183669384914, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.010759949684143, + "step": 6220, + "unclipped_grad_norm": 0.8542900681495667 + }, + { + "bits_per_byte": 1.4401650041121208, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 0.9982463121414185, + "step": 6230, + "unclipped_grad_norm": 0.8145909309387207 + }, + { + "bits_per_byte": 1.458188613939892, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0107393264770508, + "step": 6240, + "unclipped_grad_norm": 0.8962635397911072 + }, + { + "bits_per_byte": 1.4595658510080016, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0116939544677734, + "step": 6250, + "unclipped_grad_norm": 0.855821430683136 + }, + { + "bits_per_byte": 1.5147881043388425, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 1.049971103668213, + "step": 6260, + "unclipped_grad_norm": 0.8663405179977417 + }, + { + "bits_per_byte": 1.4764731214490225, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0234131813049316, + "step": 6270, + "unclipped_grad_norm": 0.8501848578453064 + }, + { + "bits_per_byte": 1.59573105103682, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.1060764789581299, + "step": 6280, + "unclipped_grad_norm": 0.9987632036209106 + }, + { + "bits_per_byte": 1.4579987450933396, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0106077194213867, + "step": 6290, + "unclipped_grad_norm": 0.8792169690132141 + }, + { + "bits_per_byte": 1.4898052165439075, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0326542854309082, + "step": 6300, + "unclipped_grad_norm": 0.8601147532463074 + }, + { + "bits_per_byte": 1.537688110249178, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.065844178199768, + "step": 6310, + "unclipped_grad_norm": 0.907733678817749 + }, + { + "bits_per_byte": 1.487509248154889, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 1.0310628414154053, + "step": 6320, + "unclipped_grad_norm": 0.8578711152076721 + }, + { + "bits_per_byte": 1.4237945775157679, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 0.986899197101593, + "step": 6330, + "unclipped_grad_norm": 0.845673143863678 + }, + { + "bits_per_byte": 1.4966876182661353, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.0374248027801514, + "step": 6340, + "unclipped_grad_norm": 0.8802160620689392 + }, + { + "bits_per_byte": 1.485631197607467, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.0297610759735107, + "step": 6350, + "unclipped_grad_norm": 0.884225606918335 + }, + { + "bits_per_byte": 1.4432266392627804, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0003684759140015, + "step": 6360, + "unclipped_grad_norm": 0.8594306707382202 + }, + { + "bits_per_byte": 1.497669639202562, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 1.0381054878234863, + "step": 6370, + "unclipped_grad_norm": 0.9075121283531189 + }, + { + "bits_per_byte": 1.4643473126672892, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0150082111358643, + "step": 6380, + "unclipped_grad_norm": 0.8724440932273865 + }, + { + "bits_per_byte": 1.4983035672536422, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0385448932647705, + "step": 6390, + "unclipped_grad_norm": 0.9329665303230286 + }, + { + "bits_per_byte": 1.4855978329731998, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.029737949371338, + "step": 6400, + "unclipped_grad_norm": 0.8683961033821106 + }, + { + "bits_per_byte": 1.4585226042478678, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0109708309173584, + "step": 6410, + "unclipped_grad_norm": 0.8944246768951416 + }, + { + "bits_per_byte": 1.4417726979323862, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 0.9993606805801392, + "step": 6420, + "unclipped_grad_norm": 0.8450288772583008 + }, + { + "bits_per_byte": 1.474598510554618, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.0221138000488281, + "step": 6430, + "unclipped_grad_norm": 0.8708812594413757 + }, + { + "bits_per_byte": 1.4773634756325393, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.024030327796936, + "step": 6440, + "unclipped_grad_norm": 0.8653560876846313 + }, + { + "bits_per_byte": 1.4567353605401014, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0097320079803467, + "step": 6450, + "unclipped_grad_norm": 0.8851431012153625 + }, + { + "bits_per_byte": 1.4433678370191387, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0004663467407227, + "step": 6460, + "unclipped_grad_norm": 0.902314305305481 + }, + { + "bits_per_byte": 1.478021997202693, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.024486780166626, + "step": 6470, + "unclipped_grad_norm": 0.8634862899780273 + }, + { + "bits_per_byte": 1.49569390250945, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.036736011505127, + "step": 6480, + "unclipped_grad_norm": 0.8877258896827698 + }, + { + "bits_per_byte": 1.4739315618345719, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0216515064239502, + "step": 6490, + "unclipped_grad_norm": 0.9130761027336121 + }, + { + "bits_per_byte": 1.4798123365981748, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0257277488708496, + "step": 6500, + "unclipped_grad_norm": 0.9244931936264038 + }, + { + "bits_per_byte": 1.429752142532981, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 0.9910286664962769, + "step": 6510, + "unclipped_grad_norm": 0.8583832383155823 + }, + { + "bits_per_byte": 1.3992985726054255, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9699198603630066, + "step": 6520, + "unclipped_grad_norm": 0.8968297243118286 + }, + { + "bits_per_byte": 1.5067831718544331, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0444225072860718, + "step": 6530, + "unclipped_grad_norm": 0.9009160399436951 + }, + { + "bits_per_byte": 1.468959715380779, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0182052850723267, + "step": 6540, + "unclipped_grad_norm": 0.8687822222709656 + }, + { + "bits_per_byte": 1.5110841139872102, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 1.0474036931991577, + "step": 6550, + "unclipped_grad_norm": 0.9212287068367004 + }, + { + "bits_per_byte": 1.4657298811975754, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.015966534614563, + "step": 6560, + "unclipped_grad_norm": 0.8736171126365662 + }, + { + "bits_per_byte": 1.3923212364599213, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 0.9650835394859314, + "step": 6570, + "unclipped_grad_norm": 0.9459646344184875 + }, + { + "bits_per_byte": 1.5011586512406159, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.040523886680603, + "step": 6580, + "unclipped_grad_norm": 0.8902279138565063 + }, + { + "bits_per_byte": 1.4631396504929306, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0141711235046387, + "step": 6590, + "unclipped_grad_norm": 0.8934556245803833 + }, + { + "bits_per_byte": 1.3892133379761795, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9629293084144592, + "step": 6600, + "unclipped_grad_norm": 0.8376922607421875 + }, + { + "bits_per_byte": 1.5024242715683152, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 1.0414011478424072, + "step": 6610, + "unclipped_grad_norm": 0.8684409856796265 + }, + { + "bits_per_byte": 1.5173028346597586, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 1.0517141819000244, + "step": 6620, + "unclipped_grad_norm": 0.9338745474815369 + }, + { + "bits_per_byte": 1.4964176055042808, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.0372376441955566, + "step": 6630, + "unclipped_grad_norm": 0.9484766721725464 + }, + { + "bits_per_byte": 1.4696447222791655, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.0186800956726074, + "step": 6640, + "unclipped_grad_norm": 0.9511991143226624 + }, + { + "bits_per_byte": 1.4476113369465278, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0034077167510986, + "step": 6650, + "unclipped_grad_norm": 0.8744685649871826 + }, + { + "bits_per_byte": 1.5102728718230902, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.0468413829803467, + "step": 6660, + "unclipped_grad_norm": 0.9144671559333801 + }, + { + "bits_per_byte": 1.4729612357184039, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0209789276123047, + "step": 6670, + "unclipped_grad_norm": 0.918013870716095 + }, + { + "bits_per_byte": 1.4976474534406006, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.0380901098251343, + "step": 6680, + "unclipped_grad_norm": 0.9484038352966309 + }, + { + "bits_per_byte": 1.4621077545877539, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.013455867767334, + "step": 6690, + "unclipped_grad_norm": 0.9366878867149353 + }, + { + "bits_per_byte": 1.513278096664267, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.048924446105957, + "step": 6700, + "unclipped_grad_norm": 0.9588149785995483 + }, + { + "bits_per_byte": 1.448871969777352, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0042815208435059, + "step": 6710, + "unclipped_grad_norm": 0.8966986536979675 + }, + { + "bits_per_byte": 1.4475960304906008, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0033971071243286, + "step": 6720, + "unclipped_grad_norm": 0.9147130250930786 + }, + { + "bits_per_byte": 1.4208115384365525, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 0.9848315119743347, + "step": 6730, + "unclipped_grad_norm": 0.9434075951576233 + }, + { + "bits_per_byte": 1.4530764296429952, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 1.0071958303451538, + "step": 6740, + "unclipped_grad_norm": 0.9062299728393555 + }, + { + "bits_per_byte": 1.4277258429405155, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 0.9896241426467896, + "step": 6750, + "unclipped_grad_norm": 0.9092971086502075 + }, + { + "bits_per_byte": 1.4128754850038137, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 0.9793306589126587, + "step": 6760, + "unclipped_grad_norm": 0.8910818099975586 + }, + { + "bits_per_byte": 1.434429210723195, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9942705631256104, + "step": 6770, + "unclipped_grad_norm": 0.8744387030601501 + }, + { + "bits_per_byte": 1.4315308730995298, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 0.9922615885734558, + "step": 6780, + "unclipped_grad_norm": 0.9047504663467407 + }, + { + "bits_per_byte": 1.53544803622169, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0642914772033691, + "step": 6790, + "unclipped_grad_norm": 0.9123643636703491 + }, + { + "bits_per_byte": 1.504298710480069, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.0427004098892212, + "step": 6800, + "unclipped_grad_norm": 0.978726327419281 + }, + { + "bits_per_byte": 1.4819994399691971, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.027243733406067, + "step": 6810, + "unclipped_grad_norm": 0.9510568380355835 + }, + { + "bits_per_byte": 1.4201767504722178, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 0.9843915104866028, + "step": 6820, + "unclipped_grad_norm": 0.9054594039916992 + }, + { + "bits_per_byte": 1.441129998766112, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 0.9989151954650879, + "step": 6830, + "unclipped_grad_norm": 0.9485244154930115 + }, + { + "bits_per_byte": 1.4736460706341397, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 1.021453619003296, + "step": 6840, + "unclipped_grad_norm": 0.9141860604286194 + }, + { + "bits_per_byte": 1.439767810179953, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 0.9979709982872009, + "step": 6850, + "unclipped_grad_norm": 0.9174992442131042 + }, + { + "bits_per_byte": 1.4496288654237992, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.0048061609268188, + "step": 6860, + "unclipped_grad_norm": 0.9317811727523804 + }, + { + "bits_per_byte": 1.4588201342338605, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 1.0111770629882812, + "step": 6870, + "unclipped_grad_norm": 0.9191097617149353 + }, + { + "bits_per_byte": 1.4995917173086046, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 1.0394377708435059, + "step": 6880, + "unclipped_grad_norm": 0.9317643642425537 + }, + { + "bits_per_byte": 1.3990970949299397, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9697802066802979, + "step": 6890, + "unclipped_grad_norm": 0.9130836725234985 + }, + { + "bits_per_byte": 1.4230204836042344, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 0.986362636089325, + "step": 6900, + "unclipped_grad_norm": 0.8910167217254639 + }, + { + "bits_per_byte": 1.5914774041330302, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.1031280755996704, + "step": 6910, + "unclipped_grad_norm": 1.0252712965011597 + }, + { + "bits_per_byte": 1.394897880535148, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 0.9668695330619812, + "step": 6920, + "unclipped_grad_norm": 0.8750832676887512 + }, + { + "bits_per_byte": 1.42465509270936, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9874956607818604, + "step": 6930, + "unclipped_grad_norm": 0.8865536451339722 + }, + { + "bits_per_byte": 1.4536008047454758, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 1.0075592994689941, + "step": 6940, + "unclipped_grad_norm": 0.913940966129303 + }, + { + "bits_per_byte": 1.4401447961506444, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9982323050498962, + "step": 6950, + "unclipped_grad_norm": 0.941871702671051 + }, + { + "bits_per_byte": 1.4963969675861772, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0372233390808105, + "step": 6960, + "unclipped_grad_norm": 0.9974261522293091 + }, + { + "bits_per_byte": 1.4704031657694705, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0192058086395264, + "step": 6970, + "unclipped_grad_norm": 0.9634589552879333 + }, + { + "bits_per_byte": 1.5157009882496222, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.0506038665771484, + "step": 6980, + "unclipped_grad_norm": 0.9805173873901367 + }, + { + "bits_per_byte": 1.4572141602401034, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.010063886642456, + "step": 6990, + "unclipped_grad_norm": 0.9425977468490601 + }, + { + "bits_per_byte": 1.4146713278441232, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 0.980575442314148, + "step": 7000, + "unclipped_grad_norm": 0.9211949110031128 + }, + { + "bits_per_byte": 1.4423556331274856, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 0.9997647404670715, + "step": 7010, + "unclipped_grad_norm": 0.9508169293403625 + }, + { + "bits_per_byte": 1.4268384124620634, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 0.9890090227127075, + "step": 7020, + "unclipped_grad_norm": 0.9370734095573425 + }, + { + "bits_per_byte": 1.4214180352548202, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 0.9852519035339355, + "step": 7030, + "unclipped_grad_norm": 0.8771427273750305 + }, + { + "bits_per_byte": 1.5093840654834314, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0462253093719482, + "step": 7040, + "unclipped_grad_norm": 0.9992957711219788 + }, + { + "bits_per_byte": 1.466185635222362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0162824392318726, + "step": 7050, + "unclipped_grad_norm": 0.9372990131378174 + }, + { + "bits_per_byte": 1.4783773133593756, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.024733066558838, + "step": 7060, + "unclipped_grad_norm": 1.0182687044143677 + }, + { + "bits_per_byte": 1.4142496263842075, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 0.9802831411361694, + "step": 7070, + "unclipped_grad_norm": 0.8880259394645691 + }, + { + "bits_per_byte": 1.3447492013960853, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9321091175079346, + "step": 7080, + "unclipped_grad_norm": 0.910698413848877 + }, + { + "bits_per_byte": 1.413691198716856, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 0.979896068572998, + "step": 7090, + "unclipped_grad_norm": 0.9244340062141418 + }, + { + "bits_per_byte": 1.4025504205679395, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 0.9721738696098328, + "step": 7100, + "unclipped_grad_norm": 0.9016135334968567 + }, + { + "bits_per_byte": 1.4533139376838367, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.0073604583740234, + "step": 7110, + "unclipped_grad_norm": 0.970280110836029 + }, + { + "bits_per_byte": 1.4606615524766484, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.0124534368515015, + "step": 7120, + "unclipped_grad_norm": 0.9771916270256042 + }, + { + "bits_per_byte": 1.396041221198084, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 0.9676620364189148, + "step": 7130, + "unclipped_grad_norm": 0.8866344094276428 + }, + { + "bits_per_byte": 1.5095894127685614, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0463676452636719, + "step": 7140, + "unclipped_grad_norm": 0.9947805404663086 + }, + { + "bits_per_byte": 1.5291790966150898, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 1.0599461793899536, + "step": 7150, + "unclipped_grad_norm": 0.9418985247612 + }, + { + "bits_per_byte": 1.4640222654571586, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0147829055786133, + "step": 7160, + "unclipped_grad_norm": 1.0358314514160156 + }, + { + "bits_per_byte": 1.4559156912260893, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0091638565063477, + "step": 7170, + "unclipped_grad_norm": 0.9539772272109985 + }, + { + "bits_per_byte": 1.3591113005869877, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 0.9420641660690308, + "step": 7180, + "unclipped_grad_norm": 0.8915972709655762 + }, + { + "bits_per_byte": 1.3840849873101015, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 0.9593746066093445, + "step": 7190, + "unclipped_grad_norm": 0.9118176102638245 + }, + { + "bits_per_byte": 1.4715360154907038, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0199910402297974, + "step": 7200, + "unclipped_grad_norm": 0.973518431186676 + }, + { + "bits_per_byte": 1.4678313372084681, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.017423152923584, + "step": 7210, + "unclipped_grad_norm": 0.9181845188140869 + }, + { + "bits_per_byte": 1.4265923052886786, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 0.9888384342193604, + "step": 7220, + "unclipped_grad_norm": 0.926552951335907 + }, + { + "bits_per_byte": 1.4337483314084294, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9937986135482788, + "step": 7230, + "unclipped_grad_norm": 0.9526586532592773 + }, + { + "bits_per_byte": 1.383381664259398, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 0.9588871002197266, + "step": 7240, + "unclipped_grad_norm": 0.8890237212181091 + }, + { + "bits_per_byte": 1.431437056563484, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 0.9921965599060059, + "step": 7250, + "unclipped_grad_norm": 0.9829798340797424 + }, + { + "bits_per_byte": 1.4272578781475178, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9892997741699219, + "step": 7260, + "unclipped_grad_norm": 0.9537968039512634 + }, + { + "bits_per_byte": 1.5069463833901018, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 1.0445356369018555, + "step": 7270, + "unclipped_grad_norm": 0.9953339099884033 + }, + { + "bits_per_byte": 1.4318632295723221, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 0.9924919605255127, + "step": 7280, + "unclipped_grad_norm": 0.9815621972084045 + }, + { + "bits_per_byte": 1.435238647069481, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.9948316216468811, + "step": 7290, + "unclipped_grad_norm": 0.9373346567153931 + }, + { + "bits_per_byte": 1.44098209368637, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 0.9988126754760742, + "step": 7300, + "unclipped_grad_norm": 0.9782799482345581 + }, + { + "bits_per_byte": 1.467732619166873, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0173547267913818, + "step": 7310, + "unclipped_grad_norm": 0.9545702338218689 + }, + { + "bits_per_byte": 1.4680886232541588, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 1.017601490020752, + "step": 7320, + "unclipped_grad_norm": 0.9820490479469299 + }, + { + "bits_per_byte": 1.4597027491980883, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 1.0117888450622559, + "step": 7330, + "unclipped_grad_norm": 1.0537965297698975 + }, + { + "bits_per_byte": 1.4358094574876945, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9952272772789001, + "step": 7340, + "unclipped_grad_norm": 0.9470505714416504 + }, + { + "bits_per_byte": 1.4763711357370608, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.023342490196228, + "step": 7350, + "unclipped_grad_norm": 0.93792724609375 + }, + { + "bits_per_byte": 1.4524729425211176, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 1.0067775249481201, + "step": 7360, + "unclipped_grad_norm": 0.959013044834137 + }, + { + "bits_per_byte": 1.4678055398108387, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0174052715301514, + "step": 7370, + "unclipped_grad_norm": 0.9769075512886047 + }, + { + "bits_per_byte": 1.5064952728968888, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 1.0442229509353638, + "step": 7380, + "unclipped_grad_norm": 1.0261726379394531 + }, + { + "bits_per_byte": 1.4743861119808024, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.021966576576233, + "step": 7390, + "unclipped_grad_norm": 0.977978527545929 + }, + { + "bits_per_byte": 1.4106443540741704, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9777841567993164, + "step": 7400, + "unclipped_grad_norm": 0.9216173887252808 + }, + { + "bits_per_byte": 1.4343313525948542, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.994202733039856, + "step": 7410, + "unclipped_grad_norm": 0.9778451919555664 + }, + { + "bits_per_byte": 1.5075760118749106, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0449720621109009, + "step": 7420, + "unclipped_grad_norm": 1.0143206119537354 + }, + { + "bits_per_byte": 1.4176087055295346, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 0.9826114773750305, + "step": 7430, + "unclipped_grad_norm": 0.9865745902061462 + }, + { + "bits_per_byte": 1.4384498211350658, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 0.9970574378967285, + "step": 7440, + "unclipped_grad_norm": 0.9753794074058533 + }, + { + "bits_per_byte": 1.385612537215065, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 0.9604334235191345, + "step": 7450, + "unclipped_grad_norm": 0.9882722496986389 + }, + { + "bits_per_byte": 1.40335280562554, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.9727300405502319, + "step": 7460, + "unclipped_grad_norm": 0.8938714861869812 + }, + { + "bits_per_byte": 1.4650283639647061, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 1.0154802799224854, + "step": 7470, + "unclipped_grad_norm": 0.95097815990448 + }, + { + "bits_per_byte": 1.378530205661209, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 0.9555243253707886, + "step": 7480, + "unclipped_grad_norm": 0.9593113660812378 + }, + { + "bits_per_byte": 1.4079058743244819, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 0.9758859872817993, + "step": 7490, + "unclipped_grad_norm": 0.9498793482780457 + }, + { + "bits_per_byte": 1.4795925427703722, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0255753993988037, + "step": 7500, + "unclipped_grad_norm": 0.9999135732650757 + }, + { + "bits_per_byte": 1.462181535144974, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 1.0135070085525513, + "step": 7510, + "unclipped_grad_norm": 0.96773761510849 + }, + { + "bits_per_byte": 1.424363066168195, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 0.9872932434082031, + "step": 7520, + "unclipped_grad_norm": 0.9349227547645569 + }, + { + "bits_per_byte": 1.4575968216382733, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0103291273117065, + "step": 7530, + "unclipped_grad_norm": 0.9571959972381592 + }, + { + "bits_per_byte": 1.484410464751643, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0289149284362793, + "step": 7540, + "unclipped_grad_norm": 0.9889418482780457 + }, + { + "bits_per_byte": 1.4190770074112757, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 0.9836292266845703, + "step": 7550, + "unclipped_grad_norm": 0.9660657048225403 + }, + { + "bits_per_byte": 1.458957376389249, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0112721920013428, + "step": 7560, + "unclipped_grad_norm": 0.9621381163597107 + }, + { + "bits_per_byte": 1.4744461339259534, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 1.0220081806182861, + "step": 7570, + "unclipped_grad_norm": 0.9809478521347046 + }, + { + "bits_per_byte": 1.424117474942763, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 0.9871230125427246, + "step": 7580, + "unclipped_grad_norm": 1.0034832954406738 + }, + { + "bits_per_byte": 1.4167208450944553, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9819960594177246, + "step": 7590, + "unclipped_grad_norm": 0.9541960954666138 + }, + { + "bits_per_byte": 1.3862604818521906, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9608825445175171, + "step": 7600, + "unclipped_grad_norm": 0.950609564781189 + }, + { + "bits_per_byte": 1.3616721222583348, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 0.9438391923904419, + "step": 7610, + "unclipped_grad_norm": 0.9224535226821899 + }, + { + "bits_per_byte": 1.4445553772233464, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 1.0012894868850708, + "step": 7620, + "unclipped_grad_norm": 0.9717100858688354 + }, + { + "bits_per_byte": 1.4175787805482842, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9825907349586487, + "step": 7630, + "unclipped_grad_norm": 0.9714697599411011 + }, + { + "bits_per_byte": 1.4152020663046858, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.9809433221817017, + "step": 7640, + "unclipped_grad_norm": 0.9795277118682861 + }, + { + "bits_per_byte": 1.5113582543326856, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.0475937128067017, + "step": 7650, + "unclipped_grad_norm": 1.0563151836395264 + }, + { + "bits_per_byte": 1.4375178471500403, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 0.9964114427566528, + "step": 7660, + "unclipped_grad_norm": 0.9650751948356628 + }, + { + "bits_per_byte": 1.4172367070557181, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 0.9823536276817322, + "step": 7670, + "unclipped_grad_norm": 0.9894714951515198 + }, + { + "bits_per_byte": 1.3937812831744212, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 0.9660955667495728, + "step": 7680, + "unclipped_grad_norm": 0.9648637175559998 + }, + { + "bits_per_byte": 1.4266868097553278, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 0.9889039397239685, + "step": 7690, + "unclipped_grad_norm": 0.9686183333396912 + }, + { + "bits_per_byte": 1.4503658110827464, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 1.005316972732544, + "step": 7700, + "unclipped_grad_norm": 0.9993582367897034 + }, + { + "bits_per_byte": 1.3691781330725912, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 0.9490419626235962, + "step": 7710, + "unclipped_grad_norm": 0.9620972275733948 + }, + { + "bits_per_byte": 1.3650053180147068, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 0.9461495876312256, + "step": 7720, + "unclipped_grad_norm": 0.9128993153572083 + }, + { + "bits_per_byte": 1.4252105106803215, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 0.9878806471824646, + "step": 7730, + "unclipped_grad_norm": 0.9594771265983582 + }, + { + "bits_per_byte": 1.4081796707046554, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9760757684707642, + "step": 7740, + "unclipped_grad_norm": 0.9255442023277283 + }, + { + "bits_per_byte": 1.433292233418341, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9934824705123901, + "step": 7750, + "unclipped_grad_norm": 0.9785766005516052 + }, + { + "bits_per_byte": 1.3999133245909345, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 0.9703459739685059, + "step": 7760, + "unclipped_grad_norm": 1.0032256841659546 + }, + { + "bits_per_byte": 1.4048038232508695, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9737358093261719, + "step": 7770, + "unclipped_grad_norm": 1.0204466581344604 + }, + { + "bits_per_byte": 1.536527399338505, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0650396347045898, + "step": 7780, + "unclipped_grad_norm": 1.0556561946868896 + }, + { + "bits_per_byte": 1.4128328333063997, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9793010950088501, + "step": 7790, + "unclipped_grad_norm": 1.0078489780426025 + }, + { + "bits_per_byte": 1.451735308931567, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0062662363052368, + "step": 7800, + "unclipped_grad_norm": 1.0289596319198608 + }, + { + "bits_per_byte": 1.3852072600983067, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9601525068283081, + "step": 7810, + "unclipped_grad_norm": 0.9891387224197388 + }, + { + "bits_per_byte": 1.3953124447150527, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.9671568870544434, + "step": 7820, + "unclipped_grad_norm": 0.9778857231140137 + }, + { + "bits_per_byte": 1.45392653988621, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0077850818634033, + "step": 7830, + "unclipped_grad_norm": 1.0185887813568115 + }, + { + "bits_per_byte": 1.4113820736550464, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9782955050468445, + "step": 7840, + "unclipped_grad_norm": 0.9330953359603882 + }, + { + "bits_per_byte": 1.380841996454107, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9571267366409302, + "step": 7850, + "unclipped_grad_norm": 0.9742368459701538 + }, + { + "bits_per_byte": 1.3846048048723343, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 0.9597349166870117, + "step": 7860, + "unclipped_grad_norm": 0.9645414352416992 + }, + { + "bits_per_byte": 1.4429803601067448, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0001977682113647, + "step": 7870, + "unclipped_grad_norm": 0.9958038926124573 + }, + { + "bits_per_byte": 1.4256185395194934, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9881634712219238, + "step": 7880, + "unclipped_grad_norm": 0.9739922881126404 + }, + { + "bits_per_byte": 1.4834710955126302, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.028263807296753, + "step": 7890, + "unclipped_grad_norm": 0.9912284016609192 + }, + { + "bits_per_byte": 1.4598313922209336, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.0118780136108398, + "step": 7900, + "unclipped_grad_norm": 0.9979069232940674 + }, + { + "bits_per_byte": 1.465345499972897, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.015700101852417, + "step": 7910, + "unclipped_grad_norm": 1.0503500699996948 + }, + { + "bits_per_byte": 1.3660396216769959, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 0.946866512298584, + "step": 7920, + "unclipped_grad_norm": 0.9824531078338623 + }, + { + "bits_per_byte": 1.3848080023743288, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.959875762462616, + "step": 7930, + "unclipped_grad_norm": 1.0091050863265991 + }, + { + "bits_per_byte": 1.4375059803471308, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9964032173156738, + "step": 7940, + "unclipped_grad_norm": 1.0658900737762451 + }, + { + "bits_per_byte": 1.4049340141175726, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 0.9738260507583618, + "step": 7950, + "unclipped_grad_norm": 1.0148085355758667 + }, + { + "bits_per_byte": 1.415199744538899, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9809417128562927, + "step": 7960, + "unclipped_grad_norm": 0.9737400412559509 + }, + { + "bits_per_byte": 1.4008050546356583, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9709640741348267, + "step": 7970, + "unclipped_grad_norm": 0.9626508951187134 + }, + { + "bits_per_byte": 1.4372590132604919, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 0.9962320327758789, + "step": 7980, + "unclipped_grad_norm": 0.9673776626586914 + }, + { + "bits_per_byte": 1.4403348369798477, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 0.9983640313148499, + "step": 7990, + "unclipped_grad_norm": 0.9839751720428467 + }, + { + "bits_per_byte": 1.3635582559903472, + "learning_rate": 3e-05, + "loss_nats": 0.9451465606689453, + "step": 8000, + "unclipped_grad_norm": 0.9773924946784973 + } + ] +} diff --git a/experiments/k3/attnres_spike/results/raw/replay-seed-2026073001.json b/experiments/k3/attnres_spike/results/raw/replay-seed-2026073001.json new file mode 100644 index 0000000..1078d2f --- /dev/null +++ b/experiments/k3/attnres_spike/results/raw/replay-seed-2026073001.json @@ -0,0 +1,86460 @@ +{ + "architecture": "block", + "artifacts": { + "protocol_sha256": "6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16", + "runner_sha256": "77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084", + "scoping_sha256": "590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27" + }, + "batch_size": 32, + "canonical_sha256_without_self": "c9ebccba0acf10b05fea65beade50e588fd9a83b1807f7ebba8ec1ab836b26fb", + "depth": 32, + "diagnostics": [ + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "initialization_negative_control": { + "comparisons": { + "detached_learned": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + }, + "uniform_value_backward": { + "attention_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "mlp_branch_output": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_attention_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "post_mlp_state": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_attention_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + }, + "pre_mlp_input": { + "max_normalized_absolute_error": 0.0, + "max_raw_relative_error": 0.0, + "passed": true + } + } + }, + "passed": true + }, + "loss_scale_gate": { + "checks": { + "attention_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "mlp_branch_output": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_attention_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "post_mlp_state": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_attention_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + }, + "pre_mlp_input": { + "batch_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "element_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "global_l2": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_mean_rms": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_mean": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_median": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + }, + "token_rms_p95": { + "cv_absolute_error": 0.0, + "max_normalized_absolute_error": 0.0, + "max_scale_ratio_error": 0.0, + "passed": true, + "spike_contrast_absolute_error": 0.0 + } + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 7.959341947457508, + "forward": { + "activation_sha256": { + "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" + }, + "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", + "loss_nats": 5.516995429992676, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0021613112185150385, + 0.0025601957459002733, + 0.0025323182344436646, + 0.0027253651060163975, + 0.002904455177485943, + 0.0029208383057266474, + 0.00299624796025455, + 0.0030937574338167906, + 0.0033804215490818024, + 0.0034370317589491606, + 0.0038969130255281925, + 0.004255065228790045, + 0.004119740333408117, + 0.003970506135374308, + 0.004385598469525576, + 0.0041999719105660915, + 0.005232699681073427, + 0.004664790350943804, + 0.004898969549685717, + 0.005131533369421959, + 0.005069608334451914, + 0.0051318020559847355, + 0.0058372304774820805, + 0.0059690922498703, + 0.00554921617731452, + 0.0058550527319312096, + 0.006255472078919411, + 0.006502739619463682, + 0.006122888531535864, + 0.007071305997669697, + 0.006241494789719582 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962477027642308, + "first_to_last_ratio": 3.7699040072151497, + "imbalance_abs_log_ratio": 1.3270495388586014, + "last_quartile_mean": 4.234186599205714e-05, + "mean": 8.617327489446325e-05, + "normalized": [ + 2.5428430303860874, + 2.3222064178412967, + 2.1011513502014654, + 1.8911470095267005, + 1.7004447966680725, + 1.5353700031893112, + 1.4229335846259188, + 1.3028620492588128, + 1.2049362956920768, + 1.1705197150696323, + 1.0773612761873523, + 1.0133200843661747, + 0.9906112963967714, + 0.9272530070793641, + 0.8550207003106389, + 0.8185600560490035, + 0.7703816559177782, + 0.7173918665339755, + 0.6760157258502337, + 0.6514535263006792, + 0.6184138115448249, + 0.6053606047045522, + 0.5838821473278404, + 0.5697017460729616, + 0.5457761369570343, + 0.5276086337508958, + 0.5104655265174319, + 0.4951752820428687, + 0.47527333481689216, + 0.465789239220051, + 0.45896414036514044, + 0.45180594922816086 + ], + "peak_layer": 1, + "peak_normalized": 2.5428430303860874, + "population_cv": 0.5705341151299013, + "rest_mean": 9.280180475562466e-05, + "spike_contrast": 0.5428688997682254, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0379213644191625e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021912511147093028, + 0.00020011213200632483, + 0.00018106309289578348, + 0.00016296633111778647, + 0.00014653289690613747, + 0.00013230786134954542, + 0.0001226188469445333, + 0.00011227188952034339, + 0.0001038333066389896, + 0.00010086751717608422, + 9.283974941354245e-05, + 8.732111018616706e-05, + 8.536421955795959e-05, + 7.990442827576771e-05, + 7.367993384832516e-05, + 7.053800072753802e-05, + 6.63863102090545e-05, + 6.182000652188435e-05, + 5.825448897667229e-05, + 5.6137883802875876e-05, + 5.329074338078499e-05, + 5.216590579948388e-05, + 5.031503678765148e-05, + 4.909306517220102e-05, + 4.703131708083674e-05, + 4.5465763832908124e-05, + 4.3988486140733585e-05, + 4.2670875700423494e-05, + 4.095585973118432e-05, + 4.013858415419236e-05, + 3.955044303438626e-05, + 3.893359826179221e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442661636683624, + "first_to_last_ratio": 1.843144421770586, + "imbalance_abs_log_ratio": 0.6114730379455638, + "last_quartile_mean": 0.0001163373926829081, + "mean": 0.00015221274247778638, + "normalized": [ + 1.7741446897870732, + 1.648461275666186, + 1.5301464224684427, + 1.4217121808823485, + 1.3265065496562456, + 1.2454420544341354, + 1.1897998831173078, + 1.1336248235930768, + 1.0869089599614004, + 1.0643003125599004, + 1.0227030055526172, + 0.9941765648180964, + 0.9788958472869692, + 0.9511102301475627, + 0.9228060651723192, + 0.9052418824754537, + 0.8846484396889256, + 0.865292475366091, + 0.8497311645646503, + 0.8388691935254817, + 0.8256303537004397, + 0.8176325372729673, + 0.8076852907347736, + 0.8000670639426607, + 0.7892588692968658, + 0.7814415503461742, + 0.7731202151361662, + 0.7664994567889658, + 0.758490072461763, + 0.7529625284402992, + 0.7485174912671357, + 0.7441725498875049 + ], + "peak_layer": 1, + "peak_normalized": 1.7741446897870732, + "population_cv": 0.27364872432853743, + "rest_mean": 0.00015762320577373935, + "spike_contrast": 0.7803181014868814, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299624067964033, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027004742878489196, + 0.0002509168116375804, + 0.0002329077833564952, + 0.00021640271006617695, + 0.00020191119983792305, + 0.00018957215070258826, + 0.0001811027032090351, + 0.00017255214333999902, + 0.00016544139361940324, + 0.00016200006939470768, + 0.00015566842921543866, + 0.0001513263414381072, + 0.00014900042151566595, + 0.0001447710965294391, + 0.00014046284195501357, + 0.0001377893495373428, + 0.00013465476513374597, + 0.00013170854072086513, + 0.00012933991092722863, + 0.0001276865805266425, + 0.00012567146040964872, + 0.00012445409083738923, + 0.00012293999316170812, + 0.00012178040196886286, + 0.00012013525702059269, + 0.00011894536146428436, + 0.00011767874821089208, + 0.00011667098442558199, + 0.00011545185407157987, + 0.00011461049143690616, + 0.00011393390013836324, + 0.00011327254469506443 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19015566632151604, + "first_to_last_ratio": 1.8431444167877922, + "imbalance_abs_log_ratio": 0.6114730352421437, + "last_quartile_mean": 0.10316916275769472, + "mean": 0.13498378219082952, + "normalized": [ + 1.7741447141792424, + 1.6484611693507762, + 1.5301464164964773, + 1.421712099711509, + 1.3265065258282553, + 1.245442086656075, + 1.1897999853143832, + 1.1336247998774232, + 1.0869090145314644, + 1.0643003539468234, + 1.0227030148964142, + 0.9941765578539046, + 0.978895844117059, + 0.9511102293531465, + 0.9228060470726628, + 0.9052419251832852, + 0.8846484748200404, + 0.8652925213936268, + 0.8497311905993837, + 0.8388692019864461, + 0.8256303029992552, + 0.8176324960789627, + 0.8076852727084025, + 0.800067049482784, + 0.789258877083647, + 0.7814415614650243, + 0.7731201949085273, + 0.7664994759042474, + 0.758490077799444, + 0.7529625178867759, + 0.7485174642151933, + 0.7441725362993381 + ], + "peak_layer": 1, + "peak_normalized": 1.7741447141792424, + "population_cv": 0.2736487184871982, + "rest_mean": 0.13978183573042904, + "spike_contrast": 0.7803180757143806, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907429307699204, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23948076367378235, + 0.2225155234336853, + 0.20654495060443878, + 0.1919080764055252, + 0.17905686795711517, + 0.16811448335647583, + 0.16060370206832886, + 0.1530209630727768, + 0.14671508967876434, + 0.14366328716278076, + 0.13804832100868225, + 0.13419771194458008, + 0.1321350634098053, + 0.12838445603847504, + 0.12456385046243668, + 0.12219297885894775, + 0.11941319704055786, + 0.116800457239151, + 0.11469992995262146, + 0.11323373764753342, + 0.11144670099020004, + 0.11036712676286697, + 0.10902441293001175, + 0.10799607634544373, + 0.10653714835643768, + 0.10548193752765656, + 0.10435868799686432, + 0.1034649983048439, + 0.10238385945558548, + 0.10163772851228714, + 0.10103771835565567, + 0.10045122355222702 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883365585963475e-05, + "first_to_last_ratio": 2.2497546817252747, + "imbalance_abs_log_ratio": 0.8108211799277664, + "last_quartile_mean": 3.059607183786284e-05, + "mean": 4.691841348858361e-05, + "normalized": [ + 1.7359303646705315, + 1.6475859007944922, + 1.5536416135247637, + 1.4921074478374423, + 1.436565604934822, + 1.343834173349448, + 1.2787439414688093, + 1.2483315192759759, + 1.191227587726315, + 1.1603086071531024, + 1.0995306735006558, + 1.059977946939621, + 1.0583984121521266, + 1.0176934649371145, + 0.9754909410255406, + 0.9631621800279858, + 0.9242301483750994, + 0.8752327792179452, + 0.8454160845678106, + 0.8232909667825862, + 0.7863453139125789, + 0.77746685672799, + 0.7497386589361914, + 0.7388508716017261, + 0.7093180450413769, + 0.6908354487677191, + 0.6685471901846501, + 0.6592354484095886, + 0.631440334976849, + 0.6235901145414114, + 0.6199506557967245, + 0.6139807028410055 + ], + "peak_layer": 1, + "peak_normalized": 1.7359303646705315, + "population_cv": 0.3220412433027962, + "rest_mean": 4.907019663530971e-05, + "spike_contrast": 0.7193528234378936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.529878449626267e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144709863699973e-05, + 7.730211655143648e-05, + 7.289439963642508e-05, + 7.000731420703232e-05, + 6.740137905580923e-05, + 6.305056740529835e-05, + 5.999663699185476e-05, + 5.856973439222202e-05, + 5.5890508519951254e-05, + 5.4439839004771784e-05, + 5.1588234782684594e-05, + 4.9732483603293076e-05, + 4.965837433701381e-05, + 4.774856279254891e-05, + 4.576848732540384e-05, + 4.519004141911864e-05, + 4.3363412260077894e-05, + 4.106453343410976e-05, + 3.966558142565191e-05, + 3.862750600092113e-05, + 3.6894074582960457e-05, + 3.647751145763323e-05, + 3.517654840834439e-05, + 3.4665710700210184e-05, + 3.328007733216509e-05, + 3.241290323785506e-05, + 3.136717350571416e-05, + 3.0930281354812905e-05, + 2.9626178729813546e-05, + 2.925785884144716e-05, + 2.9087101211189292e-05, + 2.8807000489905477e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001734125216898974, + "first_to_last_ratio": 1.495517223041696, + "imbalance_abs_log_ratio": 0.4024721155990803, + "last_quartile_mean": 0.00011595488103921525, + "mean": 0.00013990532806928968, + "normalized": [ + 1.3768889945483427, + 1.3340940553978193, + 1.2910049686488183, + 1.2527203273458163, + 1.2147536525239127, + 1.1769854337196524, + 1.14744939519443, + 1.1220955988146075, + 1.0944022472959734, + 1.0773022658134643, + 1.0508559212107247, + 1.03206209473486, + 1.0208171898483585, + 1.000561882500425, + 0.9805161605170362, + 0.9663129296065233, + 0.9487424991891081, + 0.931931984738623, + 0.9181974345490227, + 0.9075104531838204, + 0.8943640941785878, + 0.8862664021756573, + 0.8757830774574916, + 0.86790401991425, + 0.8562894022841879, + 0.8478155487572495, + 0.8387227683262297, + 0.8314011142413483, + 0.8224071457687988, + 0.8161875050523302, + 0.8112199678685613, + 0.8064334645939688 + ], + "peak_layer": 1, + "peak_normalized": 1.3768889945483427, + "population_cv": 0.16325957486062354, + "rest_mean": 0.0001431148236776978, + "spike_contrast": 0.8564734849544942, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225740517838858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926341064972803, + 0.00018664686649572104, + 0.00018061847367789596, + 0.00017526224837638438, + 0.0001699505082797259, + 0.00016466653323732316, + 0.00016053428407758474, + 0.00015698715287726372, + 0.00015311270544771105, + 0.00015072032692842185, + 0.00014702034241054207, + 0.00014439098595175892, + 0.00014281776384450495, + 0.00013998393842484802, + 0.00013717943511437625, + 0.00013519232743419707, + 0.00013273413060232997, + 0.00013038225006312132, + 0.00012846071331296116, + 0.00012696554767899215, + 0.0001251263020094484, + 0.00012399339175317436, + 0.0001225267187692225, + 0.00012142439663875848, + 0.00011979944974882528, + 0.00011861391249112785, + 0.00011734178406186402, + 0.00011631744564510882, + 0.00011505914153531194, + 0.00011418898066040128, + 0.00011349399574100971, + 0.00011282433843007311 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675117381557357, + "first_to_last_ratio": 1.2676837465168667, + "imbalance_abs_log_ratio": 0.23719141364257146, + "last_quartile_mean": 0.00011576323686313117, + "mean": 0.00013042020850662084, + "normalized": [ + 1.169988069421413, + 1.156175152925158, + 1.1417665260225645, + 1.1306726331060184, + 1.1183621029407895, + 1.1062694829504214, + 1.0940115056230622, + 1.0844989956600726, + 1.0711591662817939, + 1.0579765491085218, + 1.0443900226505625, + 1.035471661013805, + 1.0260498632741444, + 1.013992390086173, + 1.0062479316494695, + 0.9977001177294099, + 0.9863163473759529, + 0.9769959732660659, + 0.9687585669786032, + 0.9602628595878113, + 0.9503845997255937, + 0.9426104617668664, + 0.9330243108845692, + 0.9259760938667744, + 0.914797625470175, + 0.906289421438531, + 0.8956357003805738, + 0.888470885240542, + 0.8806216558595509, + 0.8756991509238615, + 0.8712858841028995, + 0.8681382926882502 + ], + "peak_layer": 1, + "peak_normalized": 1.169988069421413, + "population_cv": 0.09123006364857297, + "rest_mean": 0.00013202972385678786, + "spike_contrast": 0.9219804606102014, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012172882561571896, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015259008796419948, + 0.00015078860451467335, + 0.00014890942838974297, + 0.0001474625605624169, + 0.00014585701865144074, + 0.00014427989663090557, + 0.00014268120867200196, + 0.00014144058513920754, + 0.00013970080181024969, + 0.0001379815221298486, + 0.00013620956451632082, + 0.00013504642993211746, + 0.0001338176371064037, + 0.00013224509893916547, + 0.00013123506505507976, + 0.0001301202573813498, + 0.00012863558367826045, + 0.00012742001854348928, + 0.00012634569429792464, + 0.00012523768236860633, + 0.00012394935765769333, + 0.00012293545296415687, + 0.00012168522516731173, + 0.00012076599523425102, + 0.00011930809705518186, + 0.00011819845531135798, + 0.00011680899478960782, + 0.00011587455810513347, + 0.00011485085997264832, + 0.00011420886585256085, + 0.00011363328667357564, + 0.00011322277714498341 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030002388848515693, + "first_to_last_ratio": 2.2836575149168885, + "imbalance_abs_log_ratio": 0.8257783308640597, + "last_quartile_mean": 0.0001313786706305109, + "mean": 0.0001947244400071213, + "normalized": [ + 1.8998833386658438, + 1.7835645331795598, + 1.6713073472446418, + 1.5703972875779748, + 1.4780250735507643, + 1.3761669788219328, + 1.3066447563403325, + 1.2401013786965793, + 1.1762106459027857, + 1.150547637187856, + 1.0768735950666006, + 1.0264900078881174, + 1.0140784159549234, + 0.9666871251601796, + 0.9096841069789114, + 0.8754807117769676, + 0.8421876125778792, + 0.8073809902734768, + 0.77911667677541, + 0.7607651063686129, + 0.7403822030704592, + 0.7308855611321243, + 0.7152791523187013, + 0.7043381880055872, + 0.6944157299089508, + 0.686864629891504, + 0.680599049276497, + 0.6764772714226681, + 0.6702412094778135, + 0.6672258960044359, + 0.6635863560708457, + 0.6581114274310631 + ], + "peak_layer": 1, + "peak_normalized": 1.8998833386658438, + "population_cv": 0.3592853832280775, + "rest_mean": 0.00020492727344390005, + "spike_contrast": 0.6813594750078023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962913944851608, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699537192005664, + 0.0003473036049399525, + 0.0003254443872720003, + 0.00030579473241232336, + 0.00028780760476365685, + 0.00026797334430739284, + 0.0002544356684666127, + 0.0002414780465187505, + 0.0002290369593538344, + 0.00022403974435292184, + 0.00020969360775779933, + 0.00019988269195891917, + 0.0001974658516701311, + 0.00018823760910890996, + 0.00017713772831484675, + 0.00017047749133780599, + 0.00016399451124016196, + 0.0001572168112033978, + 0.00015171305858530104, + 0.00014813955931458622, + 0.00014417050988413393, + 0.00014232128160074353, + 0.00013928233238402754, + 0.0001371518592350185, + 0.00013521971413865685, + 0.00013374933041632175, + 0.00013252926873974502, + 0.00013172665785532445, + 0.00013051234418526292, + 0.00012992518895771354, + 0.00012921648158226162, + 0.000128150379168801 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027073738165199757, + 0.002722575329244137, + 0.0026850115973502398, + 0.0026861224323511124, + 0.00271838391199708, + 0.0026645169127732515, + 0.0026859452482312918, + 0.0027359756641089916, + 0.0026767742820084095, + 0.0027451682835817337, + 0.0026999011170119047, + 0.002663193503394723, + 0.002626648638397455, + 0.0026531771291047335, + 0.0026517712976783514, + 0.002676903735846281, + 0.002670086920261383, + 0.0026355870068073273, + 0.0026610493659973145, + 0.0026388936676084995, + 0.0026469926815479994, + 0.00264862016774714, + 0.002641494618728757, + 0.002674198243767023, + 0.002557571977376938, + 0.00252733426168561, + 0.0025468147359788418, + 0.0026856842450797558, + 0.002551089273765683, + 0.0025778315030038357, + 0.0026001501828432083, + 0.0025662274565547705 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858857113926206, + "first_to_last_ratio": 3.757867690304466, + "imbalance_abs_log_ratio": 1.3238516928952826, + "last_quartile_mean": 4.220174423608114e-05, + "mean": 8.573221236929385e-05, + "normalized": [ + 2.5461641896458365, + 2.3032502265274117, + 2.0924715218207024, + 1.8886722264368014, + 1.7031663195050915, + 1.537434754996766, + 1.4223955235407997, + 1.3049501874408618, + 1.2055297161045848, + 1.1692542365736316, + 1.0769571962730058, + 1.0144505847411098, + 0.9913601784485884, + 0.9317001386730132, + 0.857757082878185, + 0.8155634736467603, + 0.7682612945963089, + 0.7217806418748633, + 0.678338533281248, + 0.6511820384364709, + 0.6192882306865346, + 0.6053793146733001, + 0.5848721334223365, + 0.5718144052517429, + 0.5462281883927541, + 0.527515083157495, + 0.5112993585545178, + 0.496920907994256, + 0.47722358366421574, + 0.46552107689700334, + 0.4601234027970099, + 0.45317424906679377 + ], + "peak_layer": 1, + "peak_normalized": 2.5461641896458365, + "population_cv": 0.5686091737198813, + "rest_mean": 9.231269224848668e-05, + "spike_contrast": 0.5437781067692293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0197621021652594e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021828828903380781, + 0.00019746273756027222, + 0.00017939221288543195, + 0.0001619200484128669, + 0.00014601621660403907, + 0.000131807682919316, + 0.00012194511509733275, + 0.00011187626660102978, + 0.00010335272963857278, + 0.00010024275252362713, + 9.232992306351662e-05, + 8.697109296917915e-05, + 8.499150135321543e-05, + 7.987671415321529e-05, + 7.353741239057854e-05, + 6.992006092332304e-05, + 6.586474046343938e-05, + 6.187985127326101e-05, + 5.8155463193543255e-05, + 5.582727681030519e-05, + 5.309295011102222e-05, + 5.1900507969548926e-05, + 5.0142381951445714e-05, + 4.902291402686387e-05, + 4.682935104938224e-05, + 4.522503513726406e-05, + 4.3834825191879645e-05, + 4.260212881490588e-05, + 4.0913433622336015e-05, + 3.991015182691626e-05, + 3.944739728467539e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242323236947414, + "first_to_last_ratio": 1.8345651540958352, + "imbalance_abs_log_ratio": 0.6068074801791611, + "last_quartile_mean": 0.0001157894184871111, + "mean": 0.00015111673860701558, + "normalized": [ + 1.771707915810251, + 1.6366668487510905, + 1.5242909365839221, + 1.4186705636795862, + 1.3264449439106374, + 1.2453857912414694, + 1.1893415085703798, + 1.1330085309323608, + 1.0858463851390703, + 1.062861239127963, + 1.023237806768965, + 0.9942126959777651, + 0.978436257513136, + 0.9531027445378829, + 0.9232414968561398, + 0.904013237249231, + 0.8853704564278231, + 0.8680164030054869, + 0.8515402788738432, + 0.8395817783841374, + 0.8267435189825344, + 0.8185143642831039, + 0.809172703595369, + 0.8007918347797284, + 0.7909896391756025, + 0.7821052393452608, + 0.7753388667989797, + 0.7682453290925442, + 0.7604995795745633, + 0.7549901567959042, + 0.7508572350862155, + 0.7467737131490535 + ], + "peak_layer": 1, + "peak_normalized": 1.771707915810251, + "population_cv": 0.2714170004285523, + "rest_mean": 0.00015645500981783562, + "spike_contrast": 0.7816309251520459, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229007406858728, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677347220014781, + 0.00024732775636948645, + 0.00023034587502479553, + 0.00021438486874103546, + 0.00020044803386554122, + 0.0001881986390799284, + 0.00017972940986510366, + 0.00017121655400842428, + 0.00016408956435043365, + 0.00016061612404882908, + 0.0001546283601783216, + 0.00015024218009784818, + 0.00014785809617023915, + 0.0001440297783119604, + 0.00013951724395155907, + 0.000136611532070674, + 0.0001337942958343774, + 0.00013117180787958205, + 0.0001286819897359237, + 0.00012687486014328897, + 0.00012493478425312787, + 0.00012369122123345733, + 0.00012227953993715346, + 0.00012101305037504062, + 0.00011953177454415709, + 0.00011818919301731512, + 0.00011716668086592108, + 0.00011609472858253866, + 0.00011492421617731452, + 0.00011409165017539635, + 0.00011346709652571008, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837904930114746, + "first_to_last_ratio": 1.8345651581276443, + "imbalance_abs_log_ratio": 0.6068074823768531, + "last_quartile_mean": 0.10268321540206671, + "mean": 0.13401183625683188, + "normalized": [ + 1.7717078885231814, + 1.6366668119471057, + 1.524290957183596, + 1.418670514273678, + 1.326444920903592, + 1.2453858718827995, + 1.1893415500975306, + 1.1330085716119305, + 1.0858463365841577, + 1.062861212873729, + 1.0232377434953237, + 0.9942127340740732, + 0.9784362443707608, + 0.9531027271661117, + 0.9232414917376033, + 0.904013241653558, + 0.885370477646398, + 0.8680164390463408, + 0.8515402681847718, + 0.8395816969955481, + 0.8267435345782925, + 0.8185144279720658, + 0.8091727254708025, + 0.8007918405918669, + 0.7909896328038464, + 0.7821052665924377, + 0.7753388466443417, + 0.7682453528580236, + 0.7604996016741158, + 0.7549901615676585, + 0.7508572336421532, + 0.7467736753526057 + ], + "peak_layer": 1, + "peak_normalized": 1.7717078885231814, + "population_cv": 0.27141699472708714, + "rest_mean": 0.1387458680956452, + "spike_contrast": 0.7816309473985974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844806432724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23742982745170593, + 0.2193327248096466, + 0.2042730301618576, + 0.19011864066123962, + 0.1777593195438385, + 0.16689644753932953, + 0.15938584506511688, + 0.151836559176445, + 0.1455162614583969, + 0.1424359828233719, + 0.13712596893310547, + 0.13323627412319183, + 0.13112203776836395, + 0.12772704660892487, + 0.1237252876162529, + 0.1211484745144844, + 0.11865012347698212, + 0.11632447689771652, + 0.11411647498607635, + 0.11251388490200043, + 0.11079341918230057, + 0.10969062149524689, + 0.10843872278928757, + 0.10731558501720428, + 0.10600197315216064, + 0.10481136292219162, + 0.10390458256006241, + 0.1029539704322815, + 0.10191594809293747, + 0.10117761790752411, + 0.10062375664710999, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873187476230669e-05, + "first_to_last_ratio": 2.248757500145603, + "imbalance_abs_log_ratio": 0.8103778415280068, + "last_quartile_mean": 3.0564378221242805e-05, + "mean": 4.685761763312257e-05, + "normalized": [ + 1.7400006587426948, + 1.6405194725375354, + 1.5551306973978087, + 1.4944078341288562, + 1.4327724019957897, + 1.3414187773842097, + 1.2804088050749736, + 1.2499327782691996, + 1.1927955171655855, + 1.1608931065609767, + 1.0981756972550076, + 1.0577423904261343, + 1.0542786038595535, + 1.0218640087393278, + 0.9775807389332813, + 0.9570972402588515, + 0.9209982861589985, + 0.8815092249082567, + 0.8478232134430647, + 0.8204952153707635, + 0.7871780668180614, + 0.7759097745574741, + 0.7506963524238334, + 0.7421155343193898, + 0.7072927325438048, + 0.6891814936654451, + 0.6695399899996352, + 0.6616343983211914, + 0.6345111343049281, + 0.6212687927046295, + 0.6200231913288072, + 0.6148038704019305 + ], + "peak_layer": 1, + "peak_normalized": 1.7400006587426948, + "population_cv": 0.3217536161653951, + "rest_mean": 4.900405669845727e-05, + "spike_contrast": 0.719671983430414, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.526684668031521e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153228554874659e-05, + 7.687083416385576e-05, + 7.286971958819777e-05, + 7.002439087955281e-05, + 6.71363013680093e-05, + 6.285568815656006e-05, + 5.999690620228648e-05, + 5.8568872191244736e-05, + 5.5891556257847697e-05, + 5.439668530016206e-05, + 5.145789691596292e-05, + 4.9563288484932855e-05, + 4.9400983698433265e-05, + 4.7882112994557247e-05, + 4.580710447044112e-05, + 4.484729652176611e-05, + 4.3155785533599555e-05, + 4.130542220082134e-05, + 3.972697595600039e-05, + 3.844645107164979e-05, + 3.688528886414133e-05, + 3.635728353401646e-05, + 3.517584264045581e-05, + 3.477376594673842e-05, + 3.314205241622403e-05, + 3.229340290999971e-05, + 3.1373048841487616e-05, + 3.10026116494555e-05, + 2.9731680115219206e-05, + 2.9111175535945222e-05, + 2.9052809622953646e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001719075116852764, + "first_to_last_ratio": 1.4894635014227988, + "imbalance_abs_log_ratio": 0.39841598895878805, + "last_quartile_mean": 0.00011541572621354135, + "mean": 0.00013894635094402474, + "normalized": [ + 1.3738985779819257, + 1.3296082759737424, + 1.2883384004275333, + 1.2499940655466777, + 1.2130730176151365, + 1.1755503979222366, + 1.147409009170091, + 1.11990595257071, + 1.0929037170303153, + 1.075598370036652, + 1.0512762916941902, + 1.0312594730951, + 1.0194418974669357, + 1.0018996501730997, + 0.9803460168062212, + 0.9648004557293595, + 0.9493010807852901, + 0.9344788929945884, + 0.9198953859218769, + 0.908001147597105, + 0.8952913731815082, + 0.886953676542665, + 0.8771894463102521, + 0.8683888934041172, + 0.8579399877904575, + 0.8482799643655919, + 0.8408636339472488, + 0.8330870307470335, + 0.82434015220157, + 0.8181187963300758, + 0.8135579418994486, + 0.809009026741244 + ], + "peak_layer": 1, + "peak_normalized": 1.3738985779819257, + "population_cv": 0.16161423472023242, + "rest_mean": 0.0001421073115866252, + "spike_contrast": 0.8576417505420824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187716347398237, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001908981939777732, + 0.0001847442181315273, + 0.00017900991952046752, + 0.00017368211410939693, + 0.00016855206922627985, + 0.00016333843814209104, + 0.00015942829486448318, + 0.00015560684551019222, + 0.0001518549834145233, + 0.00014945046859793365, + 0.00014607100456487387, + 0.0001432897406630218, + 0.00014164773165248334, + 0.00013921030040364712, + 0.000136215501697734, + 0.0001340555027127266, + 0.0001319019211223349, + 0.00012984243221580982, + 0.0001278161071240902, + 0.00012616344611160457, + 0.00012439746933523566, + 0.00012323897681199014, + 0.00012188227265141904, + 0.00012065946793882176, + 0.00011920763063244522, + 0.00011786540562752634, + 0.00011683493357850239, + 0.00011575440294109285, + 0.0001145390560850501, + 0.00011367462138878182, + 0.00011304090730845928, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552221000485588, + "first_to_last_ratio": 1.262697055715921, + "imbalance_abs_log_ratio": 0.2332499537229619, + "last_quartile_mean": 0.00011524712863320019, + "mean": 0.00012953460168318998, + "normalized": [ + 1.1676213631634658, + 1.1525055688027706, + 1.1391580058081985, + 1.1289307982409096, + 1.1166731577821778, + 1.1054001772932092, + 1.094813144753513, + 1.0822854389705017, + 1.0679308603491235, + 1.0564894822224282, + 1.0456838360716472, + 1.0337072702557388, + 1.0234945545465333, + 1.0158819477934218, + 1.0051772952892828, + 0.9951467950359763, + 0.9866643397779161, + 0.9787559418439278, + 0.970448736962688, + 0.9607908684400951, + 0.9509939230158972, + 0.9428061355388626, + 0.934924980051271, + 0.9261034265480621, + 0.9170582603044295, + 0.9058562786860701, + 0.8981778407016658, + 0.8907564927732574, + 0.8831168120837269, + 0.8778216107141017, + 0.8745192645108896, + 0.8703053916682415 + ], + "peak_layer": 1, + "peak_normalized": 1.1676213631634658, + "population_cv": 0.08967236486753827, + "rest_mean": 0.00013110875065817877, + "spike_contrast": 0.9231588022206524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103419721825049, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124736819416285, + 0.00014928934979252517, + 0.000147560378536582, + 0.00014623560127802193, + 0.00014464781270362437, + 0.00014318757166620344, + 0.00014181618462316692, + 0.00014019341324456036, + 0.0001383339986205101, + 0.00013685194426216185, + 0.00013545223919209093, + 0.00013390085950959474, + 0.00013257795944809914, + 0.00013159186346456409, + 0.00013020524056628346, + 0.00012890594371128827, + 0.00012780717224813998, + 0.00012678276107180864, + 0.0001257066905964166, + 0.0001244556624442339, + 0.00012318661902099848, + 0.0001221260172314942, + 0.00012110513489460573, + 0.0001199624384753406, + 0.00011879077646881342, + 0.00011733973224181682, + 0.00011634510883595794, + 0.00011538378748809919, + 0.0001143941844929941, + 0.00011370827269274741, + 0.00011328050459269434, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029741139769612346, + "first_to_last_ratio": 2.278072415841521, + "imbalance_abs_log_ratio": 0.8233296537390045, + "last_quartile_mean": 0.00013055396993877366, + "mean": 0.00019328556300024502, + "normalized": [ + 1.889398576482247, + 1.780233894901915, + 1.6772594716902607, + 1.566224958959574, + 1.4762232610465842, + 1.375571144634086, + 1.3083686481730985, + 1.2364401527267181, + 1.173148897934763, + 1.1508864184295666, + 1.080457040307468, + 1.0318582947152708, + 1.0081957004611959, + 0.9698994715426978, + 0.9066127339785177, + 0.8754550048258108, + 0.8446146862114313, + 0.8115724449181009, + 0.7806677558079268, + 0.7613480996239201, + 0.7403134026279838, + 0.7294052762839086, + 0.7160186976339193, + 0.7062575707090604, + 0.6942777320753531, + 0.6866005528973866, + 0.6829938476675228, + 0.6753403086482458, + 0.6715967314167978, + 0.6686860556999406, + 0.6640286432895989, + 0.6600445236791291 + ], + "peak_layer": 1, + "peak_normalized": 1.889398576482247, + "population_cv": 0.3579154260008552, + "rest_mean": 0.00020340604748559632, + "spike_contrast": 0.681567477924495, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013863494677934796, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003651934675872326, + 0.0003440935106482357, + 0.00032419004128314555, + 0.0003027286729775369, + 0.0002853326441254467, + 0.00026587804313749075, + 0.00025288877077400684, + 0.00023898603103589267, + 0.00022675274522043765, + 0.00022244972933549434, + 0.0002088367473334074, + 0.00019944331143051386, + 0.00019486967357806861, + 0.00018746756541077048, + 0.00017523515271022916, + 0.00016921281348913908, + 0.0001632518251426518, + 0.00015686523693148047, + 0.00015089180669747293, + 0.00014715759607497603, + 0.00014309189282357693, + 0.00014098350948188454, + 0.00013839607709087431, + 0.0001365093921776861, + 0.0001341938623227179, + 0.00013270997442305088, + 0.00013201285037212074, + 0.00013053353177383542, + 0.0001298099523410201, + 0.00012924736074637622, + 0.00012834715016651899, + 0.00012757707736454904 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0040087271481752396, + 0.005379301495850086, + 0.006410556845366955, + 0.0027253651060163975, + 0.004838948138058186, + 0.006175771821290255, + 0.00742313964292407, + 0.0030937574338167906, + 0.005150026176124811, + 0.007019025273621082, + 0.008143599145114422, + 0.004255065228790045, + 0.0066224560141563416, + 0.008129785768687725, + 0.010099578648805618, + 0.0041999719105660915, + 0.00719039561226964, + 0.009030314162373543, + 0.010736005380749702, + 0.005131533369421959, + 0.00787449348717928, + 0.009912566281855106, + 0.012113703414797783, + 0.0059690922498703, + 0.00876442901790142, + 0.010806876234710217, + 0.012353328987956047, + 0.006502739619463682, + 0.009470963850617409, + 0.012123330496251583, + 0.01384514570236206 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962236011546338, + "first_to_last_ratio": 3.7698422272252903, + "imbalance_abs_log_ratio": 1.3270331510418396, + "last_quartile_mean": 4.2341920561739244e-05, + "mean": 8.61727822893954e-05, + "normalized": [ + 2.542734798705971, + 2.3221448837042376, + 2.101145461274622, + 1.8911858524935854, + 1.7004204056817789, + 1.5354142426015858, + 1.4229001770142031, + 1.3028733810328659, + 1.2049412416934622, + 1.1705258152816296, + 1.0773908232649334, + 1.0133220774376726, + 0.990609360100461, + 0.9272534949358001, + 0.8550385909364457, + 0.8185651575045715, + 0.7703972051466339, + 0.7173947009730917, + 0.6760256695667757, + 0.6514523109028573, + 0.618419077603279, + 0.6053620388084041, + 0.5838930841859182, + 0.5697043695032278, + 0.5457802700901755, + 0.5276149005446632, + 0.5104701332743111, + 0.49517988582787786, + 0.47527562953524777, + 0.46579059316028215, + 0.45896435764104826, + 0.45181000957238127 + ], + "peak_layer": 1, + "peak_normalized": 2.542734798705971, + "population_cv": 0.5705202801130907, + "rest_mean": 9.280119640472414e-05, + "spike_contrast": 0.542873885449774, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.03793460666202e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021911453222855926, + 0.00020010568550787866, + 0.00018106155039276928, + 0.00016296874673571438, + 0.00014652995741926134, + 0.00013231091725174338, + 0.0001226152671733871, + 0.00011227222421439365, + 0.00010383313929196447, + 0.00010086746624438092, + 9.284176485380158e-05, + 8.732078276807442e-05, + 8.536356472177431e-05, + 7.99040135461837e-05, + 7.368105434579775e-05, + 7.053803710732609e-05, + 6.638727063545957e-05, + 6.181989738252014e-05, + 5.825501284562051e-05, + 5.613745815935545e-05, + 5.329089253791608e-05, + 5.216573117650114e-05, + 5.0315691623836756e-05, + 4.9093010602518916e-05, + 4.703140439232811e-05, + 4.546604395727627e-05, + 4.398863165988587e-05, + 4.267102849553339e-05, + 4.095582335139625e-05, + 4.013847137684934e-05, + 3.955023566959426e-05, + 3.893372559105046e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442477373057045, + "first_to_last_ratio": 1.8431254562763404, + "imbalance_abs_log_ratio": 0.6114627481432037, + "last_quartile_mean": 0.00011633759004325839, + "mean": 0.00015221248349917005, + "normalized": [ + 1.7740805953138137, + 1.6484457246979003, + 1.5301434809496548, + 1.4217241600875643, + 1.3264958046529274, + 1.2454577490392524, + 1.1897861330408546, + 1.1336265611690741, + 1.086914920172498, + 1.064301358569379, + 1.0227209980571423, + 0.9941753882589759, + 0.9788979908211966, + 0.9511107967643203, + 0.9228143274453928, + 0.9052447611165039, + 0.8846530997418666, + 0.8652930871753776, + 0.8497381552743479, + 0.8388656494639406, + 0.8256334792987466, + 0.8176309647342147, + 0.8076951735882674, + 0.8000661307359944, + 0.7892605467734257, + 0.78144613040264, + 0.7731219607566042, + 0.7664988488799842, + 0.7584935618388989, + 0.7529639051546407, + 0.7485195296370439, + 0.7441790263875565 + ], + "peak_layer": 1, + "peak_normalized": 1.7740805953138137, + "population_cv": 0.2736411578809574, + "rest_mean": 0.00015762286892384981, + "spike_contrast": 0.7803207938393821, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299640220589936, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027003721334040165, + 0.00025091401766985655, + 0.00023290693934541196, + 0.0002164041652576998, + 0.000201909220777452, + 0.0001895742170745507, + 0.00018110030214302242, + 0.00017255211423616856, + 0.00016544201935175806, + 0.00016199995297938585, + 0.00015567090304102749, + 0.00015132590488065034, + 0.0001490004942752421, + 0.00014477093645837158, + 0.00014046386058907956, + 0.00013778955326415598, + 0.0001346552453469485, + 0.00013170840975362808, + 0.00012934075493831187, + 0.00012768582382705063, + 0.00012567172234412283, + 0.00012445363972801715, + 0.00012294128828216344, + 0.00012178005272289738, + 0.00012013530795229599, + 0.00011894585622940212, + 0.00011767881369451061, + 0.00011667069338727742, + 0.00011545218876563013, + 0.00011461050598882139, + 0.00011393401655368507, + 0.00011327333777444437 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.190154030919075, + "first_to_last_ratio": 1.84312545379361, + "imbalance_abs_log_ratio": 0.6114627467961817, + "last_quartile_mean": 0.10316933691501617, + "mean": 0.13498355145566165, + "normalized": [ + 1.774080628234443, + 1.6484456620184311, + 1.5301434020504485, + 1.4217242444591272, + 1.326495766996536, + 1.2454576834375484, + 1.189786122599316, + 1.133626627255238, + 1.0869149569673349, + 1.0643014004698326, + 1.0227209907477024, + 0.9941753870542451, + 0.9788979589698585, + 0.9511107512164827, + 0.9228143032197887, + 0.9052447972745712, + 0.8846531331847195, + 0.8652931173486506, + 0.8497382179112226, + 0.8388656682549475, + 0.8256334805754935, + 0.8176309131099365, + 0.8076951535479161, + 0.8000661540388709, + 0.7892605573866218, + 0.7814461538045975, + 0.7731219580171443, + 0.7664988542586116, + 0.7584935821787124, + 0.7529638601652359, + 0.7485195164462104, + 0.7441789968002049 + ], + "peak_layer": 1, + "peak_normalized": 1.774080628234443, + "population_cv": 0.27364115345546325, + "rest_mean": 0.13978153549962574, + "spike_contrast": 0.7803207857775153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907443761825561, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2394717037677765, + 0.22251304984092712, + 0.2065441906452179, + 0.19190938770771027, + 0.1790551096200943, + 0.16811630129814148, + 0.16060155630111694, + 0.1530209481716156, + 0.14671564102172852, + 0.1436631828546524, + 0.13805051147937775, + 0.13419732451438904, + 0.13213512301445007, + 0.1283843070268631, + 0.1245647519826889, + 0.12219315767288208, + 0.11941362172365189, + 0.11680033802986145, + 0.11470068246126175, + 0.1132330670952797, + 0.11144693940877914, + 0.11036672443151474, + 0.10902556031942368, + 0.10799577087163925, + 0.10653719305992126, + 0.10548237711191177, + 0.1043587476015091, + 0.10346473753452301, + 0.10238415747880936, + 0.10163773596286774, + 0.10103782266378403, + 0.10045192390680313 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883367768750759e-05, + "first_to_last_ratio": 2.249759357551583, + "imbalance_abs_log_ratio": 0.810823258297238, + "last_quartile_mean": 3.059601795030176e-05, + "mean": 4.691846504556452e-05, + "normalized": [ + 1.7359059710084281, + 1.6475789727938472, + 1.5536456441204052, + 1.4921194549590373, + 1.4365657321885223, + 1.3438513058553838, + 1.2787343947807137, + 1.2483299149152907, + 1.1912354282513808, + 1.1603083401306369, + 1.099547144002948, + 1.059972440022979, + 1.0583955432739494, + 1.0176800955770247, + 0.9754963047746087, + 0.9631595708780456, + 0.9242400656751482, + 0.8752301116137658, + 0.8454245377074081, + 0.8232853322610043, + 0.7863494898171388, + 0.7774642190168813, + 0.7497486129019063, + 0.7388483538626245, + 0.709315327139865, + 0.6908343794807076, + 0.6685469983108738, + 0.6592365073812129, + 0.6314341746588822, + 0.6235898169926835, + 0.6199474545603838, + 0.6139783610863121 + ], + "peak_layer": 1, + "peak_normalized": 1.7359059710084281, + "population_cv": 0.3220406003070692, + "rest_mean": 4.9070239684725595e-05, + "spike_contrast": 0.7193541792517962, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.52988819940947e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144604362314567e-05, + 7.730187644483522e-05, + 7.28946688468568e-05, + 7.000795449130237e-05, + 6.740145909134299e-05, + 6.305144052021205e-05, + 5.999625500408001e-05, + 5.8569723478285596e-05, + 5.589093780145049e-05, + 5.4439886298496276e-05, + 5.158906424185261e-05, + 4.973227987647988e-05, + 4.9658294301480055e-05, + 4.774798799189739e-05, + 4.576878927764483e-05, + 4.51899686595425e-05, + 4.3363925215089694e-05, + 4.1064453398576006e-05, + 3.9666021621087566e-05, + 3.86272840842139e-05, + 3.689431105158292e-05, + 3.6477427784120664e-05, + 3.5177054087398574e-05, + 3.466563066467643e-05, + 3.3279986382694915e-05, + 3.2412888685939834e-05, + 3.136719897156581e-05, + 3.093036502832547e-05, + 2.9625922252307646e-05, + 2.9257877031341195e-05, + 2.9086982976878062e-05, + 2.8806922273361124e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017341253987979144, + "first_to_last_ratio": 1.4955148931299893, + "imbalance_abs_log_ratio": 0.4024705576674927, + "last_quartile_mean": 0.00011595507385209203, + "mean": 0.00013990553861731314, + "normalized": [ + 1.3768818258178377, + 1.3340879911950096, + 1.2910054180628048, + 1.2527236427111486, + 1.2147542166907819, + 1.1769889670731195, + 1.1474411155803916, + 1.1220953663134672, + 1.0944075691302455, + 1.0773007485615413, + 1.0508678613623372, + 1.0320596054419082, + 1.0208161736522385, + 1.000559648637946, + 0.9805197815181176, + 0.9663128275349431, + 0.948744191771126, + 0.9319305822463463, + 0.9182019814348666, + 0.9075038868226692, + 0.8943650364968407, + 0.886262676120959, + 0.8757907045344768, + 0.8679002174812178, + 0.856288373659771, + 0.8478178092769747, + 0.8387215581135022, + 0.8313975747674917, + 0.8224084044020752, + 0.8161864847722404, + 0.8112200992010112, + 0.8064376596145924 + ], + "peak_layer": 1, + "peak_normalized": 1.3768818258178377, + "population_cv": 0.16325888721442564, + "rest_mean": 0.0001431150389921472, + "spike_contrast": 0.8564734877369171, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225742365932092, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926333934534341, + 0.00018664629897102714, + 0.00018061880837194622, + 0.0001752629759721458, + 0.00016995084297377616, + 0.0001646672753849998, + 0.00016053336730692536, + 0.00015698735660407692, + 0.00015311368042603135, + 0.00015072034148033708, + 0.00014702223415952176, + 0.00014439085498452187, + 0.0001428178366040811, + 0.00013998383656144142, + 0.00013718014815822244, + 0.00013519251660909504, + 0.00013273456715978682, + 0.00013038225006312132, + 0.00012846154277212918, + 0.00012696482008323073, + 0.00012512662215158343, + 0.0001239930570591241, + 0.00012252797023393214, + 0.000121424047392793, + 0.00011979948612861335, + 0.00011861440725624561, + 0.00011734179133782163, + 0.0001163171255029738, + 0.00011505949078127742, + 0.00011418900976423174, + 0.00011349418491590768, + 0.00011282509512966499 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675183047074825, + "first_to_last_ratio": 1.2676545313577883, + "imbalance_abs_log_ratio": 0.23716836728277432, + "last_quartile_mean": 0.00011576642282307148, + "mean": 0.00013041823922321782, + "normalized": [ + 1.1700698937843759, + 1.1560440994564782, + 1.1417398043557874, + 1.130463200949706, + 1.118403537277814, + 1.1065304334184665, + 1.0939881912841023, + 1.0846815122144253, + 1.0709468270677047, + 1.0579907390333845, + 1.0442838370166418, + 1.035550673160169, + 1.025959021745254, + 1.0140203095307772, + 1.0062047700154308, + 0.9976631870190885, + 0.9861765922450596, + 0.9770016877798172, + 0.9687702939454803, + 0.9602422119882037, + 0.9504995944091904, + 0.9425883202531964, + 0.9330014667281973, + 0.9259385264566372, + 0.9147443798098084, + 0.906202796813466, + 0.8958065504175365, + 0.8885655303249116, + 0.8806194435723184, + 0.8757905905296257, + 0.8713783728577705, + 0.8681336045391749 + ], + "peak_layer": 1, + "peak_normalized": 1.1700698937843759, + "population_cv": 0.09121429133712369, + "rest_mean": 0.0001320278307604086, + "spike_contrast": 0.921975648780332, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001217264449223876, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001525984553154558, + 0.0001507692359155044, + 0.000148903694935143, + 0.00014743302017450333, + 0.00014586022007279098, + 0.00014431175077334046, + 0.00014267601363826543, + 0.00014146225294098258, + 0.000139670999487862, + 0.00013798128929920495, + 0.00013619365927297622, + 0.00013505469541996717, + 0.00013380376913119107, + 0.00013224674330558628, + 0.00013122745440341532, + 0.00013011347618885338, + 0.00012861541472375393, + 0.00012741883983835578, + 0.0001263453159481287, + 0.0001252330985153094, + 0.0001239624834852293, + 0.00012293070903979242, + 0.00012168040848337114, + 0.00012075927224941552, + 0.00011929935135412961, + 0.00011818537313956767, + 0.00011682951299007982, + 0.00011588515189941972, + 0.00011484883725643158, + 0.00011421906674513593, + 0.00011364363308530301, + 0.00011322045611450449 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030004180916876066, + "first_to_last_ratio": 2.2837506327398005, + "imbalance_abs_log_ratio": 0.8258191057718037, + "last_quartile_mean": 0.00013138116082700435, + "mean": 0.00019473525253488333, + "normalized": [ + 1.9000096511818072, + 1.783700741635724, + 1.6708066910504589, + 1.570366884717585, + 1.4783058799917985, + 1.376185321596543, + 1.3064973297630273, + 1.2402700043336092, + 1.1767953849206394, + 1.1505341943958838, + 1.0771542571420398, + 1.026098162637436, + 1.013898063759585, + 0.9669473025647412, + 0.9095398902231254, + 0.8755205777387188, + 0.842301214161826, + 0.8073124727024382, + 0.7791991818813049, + 0.760574607765185, + 0.7402055397447304, + 0.7308143413325056, + 0.7152667122155245, + 0.7043714155678553, + 0.6942467749765324, + 0.6868749151283189, + 0.680655564574882, + 0.676363713518914, + 0.6701061029149729, + 0.6672276319278244, + 0.6636177363814285, + 0.6582317375530352 + ], + "peak_layer": 1, + "peak_normalized": 1.9000096511818072, + "population_cv": 0.35930746026536914, + "rest_mean": 0.00020494150898971216, + "spike_contrast": 0.6812747128050888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962146767880766, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699988592416048, + 0.0003473494143690914, + 0.0003253649629186839, + 0.0003058057918678969, + 0.0002878782688640058, + 0.0002679917961359024, + 0.0002544210874475539, + 0.00024152429250534624, + 0.00022916354646440595, + 0.000224049566895701, + 0.00020975990628357977, + 0.00019981748482678086, + 0.000197441695490852, + 0.0001882987271528691, + 0.00017711948021315038, + 0.00017049472080543637, + 0.00016402573965024203, + 0.00015721219824627042, + 0.00015173754945863038, + 0.00014811068831477314, + 0.00014414411270990968, + 0.0001423153153154999, + 0.0001392876438330859, + 0.00013716594548895955, + 0.00013519432104658335, + 0.00013375876005738974, + 0.00013254763325676322, + 0.0001317118585575372, + 0.00013049328117631376, + 0.00012993274140171707, + 0.00012922976748086512, + 0.00012818092363886535 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858902861509705, + "first_to_last_ratio": 3.757869257587208, + "imbalance_abs_log_ratio": 1.3238521099622325, + "last_quartile_mean": 4.220184837322449e-05, + "mean": 8.57325238712292e-05, + "normalized": [ + 2.5462257183789148, + 2.3032637538343064, + 2.0924289533379263, + 1.8886492391629932, + 1.7031256747388024, + 1.5374176267930122, + 1.4224231993460965, + 1.3049597038569594, + 1.2055618292025094, + 1.169245405308266, + 1.0769812048488787, + 1.0144334047898542, + 0.9913512297333369, + 0.9317090592952362, + 0.8577499774884012, + 0.81555804918997, + 0.7682725064221458, + 0.7217807351250844, + 0.6783324192661732, + 0.6511800118936032, + 0.6192916242842709, + 0.6053728292385717, + 0.5848720876072855, + 0.5718124973482859, + 0.5462302773879167, + 0.5275173250132701, + 0.5112998770983732, + 0.49691833866078555, + 0.47721658788927634, + 0.46551989467389665, + 0.46012635628943105, + 0.45317260249616614 + ], + "peak_layer": 1, + "peak_normalized": 2.5462257183789148, + "population_cv": 0.5686111305901927, + "rest_mean": 9.231303704803361e-05, + "spike_contrast": 0.5437775023084314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.019775271648541e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021829435718245804, + 0.00019746461475733668, + 0.0001793892151908949, + 0.0001619186659809202, + 0.00014601326256524771, + 0.00013180669338908046, + 0.00012194793089292943, + 0.00011187748896190897, + 0.00010335585830034688, + 0.00010024235962191597, + 9.233231685357168e-05, + 8.69699360919185e-05, + 8.499104296788573e-05, + 7.987776916706935e-05, + 7.353707042057067e-05, + 6.991984992055222e-05, + 6.586594099644572e-05, + 6.188008410390466e-05, + 5.8155150327365845e-05, + 5.582730591413565e-05, + 5.309343396220356e-05, + 5.190014053368941e-05, + 5.014256021240726e-05, + 4.90229285787791e-05, + 4.682970029534772e-05, + 4.522539165918715e-05, + 4.383502891869284e-05, + 4.2602063331287354e-05, + 4.0912982512963936e-05, + 3.991019548266195e-05, + 3.944779382436536e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "rest_mean": 0.00015645541295978345, + "spike_contrast": 0.7816308178118705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229037238284946, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837944976985455, + "first_to_last_ratio": 1.8345658800697515, + "imbalance_abs_log_ratio": 0.6068078758989703, + "last_quartile_mean": 0.10268339328467846, + "mean": 0.1340121782850474, + "normalized": [ + 1.7717451751532867, + 1.6366747548098712, + 1.5242756140248972, + 1.4186514377480737, + 1.3264266357207033, + 1.2453739091726768, + 1.1893513017871604, + 1.1330134634090196, + 1.0858604665419038, + 1.0628542749024097, + 1.0232520332438115, + 0.9942070832344971, + 0.9784308561813967, + 0.9531084116998004, + 0.9232390798282929, + 0.9040072094634526, + 0.8853775581660035, + 0.8680181154203164, + 0.8515367049614931, + 0.8395794986042229, + 0.8267439263823189, + 0.8185133396785964, + 0.809173050926764, + 0.8007913534932437, + 0.7909900046670136, + 0.782107217826699, + 0.7753403703776178, + 0.7682422246066585, + 0.7604971603471317, + 0.7549863443945886, + 0.7508596538022533, + 0.7467717694238248 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451751532867, + "population_cv": 0.27141865758333467, + "rest_mean": 0.13874622461972413, + "spike_contrast": 0.7816308398662988, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844832807779312, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23743543028831482, + 0.21933434903621674, + 0.20427149534225464, + 0.19011656939983368, + 0.17775732278823853, + 0.16689527034759521, + 0.15938755869865417, + 0.15183760225772858, + 0.14551852643489838, + 0.14243541657924652, + 0.13712823390960693, + 0.1332358568906784, + 0.1311216503381729, + 0.12772813439369202, + 0.1237252801656723, + 0.12114797532558441, + 0.1186513751745224, + 0.1163249984383583, + 0.11411628872156143, + 0.11251387745141983, + 0.11079375445842743, + 0.10969075560569763, + 0.10843904316425323, + 0.107315793633461, + 0.10600229352712631, + 0.104811891913414, + 0.10390505194664001, + 0.10295381397008896, + 0.1019158810377121, + 0.10117736458778381, + 0.10062433779239655, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873207303215167e-05, + "first_to_last_ratio": 2.248757814140914, + "imbalance_abs_log_ratio": 0.8103779811585758, + "last_quartile_mean": 3.0564462122129044e-05, + "mean": 4.6857780716891284e-05, + "normalized": [ + 1.7400119939228862, + 1.6405409364325143, + 1.555121247708426, + 1.4944054279879166, + 1.4327475398601994, + 1.3414100714975594, + 1.2804101716466747, + 1.2499370459040815, + 1.192804641985012, + 1.1608858830017554, + 1.098191983597808, + 1.057744066133373, + 1.0542717513613309, + 1.021869458338179, + 0.9775749297615711, + 0.9570927445976433, + 0.9210091333285576, + 0.8815125232807631, + 0.847820728512489, + 0.8204970180441978, + 0.7871838673851995, + 0.7759041238100245, + 0.7506938949785397, + 0.7421070509174572, + 0.7072987335050381, + 0.6891811912847667, + 0.6695407652827761, + 0.6616333377893114, + 0.6345027924930273, + 0.6212629426015038, + 0.6200302724102036, + 0.614801730639213 + ], + "peak_layer": 1, + "peak_normalized": 1.7400119939228862, + "population_cv": 0.32175451551289735, + "rest_mean": 4.9004235565748616e-05, + "spike_contrast": 0.7196709452950106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5266924533061685e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153310045599937e-05, + 7.687210745643824e-05, + 7.28695304132998e-05, + 7.002452184678987e-05, + 6.713537004543468e-05, + 6.28554989816621e-05, + 5.9997179050697014e-05, + 5.8569276006892323e-05, + 5.589217835222371e-05, + 5.439653614303097e-05, + 5.1458839152473956e-05, + 4.9563539505470544e-05, + 4.9400834541302174e-05, + 4.788253500009887e-05, + 4.58069916930981e-05, + 4.4847241952084005e-05, + 4.315644400776364e-05, + 4.130572051508352e-05, + 3.972699778387323e-05, + 3.8446669350378215e-05, + 3.688568904181011e-05, + 3.635714529082179e-05, + 3.5175849916413426e-05, + 3.477348946034908e-05, + 3.3142448955914006e-05, + 3.2293501135427505e-05, + 3.13731943606399e-05, + 3.1002669857116416e-05, + 2.9731392714893445e-05, + 2.9111002731951885e-05, + 2.905324254243169e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017190765356644988, + "first_to_last_ratio": 1.4894621368061003, + "imbalance_abs_log_ratio": 0.3984150727783497, + "last_quartile_mean": 0.00011541592721187044, + "mean": 0.00013894661265112518, + "normalized": [ + 1.3739040544627474, + 1.3296102750467884, + 1.2883342981472405, + 1.2499918159027226, + 1.2130627732820631, + 1.175544413473833, + 1.1474083142393565, + 1.1199112790671786, + 1.092909618037032, + 1.0755929927697745, + 1.0512861461262244, + 1.0312571117882559, + 1.0194388253067554, + 1.0019061415077835, + 0.980344379776871, + 0.9647961249933756, + 0.9493075664595542, + 0.9344817410273759, + 0.9198929201794295, + 0.9079992279088673, + 0.8952920956888009, + 0.8869530532606651, + 0.8771902552778059, + 0.8683889858357474, + 0.8579407806493952, + 0.8482823463732029, + 0.8408658204630546, + 0.8330843095877946, + 0.8243382853592501, + 0.818115422616924, + 0.8135611224181157, + 0.8090075029660142 + ], + "peak_layer": 1, + "peak_normalized": 1.3739040544627474, + "population_cv": 0.16161403917150835, + "rest_mean": 0.00014210757001786045, + "spike_contrast": 0.8576421569620596, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187744287075474, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00019089931447524577, + 0.00018474484386388212, + 0.00017900968668982387, + 0.00017368212866131216, + 0.0001685509632807225, + 0.00016333791427314281, + 0.00015942849859129637, + 0.00015560787869617343, + 0.00015185608936008066, + 0.00014945000293664634, + 0.00014607264893129468, + 0.0001432896824553609, + 0.00014164757158141583, + 0.0001392114645568654, + 0.00013621553080156446, + 0.0001340551534667611, + 0.00013190307072363794, + 0.00012984307250007987, + 0.0001278160052606836, + 0.00012616341700777411, + 0.0001243978040292859, + 0.00012323912233114243, + 0.00012188261462142691, + 0.00012065970804542303, + 0.00011920796532649547, + 0.00011786595860030502, + 0.00011683545744745061, + 0.00011575424287002534, + 0.00011453901242930442, + 0.00011367436673026532, + 0.00011304156214464456, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552358334185556, + "first_to_last_ratio": 1.262638843036328, + "imbalance_abs_log_ratio": 0.2332038508028458, + "last_quartile_mean": 0.00011525352965691127, + "mean": 0.00012953498708156985, + "normalized": [ + 1.1676304712442283, + 1.1525836984023852, + 1.1391521450612143, + 1.1288261090295841, + 1.1167600441451297, + 1.1054423860149858, + 1.094731137320412, + 1.082319740349084, + 1.067957453000188, + 1.0563935463552665, + 1.04560972624621, + 1.0337959762176818, + 1.0235535208863231, + 1.0159040893770892, + 1.0050628637064216, + 0.9950792389307415, + 0.9865814183778941, + 0.9786230528286792, + 0.9704418054120744, + 0.9608381133357455, + 0.9508407831226554, + 0.9429749854457838, + 0.9346839821902475, + 0.9262276149685831, + 0.9170542960948607, + 0.9059488475652709, + 0.8984648923618161, + 0.8908037775288885, + 0.8831483920181569, + 0.8778579808382188, + 0.8744051093280881, + 0.8703028022960919 + ], + "peak_layer": 1, + "peak_normalized": 1.1676304712442283, + "population_cv": 0.0896642712448568, + "rest_mean": 0.00013110964479252557, + "spike_contrast": 0.9231344927669946, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103183544240892, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124899800866842, + 0.00014929991448298097, + 0.00014756005839444697, + 0.00014622247545048594, + 0.00014465949789155275, + 0.00014319346519187093, + 0.00014180598373059183, + 0.00014019827358424664, + 0.00013833785487804562, + 0.00013683992438018322, + 0.0001354430423816666, + 0.00013391274842433631, + 0.0001325859921053052, + 0.00013159512309357524, + 0.00013019080506637692, + 0.00012889757636003196, + 0.00012779681128449738, + 0.00012676592450588942, + 0.00012570616672746837, + 0.00012446215259842575, + 0.00012316714855842292, + 0.00012214825255796313, + 0.00012107427755836397, + 0.00011997888213954866, + 0.00011879061639774591, + 0.00011735207226593047, + 0.0001163826382253319, + 0.0001153902558144182, + 0.00011439861555118114, + 0.00011371332220733166, + 0.00011326605454087257, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029742828155576717, + "first_to_last_ratio": 2.2781890121807993, + "imbalance_abs_log_ratio": 0.8233808344454435, + "last_quartile_mean": 0.00013055469935352448, + "mean": 0.00019329176166138495, + "normalized": [ + 1.8891850069388627, + 1.7801603926271052, + 1.6773544463608039, + 1.566500262900107, + 1.476456430986117, + 1.3754925511314788, + 1.3084504582544147, + 1.236424592515421, + 1.172965600373877, + 1.150871569148191, + 1.0802294364746856, + 1.0319209662753672, + 1.0080063247818516, + 0.9698985570693867, + 0.9067931019504507, + 0.8756163462587222, + 0.8444435806568995, + 0.8113939671049883, + 0.7809677247391495, + 0.7613442367276676, + 0.7402582678385915, + 0.7293562882295931, + 0.7162426695419845, + 0.7062419232587185, + 0.6940340549797547, + 0.6866100033591334, + 0.6831688143117768, + 0.6753320518823956, + 0.6715919825156708, + 0.6686410475337363, + 0.6640239865195754, + 0.6600233567535221 + ], + "peak_layer": 1, + "peak_normalized": 1.8891850069388627, + "population_cv": 0.357915029863326, + "rest_mean": 0.00020341356920888993, + "spike_contrast": 0.6815376252628051, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001386340009048581, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036516389809548855, + 0.0003440903383307159, + 0.0003242187958676368, + 0.0003027915954589844, + 0.00028538686456158757, + 0.00026587137836031616, + 0.0002529126941226423, + 0.00023899068764876574, + 0.00022672458726447076, + 0.00022245399304665625, + 0.0002087994507746771, + 0.00019946182146668434, + 0.00019483931828290224, + 0.00018747340072877705, + 0.00017527563613839447, + 0.00016924942610785365, + 0.00016322398732881993, + 0.00015683576930314302, + 0.0001509546273155138, + 0.00014716156874783337, + 0.0001430858246749267, + 0.0001409785618307069, + 0.00013844380737282336, + 0.00013651074550580233, + 0.00013415106514003128, + 0.00013271605712361634, + 0.00013205090363044292, + 0.00013053612201474607, + 0.00012981319741811603, + 0.00012924280599690974, + 0.00012835036613978446, + 0.00012757707736454904 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.02802751213312149, + 0.01413465291261673, + 0.014225807972252369, + 0.01433455292135477, + 0.01440478302538395, + 0.009703479707241058, + 0.00977881159633398, + 0.009858808480203152, + 0.009950309991836548, + 0.00752287870272994, + 0.007571155205368996, + 0.007693085819482803, + 0.007788818795233965, + 0.006353048607707024, + 0.006389216054230928, + 0.0064707002602517605, + 0.00656962301582098, + 0.005553483963012695, + 0.0056249531917274, + 0.005688921548426151, + 0.005768724251538515, + 0.004966245498508215, + 0.005057469941675663, + 0.005098348017781973, + 0.005216081161051989, + 0.0046758754178881645, + 0.004727107938379049, + 0.004777040798217058, + 0.004837613552808762, + 0.0044186655431985855, + 0.004510571248829365, + 0.0045849340967834 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001580166108396952, + "first_to_last_ratio": 1.9713740353643698, + "imbalance_abs_log_ratio": 0.6787307795042399, + "last_quartile_mean": 8.015557068574708e-05, + "mean": 0.00011843875881822896, + "normalized": [ + 0.8918679597819287, + 1.5814819593588525, + 1.5722239987443252, + 1.3246067564285042, + 1.1252650527356378, + 1.512743589128775, + 1.3788112428745087, + 1.2863035124436846, + 1.097095055843533, + 1.468154353173571, + 1.1523254606068964, + 1.0839888571909224, + 1.054518217028451, + 1.146335080549274, + 1.1665208527784754, + 0.8931136212023528, + 0.8203804970547371, + 1.073741223277434, + 0.919654742394771, + 0.8223971330190447, + 0.7711288002244273, + 0.8470775332584999, + 0.8298711629738247, + 0.7662487463744663, + 0.7210382462310316, + 0.7071376059209309, + 0.7189559392079661, + 0.727980518763353, + 0.6111916361855597, + 0.6986829946379066, + 0.6244678178312321, + 0.6046898327751223 + ], + "peak_layer": 2, + "peak_normalized": 1.5814819593588525, + "population_cv": 0.2904715201464359, + "rest_mean": 0.00012310891098523926, + "spike_contrast": 0.7572151875143335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.321993711637333e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010563173418631777, + 0.0001873087603598833, + 0.00018621225899551064, + 0.00015688478015363216, + 0.0001332749961875379, + 0.00017916747310664505, + 0.00016330469225067645, + 0.00015234819147735834, + 0.00012993857671972364, + 0.00017388637934345752, + 0.0001364799973089248, + 0.0001283862948184833, + 0.00012489582877606153, + 0.00013577050413005054, + 0.00013816128193866462, + 0.00010577926877886057, + 9.71648478298448e-05, + 0.00012717257777694613, + 0.00010892276623053476, + 9.74036956904456e-05, + 9.133153798757121e-05, + 0.0001003268116619438, + 9.828891052166e-05, + 9.07535504666157e-05, + 8.539887494407594e-05, + 8.375250035896897e-05, + 8.515224908478558e-05, + 8.622110908618197e-05, + 7.238877878990024e-05, + 8.275114669231698e-05, + 7.396119326585904e-05, + 7.161871326388791e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001664285200604354, + "first_to_last_ratio": 1.9719337364651075, + "imbalance_abs_log_ratio": 0.6790146534176558, + "last_quartile_mean": 8.439863722742302e-05, + "mean": 0.0001245867656507471, + "normalized": [ + 0.8984334461398115, + 1.5927952483854981, + 1.5658595485525297, + 1.3198735865176374, + 1.131213894321221, + 1.5125315092450449, + 1.3796810696180641, + 1.286366052942558, + 1.0960606282761096, + 1.4536884557404346, + 1.1543342746982923, + 1.0797559630746076, + 1.0497530570909623, + 1.1457182987306291, + 1.1551962694199982, + 0.9023721663305979, + 0.8238710779144349, + 1.0632503993742801, + 0.9211624832492573, + 0.8274335806257342, + 0.7739032412019898, + 0.850990856837025, + 0.8281317613098801, + 0.7681943949939335, + 0.7208251570035091, + 0.7089546837555952, + 0.7203162530736348, + 0.7236174125342815, + 0.6145740290254166, + 0.6994991973458914, + 0.6274381906330746, + 0.6042038120380651 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952483854981, + "population_cv": 0.2889665774838781, + "rest_mean": 0.00012946851159169995, + "spike_contrast": 0.7586812913967177, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.822533756960183e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00011193291720701382, + 0.0001984412083402276, + 0.0001950853766174987, + 0.000164438781212084, + 0.00014093428035266697, + 0.00018844140868168324, + 0.00017189000209327787, + 0.00016026418597903103, + 0.00013655464863404632, + 0.00018111034296453, + 0.00014381477376446128, + 0.0001345233031315729, + 0.00013078533811494708, + 0.00014274133718572557, + 0.00014392216689884663, + 0.00011242362961638719, + 0.00010264343291055411, + 0.0001324669283349067, + 0.00011476465442683548, + 0.00010308727360097691, + 9.641810174798593e-05, + 0.00010602219845168293, + 0.00010317425767425448, + 9.570685506332666e-05, + 8.980527491075918e-05, + 8.832637104205787e-05, + 8.974187221610919e-05, + 9.015315299620852e-05, + 7.656779052922502e-05, + 8.714834257261828e-05, + 7.817049481673166e-05, + 7.527579873567447e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.14759047608822584, + "first_to_last_ratio": 1.9719337947467823, + "imbalance_abs_log_ratio": 0.6790146829732503, + "last_quartile_mean": 0.07484555337578058, + "mean": 0.1104847895912826, + "normalized": [ + 0.8984334974418968, + 1.5927952138130326, + 1.565859646322909, + 1.3198735575362357, + 1.1312139236939245, + 1.5125315007911495, + 1.3796810264894532, + 1.286366010451999, + 1.0960606168617568, + 1.4536884922319049, + 1.1543342013148132, + 1.0797558953237962, + 1.0497530983161607, + 1.1457182562234747, + 1.1551962948455465, + 0.9023721940039697, + 0.8238711061937815, + 1.0632504861804335, + 0.9211625151546727, + 0.8274335809310764, + 0.7739032642161865, + 0.8509909070635955, + 0.8281317391228538, + 0.7681943896830757, + 0.7208251689669645, + 0.7089546589522648, + 0.7203162343708721, + 0.7236173971219663, + 0.6145740294939214, + 0.6994991437375198, + 0.6274381319743458, + 0.6042038211744466 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952138130326, + "population_cv": 0.28896658018674404, + "rest_mean": 0.11481397047086998, + "spike_contrast": 0.7586813040631772, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08710721284151077, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.09926323592662811, + 0.1759796440601349, + 0.1730036735534668, + 0.14582595229148865, + 0.12498193234205246, + 0.16711172461509705, + 0.15243376791477203, + 0.14212387800216675, + 0.12109802663326263, + 0.16061046719551086, + 0.1275363713502884, + 0.1192966029047966, + 0.11598175019025803, + 0.12658444046974182, + 0.12763161957263947, + 0.09969840198755264, + 0.0910252258181572, + 0.11747300624847412, + 0.10177444666624069, + 0.09141882508993149, + 0.08550453931093216, + 0.09402155131101608, + 0.09149596095085144, + 0.08487379550933838, + 0.0796402171254158, + 0.0783287063241005, + 0.07958398759365082, + 0.07994871586561203, + 0.06790108233690262, + 0.07728401571512222, + 0.069322369992733, + 0.06675533205270767 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.093040158499207e-05, + "first_to_last_ratio": 0.8860093039525244, + "imbalance_abs_log_ratio": 0.12102782735710801, + "last_quartile_mean": 5.748291960117058e-05, + "mean": 5.7093861016710434e-05, + "normalized": [ + 0.4337136137178416, + 0.9185569667519601, + 0.9444095926066538, + 0.8728787927927439, + 0.7849140301431445, + 1.1296292591692207, + 1.0726593944235638, + 0.9796135302182263, + 0.9386642177229508, + 1.3046353648934683, + 1.0461832118506482, + 1.0032917361864164, + 0.9662905330123907, + 1.0760080966276753, + 1.1828739682986207, + 0.9087955578115051, + 0.9194941496879511, + 1.198292373503509, + 1.0946132296674007, + 1.0673695485587213, + 0.9244260207272295, + 1.0525679458300632, + 1.125653431968099, + 0.9999504891362015, + 0.9629027708994021, + 1.0063234987710272, + 0.9975933861195996, + 1.0791525148711678, + 0.9412468232151083, + 1.085173475571988, + 1.0031556955576197, + 0.9789667796878816 + ], + "peak_layer": 10, + "peak_normalized": 1.3046353648934683, + "population_cv": 0.1441687540611157, + "rest_mean": 5.695535414647397e-05, + "spike_contrast": 1.0155638391297448, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.784179811598733e-05, + "top_five_layers": [ + 10, + 18, + 15, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.4762384782661684e-05, + 5.2443963795667514e-05, + 5.3919990023132414e-05, + 4.9836020480142906e-05, + 4.481377254705876e-05, + 6.449489592341706e-05, + 6.124226638348773e-05, + 5.592991874436848e-05, + 5.3591964388033375e-05, + 7.448667020071298e-05, + 5.973063889541663e-05, + 5.728179894504137e-05, + 5.516925739357248e-05, + 6.143345672171563e-05, + 6.75348419463262e-05, + 5.1886647270293906e-05, + 5.249747118796222e-05, + 6.84151382301934e-05, + 6.249569560168311e-05, + 6.094024865888059e-05, + 5.277905074763112e-05, + 6.009516800986603e-05, + 6.426790059776977e-05, + 5.7091034250333905e-05, + 5.4975836974335834e-05, + 5.74548939766828e-05, + 5.695645813830197e-05, + 6.161298369988799e-05, + 5.373941530706361e-05, + 6.19567435933277e-05, + 5.727403186028823e-05, + 5.58929932594765e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.801741053801379e-05, + "first_to_last_ratio": 0.954195857595535, + "imbalance_abs_log_ratio": 0.046886327142486896, + "last_quartile_mean": 6.08024129178375e-05, + "mean": 6.296396202287724e-05, + "normalized": [ + 0.4796364558374551, + 0.9568353419130547, + 0.9804638447804884, + 0.8949545354119591, + 0.8237377234766136, + 1.1597846916175047, + 1.085649359633547, + 0.9904450076993521, + 0.9595078978373608, + 1.2941748206155441, + 1.0680753760655664, + 1.0231559828609242, + 0.9904019047543832, + 1.096490963646882, + 1.1850351594596897, + 0.9422215946487361, + 0.9418631353047864, + 1.1705024179981143, + 1.0926353878857977, + 1.0382275689133567, + 0.9320833313178482, + 1.0784583323883348, + 1.0985080196909318, + 0.9917906747078287, + 0.9476388143731139, + 0.9855066582268529, + 0.978396290111904, + 1.028407839812513, + 0.8982055143685478, + 1.0282804954549911, + 0.9451124964564701, + 0.9138123627295472 + ], + "peak_layer": 10, + "peak_normalized": 1.2941748206155441, + "population_cv": 0.13431233496292436, + "rest_mean": 6.285090866084729e-05, + "spike_contrast": 1.0115120295379672, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.357445017783903e-05, + "top_five_layers": [ + 10, + 15, + 18, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.019981159013696e-05, + 6.0246144130360335e-05, + 6.173388828756288e-05, + 5.634988337988034e-05, + 5.186579073779285e-05, + 7.302463927771896e-05, + 6.835678505012766e-05, + 6.236234185053036e-05, + 6.041441884008236e-05, + 8.148637425620109e-05, + 6.725025741616264e-05, + 6.442195444833487e-05, + 6.235962791834027e-05, + 6.903941539349034e-05, + 7.461450877599418e-05, + 5.9326004702597857e-05, + 5.930343468207866e-05, + 7.369946979451925e-05, + 6.879665306769311e-05, + 6.537092122016475e-05, + 5.8687659475253895e-05, + 6.790400948375463e-05, + 6.916641723364592e-05, + 6.244707037694752e-05, + 5.966709431959316e-05, + 6.205140380188823e-05, + 6.160370685392991e-05, + 6.475263216998428e-05, + 5.655457789544016e-05, + 6.474461406469345e-05, + 5.950802733423188e-05, + 5.753724690293893e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7888660295284353e-05, + "first_to_last_ratio": 0.41855462748958633, + "imbalance_abs_log_ratio": 0.8709478659353102, + "last_quartile_mean": 4.2739129185065394e-05, + "mean": 3.1986895351110434e-05, + "normalized": [ + 0.2593955501951102, + 0.4997829486650695, + 0.48480994473028793, + 0.4968760363401999, + 0.5474125689219601, + 0.7788406177150305, + 0.7554473604850604, + 0.6514322393717875, + 0.7578135841604356, + 0.9674810188795085, + 0.9032772556875033, + 0.9019220651946442, + 0.8311356963419073, + 0.9587970177975769, + 1.1401394869948454, + 0.8953886492354074, + 1.0262272842074007, + 1.1413916918814533, + 1.174037958462901, + 1.194927715786966, + 1.0267140632187253, + 1.3434898267078448, + 1.3729067467361273, + 1.2011929483563188, + 1.1735959904166378, + 1.3266745689353452, + 1.2739952988364465, + 1.4049535244931144, + 1.2752889026856997, + 1.4680979746684197, + 1.4182292846928506, + 1.3483241791974143 + ], + "peak_layer": 30, + "peak_normalized": 1.4680979746684197, + "population_cv": 0.3184955307507873, + "rest_mean": 3.066251991657299e-05, + "spike_contrast": 1.276428773763581, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.913852269761264e-05, + "top_five_layers": [ + 30, + 31, + 28, + 23, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.297258318634704e-06, + 1.5986504877218977e-05, + 1.5507564967265353e-05, + 1.589352177688852e-05, + 1.7510028555989265e-05, + 2.491269333404489e-05, + 2.416441566310823e-05, + 2.083729486912489e-05, + 2.4240103812189773e-05, + 3.094671410508454e-05, + 2.889303505071439e-05, + 2.8849686714238487e-05, + 2.6585450541460887e-05, + 3.066893987124786e-05, + 3.6469522456172854e-05, + 2.8640703021665104e-05, + 3.282582474639639e-05, + 3.6509576602838933e-05, + 3.755382931558415e-05, + 3.822202779701911e-05, + 3.2841395295690745e-05, + 4.2974068492185324e-05, + 4.391502443468198e-05, + 3.842243313556537e-05, + 3.7539692129939795e-05, + 4.243620060151443e-05, + 4.0751154301688075e-05, + 4.494010136113502e-05, + 4.0792532672639936e-05, + 4.695989628089592e-05, + 4.5364751713350415e-05, + 4.3128704419359565e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00022091567916504573, + "first_to_last_ratio": 1.232986382535498, + "imbalance_abs_log_ratio": 0.20943917994887448, + "last_quartile_mean": 0.00017917122386279516, + "mean": 0.0002198539991695725, + "normalized": [ + 0.4672441326905094, + 1.0299890566632133, + 1.08986441571188, + 1.0084939820045402, + 0.8814237539069011, + 1.2693826047962031, + 1.1886514905905403, + 1.1035827467935222, + 1.0473307057033197, + 1.4274044616309502, + 1.1442998336482333, + 1.0911283607583675, + 1.1094038041384326, + 1.1946873974128753, + 1.2511466100570947, + 0.9923165311956169, + 0.9239703707694655, + 1.2473578196113966, + 1.1181336030529359, + 1.0160745414007064, + 0.9397394354798955, + 0.9905446518023159, + 1.0121913654482901, + 0.9359943959598221, + 0.8839446941554127, + 0.8510229499852975, + 0.8648513533654478, + 0.8786898836618169, + 0.7455559493639876, + 0.8399702482950041, + 0.7392957942269532, + 0.7163130557190521 + ], + "peak_layer": 10, + "peak_normalized": 1.4274044616309502, + "population_cv": 0.18952709251819305, + "rest_mean": 0.0002217885959436427, + "spike_contrast": 0.9441746799407325, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020940717658959329, + "top_five_layers": [ + 10, + 6, + 15, + 18, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010272549116052687, + 0.00022644721320830286, + 0.00023961105034686625, + 0.00022172143508214504, + 0.0001937845372594893, + 0.0002790788421407342, + 0.0002613297838252038, + 0.00024262708029709756, + 0.00023025984410196543, + 0.000313820579322055, + 0.00025157889467664063, + 0.00023988893372006714, + 0.00024390686303377151, + 0.000262656802078709, + 0.00027506958576850593, + 0.0002181647578254342, + 0.00020313858112785965, + 0.00027423660503700376, + 0.0002458261442370713, + 0.00022338805138133466, + 0.00020660547306761146, + 0.00021777520305477083, + 0.00022253431961871684, + 0.00020578211115207523, + 0.0001943387760547921, + 0.00018710079893935472, + 0.0001901410287246108, + 0.00019318348495289683, + 0.00016391345707233995, + 0.00018467081827111542, + 0.00016253713693004102, + 0.00015748428995721042 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014061377383768559, + 0.014165290631353855, + 0.014281988143920898, + 0.014350161887705326, + 0.00966146495193243, + 0.009741888381540775, + 0.009820535778999329, + 0.00991273857653141, + 0.0074927168898284435, + 0.007546746172010899, + 0.00766373984515667, + 0.007762390188872814, + 0.006325984839349985, + 0.006375366821885109, + 0.006445344537496567, + 0.006555322092026472, + 0.005531130358576775, + 0.005611324217170477, + 0.0056777093559503555, + 0.00574630219489336, + 0.004946626722812653, + 0.0050392793491482735, + 0.005084683652967215, + 0.005203457549214363, + 0.004667261149734259, + 0.004717131145298481, + 0.004768412560224533, + 0.004826710093766451, + 0.0044052014127373695, + 0.004504534415900707, + 0.004574235528707504, + 0.004644096828997135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.729895206561196e-05, + "first_to_last_ratio": 1.2469721805352876, + "imbalance_abs_log_ratio": 0.22071835733540474, + "last_quartile_mean": 2.9911615229138988e-05, + "mean": 3.340667609563752e-05, + "normalized": [ + 1.308752778077389, + 1.2456464596610868, + 1.0236956931411572, + 0.9922285647386262, + 1.26903181062375, + 1.1170846373362797, + 1.0287316533628064, + 0.9469237624012169, + 1.1300261751959062, + 1.1132698789554223, + 1.0126240729566007, + 0.9379202574439188, + 1.1130503370514597, + 1.070713488608195, + 0.9897642722064988, + 0.922085688720865, + 1.036831181462568, + 0.9898105546019274, + 0.9395130252451869, + 0.9001967292772246, + 0.9975952535130206, + 0.9404117749145336, + 0.9224862220158919, + 0.8885787590271026, + 0.9461670180110204, + 0.8974372095121397, + 0.8904691498559152, + 0.8684437934704528, + 0.9244147277585869, + 0.9015689206268677, + 0.8740901912628669, + 0.8604359589635168 + ], + "peak_layer": 1, + "peak_normalized": 1.308752778077389, + "population_cv": 0.1164905978879517, + "rest_mean": 3.378375206213807e-05, + "spike_contrast": 0.9285666618330386, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.137046587653458e-05, + "top_five_layers": [ + 1, + 5, + 2, + 9, + 6 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.3721080146497115e-05, + 4.161290780757554e-05, + 3.419827044126578e-05, + 3.314705827506259e-05, + 4.239413465256803e-05, + 3.73180846509058e-05, + 3.4366505133220926e-05, + 3.1633575417799875e-05, + 3.7750418414361775e-05, + 3.719064625329338e-05, + 3.3828404411906376e-05, + 3.1332798243965954e-05, + 3.718331208801828e-05, + 3.5768978705164045e-05, + 3.306473445263691e-05, + 3.080381793552078e-05, + 3.463708344497718e-05, + 3.3066280593629926e-05, + 3.138600732199848e-05, + 3.007258055731654e-05, + 3.332634150865488e-05, + 3.1416031561093405e-05, + 3.0817198421573266e-05, + 2.9684462788281962e-05, + 3.1608295103069395e-05, + 2.9980394174344838e-05, + 2.9747614462394267e-05, + 2.9011820515734144e-05, + 3.0881623388268054e-05, + 3.01184209092753e-05, + 2.9200447897892445e-05, + 2.874430538213346e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.705871965095866e-05, + "first_to_last_ratio": 0.7180297289018083, + "imbalance_abs_log_ratio": 0.33124430563536084, + "last_quartile_mean": 9.33926785364747e-05, + "mean": 8.228966771639534e-05, + "normalized": [ + 0.8685826408392191, + 0.8249006353131848, + 0.7154905339803773, + 0.6884730939760333, + 0.9470487095509537, + 0.8638069620260335, + 0.8256155016985178, + 0.7853664463613442, + 0.9851686390305121, + 0.9692511243547932, + 0.9083804947955202, + 0.8729855457896263, + 1.0422548534804916, + 1.0255492616133721, + 0.9871528463964067, + 0.9398223255303129, + 1.0958375595009802, + 1.0726325633807163, + 1.0495259774381736, + 1.0230838788771643, + 1.1576441025653552, + 1.1142532603054807, + 1.1008567438213033, + 1.056908774400763, + 1.159063578749664, + 1.1192434432436902, + 1.1032241584740294, + 1.1060435702772593, + 1.1786235078649367, + 1.1628365878275773, + 1.141840825041133, + 1.1085318534950752 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235078649367, + "population_cv": 0.1341844256762087, + "rest_mean": 8.049536754023629e-05, + "spike_contrast": 1.1426606459269575, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.197888866765424e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.14753768988885e-05, + 6.788079917896539e-05, + 5.887747829547152e-05, + 5.66542221349664e-05, + 7.793232362018898e-05, + 7.108238787623122e-05, + 6.793962529627606e-05, + 6.462754390668124e-05, + 8.106919995043427e-05, + 7.975935295689851e-05, + 7.475032907677814e-05, + 7.183769048424438e-05, + 8.576680556870997e-05, + 8.4392107964959e-05, + 8.123247971525416e-05, + 7.733766688033938e-05, + 9.017610864248127e-05, + 8.826657722238451e-05, + 8.636514394311234e-05, + 8.418923243880272e-05, + 9.526214853394777e-05, + 9.169153054244816e-05, + 9.0589135652408e-05, + 8.697267185198143e-05, + 9.537895675748587e-05, + 9.210217103827745e-05, + 9.078394941752777e-05, + 9.101595787797123e-05, + 9.698853682493791e-05, + 9.56894364207983e-05, + 9.396170207764953e-05, + 9.122071787714958e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05946834245696664, + "first_to_last_ratio": 0.7180297222365345, + "imbalance_abs_log_ratio": 0.33124431491808765, + "last_quartile_mean": 0.08282156102359295, + "mean": 0.07297530013602227, + "normalized": [ + 0.8685826122359729, + 0.8249005917343738, + 0.71549052911611, + 0.6884731350635278, + 0.9470486699630107, + 0.8638070113577097, + 0.8256154769796886, + 0.7853664269052657, + 0.9851686341386658, + 0.9692511556795396, + 0.9083805454214465, + 0.8729855579949275, + 1.0422548040385637, + 1.0255492365750694, + 0.9871528950273681, + 0.9398223256645052, + 1.0958375053304956, + 1.07263252629438, + 1.0495259690486791, + 1.0230837911446407, + 1.1576441435704576, + 1.1142533045483842, + 1.1008568124470575, + 1.0569088284972943, + 1.1590636022344982, + 1.119243411839159, + 1.1032241423789504, + 1.1060435591240758, + 1.1786235037173105, + 1.1628366076958818, + 1.141840809884862, + 1.1085318743481278 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235037173105, + "population_cv": 0.13418442668798974, + "rest_mean": 0.07138409617322462, + "spike_contrast": 1.1426607032635474, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08156780153512955, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.06338507682085037, + 0.0601973682641983, + 0.05221313610672951, + 0.050241533666849136, + 0.0691111609339714, + 0.06303657591342926, + 0.060249537229537964, + 0.05731235072016716, + 0.07189297676086426, + 0.07073139399290085, + 0.06628934293985367, + 0.06370638310909271, + 0.07605885714292526, + 0.0748397633433342, + 0.07203777879476547, + 0.06858381628990173, + 0.07996907085180283, + 0.07827568054199219, + 0.07658947259187698, + 0.07465984672307968, + 0.0844794288277626, + 0.08131296932697296, + 0.0803353562951088, + 0.07712823897600174, + 0.08458301424980164, + 0.08167712390422821, + 0.08050811290740967, + 0.08071386069059372, + 0.0860104039311409, + 0.08485835045576096, + 0.08332617580890656, + 0.08089544624090195 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.408341275549901e-06, + "first_to_last_ratio": 0.49124437919492936, + "imbalance_abs_log_ratio": 0.7108135577065074, + "last_quartile_mean": 1.5080765479069669e-05, + "mean": 1.1844782648040564e-05, + "normalized": [ + 0.5657990507718705, + 0.5382233871697658, + 0.49942145823009426, + 0.48906984415931193, + 0.8062921756393574, + 0.7122953827323945, + 0.7013508394120195, + 0.6911626993599347, + 0.8940675881372303, + 0.868722127857376, + 0.9735911331661544, + 0.854855860776803, + 0.989483587379368, + 1.0059088486455374, + 1.0303729006301217, + 0.9941025538400318, + 1.1620912657632405, + 1.127012997009919, + 1.113401270263767, + 1.066097770938846, + 1.1826034542812638, + 1.1643547935214302, + 1.2049778184223279, + 1.1791491538213539, + 1.2365172470752577, + 1.219921447624247, + 1.2337761202582145, + 1.2288278254433027, + 1.3039217543779833, + 1.3631114810840155, + 1.2856635723507386, + 1.3138525898567213 + ], + "peak_layer": 30, + "peak_normalized": 1.3631114810840155, + "population_cv": 0.2549101540724733, + "rest_mean": 1.1420299651271543e-05, + "spike_contrast": 1.237882653019464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4136990830593277e-05, + "top_five_layers": [ + 30, + 32, + 29, + 31, + 25 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.701766778860474e-06, + 6.3751390371180605e-06, + 5.915538622502936e-06, + 5.7929260037781205e-06, + 9.550355571263935e-06, + 8.436983989668079e-06, + 8.307348252856173e-06, + 8.186671948351432e-06, + 1.0590036254143342e-05, + 1.0289824786013924e-05, + 1.1531975360412616e-05, + 1.0125581866304856e-05, + 1.1720218026312068e-05, + 1.1914771675947122e-05, + 1.220454305439489e-05, + 1.1774928680097219e-05, + 1.3764718460151926e-05, + 1.334922399109928e-05, + 1.3187996046326589e-05, + 1.2627696378331166e-05, + 1.4007680874783546e-05, + 1.379152945446549e-05, + 1.4272700354922563e-05, + 1.3966765436634887e-05, + 1.4646278032159898e-05, + 1.4449704394792207e-05, + 1.4613809980801307e-05, + 1.4555198504240252e-05, + 1.5444669770658948e-05, + 1.614575921848882e-05, + 1.5228405572997872e-05, + 1.5562298358418047e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6029601637419546e-05, + "first_to_last_ratio": 0.6070213107493427, + "imbalance_abs_log_ratio": 0.49919138022039955, + "last_quartile_mean": 9.230252817360451e-05, + "mean": 7.840498903988191e-05, + "normalized": [ + 0.6598133090619259, + 0.6419677920829977, + 0.6005256373772865, + 0.5913463573822231, + 0.8562376513989928, + 0.8062968559250673, + 0.7923156566726156, + 0.768439227921366, + 0.9790122729268586, + 0.9665730330981341, + 0.9163710065350301, + 0.8881686280668529, + 1.0631190324760253, + 1.0511030519715439, + 1.0188080221730096, + 0.9712812246823179, + 1.1361272291219966, + 1.1138561419734825, + 1.0918134348286874, + 1.0639217496772315, + 1.203414696914147, + 1.1592426077267617, + 1.1442664103435203, + 1.097952968622282, + 1.2042281788778202, + 1.1625445130141736, + 1.1452986211445626, + 1.1473932954411665, + 1.2233942816523278, + 1.205318760675343, + 1.1829469858778916, + 1.146901364356358 + ], + "peak_layer": 29, + "peak_normalized": 1.2233942816523278, + "population_cv": 0.1898597249088972, + "rest_mean": 7.605543874820728e-05, + "spike_contrast": 1.1977126437531995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.109256061492487e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.173265526536852e-05, + 5.033347770222463e-05, + 4.708420601673424e-05, + 4.636450466932729e-05, + 6.713330367347226e-05, + 6.321769615169615e-05, + 6.212150037754327e-05, + 6.024946924299002e-05, + 7.675944652874023e-05, + 7.578414806630462e-05, + 7.184805872384459e-05, + 6.963685154914856e-05, + 8.335383608937263e-05, + 8.241172326961532e-05, + 7.987963181221858e-05, + 7.615329377586022e-05, + 8.907804294722155e-05, + 8.733187860343605e-05, + 8.560362039133906e-05, + 8.341677312273532e-05, + 9.435371612198651e-05, + 9.089040395338088e-05, + 8.971619536168873e-05, + 8.608499047113582e-05, + 9.441749716643244e-05, + 9.114928980125114e-05, + 8.97971258382313e-05, + 8.996135875349864e-05, + 9.592021524440497e-05, + 9.450300422031432e-05, + 9.274894546251744e-05, + 8.992278890218586e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.8511469685763586e-05, + "first_to_last_ratio": 0.5271352332510719, + "imbalance_abs_log_ratio": 0.6402981537748061, + "last_quartile_mean": 9.202850924339145e-05, + "mean": 7.532146378252946e-05, + "normalized": [ + 0.548104082267993, + 0.5416830199005969, + 0.528884702689083, + 0.5247448675895723, + 0.7777813723908718, + 0.7487613756360456, + 0.7496059866168319, + 0.7329069161583202, + 0.9480676391902623, + 0.9399123881532756, + 0.9000936195631334, + 0.883441930780847, + 1.0560589316963704, + 1.0534552095740957, + 1.032095781768693, + 0.9835620691098637, + 1.1601754560133901, + 1.1467788545782247, + 1.1233340534265408, + 1.0967375255670042, + 1.2396083970871608, + 1.1951467072205837, + 1.1828066060767042, + 1.1317734013234224, + 1.244698956870482, + 1.2025862582796845, + 1.1881970083126314, + 1.1882256015362047, + 1.2689460104605979, + 1.2589162611011824, + 1.2307315535498626, + 1.1921774555104672 + ], + "peak_layer": 29, + "peak_normalized": 1.2689460104605979, + "population_cv": 0.23486334953575583, + "rest_mean": 7.254842300123225e-05, + "spike_contrast": 1.244629176846487, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.029588400153444e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 4.1284001781605184e-05, + 4.080035796505399e-05, + 3.9836369978729635e-05, + 3.952455153921619e-05, + 5.858363147126511e-05, + 5.639780283672735e-05, + 5.6461420172126964e-05, + 5.520362174138427e-05, + 7.140984234865755e-05, + 7.079557690303773e-05, + 6.77963689668104e-05, + 6.654213939327747e-05, + 7.954390457598493e-05, + 7.934778841445222e-05, + 7.773896504659206e-05, + 7.408333476632833e-05, + 8.738611359149218e-05, + 8.637706196168438e-05, + 8.461116522084922e-05, + 8.26078758109361e-05, + 9.336911898571998e-05, + 9.002019942272455e-05, + 8.909072494134307e-05, + 8.524682925781235e-05, + 9.375254740007222e-05, + 9.058055729838088e-05, + 8.949673792812973e-05, + 8.949889161158353e-05, + 9.557887096889317e-05, + 9.482341556577012e-05, + 9.270050213672221e-05, + 8.979655103757977e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.61669284151867e-05, + "first_to_last_ratio": 0.8297833624837898, + "imbalance_abs_log_ratio": 0.18659062131596024, + "last_quartile_mean": 0.00011589401856326731, + "mean": 0.00010716335873439675, + "normalized": [ + 0.9380742209455786, + 0.8898820858744699, + 0.7837417114644826, + 0.7583102664636657, + 1.0822070619439073, + 0.9520749081713696, + 0.9107982467806989, + 0.8640017807748708, + 1.0669538303069324, + 1.0266950649381863, + 0.9594029855154438, + 0.8901506143407661, + 1.0795289063552718, + 1.0379165674077444, + 0.9706971366471856, + 0.914979822300863, + 1.052939971267624, + 1.0225707935964177, + 0.9851238030379684, + 0.9636652893153618, + 1.0990044510799908, + 1.0523904893769196, + 1.0422234788441622, + 1.0049019599938585, + 1.0962247910521359, + 1.0599378706264646, + 1.05039930496957, + 1.0583562345651827, + 1.1199788667078585, + 1.1100529546792268, + 1.0850973923636869, + 1.0717171382921347 + ], + "peak_layer": 29, + "peak_normalized": 1.1199788667078585, + "population_cv": 0.09156717959150794, + "rest_mean": 0.00010599351123716958, + "spike_contrast": 1.0706366257223145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011348053521942347, + "top_five_layers": [ + 29, + 30, + 21, + 25, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010052718425868079, + 9.536275319987908e-05, + 8.398839418077841e-05, + 8.12630751170218e-05, + 0.00011597294360399246, + 0.00010202754492638633, + 9.760419925441965e-05, + 9.25893327803351e-05, + 0.00011433835607022047, + 0.00011002409155480564, + 0.00010281284630764276, + 9.539152961224318e-05, + 0.000115685943455901, + 0.0001112266254494898, + 0.0001040231654769741, + 9.805231093196198e-05, + 0.00011283658386673778, + 0.00010958212078548968, + 0.00010556917550275102, + 0.00010326960909878835, + 0.00011777300824178383, + 0.0001127776995417662, + 0.00011168816854478791, + 0.00010768866923172027, + 0.00011747513053705916, + 0.00011358650226611644, + 0.00011256431753281504, + 0.00011341700883349404, + 0.00012002069706795737, + 0.00011895700299646705, + 0.00011628268111962825, + 0.00011484880815260112 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 7.959341947457508, + "forward": { + "activation_sha256": { + "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" + }, + "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", + "loss_nats": 5.516995429992676, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0021613112185150385, + 0.0025601957459002733, + 0.0025323182344436646, + 0.0027253651060163975, + 0.002904455177485943, + 0.0029208383057266474, + 0.00299624796025455, + 0.0030937574338167906, + 0.0033804215490818024, + 0.0034370317589491606, + 0.0038969130255281925, + 0.004255065228790045, + 0.004119740333408117, + 0.003970506135374308, + 0.004385598469525576, + 0.0041999719105660915, + 0.005232699681073427, + 0.004664790350943804, + 0.004898969549685717, + 0.005131533369421959, + 0.005069608334451914, + 0.0051318020559847355, + 0.0058372304774820805, + 0.0059690922498703, + 0.00554921617731452, + 0.0058550527319312096, + 0.006255472078919411, + 0.006502739619463682, + 0.006122888531535864, + 0.007071305997669697, + 0.006241494789719582 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962477027642308, + "first_to_last_ratio": 3.7699040072151497, + "imbalance_abs_log_ratio": 1.3270495388586014, + "last_quartile_mean": 4.234186599205714e-05, + "mean": 8.617327489446325e-05, + "normalized": [ + 2.5428430303860874, + 2.3222064178412967, + 2.1011513502014654, + 1.8911470095267005, + 1.7004447966680725, + 1.5353700031893112, + 1.4229335846259188, + 1.3028620492588128, + 1.2049362956920768, + 1.1705197150696323, + 1.0773612761873523, + 1.0133200843661747, + 0.9906112963967714, + 0.9272530070793641, + 0.8550207003106389, + 0.8185600560490035, + 0.7703816559177782, + 0.7173918665339755, + 0.6760157258502337, + 0.6514535263006792, + 0.6184138115448249, + 0.6053606047045522, + 0.5838821473278404, + 0.5697017460729616, + 0.5457761369570343, + 0.5276086337508958, + 0.5104655265174319, + 0.4951752820428687, + 0.47527333481689216, + 0.465789239220051, + 0.45896414036514044, + 0.45180594922816086 + ], + "peak_layer": 1, + "peak_normalized": 2.5428430303860874, + "population_cv": 0.5705341151299013, + "rest_mean": 9.280180475562466e-05, + "spike_contrast": 0.5428688997682254, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0379213644191625e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021912511147093028, + 0.00020011213200632483, + 0.00018106309289578348, + 0.00016296633111778647, + 0.00014653289690613747, + 0.00013230786134954542, + 0.0001226188469445333, + 0.00011227188952034339, + 0.0001038333066389896, + 0.00010086751717608422, + 9.283974941354245e-05, + 8.732111018616706e-05, + 8.536421955795959e-05, + 7.990442827576771e-05, + 7.367993384832516e-05, + 7.053800072753802e-05, + 6.63863102090545e-05, + 6.182000652188435e-05, + 5.825448897667229e-05, + 5.6137883802875876e-05, + 5.329074338078499e-05, + 5.216590579948388e-05, + 5.031503678765148e-05, + 4.909306517220102e-05, + 4.703131708083674e-05, + 4.5465763832908124e-05, + 4.3988486140733585e-05, + 4.2670875700423494e-05, + 4.095585973118432e-05, + 4.013858415419236e-05, + 3.955044303438626e-05, + 3.893359826179221e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442661636683624, + "first_to_last_ratio": 1.843144421770586, + "imbalance_abs_log_ratio": 0.6114730379455638, + "last_quartile_mean": 0.0001163373926829081, + "mean": 0.00015221274247778638, + "normalized": [ + 1.7741446897870732, + 1.648461275666186, + 1.5301464224684427, + 1.4217121808823485, + 1.3265065496562456, + 1.2454420544341354, + 1.1897998831173078, + 1.1336248235930768, + 1.0869089599614004, + 1.0643003125599004, + 1.0227030055526172, + 0.9941765648180964, + 0.9788958472869692, + 0.9511102301475627, + 0.9228060651723192, + 0.9052418824754537, + 0.8846484396889256, + 0.865292475366091, + 0.8497311645646503, + 0.8388691935254817, + 0.8256303537004397, + 0.8176325372729673, + 0.8076852907347736, + 0.8000670639426607, + 0.7892588692968658, + 0.7814415503461742, + 0.7731202151361662, + 0.7664994567889658, + 0.758490072461763, + 0.7529625284402992, + 0.7485174912671357, + 0.7441725498875049 + ], + "peak_layer": 1, + "peak_normalized": 1.7741446897870732, + "population_cv": 0.27364872432853743, + "rest_mean": 0.00015762320577373935, + "spike_contrast": 0.7803181014868814, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299624067964033, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027004742878489196, + 0.0002509168116375804, + 0.0002329077833564952, + 0.00021640271006617695, + 0.00020191119983792305, + 0.00018957215070258826, + 0.0001811027032090351, + 0.00017255214333999902, + 0.00016544139361940324, + 0.00016200006939470768, + 0.00015566842921543866, + 0.0001513263414381072, + 0.00014900042151566595, + 0.0001447710965294391, + 0.00014046284195501357, + 0.0001377893495373428, + 0.00013465476513374597, + 0.00013170854072086513, + 0.00012933991092722863, + 0.0001276865805266425, + 0.00012567146040964872, + 0.00012445409083738923, + 0.00012293999316170812, + 0.00012178040196886286, + 0.00012013525702059269, + 0.00011894536146428436, + 0.00011767874821089208, + 0.00011667098442558199, + 0.00011545185407157987, + 0.00011461049143690616, + 0.00011393390013836324, + 0.00011327254469506443 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19015566632151604, + "first_to_last_ratio": 1.8431444167877922, + "imbalance_abs_log_ratio": 0.6114730352421437, + "last_quartile_mean": 0.10316916275769472, + "mean": 0.13498378219082952, + "normalized": [ + 1.7741447141792424, + 1.6484611693507762, + 1.5301464164964773, + 1.421712099711509, + 1.3265065258282553, + 1.245442086656075, + 1.1897999853143832, + 1.1336247998774232, + 1.0869090145314644, + 1.0643003539468234, + 1.0227030148964142, + 0.9941765578539046, + 0.978895844117059, + 0.9511102293531465, + 0.9228060470726628, + 0.9052419251832852, + 0.8846484748200404, + 0.8652925213936268, + 0.8497311905993837, + 0.8388692019864461, + 0.8256303029992552, + 0.8176324960789627, + 0.8076852727084025, + 0.800067049482784, + 0.789258877083647, + 0.7814415614650243, + 0.7731201949085273, + 0.7664994759042474, + 0.758490077799444, + 0.7529625178867759, + 0.7485174642151933, + 0.7441725362993381 + ], + "peak_layer": 1, + "peak_normalized": 1.7741447141792424, + "population_cv": 0.2736487184871982, + "rest_mean": 0.13978183573042904, + "spike_contrast": 0.7803180757143806, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907429307699204, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23948076367378235, + 0.2225155234336853, + 0.20654495060443878, + 0.1919080764055252, + 0.17905686795711517, + 0.16811448335647583, + 0.16060370206832886, + 0.1530209630727768, + 0.14671508967876434, + 0.14366328716278076, + 0.13804832100868225, + 0.13419771194458008, + 0.1321350634098053, + 0.12838445603847504, + 0.12456385046243668, + 0.12219297885894775, + 0.11941319704055786, + 0.116800457239151, + 0.11469992995262146, + 0.11323373764753342, + 0.11144670099020004, + 0.11036712676286697, + 0.10902441293001175, + 0.10799607634544373, + 0.10653714835643768, + 0.10548193752765656, + 0.10435868799686432, + 0.1034649983048439, + 0.10238385945558548, + 0.10163772851228714, + 0.10103771835565567, + 0.10045122355222702 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883365585963475e-05, + "first_to_last_ratio": 2.2497546817252747, + "imbalance_abs_log_ratio": 0.8108211799277664, + "last_quartile_mean": 3.059607183786284e-05, + "mean": 4.691841348858361e-05, + "normalized": [ + 1.7359303646705315, + 1.6475859007944922, + 1.5536416135247637, + 1.4921074478374423, + 1.436565604934822, + 1.343834173349448, + 1.2787439414688093, + 1.2483315192759759, + 1.191227587726315, + 1.1603086071531024, + 1.0995306735006558, + 1.059977946939621, + 1.0583984121521266, + 1.0176934649371145, + 0.9754909410255406, + 0.9631621800279858, + 0.9242301483750994, + 0.8752327792179452, + 0.8454160845678106, + 0.8232909667825862, + 0.7863453139125789, + 0.77746685672799, + 0.7497386589361914, + 0.7388508716017261, + 0.7093180450413769, + 0.6908354487677191, + 0.6685471901846501, + 0.6592354484095886, + 0.631440334976849, + 0.6235901145414114, + 0.6199506557967245, + 0.6139807028410055 + ], + "peak_layer": 1, + "peak_normalized": 1.7359303646705315, + "population_cv": 0.3220412433027962, + "rest_mean": 4.907019663530971e-05, + "spike_contrast": 0.7193528234378936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.529878449626267e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144709863699973e-05, + 7.730211655143648e-05, + 7.289439963642508e-05, + 7.000731420703232e-05, + 6.740137905580923e-05, + 6.305056740529835e-05, + 5.999663699185476e-05, + 5.856973439222202e-05, + 5.5890508519951254e-05, + 5.4439839004771784e-05, + 5.1588234782684594e-05, + 4.9732483603293076e-05, + 4.965837433701381e-05, + 4.774856279254891e-05, + 4.576848732540384e-05, + 4.519004141911864e-05, + 4.3363412260077894e-05, + 4.106453343410976e-05, + 3.966558142565191e-05, + 3.862750600092113e-05, + 3.6894074582960457e-05, + 3.647751145763323e-05, + 3.517654840834439e-05, + 3.4665710700210184e-05, + 3.328007733216509e-05, + 3.241290323785506e-05, + 3.136717350571416e-05, + 3.0930281354812905e-05, + 2.9626178729813546e-05, + 2.925785884144716e-05, + 2.9087101211189292e-05, + 2.8807000489905477e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001734125216898974, + "first_to_last_ratio": 1.495517223041696, + "imbalance_abs_log_ratio": 0.4024721155990803, + "last_quartile_mean": 0.00011595488103921525, + "mean": 0.00013990532806928968, + "normalized": [ + 1.3768889945483427, + 1.3340940553978193, + 1.2910049686488183, + 1.2527203273458163, + 1.2147536525239127, + 1.1769854337196524, + 1.14744939519443, + 1.1220955988146075, + 1.0944022472959734, + 1.0773022658134643, + 1.0508559212107247, + 1.03206209473486, + 1.0208171898483585, + 1.000561882500425, + 0.9805161605170362, + 0.9663129296065233, + 0.9487424991891081, + 0.931931984738623, + 0.9181974345490227, + 0.9075104531838204, + 0.8943640941785878, + 0.8862664021756573, + 0.8757830774574916, + 0.86790401991425, + 0.8562894022841879, + 0.8478155487572495, + 0.8387227683262297, + 0.8314011142413483, + 0.8224071457687988, + 0.8161875050523302, + 0.8112199678685613, + 0.8064334645939688 + ], + "peak_layer": 1, + "peak_normalized": 1.3768889945483427, + "population_cv": 0.16325957486062354, + "rest_mean": 0.0001431148236776978, + "spike_contrast": 0.8564734849544942, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225740517838858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926341064972803, + 0.00018664686649572104, + 0.00018061847367789596, + 0.00017526224837638438, + 0.0001699505082797259, + 0.00016466653323732316, + 0.00016053428407758474, + 0.00015698715287726372, + 0.00015311270544771105, + 0.00015072032692842185, + 0.00014702034241054207, + 0.00014439098595175892, + 0.00014281776384450495, + 0.00013998393842484802, + 0.00013717943511437625, + 0.00013519232743419707, + 0.00013273413060232997, + 0.00013038225006312132, + 0.00012846071331296116, + 0.00012696554767899215, + 0.0001251263020094484, + 0.00012399339175317436, + 0.0001225267187692225, + 0.00012142439663875848, + 0.00011979944974882528, + 0.00011861391249112785, + 0.00011734178406186402, + 0.00011631744564510882, + 0.00011505914153531194, + 0.00011418898066040128, + 0.00011349399574100971, + 0.00011282433843007311 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675117381557357, + "first_to_last_ratio": 1.2676837465168667, + "imbalance_abs_log_ratio": 0.23719141364257146, + "last_quartile_mean": 0.00011576323686313117, + "mean": 0.00013042020850662084, + "normalized": [ + 1.169988069421413, + 1.156175152925158, + 1.1417665260225645, + 1.1306726331060184, + 1.1183621029407895, + 1.1062694829504214, + 1.0940115056230622, + 1.0844989956600726, + 1.0711591662817939, + 1.0579765491085218, + 1.0443900226505625, + 1.035471661013805, + 1.0260498632741444, + 1.013992390086173, + 1.0062479316494695, + 0.9977001177294099, + 0.9863163473759529, + 0.9769959732660659, + 0.9687585669786032, + 0.9602628595878113, + 0.9503845997255937, + 0.9426104617668664, + 0.9330243108845692, + 0.9259760938667744, + 0.914797625470175, + 0.906289421438531, + 0.8956357003805738, + 0.888470885240542, + 0.8806216558595509, + 0.8756991509238615, + 0.8712858841028995, + 0.8681382926882502 + ], + "peak_layer": 1, + "peak_normalized": 1.169988069421413, + "population_cv": 0.09123006364857297, + "rest_mean": 0.00013202972385678786, + "spike_contrast": 0.9219804606102014, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012172882561571896, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015259008796419948, + 0.00015078860451467335, + 0.00014890942838974297, + 0.0001474625605624169, + 0.00014585701865144074, + 0.00014427989663090557, + 0.00014268120867200196, + 0.00014144058513920754, + 0.00013970080181024969, + 0.0001379815221298486, + 0.00013620956451632082, + 0.00013504642993211746, + 0.0001338176371064037, + 0.00013224509893916547, + 0.00013123506505507976, + 0.0001301202573813498, + 0.00012863558367826045, + 0.00012742001854348928, + 0.00012634569429792464, + 0.00012523768236860633, + 0.00012394935765769333, + 0.00012293545296415687, + 0.00012168522516731173, + 0.00012076599523425102, + 0.00011930809705518186, + 0.00011819845531135798, + 0.00011680899478960782, + 0.00011587455810513347, + 0.00011485085997264832, + 0.00011420886585256085, + 0.00011363328667357564, + 0.00011322277714498341 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030002388848515693, + "first_to_last_ratio": 2.2836575149168885, + "imbalance_abs_log_ratio": 0.8257783308640597, + "last_quartile_mean": 0.0001313786706305109, + "mean": 0.0001947244400071213, + "normalized": [ + 1.8998833386658438, + 1.7835645331795598, + 1.6713073472446418, + 1.5703972875779748, + 1.4780250735507643, + 1.3761669788219328, + 1.3066447563403325, + 1.2401013786965793, + 1.1762106459027857, + 1.150547637187856, + 1.0768735950666006, + 1.0264900078881174, + 1.0140784159549234, + 0.9666871251601796, + 0.9096841069789114, + 0.8754807117769676, + 0.8421876125778792, + 0.8073809902734768, + 0.77911667677541, + 0.7607651063686129, + 0.7403822030704592, + 0.7308855611321243, + 0.7152791523187013, + 0.7043381880055872, + 0.6944157299089508, + 0.686864629891504, + 0.680599049276497, + 0.6764772714226681, + 0.6702412094778135, + 0.6672258960044359, + 0.6635863560708457, + 0.6581114274310631 + ], + "peak_layer": 1, + "peak_normalized": 1.8998833386658438, + "population_cv": 0.3592853832280775, + "rest_mean": 0.00020492727344390005, + "spike_contrast": 0.6813594750078023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962913944851608, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699537192005664, + 0.0003473036049399525, + 0.0003254443872720003, + 0.00030579473241232336, + 0.00028780760476365685, + 0.00026797334430739284, + 0.0002544356684666127, + 0.0002414780465187505, + 0.0002290369593538344, + 0.00022403974435292184, + 0.00020969360775779933, + 0.00019988269195891917, + 0.0001974658516701311, + 0.00018823760910890996, + 0.00017713772831484675, + 0.00017047749133780599, + 0.00016399451124016196, + 0.0001572168112033978, + 0.00015171305858530104, + 0.00014813955931458622, + 0.00014417050988413393, + 0.00014232128160074353, + 0.00013928233238402754, + 0.0001371518592350185, + 0.00013521971413865685, + 0.00013374933041632175, + 0.00013252926873974502, + 0.00013172665785532445, + 0.00013051234418526292, + 0.00012992518895771354, + 0.00012921648158226162, + 0.000128150379168801 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027073738165199757, + 0.002722575329244137, + 0.0026850115973502398, + 0.0026861224323511124, + 0.00271838391199708, + 0.0026645169127732515, + 0.0026859452482312918, + 0.0027359756641089916, + 0.0026767742820084095, + 0.0027451682835817337, + 0.0026999011170119047, + 0.002663193503394723, + 0.002626648638397455, + 0.0026531771291047335, + 0.0026517712976783514, + 0.002676903735846281, + 0.002670086920261383, + 0.0026355870068073273, + 0.0026610493659973145, + 0.0026388936676084995, + 0.0026469926815479994, + 0.00264862016774714, + 0.002641494618728757, + 0.002674198243767023, + 0.002557571977376938, + 0.00252733426168561, + 0.0025468147359788418, + 0.0026856842450797558, + 0.002551089273765683, + 0.0025778315030038357, + 0.0026001501828432083, + 0.0025662274565547705 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858857113926206, + "first_to_last_ratio": 3.757867690304466, + "imbalance_abs_log_ratio": 1.3238516928952826, + "last_quartile_mean": 4.220174423608114e-05, + "mean": 8.573221236929385e-05, + "normalized": [ + 2.5461641896458365, + 2.3032502265274117, + 2.0924715218207024, + 1.8886722264368014, + 1.7031663195050915, + 1.537434754996766, + 1.4223955235407997, + 1.3049501874408618, + 1.2055297161045848, + 1.1692542365736316, + 1.0769571962730058, + 1.0144505847411098, + 0.9913601784485884, + 0.9317001386730132, + 0.857757082878185, + 0.8155634736467603, + 0.7682612945963089, + 0.7217806418748633, + 0.678338533281248, + 0.6511820384364709, + 0.6192882306865346, + 0.6053793146733001, + 0.5848721334223365, + 0.5718144052517429, + 0.5462281883927541, + 0.527515083157495, + 0.5112993585545178, + 0.496920907994256, + 0.47722358366421574, + 0.46552107689700334, + 0.4601234027970099, + 0.45317424906679377 + ], + "peak_layer": 1, + "peak_normalized": 2.5461641896458365, + "population_cv": 0.5686091737198813, + "rest_mean": 9.231269224848668e-05, + "spike_contrast": 0.5437781067692293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0197621021652594e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021828828903380781, + 0.00019746273756027222, + 0.00017939221288543195, + 0.0001619200484128669, + 0.00014601621660403907, + 0.000131807682919316, + 0.00012194511509733275, + 0.00011187626660102978, + 0.00010335272963857278, + 0.00010024275252362713, + 9.232992306351662e-05, + 8.697109296917915e-05, + 8.499150135321543e-05, + 7.987671415321529e-05, + 7.353741239057854e-05, + 6.992006092332304e-05, + 6.586474046343938e-05, + 6.187985127326101e-05, + 5.8155463193543255e-05, + 5.582727681030519e-05, + 5.309295011102222e-05, + 5.1900507969548926e-05, + 5.0142381951445714e-05, + 4.902291402686387e-05, + 4.682935104938224e-05, + 4.522503513726406e-05, + 4.3834825191879645e-05, + 4.260212881490588e-05, + 4.0913433622336015e-05, + 3.991015182691626e-05, + 3.944739728467539e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242323236947414, + "first_to_last_ratio": 1.8345651540958352, + "imbalance_abs_log_ratio": 0.6068074801791611, + "last_quartile_mean": 0.0001157894184871111, + "mean": 0.00015111673860701558, + "normalized": [ + 1.771707915810251, + 1.6366668487510905, + 1.5242909365839221, + 1.4186705636795862, + 1.3264449439106374, + 1.2453857912414694, + 1.1893415085703798, + 1.1330085309323608, + 1.0858463851390703, + 1.062861239127963, + 1.023237806768965, + 0.9942126959777651, + 0.978436257513136, + 0.9531027445378829, + 0.9232414968561398, + 0.904013237249231, + 0.8853704564278231, + 0.8680164030054869, + 0.8515402788738432, + 0.8395817783841374, + 0.8267435189825344, + 0.8185143642831039, + 0.809172703595369, + 0.8007918347797284, + 0.7909896391756025, + 0.7821052393452608, + 0.7753388667989797, + 0.7682453290925442, + 0.7604995795745633, + 0.7549901567959042, + 0.7508572350862155, + 0.7467737131490535 + ], + "peak_layer": 1, + "peak_normalized": 1.771707915810251, + "population_cv": 0.2714170004285523, + "rest_mean": 0.00015645500981783562, + "spike_contrast": 0.7816309251520459, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229007406858728, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677347220014781, + 0.00024732775636948645, + 0.00023034587502479553, + 0.00021438486874103546, + 0.00020044803386554122, + 0.0001881986390799284, + 0.00017972940986510366, + 0.00017121655400842428, + 0.00016408956435043365, + 0.00016061612404882908, + 0.0001546283601783216, + 0.00015024218009784818, + 0.00014785809617023915, + 0.0001440297783119604, + 0.00013951724395155907, + 0.000136611532070674, + 0.0001337942958343774, + 0.00013117180787958205, + 0.0001286819897359237, + 0.00012687486014328897, + 0.00012493478425312787, + 0.00012369122123345733, + 0.00012227953993715346, + 0.00012101305037504062, + 0.00011953177454415709, + 0.00011818919301731512, + 0.00011716668086592108, + 0.00011609472858253866, + 0.00011492421617731452, + 0.00011409165017539635, + 0.00011346709652571008, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837904930114746, + "first_to_last_ratio": 1.8345651581276443, + "imbalance_abs_log_ratio": 0.6068074823768531, + "last_quartile_mean": 0.10268321540206671, + "mean": 0.13401183625683188, + "normalized": [ + 1.7717078885231814, + 1.6366668119471057, + 1.524290957183596, + 1.418670514273678, + 1.326444920903592, + 1.2453858718827995, + 1.1893415500975306, + 1.1330085716119305, + 1.0858463365841577, + 1.062861212873729, + 1.0232377434953237, + 0.9942127340740732, + 0.9784362443707608, + 0.9531027271661117, + 0.9232414917376033, + 0.904013241653558, + 0.885370477646398, + 0.8680164390463408, + 0.8515402681847718, + 0.8395816969955481, + 0.8267435345782925, + 0.8185144279720658, + 0.8091727254708025, + 0.8007918405918669, + 0.7909896328038464, + 0.7821052665924377, + 0.7753388466443417, + 0.7682453528580236, + 0.7604996016741158, + 0.7549901615676585, + 0.7508572336421532, + 0.7467736753526057 + ], + "peak_layer": 1, + "peak_normalized": 1.7717078885231814, + "population_cv": 0.27141699472708714, + "rest_mean": 0.1387458680956452, + "spike_contrast": 0.7816309473985974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844806432724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23742982745170593, + 0.2193327248096466, + 0.2042730301618576, + 0.19011864066123962, + 0.1777593195438385, + 0.16689644753932953, + 0.15938584506511688, + 0.151836559176445, + 0.1455162614583969, + 0.1424359828233719, + 0.13712596893310547, + 0.13323627412319183, + 0.13112203776836395, + 0.12772704660892487, + 0.1237252876162529, + 0.1211484745144844, + 0.11865012347698212, + 0.11632447689771652, + 0.11411647498607635, + 0.11251388490200043, + 0.11079341918230057, + 0.10969062149524689, + 0.10843872278928757, + 0.10731558501720428, + 0.10600197315216064, + 0.10481136292219162, + 0.10390458256006241, + 0.1029539704322815, + 0.10191594809293747, + 0.10117761790752411, + 0.10062375664710999, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873187476230669e-05, + "first_to_last_ratio": 2.248757500145603, + "imbalance_abs_log_ratio": 0.8103778415280068, + "last_quartile_mean": 3.0564378221242805e-05, + "mean": 4.685761763312257e-05, + "normalized": [ + 1.7400006587426948, + 1.6405194725375354, + 1.5551306973978087, + 1.4944078341288562, + 1.4327724019957897, + 1.3414187773842097, + 1.2804088050749736, + 1.2499327782691996, + 1.1927955171655855, + 1.1608931065609767, + 1.0981756972550076, + 1.0577423904261343, + 1.0542786038595535, + 1.0218640087393278, + 0.9775807389332813, + 0.9570972402588515, + 0.9209982861589985, + 0.8815092249082567, + 0.8478232134430647, + 0.8204952153707635, + 0.7871780668180614, + 0.7759097745574741, + 0.7506963524238334, + 0.7421155343193898, + 0.7072927325438048, + 0.6891814936654451, + 0.6695399899996352, + 0.6616343983211914, + 0.6345111343049281, + 0.6212687927046295, + 0.6200231913288072, + 0.6148038704019305 + ], + "peak_layer": 1, + "peak_normalized": 1.7400006587426948, + "population_cv": 0.3217536161653951, + "rest_mean": 4.900405669845727e-05, + "spike_contrast": 0.719671983430414, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.526684668031521e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153228554874659e-05, + 7.687083416385576e-05, + 7.286971958819777e-05, + 7.002439087955281e-05, + 6.71363013680093e-05, + 6.285568815656006e-05, + 5.999690620228648e-05, + 5.8568872191244736e-05, + 5.5891556257847697e-05, + 5.439668530016206e-05, + 5.145789691596292e-05, + 4.9563288484932855e-05, + 4.9400983698433265e-05, + 4.7882112994557247e-05, + 4.580710447044112e-05, + 4.484729652176611e-05, + 4.3155785533599555e-05, + 4.130542220082134e-05, + 3.972697595600039e-05, + 3.844645107164979e-05, + 3.688528886414133e-05, + 3.635728353401646e-05, + 3.517584264045581e-05, + 3.477376594673842e-05, + 3.314205241622403e-05, + 3.229340290999971e-05, + 3.1373048841487616e-05, + 3.10026116494555e-05, + 2.9731680115219206e-05, + 2.9111175535945222e-05, + 2.9052809622953646e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001719075116852764, + "first_to_last_ratio": 1.4894635014227988, + "imbalance_abs_log_ratio": 0.39841598895878805, + "last_quartile_mean": 0.00011541572621354135, + "mean": 0.00013894635094402474, + "normalized": [ + 1.3738985779819257, + 1.3296082759737424, + 1.2883384004275333, + 1.2499940655466777, + 1.2130730176151365, + 1.1755503979222366, + 1.147409009170091, + 1.11990595257071, + 1.0929037170303153, + 1.075598370036652, + 1.0512762916941902, + 1.0312594730951, + 1.0194418974669357, + 1.0018996501730997, + 0.9803460168062212, + 0.9648004557293595, + 0.9493010807852901, + 0.9344788929945884, + 0.9198953859218769, + 0.908001147597105, + 0.8952913731815082, + 0.886953676542665, + 0.8771894463102521, + 0.8683888934041172, + 0.8579399877904575, + 0.8482799643655919, + 0.8408636339472488, + 0.8330870307470335, + 0.82434015220157, + 0.8181187963300758, + 0.8135579418994486, + 0.809009026741244 + ], + "peak_layer": 1, + "peak_normalized": 1.3738985779819257, + "population_cv": 0.16161423472023242, + "rest_mean": 0.0001421073115866252, + "spike_contrast": 0.8576417505420824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187716347398237, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001908981939777732, + 0.0001847442181315273, + 0.00017900991952046752, + 0.00017368211410939693, + 0.00016855206922627985, + 0.00016333843814209104, + 0.00015942829486448318, + 0.00015560684551019222, + 0.0001518549834145233, + 0.00014945046859793365, + 0.00014607100456487387, + 0.0001432897406630218, + 0.00014164773165248334, + 0.00013921030040364712, + 0.000136215501697734, + 0.0001340555027127266, + 0.0001319019211223349, + 0.00012984243221580982, + 0.0001278161071240902, + 0.00012616344611160457, + 0.00012439746933523566, + 0.00012323897681199014, + 0.00012188227265141904, + 0.00012065946793882176, + 0.00011920763063244522, + 0.00011786540562752634, + 0.00011683493357850239, + 0.00011575440294109285, + 0.0001145390560850501, + 0.00011367462138878182, + 0.00011304090730845928, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552221000485588, + "first_to_last_ratio": 1.262697055715921, + "imbalance_abs_log_ratio": 0.2332499537229619, + "last_quartile_mean": 0.00011524712863320019, + "mean": 0.00012953460168318998, + "normalized": [ + 1.1676213631634658, + 1.1525055688027706, + 1.1391580058081985, + 1.1289307982409096, + 1.1166731577821778, + 1.1054001772932092, + 1.094813144753513, + 1.0822854389705017, + 1.0679308603491235, + 1.0564894822224282, + 1.0456838360716472, + 1.0337072702557388, + 1.0234945545465333, + 1.0158819477934218, + 1.0051772952892828, + 0.9951467950359763, + 0.9866643397779161, + 0.9787559418439278, + 0.970448736962688, + 0.9607908684400951, + 0.9509939230158972, + 0.9428061355388626, + 0.934924980051271, + 0.9261034265480621, + 0.9170582603044295, + 0.9058562786860701, + 0.8981778407016658, + 0.8907564927732574, + 0.8831168120837269, + 0.8778216107141017, + 0.8745192645108896, + 0.8703053916682415 + ], + "peak_layer": 1, + "peak_normalized": 1.1676213631634658, + "population_cv": 0.08967236486753827, + "rest_mean": 0.00013110875065817877, + "spike_contrast": 0.9231588022206524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103419721825049, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124736819416285, + 0.00014928934979252517, + 0.000147560378536582, + 0.00014623560127802193, + 0.00014464781270362437, + 0.00014318757166620344, + 0.00014181618462316692, + 0.00014019341324456036, + 0.0001383339986205101, + 0.00013685194426216185, + 0.00013545223919209093, + 0.00013390085950959474, + 0.00013257795944809914, + 0.00013159186346456409, + 0.00013020524056628346, + 0.00012890594371128827, + 0.00012780717224813998, + 0.00012678276107180864, + 0.0001257066905964166, + 0.0001244556624442339, + 0.00012318661902099848, + 0.0001221260172314942, + 0.00012110513489460573, + 0.0001199624384753406, + 0.00011879077646881342, + 0.00011733973224181682, + 0.00011634510883595794, + 0.00011538378748809919, + 0.0001143941844929941, + 0.00011370827269274741, + 0.00011328050459269434, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029741139769612346, + "first_to_last_ratio": 2.278072415841521, + "imbalance_abs_log_ratio": 0.8233296537390045, + "last_quartile_mean": 0.00013055396993877366, + "mean": 0.00019328556300024502, + "normalized": [ + 1.889398576482247, + 1.780233894901915, + 1.6772594716902607, + 1.566224958959574, + 1.4762232610465842, + 1.375571144634086, + 1.3083686481730985, + 1.2364401527267181, + 1.173148897934763, + 1.1508864184295666, + 1.080457040307468, + 1.0318582947152708, + 1.0081957004611959, + 0.9698994715426978, + 0.9066127339785177, + 0.8754550048258108, + 0.8446146862114313, + 0.8115724449181009, + 0.7806677558079268, + 0.7613480996239201, + 0.7403134026279838, + 0.7294052762839086, + 0.7160186976339193, + 0.7062575707090604, + 0.6942777320753531, + 0.6866005528973866, + 0.6829938476675228, + 0.6753403086482458, + 0.6715967314167978, + 0.6686860556999406, + 0.6640286432895989, + 0.6600445236791291 + ], + "peak_layer": 1, + "peak_normalized": 1.889398576482247, + "population_cv": 0.3579154260008552, + "rest_mean": 0.00020340604748559632, + "spike_contrast": 0.681567477924495, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013863494677934796, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003651934675872326, + 0.0003440935106482357, + 0.00032419004128314555, + 0.0003027286729775369, + 0.0002853326441254467, + 0.00026587804313749075, + 0.00025288877077400684, + 0.00023898603103589267, + 0.00022675274522043765, + 0.00022244972933549434, + 0.0002088367473334074, + 0.00019944331143051386, + 0.00019486967357806861, + 0.00018746756541077048, + 0.00017523515271022916, + 0.00016921281348913908, + 0.0001632518251426518, + 0.00015686523693148047, + 0.00015089180669747293, + 0.00014715759607497603, + 0.00014309189282357693, + 0.00014098350948188454, + 0.00013839607709087431, + 0.0001365093921776861, + 0.0001341938623227179, + 0.00013270997442305088, + 0.00013201285037212074, + 0.00013053353177383542, + 0.0001298099523410201, + 0.00012924736074637622, + 0.00012834715016651899, + 0.00012757707736454904 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0040087271481752396, + 0.005379301495850086, + 0.006410556845366955, + 0.0027253651060163975, + 0.004838948138058186, + 0.006175771821290255, + 0.00742313964292407, + 0.0030937574338167906, + 0.005150026176124811, + 0.007019025273621082, + 0.008143599145114422, + 0.004255065228790045, + 0.0066224560141563416, + 0.008129785768687725, + 0.010099578648805618, + 0.0041999719105660915, + 0.00719039561226964, + 0.009030314162373543, + 0.010736005380749702, + 0.005131533369421959, + 0.00787449348717928, + 0.009912566281855106, + 0.012113703414797783, + 0.0059690922498703, + 0.00876442901790142, + 0.010806876234710217, + 0.012353328987956047, + 0.006502739619463682, + 0.009470963850617409, + 0.012123330496251583, + 0.01384514570236206 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962236011546338, + "first_to_last_ratio": 3.7698422272252903, + "imbalance_abs_log_ratio": 1.3270331510418396, + "last_quartile_mean": 4.2341920561739244e-05, + "mean": 8.61727822893954e-05, + "normalized": [ + 2.542734798705971, + 2.3221448837042376, + 2.101145461274622, + 1.8911858524935854, + 1.7004204056817789, + 1.5354142426015858, + 1.4229001770142031, + 1.3028733810328659, + 1.2049412416934622, + 1.1705258152816296, + 1.0773908232649334, + 1.0133220774376726, + 0.990609360100461, + 0.9272534949358001, + 0.8550385909364457, + 0.8185651575045715, + 0.7703972051466339, + 0.7173947009730917, + 0.6760256695667757, + 0.6514523109028573, + 0.618419077603279, + 0.6053620388084041, + 0.5838930841859182, + 0.5697043695032278, + 0.5457802700901755, + 0.5276149005446632, + 0.5104701332743111, + 0.49517988582787786, + 0.47527562953524777, + 0.46579059316028215, + 0.45896435764104826, + 0.45181000957238127 + ], + "peak_layer": 1, + "peak_normalized": 2.542734798705971, + "population_cv": 0.5705202801130907, + "rest_mean": 9.280119640472414e-05, + "spike_contrast": 0.542873885449774, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.03793460666202e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021911453222855926, + 0.00020010568550787866, + 0.00018106155039276928, + 0.00016296874673571438, + 0.00014652995741926134, + 0.00013231091725174338, + 0.0001226152671733871, + 0.00011227222421439365, + 0.00010383313929196447, + 0.00010086746624438092, + 9.284176485380158e-05, + 8.732078276807442e-05, + 8.536356472177431e-05, + 7.99040135461837e-05, + 7.368105434579775e-05, + 7.053803710732609e-05, + 6.638727063545957e-05, + 6.181989738252014e-05, + 5.825501284562051e-05, + 5.613745815935545e-05, + 5.329089253791608e-05, + 5.216573117650114e-05, + 5.0315691623836756e-05, + 4.9093010602518916e-05, + 4.703140439232811e-05, + 4.546604395727627e-05, + 4.398863165988587e-05, + 4.267102849553339e-05, + 4.095582335139625e-05, + 4.013847137684934e-05, + 3.955023566959426e-05, + 3.893372559105046e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442477373057045, + "first_to_last_ratio": 1.8431254562763404, + "imbalance_abs_log_ratio": 0.6114627481432037, + "last_quartile_mean": 0.00011633759004325839, + "mean": 0.00015221248349917005, + "normalized": [ + 1.7740805953138137, + 1.6484457246979003, + 1.5301434809496548, + 1.4217241600875643, + 1.3264958046529274, + 1.2454577490392524, + 1.1897861330408546, + 1.1336265611690741, + 1.086914920172498, + 1.064301358569379, + 1.0227209980571423, + 0.9941753882589759, + 0.9788979908211966, + 0.9511107967643203, + 0.9228143274453928, + 0.9052447611165039, + 0.8846530997418666, + 0.8652930871753776, + 0.8497381552743479, + 0.8388656494639406, + 0.8256334792987466, + 0.8176309647342147, + 0.8076951735882674, + 0.8000661307359944, + 0.7892605467734257, + 0.78144613040264, + 0.7731219607566042, + 0.7664988488799842, + 0.7584935618388989, + 0.7529639051546407, + 0.7485195296370439, + 0.7441790263875565 + ], + "peak_layer": 1, + "peak_normalized": 1.7740805953138137, + "population_cv": 0.2736411578809574, + "rest_mean": 0.00015762286892384981, + "spike_contrast": 0.7803207938393821, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299640220589936, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027003721334040165, + 0.00025091401766985655, + 0.00023290693934541196, + 0.0002164041652576998, + 0.000201909220777452, + 0.0001895742170745507, + 0.00018110030214302242, + 0.00017255211423616856, + 0.00016544201935175806, + 0.00016199995297938585, + 0.00015567090304102749, + 0.00015132590488065034, + 0.0001490004942752421, + 0.00014477093645837158, + 0.00014046386058907956, + 0.00013778955326415598, + 0.0001346552453469485, + 0.00013170840975362808, + 0.00012934075493831187, + 0.00012768582382705063, + 0.00012567172234412283, + 0.00012445363972801715, + 0.00012294128828216344, + 0.00012178005272289738, + 0.00012013530795229599, + 0.00011894585622940212, + 0.00011767881369451061, + 0.00011667069338727742, + 0.00011545218876563013, + 0.00011461050598882139, + 0.00011393401655368507, + 0.00011327333777444437 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.190154030919075, + "first_to_last_ratio": 1.84312545379361, + "imbalance_abs_log_ratio": 0.6114627467961817, + "last_quartile_mean": 0.10316933691501617, + "mean": 0.13498355145566165, + "normalized": [ + 1.774080628234443, + 1.6484456620184311, + 1.5301434020504485, + 1.4217242444591272, + 1.326495766996536, + 1.2454576834375484, + 1.189786122599316, + 1.133626627255238, + 1.0869149569673349, + 1.0643014004698326, + 1.0227209907477024, + 0.9941753870542451, + 0.9788979589698585, + 0.9511107512164827, + 0.9228143032197887, + 0.9052447972745712, + 0.8846531331847195, + 0.8652931173486506, + 0.8497382179112226, + 0.8388656682549475, + 0.8256334805754935, + 0.8176309131099365, + 0.8076951535479161, + 0.8000661540388709, + 0.7892605573866218, + 0.7814461538045975, + 0.7731219580171443, + 0.7664988542586116, + 0.7584935821787124, + 0.7529638601652359, + 0.7485195164462104, + 0.7441789968002049 + ], + "peak_layer": 1, + "peak_normalized": 1.774080628234443, + "population_cv": 0.27364115345546325, + "rest_mean": 0.13978153549962574, + "spike_contrast": 0.7803207857775153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907443761825561, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2394717037677765, + 0.22251304984092712, + 0.2065441906452179, + 0.19190938770771027, + 0.1790551096200943, + 0.16811630129814148, + 0.16060155630111694, + 0.1530209481716156, + 0.14671564102172852, + 0.1436631828546524, + 0.13805051147937775, + 0.13419732451438904, + 0.13213512301445007, + 0.1283843070268631, + 0.1245647519826889, + 0.12219315767288208, + 0.11941362172365189, + 0.11680033802986145, + 0.11470068246126175, + 0.1132330670952797, + 0.11144693940877914, + 0.11036672443151474, + 0.10902556031942368, + 0.10799577087163925, + 0.10653719305992126, + 0.10548237711191177, + 0.1043587476015091, + 0.10346473753452301, + 0.10238415747880936, + 0.10163773596286774, + 0.10103782266378403, + 0.10045192390680313 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883367768750759e-05, + "first_to_last_ratio": 2.249759357551583, + "imbalance_abs_log_ratio": 0.810823258297238, + "last_quartile_mean": 3.059601795030176e-05, + "mean": 4.691846504556452e-05, + "normalized": [ + 1.7359059710084281, + 1.6475789727938472, + 1.5536456441204052, + 1.4921194549590373, + 1.4365657321885223, + 1.3438513058553838, + 1.2787343947807137, + 1.2483299149152907, + 1.1912354282513808, + 1.1603083401306369, + 1.099547144002948, + 1.059972440022979, + 1.0583955432739494, + 1.0176800955770247, + 0.9754963047746087, + 0.9631595708780456, + 0.9242400656751482, + 0.8752301116137658, + 0.8454245377074081, + 0.8232853322610043, + 0.7863494898171388, + 0.7774642190168813, + 0.7497486129019063, + 0.7388483538626245, + 0.709315327139865, + 0.6908343794807076, + 0.6685469983108738, + 0.6592365073812129, + 0.6314341746588822, + 0.6235898169926835, + 0.6199474545603838, + 0.6139783610863121 + ], + "peak_layer": 1, + "peak_normalized": 1.7359059710084281, + "population_cv": 0.3220406003070692, + "rest_mean": 4.9070239684725595e-05, + "spike_contrast": 0.7193541792517962, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.52988819940947e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144604362314567e-05, + 7.730187644483522e-05, + 7.28946688468568e-05, + 7.000795449130237e-05, + 6.740145909134299e-05, + 6.305144052021205e-05, + 5.999625500408001e-05, + 5.8569723478285596e-05, + 5.589093780145049e-05, + 5.4439886298496276e-05, + 5.158906424185261e-05, + 4.973227987647988e-05, + 4.9658294301480055e-05, + 4.774798799189739e-05, + 4.576878927764483e-05, + 4.51899686595425e-05, + 4.3363925215089694e-05, + 4.1064453398576006e-05, + 3.9666021621087566e-05, + 3.86272840842139e-05, + 3.689431105158292e-05, + 3.6477427784120664e-05, + 3.5177054087398574e-05, + 3.466563066467643e-05, + 3.3279986382694915e-05, + 3.2412888685939834e-05, + 3.136719897156581e-05, + 3.093036502832547e-05, + 2.9625922252307646e-05, + 2.9257877031341195e-05, + 2.9086982976878062e-05, + 2.8806922273361124e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017341253987979144, + "first_to_last_ratio": 1.4955148931299893, + "imbalance_abs_log_ratio": 0.4024705576674927, + "last_quartile_mean": 0.00011595507385209203, + "mean": 0.00013990553861731314, + "normalized": [ + 1.3768818258178377, + 1.3340879911950096, + 1.2910054180628048, + 1.2527236427111486, + 1.2147542166907819, + 1.1769889670731195, + 1.1474411155803916, + 1.1220953663134672, + 1.0944075691302455, + 1.0773007485615413, + 1.0508678613623372, + 1.0320596054419082, + 1.0208161736522385, + 1.000559648637946, + 0.9805197815181176, + 0.9663128275349431, + 0.948744191771126, + 0.9319305822463463, + 0.9182019814348666, + 0.9075038868226692, + 0.8943650364968407, + 0.886262676120959, + 0.8757907045344768, + 0.8679002174812178, + 0.856288373659771, + 0.8478178092769747, + 0.8387215581135022, + 0.8313975747674917, + 0.8224084044020752, + 0.8161864847722404, + 0.8112200992010112, + 0.8064376596145924 + ], + "peak_layer": 1, + "peak_normalized": 1.3768818258178377, + "population_cv": 0.16325888721442564, + "rest_mean": 0.0001431150389921472, + "spike_contrast": 0.8564734877369171, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225742365932092, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926333934534341, + 0.00018664629897102714, + 0.00018061880837194622, + 0.0001752629759721458, + 0.00016995084297377616, + 0.0001646672753849998, + 0.00016053336730692536, + 0.00015698735660407692, + 0.00015311368042603135, + 0.00015072034148033708, + 0.00014702223415952176, + 0.00014439085498452187, + 0.0001428178366040811, + 0.00013998383656144142, + 0.00013718014815822244, + 0.00013519251660909504, + 0.00013273456715978682, + 0.00013038225006312132, + 0.00012846154277212918, + 0.00012696482008323073, + 0.00012512662215158343, + 0.0001239930570591241, + 0.00012252797023393214, + 0.000121424047392793, + 0.00011979948612861335, + 0.00011861440725624561, + 0.00011734179133782163, + 0.0001163171255029738, + 0.00011505949078127742, + 0.00011418900976423174, + 0.00011349418491590768, + 0.00011282509512966499 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675183047074825, + "first_to_last_ratio": 1.2676545313577883, + "imbalance_abs_log_ratio": 0.23716836728277432, + "last_quartile_mean": 0.00011576642282307148, + "mean": 0.00013041823922321782, + "normalized": [ + 1.1700698937843759, + 1.1560440994564782, + 1.1417398043557874, + 1.130463200949706, + 1.118403537277814, + 1.1065304334184665, + 1.0939881912841023, + 1.0846815122144253, + 1.0709468270677047, + 1.0579907390333845, + 1.0442838370166418, + 1.035550673160169, + 1.025959021745254, + 1.0140203095307772, + 1.0062047700154308, + 0.9976631870190885, + 0.9861765922450596, + 0.9770016877798172, + 0.9687702939454803, + 0.9602422119882037, + 0.9504995944091904, + 0.9425883202531964, + 0.9330014667281973, + 0.9259385264566372, + 0.9147443798098084, + 0.906202796813466, + 0.8958065504175365, + 0.8885655303249116, + 0.8806194435723184, + 0.8757905905296257, + 0.8713783728577705, + 0.8681336045391749 + ], + "peak_layer": 1, + "peak_normalized": 1.1700698937843759, + "population_cv": 0.09121429133712369, + "rest_mean": 0.0001320278307604086, + "spike_contrast": 0.921975648780332, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001217264449223876, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001525984553154558, + 0.0001507692359155044, + 0.000148903694935143, + 0.00014743302017450333, + 0.00014586022007279098, + 0.00014431175077334046, + 0.00014267601363826543, + 0.00014146225294098258, + 0.000139670999487862, + 0.00013798128929920495, + 0.00013619365927297622, + 0.00013505469541996717, + 0.00013380376913119107, + 0.00013224674330558628, + 0.00013122745440341532, + 0.00013011347618885338, + 0.00012861541472375393, + 0.00012741883983835578, + 0.0001263453159481287, + 0.0001252330985153094, + 0.0001239624834852293, + 0.00012293070903979242, + 0.00012168040848337114, + 0.00012075927224941552, + 0.00011929935135412961, + 0.00011818537313956767, + 0.00011682951299007982, + 0.00011588515189941972, + 0.00011484883725643158, + 0.00011421906674513593, + 0.00011364363308530301, + 0.00011322045611450449 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030004180916876066, + "first_to_last_ratio": 2.2837506327398005, + "imbalance_abs_log_ratio": 0.8258191057718037, + "last_quartile_mean": 0.00013138116082700435, + "mean": 0.00019473525253488333, + "normalized": [ + 1.9000096511818072, + 1.783700741635724, + 1.6708066910504589, + 1.570366884717585, + 1.4783058799917985, + 1.376185321596543, + 1.3064973297630273, + 1.2402700043336092, + 1.1767953849206394, + 1.1505341943958838, + 1.0771542571420398, + 1.026098162637436, + 1.013898063759585, + 0.9669473025647412, + 0.9095398902231254, + 0.8755205777387188, + 0.842301214161826, + 0.8073124727024382, + 0.7791991818813049, + 0.760574607765185, + 0.7402055397447304, + 0.7308143413325056, + 0.7152667122155245, + 0.7043714155678553, + 0.6942467749765324, + 0.6868749151283189, + 0.680655564574882, + 0.676363713518914, + 0.6701061029149729, + 0.6672276319278244, + 0.6636177363814285, + 0.6582317375530352 + ], + "peak_layer": 1, + "peak_normalized": 1.9000096511818072, + "population_cv": 0.35930746026536914, + "rest_mean": 0.00020494150898971216, + "spike_contrast": 0.6812747128050888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962146767880766, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699988592416048, + 0.0003473494143690914, + 0.0003253649629186839, + 0.0003058057918678969, + 0.0002878782688640058, + 0.0002679917961359024, + 0.0002544210874475539, + 0.00024152429250534624, + 0.00022916354646440595, + 0.000224049566895701, + 0.00020975990628357977, + 0.00019981748482678086, + 0.000197441695490852, + 0.0001882987271528691, + 0.00017711948021315038, + 0.00017049472080543637, + 0.00016402573965024203, + 0.00015721219824627042, + 0.00015173754945863038, + 0.00014811068831477314, + 0.00014414411270990968, + 0.0001423153153154999, + 0.0001392876438330859, + 0.00013716594548895955, + 0.00013519432104658335, + 0.00013375876005738974, + 0.00013254763325676322, + 0.0001317118585575372, + 0.00013049328117631376, + 0.00012993274140171707, + 0.00012922976748086512, + 0.00012818092363886535 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858902861509705, + "first_to_last_ratio": 3.757869257587208, + "imbalance_abs_log_ratio": 1.3238521099622325, + "last_quartile_mean": 4.220184837322449e-05, + "mean": 8.57325238712292e-05, + "normalized": [ + 2.5462257183789148, + 2.3032637538343064, + 2.0924289533379263, + 1.8886492391629932, + 1.7031256747388024, + 1.5374176267930122, + 1.4224231993460965, + 1.3049597038569594, + 1.2055618292025094, + 1.169245405308266, + 1.0769812048488787, + 1.0144334047898542, + 0.9913512297333369, + 0.9317090592952362, + 0.8577499774884012, + 0.81555804918997, + 0.7682725064221458, + 0.7217807351250844, + 0.6783324192661732, + 0.6511800118936032, + 0.6192916242842709, + 0.6053728292385717, + 0.5848720876072855, + 0.5718124973482859, + 0.5462302773879167, + 0.5275173250132701, + 0.5112998770983732, + 0.49691833866078555, + 0.47721658788927634, + 0.46551989467389665, + 0.46012635628943105, + 0.45317260249616614 + ], + "peak_layer": 1, + "peak_normalized": 2.5462257183789148, + "population_cv": 0.5686111305901927, + "rest_mean": 9.231303704803361e-05, + "spike_contrast": 0.5437775023084314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.019775271648541e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021829435718245804, + 0.00019746461475733668, + 0.0001793892151908949, + 0.0001619186659809202, + 0.00014601326256524771, + 0.00013180669338908046, + 0.00012194793089292943, + 0.00011187748896190897, + 0.00010335585830034688, + 0.00010024235962191597, + 9.233231685357168e-05, + 8.69699360919185e-05, + 8.499104296788573e-05, + 7.987776916706935e-05, + 7.353707042057067e-05, + 6.991984992055222e-05, + 6.586594099644572e-05, + 6.188008410390466e-05, + 5.8155150327365845e-05, + 5.582730591413565e-05, + 5.309343396220356e-05, + 5.190014053368941e-05, + 5.014256021240726e-05, + 4.90229285787791e-05, + 4.682970029534772e-05, + 4.522539165918715e-05, + 4.383502891869284e-05, + 4.2602063331287354e-05, + 4.0912982512963936e-05, + 3.991019548266195e-05, + 3.944779382436536e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "rest_mean": 0.00015645541295978345, + "spike_contrast": 0.7816308178118705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229037238284946, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837944976985455, + "first_to_last_ratio": 1.8345658800697515, + "imbalance_abs_log_ratio": 0.6068078758989703, + "last_quartile_mean": 0.10268339328467846, + "mean": 0.1340121782850474, + "normalized": [ + 1.7717451751532867, + 1.6366747548098712, + 1.5242756140248972, + 1.4186514377480737, + 1.3264266357207033, + 1.2453739091726768, + 1.1893513017871604, + 1.1330134634090196, + 1.0858604665419038, + 1.0628542749024097, + 1.0232520332438115, + 0.9942070832344971, + 0.9784308561813967, + 0.9531084116998004, + 0.9232390798282929, + 0.9040072094634526, + 0.8853775581660035, + 0.8680181154203164, + 0.8515367049614931, + 0.8395794986042229, + 0.8267439263823189, + 0.8185133396785964, + 0.809173050926764, + 0.8007913534932437, + 0.7909900046670136, + 0.782107217826699, + 0.7753403703776178, + 0.7682422246066585, + 0.7604971603471317, + 0.7549863443945886, + 0.7508596538022533, + 0.7467717694238248 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451751532867, + "population_cv": 0.27141865758333467, + "rest_mean": 0.13874622461972413, + "spike_contrast": 0.7816308398662988, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844832807779312, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23743543028831482, + 0.21933434903621674, + 0.20427149534225464, + 0.19011656939983368, + 0.17775732278823853, + 0.16689527034759521, + 0.15938755869865417, + 0.15183760225772858, + 0.14551852643489838, + 0.14243541657924652, + 0.13712823390960693, + 0.1332358568906784, + 0.1311216503381729, + 0.12772813439369202, + 0.1237252801656723, + 0.12114797532558441, + 0.1186513751745224, + 0.1163249984383583, + 0.11411628872156143, + 0.11251387745141983, + 0.11079375445842743, + 0.10969075560569763, + 0.10843904316425323, + 0.107315793633461, + 0.10600229352712631, + 0.104811891913414, + 0.10390505194664001, + 0.10295381397008896, + 0.1019158810377121, + 0.10117736458778381, + 0.10062433779239655, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873207303215167e-05, + "first_to_last_ratio": 2.248757814140914, + "imbalance_abs_log_ratio": 0.8103779811585758, + "last_quartile_mean": 3.0564462122129044e-05, + "mean": 4.6857780716891284e-05, + "normalized": [ + 1.7400119939228862, + 1.6405409364325143, + 1.555121247708426, + 1.4944054279879166, + 1.4327475398601994, + 1.3414100714975594, + 1.2804101716466747, + 1.2499370459040815, + 1.192804641985012, + 1.1608858830017554, + 1.098191983597808, + 1.057744066133373, + 1.0542717513613309, + 1.021869458338179, + 0.9775749297615711, + 0.9570927445976433, + 0.9210091333285576, + 0.8815125232807631, + 0.847820728512489, + 0.8204970180441978, + 0.7871838673851995, + 0.7759041238100245, + 0.7506938949785397, + 0.7421070509174572, + 0.7072987335050381, + 0.6891811912847667, + 0.6695407652827761, + 0.6616333377893114, + 0.6345027924930273, + 0.6212629426015038, + 0.6200302724102036, + 0.614801730639213 + ], + "peak_layer": 1, + "peak_normalized": 1.7400119939228862, + "population_cv": 0.32175451551289735, + "rest_mean": 4.9004235565748616e-05, + "spike_contrast": 0.7196709452950106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5266924533061685e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153310045599937e-05, + 7.687210745643824e-05, + 7.28695304132998e-05, + 7.002452184678987e-05, + 6.713537004543468e-05, + 6.28554989816621e-05, + 5.9997179050697014e-05, + 5.8569276006892323e-05, + 5.589217835222371e-05, + 5.439653614303097e-05, + 5.1458839152473956e-05, + 4.9563539505470544e-05, + 4.9400834541302174e-05, + 4.788253500009887e-05, + 4.58069916930981e-05, + 4.4847241952084005e-05, + 4.315644400776364e-05, + 4.130572051508352e-05, + 3.972699778387323e-05, + 3.8446669350378215e-05, + 3.688568904181011e-05, + 3.635714529082179e-05, + 3.5175849916413426e-05, + 3.477348946034908e-05, + 3.3142448955914006e-05, + 3.2293501135427505e-05, + 3.13731943606399e-05, + 3.1002669857116416e-05, + 2.9731392714893445e-05, + 2.9111002731951885e-05, + 2.905324254243169e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017190765356644988, + "first_to_last_ratio": 1.4894621368061003, + "imbalance_abs_log_ratio": 0.3984150727783497, + "last_quartile_mean": 0.00011541592721187044, + "mean": 0.00013894661265112518, + "normalized": [ + 1.3739040544627474, + 1.3296102750467884, + 1.2883342981472405, + 1.2499918159027226, + 1.2130627732820631, + 1.175544413473833, + 1.1474083142393565, + 1.1199112790671786, + 1.092909618037032, + 1.0755929927697745, + 1.0512861461262244, + 1.0312571117882559, + 1.0194388253067554, + 1.0019061415077835, + 0.980344379776871, + 0.9647961249933756, + 0.9493075664595542, + 0.9344817410273759, + 0.9198929201794295, + 0.9079992279088673, + 0.8952920956888009, + 0.8869530532606651, + 0.8771902552778059, + 0.8683889858357474, + 0.8579407806493952, + 0.8482823463732029, + 0.8408658204630546, + 0.8330843095877946, + 0.8243382853592501, + 0.818115422616924, + 0.8135611224181157, + 0.8090075029660142 + ], + "peak_layer": 1, + "peak_normalized": 1.3739040544627474, + "population_cv": 0.16161403917150835, + "rest_mean": 0.00014210757001786045, + "spike_contrast": 0.8576421569620596, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187744287075474, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00019089931447524577, + 0.00018474484386388212, + 0.00017900968668982387, + 0.00017368212866131216, + 0.0001685509632807225, + 0.00016333791427314281, + 0.00015942849859129637, + 0.00015560787869617343, + 0.00015185608936008066, + 0.00014945000293664634, + 0.00014607264893129468, + 0.0001432896824553609, + 0.00014164757158141583, + 0.0001392114645568654, + 0.00013621553080156446, + 0.0001340551534667611, + 0.00013190307072363794, + 0.00012984307250007987, + 0.0001278160052606836, + 0.00012616341700777411, + 0.0001243978040292859, + 0.00012323912233114243, + 0.00012188261462142691, + 0.00012065970804542303, + 0.00011920796532649547, + 0.00011786595860030502, + 0.00011683545744745061, + 0.00011575424287002534, + 0.00011453901242930442, + 0.00011367436673026532, + 0.00011304156214464456, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552358334185556, + "first_to_last_ratio": 1.262638843036328, + "imbalance_abs_log_ratio": 0.2332038508028458, + "last_quartile_mean": 0.00011525352965691127, + "mean": 0.00012953498708156985, + "normalized": [ + 1.1676304712442283, + 1.1525836984023852, + 1.1391521450612143, + 1.1288261090295841, + 1.1167600441451297, + 1.1054423860149858, + 1.094731137320412, + 1.082319740349084, + 1.067957453000188, + 1.0563935463552665, + 1.04560972624621, + 1.0337959762176818, + 1.0235535208863231, + 1.0159040893770892, + 1.0050628637064216, + 0.9950792389307415, + 0.9865814183778941, + 0.9786230528286792, + 0.9704418054120744, + 0.9608381133357455, + 0.9508407831226554, + 0.9429749854457838, + 0.9346839821902475, + 0.9262276149685831, + 0.9170542960948607, + 0.9059488475652709, + 0.8984648923618161, + 0.8908037775288885, + 0.8831483920181569, + 0.8778579808382188, + 0.8744051093280881, + 0.8703028022960919 + ], + "peak_layer": 1, + "peak_normalized": 1.1676304712442283, + "population_cv": 0.0896642712448568, + "rest_mean": 0.00013110964479252557, + "spike_contrast": 0.9231344927669946, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103183544240892, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124899800866842, + 0.00014929991448298097, + 0.00014756005839444697, + 0.00014622247545048594, + 0.00014465949789155275, + 0.00014319346519187093, + 0.00014180598373059183, + 0.00014019827358424664, + 0.00013833785487804562, + 0.00013683992438018322, + 0.0001354430423816666, + 0.00013391274842433631, + 0.0001325859921053052, + 0.00013159512309357524, + 0.00013019080506637692, + 0.00012889757636003196, + 0.00012779681128449738, + 0.00012676592450588942, + 0.00012570616672746837, + 0.00012446215259842575, + 0.00012316714855842292, + 0.00012214825255796313, + 0.00012107427755836397, + 0.00011997888213954866, + 0.00011879061639774591, + 0.00011735207226593047, + 0.0001163826382253319, + 0.0001153902558144182, + 0.00011439861555118114, + 0.00011371332220733166, + 0.00011326605454087257, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029742828155576717, + "first_to_last_ratio": 2.2781890121807993, + "imbalance_abs_log_ratio": 0.8233808344454435, + "last_quartile_mean": 0.00013055469935352448, + "mean": 0.00019329176166138495, + "normalized": [ + 1.8891850069388627, + 1.7801603926271052, + 1.6773544463608039, + 1.566500262900107, + 1.476456430986117, + 1.3754925511314788, + 1.3084504582544147, + 1.236424592515421, + 1.172965600373877, + 1.150871569148191, + 1.0802294364746856, + 1.0319209662753672, + 1.0080063247818516, + 0.9698985570693867, + 0.9067931019504507, + 0.8756163462587222, + 0.8444435806568995, + 0.8113939671049883, + 0.7809677247391495, + 0.7613442367276676, + 0.7402582678385915, + 0.7293562882295931, + 0.7162426695419845, + 0.7062419232587185, + 0.6940340549797547, + 0.6866100033591334, + 0.6831688143117768, + 0.6753320518823956, + 0.6715919825156708, + 0.6686410475337363, + 0.6640239865195754, + 0.6600233567535221 + ], + "peak_layer": 1, + "peak_normalized": 1.8891850069388627, + "population_cv": 0.357915029863326, + "rest_mean": 0.00020341356920888993, + "spike_contrast": 0.6815376252628051, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001386340009048581, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036516389809548855, + 0.0003440903383307159, + 0.0003242187958676368, + 0.0003027915954589844, + 0.00028538686456158757, + 0.00026587137836031616, + 0.0002529126941226423, + 0.00023899068764876574, + 0.00022672458726447076, + 0.00022245399304665625, + 0.0002087994507746771, + 0.00019946182146668434, + 0.00019483931828290224, + 0.00018747340072877705, + 0.00017527563613839447, + 0.00016924942610785365, + 0.00016322398732881993, + 0.00015683576930314302, + 0.0001509546273155138, + 0.00014716156874783337, + 0.0001430858246749267, + 0.0001409785618307069, + 0.00013844380737282336, + 0.00013651074550580233, + 0.00013415106514003128, + 0.00013271605712361634, + 0.00013205090363044292, + 0.00013053612201474607, + 0.00012981319741811603, + 0.00012924280599690974, + 0.00012835036613978446, + 0.00012757707736454904 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.02802751213312149, + 0.01413465291261673, + 0.014225807972252369, + 0.01433455292135477, + 0.01440478302538395, + 0.009703479707241058, + 0.00977881159633398, + 0.009858808480203152, + 0.009950309991836548, + 0.00752287870272994, + 0.007571155205368996, + 0.007693085819482803, + 0.007788818795233965, + 0.006353048607707024, + 0.006389216054230928, + 0.0064707002602517605, + 0.00656962301582098, + 0.005553483963012695, + 0.0056249531917274, + 0.005688921548426151, + 0.005768724251538515, + 0.004966245498508215, + 0.005057469941675663, + 0.005098348017781973, + 0.005216081161051989, + 0.0046758754178881645, + 0.004727107938379049, + 0.004777040798217058, + 0.004837613552808762, + 0.0044186655431985855, + 0.004510571248829365, + 0.0045849340967834 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001580166108396952, + "first_to_last_ratio": 1.9713740353643698, + "imbalance_abs_log_ratio": 0.6787307795042399, + "last_quartile_mean": 8.015557068574708e-05, + "mean": 0.00011843875881822896, + "normalized": [ + 0.8918679597819287, + 1.5814819593588525, + 1.5722239987443252, + 1.3246067564285042, + 1.1252650527356378, + 1.512743589128775, + 1.3788112428745087, + 1.2863035124436846, + 1.097095055843533, + 1.468154353173571, + 1.1523254606068964, + 1.0839888571909224, + 1.054518217028451, + 1.146335080549274, + 1.1665208527784754, + 0.8931136212023528, + 0.8203804970547371, + 1.073741223277434, + 0.919654742394771, + 0.8223971330190447, + 0.7711288002244273, + 0.8470775332584999, + 0.8298711629738247, + 0.7662487463744663, + 0.7210382462310316, + 0.7071376059209309, + 0.7189559392079661, + 0.727980518763353, + 0.6111916361855597, + 0.6986829946379066, + 0.6244678178312321, + 0.6046898327751223 + ], + "peak_layer": 2, + "peak_normalized": 1.5814819593588525, + "population_cv": 0.2904715201464359, + "rest_mean": 0.00012310891098523926, + "spike_contrast": 0.7572151875143335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.321993711637333e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010563173418631777, + 0.0001873087603598833, + 0.00018621225899551064, + 0.00015688478015363216, + 0.0001332749961875379, + 0.00017916747310664505, + 0.00016330469225067645, + 0.00015234819147735834, + 0.00012993857671972364, + 0.00017388637934345752, + 0.0001364799973089248, + 0.0001283862948184833, + 0.00012489582877606153, + 0.00013577050413005054, + 0.00013816128193866462, + 0.00010577926877886057, + 9.71648478298448e-05, + 0.00012717257777694613, + 0.00010892276623053476, + 9.74036956904456e-05, + 9.133153798757121e-05, + 0.0001003268116619438, + 9.828891052166e-05, + 9.07535504666157e-05, + 8.539887494407594e-05, + 8.375250035896897e-05, + 8.515224908478558e-05, + 8.622110908618197e-05, + 7.238877878990024e-05, + 8.275114669231698e-05, + 7.396119326585904e-05, + 7.161871326388791e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001664285200604354, + "first_to_last_ratio": 1.9719337364651075, + "imbalance_abs_log_ratio": 0.6790146534176558, + "last_quartile_mean": 8.439863722742302e-05, + "mean": 0.0001245867656507471, + "normalized": [ + 0.8984334461398115, + 1.5927952483854981, + 1.5658595485525297, + 1.3198735865176374, + 1.131213894321221, + 1.5125315092450449, + 1.3796810696180641, + 1.286366052942558, + 1.0960606282761096, + 1.4536884557404346, + 1.1543342746982923, + 1.0797559630746076, + 1.0497530570909623, + 1.1457182987306291, + 1.1551962694199982, + 0.9023721663305979, + 0.8238710779144349, + 1.0632503993742801, + 0.9211624832492573, + 0.8274335806257342, + 0.7739032412019898, + 0.850990856837025, + 0.8281317613098801, + 0.7681943949939335, + 0.7208251570035091, + 0.7089546837555952, + 0.7203162530736348, + 0.7236174125342815, + 0.6145740290254166, + 0.6994991973458914, + 0.6274381906330746, + 0.6042038120380651 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952483854981, + "population_cv": 0.2889665774838781, + "rest_mean": 0.00012946851159169995, + "spike_contrast": 0.7586812913967177, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.822533756960183e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00011193291720701382, + 0.0001984412083402276, + 0.0001950853766174987, + 0.000164438781212084, + 0.00014093428035266697, + 0.00018844140868168324, + 0.00017189000209327787, + 0.00016026418597903103, + 0.00013655464863404632, + 0.00018111034296453, + 0.00014381477376446128, + 0.0001345233031315729, + 0.00013078533811494708, + 0.00014274133718572557, + 0.00014392216689884663, + 0.00011242362961638719, + 0.00010264343291055411, + 0.0001324669283349067, + 0.00011476465442683548, + 0.00010308727360097691, + 9.641810174798593e-05, + 0.00010602219845168293, + 0.00010317425767425448, + 9.570685506332666e-05, + 8.980527491075918e-05, + 8.832637104205787e-05, + 8.974187221610919e-05, + 9.015315299620852e-05, + 7.656779052922502e-05, + 8.714834257261828e-05, + 7.817049481673166e-05, + 7.527579873567447e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.14759047608822584, + "first_to_last_ratio": 1.9719337947467823, + "imbalance_abs_log_ratio": 0.6790146829732503, + "last_quartile_mean": 0.07484555337578058, + "mean": 0.1104847895912826, + "normalized": [ + 0.8984334974418968, + 1.5927952138130326, + 1.565859646322909, + 1.3198735575362357, + 1.1312139236939245, + 1.5125315007911495, + 1.3796810264894532, + 1.286366010451999, + 1.0960606168617568, + 1.4536884922319049, + 1.1543342013148132, + 1.0797558953237962, + 1.0497530983161607, + 1.1457182562234747, + 1.1551962948455465, + 0.9023721940039697, + 0.8238711061937815, + 1.0632504861804335, + 0.9211625151546727, + 0.8274335809310764, + 0.7739032642161865, + 0.8509909070635955, + 0.8281317391228538, + 0.7681943896830757, + 0.7208251689669645, + 0.7089546589522648, + 0.7203162343708721, + 0.7236173971219663, + 0.6145740294939214, + 0.6994991437375198, + 0.6274381319743458, + 0.6042038211744466 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952138130326, + "population_cv": 0.28896658018674404, + "rest_mean": 0.11481397047086998, + "spike_contrast": 0.7586813040631772, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08710721284151077, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.09926323592662811, + 0.1759796440601349, + 0.1730036735534668, + 0.14582595229148865, + 0.12498193234205246, + 0.16711172461509705, + 0.15243376791477203, + 0.14212387800216675, + 0.12109802663326263, + 0.16061046719551086, + 0.1275363713502884, + 0.1192966029047966, + 0.11598175019025803, + 0.12658444046974182, + 0.12763161957263947, + 0.09969840198755264, + 0.0910252258181572, + 0.11747300624847412, + 0.10177444666624069, + 0.09141882508993149, + 0.08550453931093216, + 0.09402155131101608, + 0.09149596095085144, + 0.08487379550933838, + 0.0796402171254158, + 0.0783287063241005, + 0.07958398759365082, + 0.07994871586561203, + 0.06790108233690262, + 0.07728401571512222, + 0.069322369992733, + 0.06675533205270767 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.093040158499207e-05, + "first_to_last_ratio": 0.8860093039525244, + "imbalance_abs_log_ratio": 0.12102782735710801, + "last_quartile_mean": 5.748291960117058e-05, + "mean": 5.7093861016710434e-05, + "normalized": [ + 0.4337136137178416, + 0.9185569667519601, + 0.9444095926066538, + 0.8728787927927439, + 0.7849140301431445, + 1.1296292591692207, + 1.0726593944235638, + 0.9796135302182263, + 0.9386642177229508, + 1.3046353648934683, + 1.0461832118506482, + 1.0032917361864164, + 0.9662905330123907, + 1.0760080966276753, + 1.1828739682986207, + 0.9087955578115051, + 0.9194941496879511, + 1.198292373503509, + 1.0946132296674007, + 1.0673695485587213, + 0.9244260207272295, + 1.0525679458300632, + 1.125653431968099, + 0.9999504891362015, + 0.9629027708994021, + 1.0063234987710272, + 0.9975933861195996, + 1.0791525148711678, + 0.9412468232151083, + 1.085173475571988, + 1.0031556955576197, + 0.9789667796878816 + ], + "peak_layer": 10, + "peak_normalized": 1.3046353648934683, + "population_cv": 0.1441687540611157, + "rest_mean": 5.695535414647397e-05, + "spike_contrast": 1.0155638391297448, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.784179811598733e-05, + "top_five_layers": [ + 10, + 18, + 15, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.4762384782661684e-05, + 5.2443963795667514e-05, + 5.3919990023132414e-05, + 4.9836020480142906e-05, + 4.481377254705876e-05, + 6.449489592341706e-05, + 6.124226638348773e-05, + 5.592991874436848e-05, + 5.3591964388033375e-05, + 7.448667020071298e-05, + 5.973063889541663e-05, + 5.728179894504137e-05, + 5.516925739357248e-05, + 6.143345672171563e-05, + 6.75348419463262e-05, + 5.1886647270293906e-05, + 5.249747118796222e-05, + 6.84151382301934e-05, + 6.249569560168311e-05, + 6.094024865888059e-05, + 5.277905074763112e-05, + 6.009516800986603e-05, + 6.426790059776977e-05, + 5.7091034250333905e-05, + 5.4975836974335834e-05, + 5.74548939766828e-05, + 5.695645813830197e-05, + 6.161298369988799e-05, + 5.373941530706361e-05, + 6.19567435933277e-05, + 5.727403186028823e-05, + 5.58929932594765e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.801741053801379e-05, + "first_to_last_ratio": 0.954195857595535, + "imbalance_abs_log_ratio": 0.046886327142486896, + "last_quartile_mean": 6.08024129178375e-05, + "mean": 6.296396202287724e-05, + "normalized": [ + 0.4796364558374551, + 0.9568353419130547, + 0.9804638447804884, + 0.8949545354119591, + 0.8237377234766136, + 1.1597846916175047, + 1.085649359633547, + 0.9904450076993521, + 0.9595078978373608, + 1.2941748206155441, + 1.0680753760655664, + 1.0231559828609242, + 0.9904019047543832, + 1.096490963646882, + 1.1850351594596897, + 0.9422215946487361, + 0.9418631353047864, + 1.1705024179981143, + 1.0926353878857977, + 1.0382275689133567, + 0.9320833313178482, + 1.0784583323883348, + 1.0985080196909318, + 0.9917906747078287, + 0.9476388143731139, + 0.9855066582268529, + 0.978396290111904, + 1.028407839812513, + 0.8982055143685478, + 1.0282804954549911, + 0.9451124964564701, + 0.9138123627295472 + ], + "peak_layer": 10, + "peak_normalized": 1.2941748206155441, + "population_cv": 0.13431233496292436, + "rest_mean": 6.285090866084729e-05, + "spike_contrast": 1.0115120295379672, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.357445017783903e-05, + "top_five_layers": [ + 10, + 15, + 18, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.019981159013696e-05, + 6.0246144130360335e-05, + 6.173388828756288e-05, + 5.634988337988034e-05, + 5.186579073779285e-05, + 7.302463927771896e-05, + 6.835678505012766e-05, + 6.236234185053036e-05, + 6.041441884008236e-05, + 8.148637425620109e-05, + 6.725025741616264e-05, + 6.442195444833487e-05, + 6.235962791834027e-05, + 6.903941539349034e-05, + 7.461450877599418e-05, + 5.9326004702597857e-05, + 5.930343468207866e-05, + 7.369946979451925e-05, + 6.879665306769311e-05, + 6.537092122016475e-05, + 5.8687659475253895e-05, + 6.790400948375463e-05, + 6.916641723364592e-05, + 6.244707037694752e-05, + 5.966709431959316e-05, + 6.205140380188823e-05, + 6.160370685392991e-05, + 6.475263216998428e-05, + 5.655457789544016e-05, + 6.474461406469345e-05, + 5.950802733423188e-05, + 5.753724690293893e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7888660295284353e-05, + "first_to_last_ratio": 0.41855462748958633, + "imbalance_abs_log_ratio": 0.8709478659353102, + "last_quartile_mean": 4.2739129185065394e-05, + "mean": 3.1986895351110434e-05, + "normalized": [ + 0.2593955501951102, + 0.4997829486650695, + 0.48480994473028793, + 0.4968760363401999, + 0.5474125689219601, + 0.7788406177150305, + 0.7554473604850604, + 0.6514322393717875, + 0.7578135841604356, + 0.9674810188795085, + 0.9032772556875033, + 0.9019220651946442, + 0.8311356963419073, + 0.9587970177975769, + 1.1401394869948454, + 0.8953886492354074, + 1.0262272842074007, + 1.1413916918814533, + 1.174037958462901, + 1.194927715786966, + 1.0267140632187253, + 1.3434898267078448, + 1.3729067467361273, + 1.2011929483563188, + 1.1735959904166378, + 1.3266745689353452, + 1.2739952988364465, + 1.4049535244931144, + 1.2752889026856997, + 1.4680979746684197, + 1.4182292846928506, + 1.3483241791974143 + ], + "peak_layer": 30, + "peak_normalized": 1.4680979746684197, + "population_cv": 0.3184955307507873, + "rest_mean": 3.066251991657299e-05, + "spike_contrast": 1.276428773763581, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.913852269761264e-05, + "top_five_layers": [ + 30, + 31, + 28, + 23, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.297258318634704e-06, + 1.5986504877218977e-05, + 1.5507564967265353e-05, + 1.589352177688852e-05, + 1.7510028555989265e-05, + 2.491269333404489e-05, + 2.416441566310823e-05, + 2.083729486912489e-05, + 2.4240103812189773e-05, + 3.094671410508454e-05, + 2.889303505071439e-05, + 2.8849686714238487e-05, + 2.6585450541460887e-05, + 3.066893987124786e-05, + 3.6469522456172854e-05, + 2.8640703021665104e-05, + 3.282582474639639e-05, + 3.6509576602838933e-05, + 3.755382931558415e-05, + 3.822202779701911e-05, + 3.2841395295690745e-05, + 4.2974068492185324e-05, + 4.391502443468198e-05, + 3.842243313556537e-05, + 3.7539692129939795e-05, + 4.243620060151443e-05, + 4.0751154301688075e-05, + 4.494010136113502e-05, + 4.0792532672639936e-05, + 4.695989628089592e-05, + 4.5364751713350415e-05, + 4.3128704419359565e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00022091567916504573, + "first_to_last_ratio": 1.232986382535498, + "imbalance_abs_log_ratio": 0.20943917994887448, + "last_quartile_mean": 0.00017917122386279516, + "mean": 0.0002198539991695725, + "normalized": [ + 0.4672441326905094, + 1.0299890566632133, + 1.08986441571188, + 1.0084939820045402, + 0.8814237539069011, + 1.2693826047962031, + 1.1886514905905403, + 1.1035827467935222, + 1.0473307057033197, + 1.4274044616309502, + 1.1442998336482333, + 1.0911283607583675, + 1.1094038041384326, + 1.1946873974128753, + 1.2511466100570947, + 0.9923165311956169, + 0.9239703707694655, + 1.2473578196113966, + 1.1181336030529359, + 1.0160745414007064, + 0.9397394354798955, + 0.9905446518023159, + 1.0121913654482901, + 0.9359943959598221, + 0.8839446941554127, + 0.8510229499852975, + 0.8648513533654478, + 0.8786898836618169, + 0.7455559493639876, + 0.8399702482950041, + 0.7392957942269532, + 0.7163130557190521 + ], + "peak_layer": 10, + "peak_normalized": 1.4274044616309502, + "population_cv": 0.18952709251819305, + "rest_mean": 0.0002217885959436427, + "spike_contrast": 0.9441746799407325, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020940717658959329, + "top_five_layers": [ + 10, + 6, + 15, + 18, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010272549116052687, + 0.00022644721320830286, + 0.00023961105034686625, + 0.00022172143508214504, + 0.0001937845372594893, + 0.0002790788421407342, + 0.0002613297838252038, + 0.00024262708029709756, + 0.00023025984410196543, + 0.000313820579322055, + 0.00025157889467664063, + 0.00023988893372006714, + 0.00024390686303377151, + 0.000262656802078709, + 0.00027506958576850593, + 0.0002181647578254342, + 0.00020313858112785965, + 0.00027423660503700376, + 0.0002458261442370713, + 0.00022338805138133466, + 0.00020660547306761146, + 0.00021777520305477083, + 0.00022253431961871684, + 0.00020578211115207523, + 0.0001943387760547921, + 0.00018710079893935472, + 0.0001901410287246108, + 0.00019318348495289683, + 0.00016391345707233995, + 0.00018467081827111542, + 0.00016253713693004102, + 0.00015748428995721042 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014061377383768559, + 0.014165290631353855, + 0.014281988143920898, + 0.014350161887705326, + 0.00966146495193243, + 0.009741888381540775, + 0.009820535778999329, + 0.00991273857653141, + 0.0074927168898284435, + 0.007546746172010899, + 0.00766373984515667, + 0.007762390188872814, + 0.006325984839349985, + 0.006375366821885109, + 0.006445344537496567, + 0.006555322092026472, + 0.005531130358576775, + 0.005611324217170477, + 0.0056777093559503555, + 0.00574630219489336, + 0.004946626722812653, + 0.0050392793491482735, + 0.005084683652967215, + 0.005203457549214363, + 0.004667261149734259, + 0.004717131145298481, + 0.004768412560224533, + 0.004826710093766451, + 0.0044052014127373695, + 0.004504534415900707, + 0.004574235528707504, + 0.004644096828997135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.729895206561196e-05, + "first_to_last_ratio": 1.2469721805352876, + "imbalance_abs_log_ratio": 0.22071835733540474, + "last_quartile_mean": 2.9911615229138988e-05, + "mean": 3.340667609563752e-05, + "normalized": [ + 1.308752778077389, + 1.2456464596610868, + 1.0236956931411572, + 0.9922285647386262, + 1.26903181062375, + 1.1170846373362797, + 1.0287316533628064, + 0.9469237624012169, + 1.1300261751959062, + 1.1132698789554223, + 1.0126240729566007, + 0.9379202574439188, + 1.1130503370514597, + 1.070713488608195, + 0.9897642722064988, + 0.922085688720865, + 1.036831181462568, + 0.9898105546019274, + 0.9395130252451869, + 0.9001967292772246, + 0.9975952535130206, + 0.9404117749145336, + 0.9224862220158919, + 0.8885787590271026, + 0.9461670180110204, + 0.8974372095121397, + 0.8904691498559152, + 0.8684437934704528, + 0.9244147277585869, + 0.9015689206268677, + 0.8740901912628669, + 0.8604359589635168 + ], + "peak_layer": 1, + "peak_normalized": 1.308752778077389, + "population_cv": 0.1164905978879517, + "rest_mean": 3.378375206213807e-05, + "spike_contrast": 0.9285666618330386, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.137046587653458e-05, + "top_five_layers": [ + 1, + 5, + 2, + 9, + 6 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.3721080146497115e-05, + 4.161290780757554e-05, + 3.419827044126578e-05, + 3.314705827506259e-05, + 4.239413465256803e-05, + 3.73180846509058e-05, + 3.4366505133220926e-05, + 3.1633575417799875e-05, + 3.7750418414361775e-05, + 3.719064625329338e-05, + 3.3828404411906376e-05, + 3.1332798243965954e-05, + 3.718331208801828e-05, + 3.5768978705164045e-05, + 3.306473445263691e-05, + 3.080381793552078e-05, + 3.463708344497718e-05, + 3.3066280593629926e-05, + 3.138600732199848e-05, + 3.007258055731654e-05, + 3.332634150865488e-05, + 3.1416031561093405e-05, + 3.0817198421573266e-05, + 2.9684462788281962e-05, + 3.1608295103069395e-05, + 2.9980394174344838e-05, + 2.9747614462394267e-05, + 2.9011820515734144e-05, + 3.0881623388268054e-05, + 3.01184209092753e-05, + 2.9200447897892445e-05, + 2.874430538213346e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.705871965095866e-05, + "first_to_last_ratio": 0.7180297289018083, + "imbalance_abs_log_ratio": 0.33124430563536084, + "last_quartile_mean": 9.33926785364747e-05, + "mean": 8.228966771639534e-05, + "normalized": [ + 0.8685826408392191, + 0.8249006353131848, + 0.7154905339803773, + 0.6884730939760333, + 0.9470487095509537, + 0.8638069620260335, + 0.8256155016985178, + 0.7853664463613442, + 0.9851686390305121, + 0.9692511243547932, + 0.9083804947955202, + 0.8729855457896263, + 1.0422548534804916, + 1.0255492616133721, + 0.9871528463964067, + 0.9398223255303129, + 1.0958375595009802, + 1.0726325633807163, + 1.0495259774381736, + 1.0230838788771643, + 1.1576441025653552, + 1.1142532603054807, + 1.1008567438213033, + 1.056908774400763, + 1.159063578749664, + 1.1192434432436902, + 1.1032241584740294, + 1.1060435702772593, + 1.1786235078649367, + 1.1628365878275773, + 1.141840825041133, + 1.1085318534950752 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235078649367, + "population_cv": 0.1341844256762087, + "rest_mean": 8.049536754023629e-05, + "spike_contrast": 1.1426606459269575, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.197888866765424e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.14753768988885e-05, + 6.788079917896539e-05, + 5.887747829547152e-05, + 5.66542221349664e-05, + 7.793232362018898e-05, + 7.108238787623122e-05, + 6.793962529627606e-05, + 6.462754390668124e-05, + 8.106919995043427e-05, + 7.975935295689851e-05, + 7.475032907677814e-05, + 7.183769048424438e-05, + 8.576680556870997e-05, + 8.4392107964959e-05, + 8.123247971525416e-05, + 7.733766688033938e-05, + 9.017610864248127e-05, + 8.826657722238451e-05, + 8.636514394311234e-05, + 8.418923243880272e-05, + 9.526214853394777e-05, + 9.169153054244816e-05, + 9.0589135652408e-05, + 8.697267185198143e-05, + 9.537895675748587e-05, + 9.210217103827745e-05, + 9.078394941752777e-05, + 9.101595787797123e-05, + 9.698853682493791e-05, + 9.56894364207983e-05, + 9.396170207764953e-05, + 9.122071787714958e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05946834245696664, + "first_to_last_ratio": 0.7180297222365345, + "imbalance_abs_log_ratio": 0.33124431491808765, + "last_quartile_mean": 0.08282156102359295, + "mean": 0.07297530013602227, + "normalized": [ + 0.8685826122359729, + 0.8249005917343738, + 0.71549052911611, + 0.6884731350635278, + 0.9470486699630107, + 0.8638070113577097, + 0.8256154769796886, + 0.7853664269052657, + 0.9851686341386658, + 0.9692511556795396, + 0.9083805454214465, + 0.8729855579949275, + 1.0422548040385637, + 1.0255492365750694, + 0.9871528950273681, + 0.9398223256645052, + 1.0958375053304956, + 1.07263252629438, + 1.0495259690486791, + 1.0230837911446407, + 1.1576441435704576, + 1.1142533045483842, + 1.1008568124470575, + 1.0569088284972943, + 1.1590636022344982, + 1.119243411839159, + 1.1032241423789504, + 1.1060435591240758, + 1.1786235037173105, + 1.1628366076958818, + 1.141840809884862, + 1.1085318743481278 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235037173105, + "population_cv": 0.13418442668798974, + "rest_mean": 0.07138409617322462, + "spike_contrast": 1.1426607032635474, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08156780153512955, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.06338507682085037, + 0.0601973682641983, + 0.05221313610672951, + 0.050241533666849136, + 0.0691111609339714, + 0.06303657591342926, + 0.060249537229537964, + 0.05731235072016716, + 0.07189297676086426, + 0.07073139399290085, + 0.06628934293985367, + 0.06370638310909271, + 0.07605885714292526, + 0.0748397633433342, + 0.07203777879476547, + 0.06858381628990173, + 0.07996907085180283, + 0.07827568054199219, + 0.07658947259187698, + 0.07465984672307968, + 0.0844794288277626, + 0.08131296932697296, + 0.0803353562951088, + 0.07712823897600174, + 0.08458301424980164, + 0.08167712390422821, + 0.08050811290740967, + 0.08071386069059372, + 0.0860104039311409, + 0.08485835045576096, + 0.08332617580890656, + 0.08089544624090195 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.408341275549901e-06, + "first_to_last_ratio": 0.49124437919492936, + "imbalance_abs_log_ratio": 0.7108135577065074, + "last_quartile_mean": 1.5080765479069669e-05, + "mean": 1.1844782648040564e-05, + "normalized": [ + 0.5657990507718705, + 0.5382233871697658, + 0.49942145823009426, + 0.48906984415931193, + 0.8062921756393574, + 0.7122953827323945, + 0.7013508394120195, + 0.6911626993599347, + 0.8940675881372303, + 0.868722127857376, + 0.9735911331661544, + 0.854855860776803, + 0.989483587379368, + 1.0059088486455374, + 1.0303729006301217, + 0.9941025538400318, + 1.1620912657632405, + 1.127012997009919, + 1.113401270263767, + 1.066097770938846, + 1.1826034542812638, + 1.1643547935214302, + 1.2049778184223279, + 1.1791491538213539, + 1.2365172470752577, + 1.219921447624247, + 1.2337761202582145, + 1.2288278254433027, + 1.3039217543779833, + 1.3631114810840155, + 1.2856635723507386, + 1.3138525898567213 + ], + "peak_layer": 30, + "peak_normalized": 1.3631114810840155, + "population_cv": 0.2549101540724733, + "rest_mean": 1.1420299651271543e-05, + "spike_contrast": 1.237882653019464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4136990830593277e-05, + "top_five_layers": [ + 30, + 32, + 29, + 31, + 25 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.701766778860474e-06, + 6.3751390371180605e-06, + 5.915538622502936e-06, + 5.7929260037781205e-06, + 9.550355571263935e-06, + 8.436983989668079e-06, + 8.307348252856173e-06, + 8.186671948351432e-06, + 1.0590036254143342e-05, + 1.0289824786013924e-05, + 1.1531975360412616e-05, + 1.0125581866304856e-05, + 1.1720218026312068e-05, + 1.1914771675947122e-05, + 1.220454305439489e-05, + 1.1774928680097219e-05, + 1.3764718460151926e-05, + 1.334922399109928e-05, + 1.3187996046326589e-05, + 1.2627696378331166e-05, + 1.4007680874783546e-05, + 1.379152945446549e-05, + 1.4272700354922563e-05, + 1.3966765436634887e-05, + 1.4646278032159898e-05, + 1.4449704394792207e-05, + 1.4613809980801307e-05, + 1.4555198504240252e-05, + 1.5444669770658948e-05, + 1.614575921848882e-05, + 1.5228405572997872e-05, + 1.5562298358418047e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6029601637419546e-05, + "first_to_last_ratio": 0.6070213107493427, + "imbalance_abs_log_ratio": 0.49919138022039955, + "last_quartile_mean": 9.230252817360451e-05, + "mean": 7.840498903988191e-05, + "normalized": [ + 0.6598133090619259, + 0.6419677920829977, + 0.6005256373772865, + 0.5913463573822231, + 0.8562376513989928, + 0.8062968559250673, + 0.7923156566726156, + 0.768439227921366, + 0.9790122729268586, + 0.9665730330981341, + 0.9163710065350301, + 0.8881686280668529, + 1.0631190324760253, + 1.0511030519715439, + 1.0188080221730096, + 0.9712812246823179, + 1.1361272291219966, + 1.1138561419734825, + 1.0918134348286874, + 1.0639217496772315, + 1.203414696914147, + 1.1592426077267617, + 1.1442664103435203, + 1.097952968622282, + 1.2042281788778202, + 1.1625445130141736, + 1.1452986211445626, + 1.1473932954411665, + 1.2233942816523278, + 1.205318760675343, + 1.1829469858778916, + 1.146901364356358 + ], + "peak_layer": 29, + "peak_normalized": 1.2233942816523278, + "population_cv": 0.1898597249088972, + "rest_mean": 7.605543874820728e-05, + "spike_contrast": 1.1977126437531995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.109256061492487e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.173265526536852e-05, + 5.033347770222463e-05, + 4.708420601673424e-05, + 4.636450466932729e-05, + 6.713330367347226e-05, + 6.321769615169615e-05, + 6.212150037754327e-05, + 6.024946924299002e-05, + 7.675944652874023e-05, + 7.578414806630462e-05, + 7.184805872384459e-05, + 6.963685154914856e-05, + 8.335383608937263e-05, + 8.241172326961532e-05, + 7.987963181221858e-05, + 7.615329377586022e-05, + 8.907804294722155e-05, + 8.733187860343605e-05, + 8.560362039133906e-05, + 8.341677312273532e-05, + 9.435371612198651e-05, + 9.089040395338088e-05, + 8.971619536168873e-05, + 8.608499047113582e-05, + 9.441749716643244e-05, + 9.114928980125114e-05, + 8.97971258382313e-05, + 8.996135875349864e-05, + 9.592021524440497e-05, + 9.450300422031432e-05, + 9.274894546251744e-05, + 8.992278890218586e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.8511469685763586e-05, + "first_to_last_ratio": 0.5271352332510719, + "imbalance_abs_log_ratio": 0.6402981537748061, + "last_quartile_mean": 9.202850924339145e-05, + "mean": 7.532146378252946e-05, + "normalized": [ + 0.548104082267993, + 0.5416830199005969, + 0.528884702689083, + 0.5247448675895723, + 0.7777813723908718, + 0.7487613756360456, + 0.7496059866168319, + 0.7329069161583202, + 0.9480676391902623, + 0.9399123881532756, + 0.9000936195631334, + 0.883441930780847, + 1.0560589316963704, + 1.0534552095740957, + 1.032095781768693, + 0.9835620691098637, + 1.1601754560133901, + 1.1467788545782247, + 1.1233340534265408, + 1.0967375255670042, + 1.2396083970871608, + 1.1951467072205837, + 1.1828066060767042, + 1.1317734013234224, + 1.244698956870482, + 1.2025862582796845, + 1.1881970083126314, + 1.1882256015362047, + 1.2689460104605979, + 1.2589162611011824, + 1.2307315535498626, + 1.1921774555104672 + ], + "peak_layer": 29, + "peak_normalized": 1.2689460104605979, + "population_cv": 0.23486334953575583, + "rest_mean": 7.254842300123225e-05, + "spike_contrast": 1.244629176846487, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.029588400153444e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 4.1284001781605184e-05, + 4.080035796505399e-05, + 3.9836369978729635e-05, + 3.952455153921619e-05, + 5.858363147126511e-05, + 5.639780283672735e-05, + 5.6461420172126964e-05, + 5.520362174138427e-05, + 7.140984234865755e-05, + 7.079557690303773e-05, + 6.77963689668104e-05, + 6.654213939327747e-05, + 7.954390457598493e-05, + 7.934778841445222e-05, + 7.773896504659206e-05, + 7.408333476632833e-05, + 8.738611359149218e-05, + 8.637706196168438e-05, + 8.461116522084922e-05, + 8.26078758109361e-05, + 9.336911898571998e-05, + 9.002019942272455e-05, + 8.909072494134307e-05, + 8.524682925781235e-05, + 9.375254740007222e-05, + 9.058055729838088e-05, + 8.949673792812973e-05, + 8.949889161158353e-05, + 9.557887096889317e-05, + 9.482341556577012e-05, + 9.270050213672221e-05, + 8.979655103757977e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.61669284151867e-05, + "first_to_last_ratio": 0.8297833624837898, + "imbalance_abs_log_ratio": 0.18659062131596024, + "last_quartile_mean": 0.00011589401856326731, + "mean": 0.00010716335873439675, + "normalized": [ + 0.9380742209455786, + 0.8898820858744699, + 0.7837417114644826, + 0.7583102664636657, + 1.0822070619439073, + 0.9520749081713696, + 0.9107982467806989, + 0.8640017807748708, + 1.0669538303069324, + 1.0266950649381863, + 0.9594029855154438, + 0.8901506143407661, + 1.0795289063552718, + 1.0379165674077444, + 0.9706971366471856, + 0.914979822300863, + 1.052939971267624, + 1.0225707935964177, + 0.9851238030379684, + 0.9636652893153618, + 1.0990044510799908, + 1.0523904893769196, + 1.0422234788441622, + 1.0049019599938585, + 1.0962247910521359, + 1.0599378706264646, + 1.05039930496957, + 1.0583562345651827, + 1.1199788667078585, + 1.1100529546792268, + 1.0850973923636869, + 1.0717171382921347 + ], + "peak_layer": 29, + "peak_normalized": 1.1199788667078585, + "population_cv": 0.09156717959150794, + "rest_mean": 0.00010599351123716958, + "spike_contrast": 1.0706366257223145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011348053521942347, + "top_five_layers": [ + 29, + 30, + 21, + 25, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010052718425868079, + 9.536275319987908e-05, + 8.398839418077841e-05, + 8.12630751170218e-05, + 0.00011597294360399246, + 0.00010202754492638633, + 9.760419925441965e-05, + 9.25893327803351e-05, + 0.00011433835607022047, + 0.00011002409155480564, + 0.00010281284630764276, + 9.539152961224318e-05, + 0.000115685943455901, + 0.0001112266254494898, + 0.0001040231654769741, + 9.805231093196198e-05, + 0.00011283658386673778, + 0.00010958212078548968, + 0.00010556917550275102, + 0.00010326960909878835, + 0.00011777300824178383, + 0.0001127776995417662, + 0.00011168816854478791, + 0.00010768866923172027, + 0.00011747513053705916, + 0.00011358650226611644, + 0.00011256431753281504, + 0.00011341700883349404, + 0.00012002069706795737, + 0.00011895700299646705, + 0.00011628268111962825, + 0.00011484880815260112 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 7.959341947457508, + "forward": { + "activation_sha256": { + "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" + }, + "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", + "loss_nats": 5.516995429992676, + "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" + }, + "loss_nats": 5.516995429992676, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 1, + "max_source_mass": 0.5, + "mixer_index": 1, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 2, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 2, + "max_source_mass": 0.5, + "mixer_index": 3, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 4, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 3, + "max_source_mass": 0.5, + "mixer_index": 5, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 6, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6931471824645996, + "group": 1, + "latest_source_mass": 0.5, + "layer": 4, + "max_source_mass": 0.5, + "mixer_index": 7, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "current_group_partial", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6931471824645996, + "group": 2, + "latest_source_mass": 0.5, + "layer": 5, + "max_source_mass": 0.5, + "mixer_index": 8, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + }, + { + "label": "completed_group_1", + "mean": 0.5, + "median": 0.5, + "p05": 0.5, + "p95": 0.5 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 5, + "max_source_mass": 0.3333333432674408, + "mixer_index": 9, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 10, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 6, + "max_source_mass": 0.3333333432674408, + "mixer_index": 11, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 12, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 7, + "max_source_mass": 0.3333333432674408, + "mixer_index": 13, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 14, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0986123085021973, + "group": 2, + "latest_source_mass": 0.3333333432674408, + "layer": 8, + "max_source_mass": 0.3333333432674408, + "mixer_index": 15, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "current_group_partial", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0986123085021973, + "group": 3, + "latest_source_mass": 0.3333333432674408, + "layer": 9, + "max_source_mass": 0.3333333432674408, + "mixer_index": 16, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_1", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + }, + { + "label": "completed_group_2", + "mean": 0.3333333432674408, + "median": 0.3333333432674408, + "p05": 0.3333333432674408, + "p95": 0.3333333432674408 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 9, + "max_source_mass": 0.25, + "mixer_index": 17, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 18, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 10, + "max_source_mass": 0.25, + "mixer_index": 19, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 20, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 11, + "max_source_mass": 0.25, + "mixer_index": 21, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 22, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3862943649291992, + "group": 3, + "latest_source_mass": 0.25, + "layer": 12, + "max_source_mass": 0.25, + "mixer_index": 23, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "current_group_partial", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862943649291992, + "group": 4, + "latest_source_mass": 0.25, + "layer": 13, + "max_source_mass": 0.25, + "mixer_index": 24, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_1", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_2", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + }, + { + "label": "completed_group_3", + "mean": 0.25, + "median": 0.25, + "p05": 0.25, + "p95": 0.25 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 13, + "max_source_mass": 0.20000000298023224, + "mixer_index": 25, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 26, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 14, + "max_source_mass": 0.20000000298023224, + "mixer_index": 27, + "normalized_entropy": 1.0000000186840277, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 28, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 15, + "max_source_mass": 0.20000000298023224, + "mixer_index": 29, + "normalized_entropy": 1.0000000186840277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 30, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6094379425048828, + "group": 4, + "latest_source_mass": 0.20000000298023224, + "layer": 16, + "max_source_mass": 0.20000000298023224, + "mixer_index": 31, + "normalized_entropy": 1.0000000186840277, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "current_group_partial", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6094379425048828, + "group": 5, + "latest_source_mass": 0.20000000298023224, + "layer": 17, + "max_source_mass": 0.20000000298023224, + "mixer_index": 32, + "normalized_entropy": 1.0000000186840277, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_1", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_2", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_3", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + }, + { + "label": "completed_group_4", + "mean": 0.20000000298023224, + "median": 0.20000000298023224, + "p05": 0.20000000298023224, + "p95": 0.20000000298023224 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 17, + "max_source_mass": 0.1666666716337204, + "mixer_index": 33, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 34, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 18, + "max_source_mass": 0.1666666716337204, + "mixer_index": 35, + "normalized_entropy": 1.0000001451965654, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 36, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 19, + "max_source_mass": 0.1666666716337204, + "mixer_index": 37, + "normalized_entropy": 1.0000001451965654, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 38, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.791759729385376, + "group": 5, + "latest_source_mass": 0.1666666716337204, + "layer": 20, + "max_source_mass": 0.1666666716337204, + "mixer_index": 39, + "normalized_entropy": 1.0000001451965654, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "current_group_partial", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.791759729385376, + "group": 6, + "latest_source_mass": 0.1666666716337204, + "layer": 21, + "max_source_mass": 0.1666666716337204, + "mixer_index": 40, + "normalized_entropy": 1.0000001451965654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_1", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_2", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_3", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_4", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + }, + { + "label": "completed_group_5", + "mean": 0.1666666716337204, + "median": 0.1666666716337204, + "p05": 0.1666666716337204, + "p95": 0.1666666716337204 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 21, + "max_source_mass": 0.1428571492433548, + "mixer_index": 41, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 42, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 22, + "max_source_mass": 0.1428571492433548, + "mixer_index": 43, + "normalized_entropy": 1.00000003408302, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 44, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 23, + "max_source_mass": 0.1428571492433548, + "mixer_index": 45, + "normalized_entropy": 1.00000003408302, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 46, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9459102153778076, + "group": 6, + "latest_source_mass": 0.1428571492433548, + "layer": 24, + "max_source_mass": 0.1428571492433548, + "mixer_index": 47, + "normalized_entropy": 1.00000003408302, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "current_group_partial", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9459102153778076, + "group": 7, + "latest_source_mass": 0.1428571492433548, + "layer": 25, + "max_source_mass": 0.1428571492433548, + "mixer_index": 48, + "normalized_entropy": 1.00000003408302, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_1", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_2", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_3", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_4", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_5", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + }, + { + "label": "completed_group_6", + "mean": 0.1428571492433548, + "median": 0.1428571492433548, + "p05": 0.1428571492433548, + "p95": 0.1428571492433548 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 25, + "max_source_mass": 0.125, + "mixer_index": 49, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 50, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 26, + "max_source_mass": 0.125, + "mixer_index": 51, + "normalized_entropy": 1.0000000027478353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 52, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 27, + "max_source_mass": 0.125, + "mixer_index": 53, + "normalized_entropy": 1.0000000027478353, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 54, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.079441547393799, + "group": 7, + "latest_source_mass": 0.125, + "layer": 28, + "max_source_mass": 0.125, + "mixer_index": 55, + "normalized_entropy": 1.0000000027478353, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "current_group_partial", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079441547393799, + "group": 8, + "latest_source_mass": 0.125, + "layer": 29, + "max_source_mass": 0.125, + "mixer_index": 56, + "normalized_entropy": 1.0000000027478353, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_1", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_2", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_3", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_4", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_5", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_6", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + }, + { + "label": "completed_group_7", + "mean": 0.125, + "median": 0.125, + "p05": 0.125, + "p95": 0.125 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 29, + "max_source_mass": 0.1111111119389534, + "mixer_index": 57, + "normalized_entropy": 1.0000000180537645, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 58, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 30, + "max_source_mass": 0.1111111119389534, + "mixer_index": 59, + "normalized_entropy": 1.0000000180537645, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 60, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 31, + "max_source_mass": 0.1111111119389534, + "mixer_index": 61, + "normalized_entropy": 1.0000000180537645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 62, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1972246170043945, + "group": 8, + "latest_source_mass": 0.1111111119389534, + "layer": 32, + "max_source_mass": 0.1111111119389534, + "mixer_index": 63, + "normalized_entropy": 1.0000000180537645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "current_group_partial", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1972246170043945, + "group": null, + "latest_source_mass": 0.1111111119389534, + "layer": null, + "max_source_mass": 0.1111111119389534, + "mixer_index": null, + "normalized_entropy": 1.0000000180537645, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_1", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_2", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_3", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_4", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_5", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_6", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_7", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + }, + { + "label": "completed_group_8", + "mean": 0.1111111119389534, + "median": 0.1111111119389534, + "p05": 0.1111111119389534, + "p95": 0.1111111119389534 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0021613112185150385, + 0.0025601957459002733, + 0.0025323182344436646, + 0.0027253651060163975, + 0.002904455177485943, + 0.0029208383057266474, + 0.00299624796025455, + 0.0030937574338167906, + 0.0033804215490818024, + 0.0034370317589491606, + 0.0038969130255281925, + 0.004255065228790045, + 0.004119740333408117, + 0.003970506135374308, + 0.004385598469525576, + 0.0041999719105660915, + 0.005232699681073427, + 0.004664790350943804, + 0.004898969549685717, + 0.005131533369421959, + 0.005069608334451914, + 0.0051318020559847355, + 0.0058372304774820805, + 0.0059690922498703, + 0.00554921617731452, + 0.0058550527319312096, + 0.006255472078919411, + 0.006502739619463682, + 0.006122888531535864, + 0.007071305997669697, + 0.006241494789719582 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962477027642308, + "first_to_last_ratio": 3.7699040072151497, + "imbalance_abs_log_ratio": 1.3270495388586014, + "last_quartile_mean": 4.234186599205714e-05, + "mean": 8.617327489446325e-05, + "normalized": [ + 2.5428430303860874, + 2.3222064178412967, + 2.1011513502014654, + 1.8911470095267005, + 1.7004447966680725, + 1.5353700031893112, + 1.4229335846259188, + 1.3028620492588128, + 1.2049362956920768, + 1.1705197150696323, + 1.0773612761873523, + 1.0133200843661747, + 0.9906112963967714, + 0.9272530070793641, + 0.8550207003106389, + 0.8185600560490035, + 0.7703816559177782, + 0.7173918665339755, + 0.6760157258502337, + 0.6514535263006792, + 0.6184138115448249, + 0.6053606047045522, + 0.5838821473278404, + 0.5697017460729616, + 0.5457761369570343, + 0.5276086337508958, + 0.5104655265174319, + 0.4951752820428687, + 0.47527333481689216, + 0.465789239220051, + 0.45896414036514044, + 0.45180594922816086 + ], + "peak_layer": 1, + "peak_normalized": 2.5428430303860874, + "population_cv": 0.5705341151299013, + "rest_mean": 9.280180475562466e-05, + "spike_contrast": 0.5428688997682254, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0379213644191625e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021912511147093028, + 0.00020011213200632483, + 0.00018106309289578348, + 0.00016296633111778647, + 0.00014653289690613747, + 0.00013230786134954542, + 0.0001226188469445333, + 0.00011227188952034339, + 0.0001038333066389896, + 0.00010086751717608422, + 9.283974941354245e-05, + 8.732111018616706e-05, + 8.536421955795959e-05, + 7.990442827576771e-05, + 7.367993384832516e-05, + 7.053800072753802e-05, + 6.63863102090545e-05, + 6.182000652188435e-05, + 5.825448897667229e-05, + 5.6137883802875876e-05, + 5.329074338078499e-05, + 5.216590579948388e-05, + 5.031503678765148e-05, + 4.909306517220102e-05, + 4.703131708083674e-05, + 4.5465763832908124e-05, + 4.3988486140733585e-05, + 4.2670875700423494e-05, + 4.095585973118432e-05, + 4.013858415419236e-05, + 3.955044303438626e-05, + 3.893359826179221e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442661636683624, + "first_to_last_ratio": 1.843144421770586, + "imbalance_abs_log_ratio": 0.6114730379455638, + "last_quartile_mean": 0.0001163373926829081, + "mean": 0.00015221274247778638, + "normalized": [ + 1.7741446897870732, + 1.648461275666186, + 1.5301464224684427, + 1.4217121808823485, + 1.3265065496562456, + 1.2454420544341354, + 1.1897998831173078, + 1.1336248235930768, + 1.0869089599614004, + 1.0643003125599004, + 1.0227030055526172, + 0.9941765648180964, + 0.9788958472869692, + 0.9511102301475627, + 0.9228060651723192, + 0.9052418824754537, + 0.8846484396889256, + 0.865292475366091, + 0.8497311645646503, + 0.8388691935254817, + 0.8256303537004397, + 0.8176325372729673, + 0.8076852907347736, + 0.8000670639426607, + 0.7892588692968658, + 0.7814415503461742, + 0.7731202151361662, + 0.7664994567889658, + 0.758490072461763, + 0.7529625284402992, + 0.7485174912671357, + 0.7441725498875049 + ], + "peak_layer": 1, + "peak_normalized": 1.7741446897870732, + "population_cv": 0.27364872432853743, + "rest_mean": 0.00015762320577373935, + "spike_contrast": 0.7803181014868814, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299624067964033, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027004742878489196, + 0.0002509168116375804, + 0.0002329077833564952, + 0.00021640271006617695, + 0.00020191119983792305, + 0.00018957215070258826, + 0.0001811027032090351, + 0.00017255214333999902, + 0.00016544139361940324, + 0.00016200006939470768, + 0.00015566842921543866, + 0.0001513263414381072, + 0.00014900042151566595, + 0.0001447710965294391, + 0.00014046284195501357, + 0.0001377893495373428, + 0.00013465476513374597, + 0.00013170854072086513, + 0.00012933991092722863, + 0.0001276865805266425, + 0.00012567146040964872, + 0.00012445409083738923, + 0.00012293999316170812, + 0.00012178040196886286, + 0.00012013525702059269, + 0.00011894536146428436, + 0.00011767874821089208, + 0.00011667098442558199, + 0.00011545185407157987, + 0.00011461049143690616, + 0.00011393390013836324, + 0.00011327254469506443 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.19015566632151604, + "first_to_last_ratio": 1.8431444167877922, + "imbalance_abs_log_ratio": 0.6114730352421437, + "last_quartile_mean": 0.10316916275769472, + "mean": 0.13498378219082952, + "normalized": [ + 1.7741447141792424, + 1.6484611693507762, + 1.5301464164964773, + 1.421712099711509, + 1.3265065258282553, + 1.245442086656075, + 1.1897999853143832, + 1.1336247998774232, + 1.0869090145314644, + 1.0643003539468234, + 1.0227030148964142, + 0.9941765578539046, + 0.978895844117059, + 0.9511102293531465, + 0.9228060470726628, + 0.9052419251832852, + 0.8846484748200404, + 0.8652925213936268, + 0.8497311905993837, + 0.8388692019864461, + 0.8256303029992552, + 0.8176324960789627, + 0.8076852727084025, + 0.800067049482784, + 0.789258877083647, + 0.7814415614650243, + 0.7731201949085273, + 0.7664994759042474, + 0.758490077799444, + 0.7529625178867759, + 0.7485174642151933, + 0.7441725362993381 + ], + "peak_layer": 1, + "peak_normalized": 1.7741447141792424, + "population_cv": 0.2736487184871982, + "rest_mean": 0.13978183573042904, + "spike_contrast": 0.7803180757143806, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907429307699204, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23948076367378235, + 0.2225155234336853, + 0.20654495060443878, + 0.1919080764055252, + 0.17905686795711517, + 0.16811448335647583, + 0.16060370206832886, + 0.1530209630727768, + 0.14671508967876434, + 0.14366328716278076, + 0.13804832100868225, + 0.13419771194458008, + 0.1321350634098053, + 0.12838445603847504, + 0.12456385046243668, + 0.12219297885894775, + 0.11941319704055786, + 0.116800457239151, + 0.11469992995262146, + 0.11323373764753342, + 0.11144670099020004, + 0.11036712676286697, + 0.10902441293001175, + 0.10799607634544373, + 0.10653714835643768, + 0.10548193752765656, + 0.10435868799686432, + 0.1034649983048439, + 0.10238385945558548, + 0.10163772851228714, + 0.10103771835565567, + 0.10045122355222702 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883365585963475e-05, + "first_to_last_ratio": 2.2497546817252747, + "imbalance_abs_log_ratio": 0.8108211799277664, + "last_quartile_mean": 3.059607183786284e-05, + "mean": 4.691841348858361e-05, + "normalized": [ + 1.7359303646705315, + 1.6475859007944922, + 1.5536416135247637, + 1.4921074478374423, + 1.436565604934822, + 1.343834173349448, + 1.2787439414688093, + 1.2483315192759759, + 1.191227587726315, + 1.1603086071531024, + 1.0995306735006558, + 1.059977946939621, + 1.0583984121521266, + 1.0176934649371145, + 0.9754909410255406, + 0.9631621800279858, + 0.9242301483750994, + 0.8752327792179452, + 0.8454160845678106, + 0.8232909667825862, + 0.7863453139125789, + 0.77746685672799, + 0.7497386589361914, + 0.7388508716017261, + 0.7093180450413769, + 0.6908354487677191, + 0.6685471901846501, + 0.6592354484095886, + 0.631440334976849, + 0.6235901145414114, + 0.6199506557967245, + 0.6139807028410055 + ], + "peak_layer": 1, + "peak_normalized": 1.7359303646705315, + "population_cv": 0.3220412433027962, + "rest_mean": 4.907019663530971e-05, + "spike_contrast": 0.7193528234378936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.529878449626267e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144709863699973e-05, + 7.730211655143648e-05, + 7.289439963642508e-05, + 7.000731420703232e-05, + 6.740137905580923e-05, + 6.305056740529835e-05, + 5.999663699185476e-05, + 5.856973439222202e-05, + 5.5890508519951254e-05, + 5.4439839004771784e-05, + 5.1588234782684594e-05, + 4.9732483603293076e-05, + 4.965837433701381e-05, + 4.774856279254891e-05, + 4.576848732540384e-05, + 4.519004141911864e-05, + 4.3363412260077894e-05, + 4.106453343410976e-05, + 3.966558142565191e-05, + 3.862750600092113e-05, + 3.6894074582960457e-05, + 3.647751145763323e-05, + 3.517654840834439e-05, + 3.4665710700210184e-05, + 3.328007733216509e-05, + 3.241290323785506e-05, + 3.136717350571416e-05, + 3.0930281354812905e-05, + 2.9626178729813546e-05, + 2.925785884144716e-05, + 2.9087101211189292e-05, + 2.8807000489905477e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001734125216898974, + "first_to_last_ratio": 1.495517223041696, + "imbalance_abs_log_ratio": 0.4024721155990803, + "last_quartile_mean": 0.00011595488103921525, + "mean": 0.00013990532806928968, + "normalized": [ + 1.3768889945483427, + 1.3340940553978193, + 1.2910049686488183, + 1.2527203273458163, + 1.2147536525239127, + 1.1769854337196524, + 1.14744939519443, + 1.1220955988146075, + 1.0944022472959734, + 1.0773022658134643, + 1.0508559212107247, + 1.03206209473486, + 1.0208171898483585, + 1.000561882500425, + 0.9805161605170362, + 0.9663129296065233, + 0.9487424991891081, + 0.931931984738623, + 0.9181974345490227, + 0.9075104531838204, + 0.8943640941785878, + 0.8862664021756573, + 0.8757830774574916, + 0.86790401991425, + 0.8562894022841879, + 0.8478155487572495, + 0.8387227683262297, + 0.8314011142413483, + 0.8224071457687988, + 0.8161875050523302, + 0.8112199678685613, + 0.8064334645939688 + ], + "peak_layer": 1, + "peak_normalized": 1.3768889945483427, + "population_cv": 0.16325957486062354, + "rest_mean": 0.0001431148236776978, + "spike_contrast": 0.8564734849544942, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225740517838858, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926341064972803, + 0.00018664686649572104, + 0.00018061847367789596, + 0.00017526224837638438, + 0.0001699505082797259, + 0.00016466653323732316, + 0.00016053428407758474, + 0.00015698715287726372, + 0.00015311270544771105, + 0.00015072032692842185, + 0.00014702034241054207, + 0.00014439098595175892, + 0.00014281776384450495, + 0.00013998393842484802, + 0.00013717943511437625, + 0.00013519232743419707, + 0.00013273413060232997, + 0.00013038225006312132, + 0.00012846071331296116, + 0.00012696554767899215, + 0.0001251263020094484, + 0.00012399339175317436, + 0.0001225267187692225, + 0.00012142439663875848, + 0.00011979944974882528, + 0.00011861391249112785, + 0.00011734178406186402, + 0.00011631744564510882, + 0.00011505914153531194, + 0.00011418898066040128, + 0.00011349399574100971, + 0.00011282433843007311 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675117381557357, + "first_to_last_ratio": 1.2676837465168667, + "imbalance_abs_log_ratio": 0.23719141364257146, + "last_quartile_mean": 0.00011576323686313117, + "mean": 0.00013042020850662084, + "normalized": [ + 1.169988069421413, + 1.156175152925158, + 1.1417665260225645, + 1.1306726331060184, + 1.1183621029407895, + 1.1062694829504214, + 1.0940115056230622, + 1.0844989956600726, + 1.0711591662817939, + 1.0579765491085218, + 1.0443900226505625, + 1.035471661013805, + 1.0260498632741444, + 1.013992390086173, + 1.0062479316494695, + 0.9977001177294099, + 0.9863163473759529, + 0.9769959732660659, + 0.9687585669786032, + 0.9602628595878113, + 0.9503845997255937, + 0.9426104617668664, + 0.9330243108845692, + 0.9259760938667744, + 0.914797625470175, + 0.906289421438531, + 0.8956357003805738, + 0.888470885240542, + 0.8806216558595509, + 0.8756991509238615, + 0.8712858841028995, + 0.8681382926882502 + ], + "peak_layer": 1, + "peak_normalized": 1.169988069421413, + "population_cv": 0.09123006364857297, + "rest_mean": 0.00013202972385678786, + "spike_contrast": 0.9219804606102014, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012172882561571896, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015259008796419948, + 0.00015078860451467335, + 0.00014890942838974297, + 0.0001474625605624169, + 0.00014585701865144074, + 0.00014427989663090557, + 0.00014268120867200196, + 0.00014144058513920754, + 0.00013970080181024969, + 0.0001379815221298486, + 0.00013620956451632082, + 0.00013504642993211746, + 0.0001338176371064037, + 0.00013224509893916547, + 0.00013123506505507976, + 0.0001301202573813498, + 0.00012863558367826045, + 0.00012742001854348928, + 0.00012634569429792464, + 0.00012523768236860633, + 0.00012394935765769333, + 0.00012293545296415687, + 0.00012168522516731173, + 0.00012076599523425102, + 0.00011930809705518186, + 0.00011819845531135798, + 0.00011680899478960782, + 0.00011587455810513347, + 0.00011485085997264832, + 0.00011420886585256085, + 0.00011363328667357564, + 0.00011322277714498341 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030002388848515693, + "first_to_last_ratio": 2.2836575149168885, + "imbalance_abs_log_ratio": 0.8257783308640597, + "last_quartile_mean": 0.0001313786706305109, + "mean": 0.0001947244400071213, + "normalized": [ + 1.8998833386658438, + 1.7835645331795598, + 1.6713073472446418, + 1.5703972875779748, + 1.4780250735507643, + 1.3761669788219328, + 1.3066447563403325, + 1.2401013786965793, + 1.1762106459027857, + 1.150547637187856, + 1.0768735950666006, + 1.0264900078881174, + 1.0140784159549234, + 0.9666871251601796, + 0.9096841069789114, + 0.8754807117769676, + 0.8421876125778792, + 0.8073809902734768, + 0.77911667677541, + 0.7607651063686129, + 0.7403822030704592, + 0.7308855611321243, + 0.7152791523187013, + 0.7043381880055872, + 0.6944157299089508, + 0.686864629891504, + 0.680599049276497, + 0.6764772714226681, + 0.6702412094778135, + 0.6672258960044359, + 0.6635863560708457, + 0.6581114274310631 + ], + "peak_layer": 1, + "peak_normalized": 1.8998833386658438, + "population_cv": 0.3592853832280775, + "rest_mean": 0.00020492727344390005, + "spike_contrast": 0.6813594750078023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962913944851608, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699537192005664, + 0.0003473036049399525, + 0.0003254443872720003, + 0.00030579473241232336, + 0.00028780760476365685, + 0.00026797334430739284, + 0.0002544356684666127, + 0.0002414780465187505, + 0.0002290369593538344, + 0.00022403974435292184, + 0.00020969360775779933, + 0.00019988269195891917, + 0.0001974658516701311, + 0.00018823760910890996, + 0.00017713772831484675, + 0.00017047749133780599, + 0.00016399451124016196, + 0.0001572168112033978, + 0.00015171305858530104, + 0.00014813955931458622, + 0.00014417050988413393, + 0.00014232128160074353, + 0.00013928233238402754, + 0.0001371518592350185, + 0.00013521971413865685, + 0.00013374933041632175, + 0.00013252926873974502, + 0.00013172665785532445, + 0.00013051234418526292, + 0.00012992518895771354, + 0.00012921648158226162, + 0.000128150379168801 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0027073738165199757, + 0.002722575329244137, + 0.0026850115973502398, + 0.0026861224323511124, + 0.00271838391199708, + 0.0026645169127732515, + 0.0026859452482312918, + 0.0027359756641089916, + 0.0026767742820084095, + 0.0027451682835817337, + 0.0026999011170119047, + 0.002663193503394723, + 0.002626648638397455, + 0.0026531771291047335, + 0.0026517712976783514, + 0.002676903735846281, + 0.002670086920261383, + 0.0026355870068073273, + 0.0026610493659973145, + 0.0026388936676084995, + 0.0026469926815479994, + 0.00264862016774714, + 0.002641494618728757, + 0.002674198243767023, + 0.002557571977376938, + 0.00252733426168561, + 0.0025468147359788418, + 0.0026856842450797558, + 0.002551089273765683, + 0.0025778315030038357, + 0.0026001501828432083, + 0.0025662274565547705 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858857113926206, + "first_to_last_ratio": 3.757867690304466, + "imbalance_abs_log_ratio": 1.3238516928952826, + "last_quartile_mean": 4.220174423608114e-05, + "mean": 8.573221236929385e-05, + "normalized": [ + 2.5461641896458365, + 2.3032502265274117, + 2.0924715218207024, + 1.8886722264368014, + 1.7031663195050915, + 1.537434754996766, + 1.4223955235407997, + 1.3049501874408618, + 1.2055297161045848, + 1.1692542365736316, + 1.0769571962730058, + 1.0144505847411098, + 0.9913601784485884, + 0.9317001386730132, + 0.857757082878185, + 0.8155634736467603, + 0.7682612945963089, + 0.7217806418748633, + 0.678338533281248, + 0.6511820384364709, + 0.6192882306865346, + 0.6053793146733001, + 0.5848721334223365, + 0.5718144052517429, + 0.5462281883927541, + 0.527515083157495, + 0.5112993585545178, + 0.496920907994256, + 0.47722358366421574, + 0.46552107689700334, + 0.4601234027970099, + 0.45317424906679377 + ], + "peak_layer": 1, + "peak_normalized": 2.5461641896458365, + "population_cv": 0.5686091737198813, + "rest_mean": 9.231269224848668e-05, + "spike_contrast": 0.5437781067692293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0197621021652594e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021828828903380781, + 0.00019746273756027222, + 0.00017939221288543195, + 0.0001619200484128669, + 0.00014601621660403907, + 0.000131807682919316, + 0.00012194511509733275, + 0.00011187626660102978, + 0.00010335272963857278, + 0.00010024275252362713, + 9.232992306351662e-05, + 8.697109296917915e-05, + 8.499150135321543e-05, + 7.987671415321529e-05, + 7.353741239057854e-05, + 6.992006092332304e-05, + 6.586474046343938e-05, + 6.187985127326101e-05, + 5.8155463193543255e-05, + 5.582727681030519e-05, + 5.309295011102222e-05, + 5.1900507969548926e-05, + 5.0142381951445714e-05, + 4.902291402686387e-05, + 4.682935104938224e-05, + 4.522503513726406e-05, + 4.3834825191879645e-05, + 4.260212881490588e-05, + 4.0913433622336015e-05, + 3.991015182691626e-05, + 3.944739728467539e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242323236947414, + "first_to_last_ratio": 1.8345651540958352, + "imbalance_abs_log_ratio": 0.6068074801791611, + "last_quartile_mean": 0.0001157894184871111, + "mean": 0.00015111673860701558, + "normalized": [ + 1.771707915810251, + 1.6366668487510905, + 1.5242909365839221, + 1.4186705636795862, + 1.3264449439106374, + 1.2453857912414694, + 1.1893415085703798, + 1.1330085309323608, + 1.0858463851390703, + 1.062861239127963, + 1.023237806768965, + 0.9942126959777651, + 0.978436257513136, + 0.9531027445378829, + 0.9232414968561398, + 0.904013237249231, + 0.8853704564278231, + 0.8680164030054869, + 0.8515402788738432, + 0.8395817783841374, + 0.8267435189825344, + 0.8185143642831039, + 0.809172703595369, + 0.8007918347797284, + 0.7909896391756025, + 0.7821052393452608, + 0.7753388667989797, + 0.7682453290925442, + 0.7604995795745633, + 0.7549901567959042, + 0.7508572350862155, + 0.7467737131490535 + ], + "peak_layer": 1, + "peak_normalized": 1.771707915810251, + "population_cv": 0.2714170004285523, + "rest_mean": 0.00015645500981783562, + "spike_contrast": 0.7816309251520459, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229007406858728, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677347220014781, + 0.00024732775636948645, + 0.00023034587502479553, + 0.00021438486874103546, + 0.00020044803386554122, + 0.0001881986390799284, + 0.00017972940986510366, + 0.00017121655400842428, + 0.00016408956435043365, + 0.00016061612404882908, + 0.0001546283601783216, + 0.00015024218009784818, + 0.00014785809617023915, + 0.0001440297783119604, + 0.00013951724395155907, + 0.000136611532070674, + 0.0001337942958343774, + 0.00013117180787958205, + 0.0001286819897359237, + 0.00012687486014328897, + 0.00012493478425312787, + 0.00012369122123345733, + 0.00012227953993715346, + 0.00012101305037504062, + 0.00011953177454415709, + 0.00011818919301731512, + 0.00011716668086592108, + 0.00011609472858253866, + 0.00011492421617731452, + 0.00011409165017539635, + 0.00011346709652571008, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837904930114746, + "first_to_last_ratio": 1.8345651581276443, + "imbalance_abs_log_ratio": 0.6068074823768531, + "last_quartile_mean": 0.10268321540206671, + "mean": 0.13401183625683188, + "normalized": [ + 1.7717078885231814, + 1.6366668119471057, + 1.524290957183596, + 1.418670514273678, + 1.326444920903592, + 1.2453858718827995, + 1.1893415500975306, + 1.1330085716119305, + 1.0858463365841577, + 1.062861212873729, + 1.0232377434953237, + 0.9942127340740732, + 0.9784362443707608, + 0.9531027271661117, + 0.9232414917376033, + 0.904013241653558, + 0.885370477646398, + 0.8680164390463408, + 0.8515402681847718, + 0.8395816969955481, + 0.8267435345782925, + 0.8185144279720658, + 0.8091727254708025, + 0.8007918405918669, + 0.7909896328038464, + 0.7821052665924377, + 0.7753388466443417, + 0.7682453528580236, + 0.7604996016741158, + 0.7549901615676585, + 0.7508572336421532, + 0.7467736753526057 + ], + "peak_layer": 1, + "peak_normalized": 1.7717078885231814, + "population_cv": 0.27141699472708714, + "rest_mean": 0.1387458680956452, + "spike_contrast": 0.7816309473985974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844806432724, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23742982745170593, + 0.2193327248096466, + 0.2042730301618576, + 0.19011864066123962, + 0.1777593195438385, + 0.16689644753932953, + 0.15938584506511688, + 0.151836559176445, + 0.1455162614583969, + 0.1424359828233719, + 0.13712596893310547, + 0.13323627412319183, + 0.13112203776836395, + 0.12772704660892487, + 0.1237252876162529, + 0.1211484745144844, + 0.11865012347698212, + 0.11632447689771652, + 0.11411647498607635, + 0.11251388490200043, + 0.11079341918230057, + 0.10969062149524689, + 0.10843872278928757, + 0.10731558501720428, + 0.10600197315216064, + 0.10481136292219162, + 0.10390458256006241, + 0.1029539704322815, + 0.10191594809293747, + 0.10117761790752411, + 0.10062375664710999, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873187476230669e-05, + "first_to_last_ratio": 2.248757500145603, + "imbalance_abs_log_ratio": 0.8103778415280068, + "last_quartile_mean": 3.0564378221242805e-05, + "mean": 4.685761763312257e-05, + "normalized": [ + 1.7400006587426948, + 1.6405194725375354, + 1.5551306973978087, + 1.4944078341288562, + 1.4327724019957897, + 1.3414187773842097, + 1.2804088050749736, + 1.2499327782691996, + 1.1927955171655855, + 1.1608931065609767, + 1.0981756972550076, + 1.0577423904261343, + 1.0542786038595535, + 1.0218640087393278, + 0.9775807389332813, + 0.9570972402588515, + 0.9209982861589985, + 0.8815092249082567, + 0.8478232134430647, + 0.8204952153707635, + 0.7871780668180614, + 0.7759097745574741, + 0.7506963524238334, + 0.7421155343193898, + 0.7072927325438048, + 0.6891814936654451, + 0.6695399899996352, + 0.6616343983211914, + 0.6345111343049281, + 0.6212687927046295, + 0.6200231913288072, + 0.6148038704019305 + ], + "peak_layer": 1, + "peak_normalized": 1.7400006587426948, + "population_cv": 0.3217536161653951, + "rest_mean": 4.900405669845727e-05, + "spike_contrast": 0.719671983430414, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.526684668031521e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153228554874659e-05, + 7.687083416385576e-05, + 7.286971958819777e-05, + 7.002439087955281e-05, + 6.71363013680093e-05, + 6.285568815656006e-05, + 5.999690620228648e-05, + 5.8568872191244736e-05, + 5.5891556257847697e-05, + 5.439668530016206e-05, + 5.145789691596292e-05, + 4.9563288484932855e-05, + 4.9400983698433265e-05, + 4.7882112994557247e-05, + 4.580710447044112e-05, + 4.484729652176611e-05, + 4.3155785533599555e-05, + 4.130542220082134e-05, + 3.972697595600039e-05, + 3.844645107164979e-05, + 3.688528886414133e-05, + 3.635728353401646e-05, + 3.517584264045581e-05, + 3.477376594673842e-05, + 3.314205241622403e-05, + 3.229340290999971e-05, + 3.1373048841487616e-05, + 3.10026116494555e-05, + 2.9731680115219206e-05, + 2.9111175535945222e-05, + 2.9052809622953646e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001719075116852764, + "first_to_last_ratio": 1.4894635014227988, + "imbalance_abs_log_ratio": 0.39841598895878805, + "last_quartile_mean": 0.00011541572621354135, + "mean": 0.00013894635094402474, + "normalized": [ + 1.3738985779819257, + 1.3296082759737424, + 1.2883384004275333, + 1.2499940655466777, + 1.2130730176151365, + 1.1755503979222366, + 1.147409009170091, + 1.11990595257071, + 1.0929037170303153, + 1.075598370036652, + 1.0512762916941902, + 1.0312594730951, + 1.0194418974669357, + 1.0018996501730997, + 0.9803460168062212, + 0.9648004557293595, + 0.9493010807852901, + 0.9344788929945884, + 0.9198953859218769, + 0.908001147597105, + 0.8952913731815082, + 0.886953676542665, + 0.8771894463102521, + 0.8683888934041172, + 0.8579399877904575, + 0.8482799643655919, + 0.8408636339472488, + 0.8330870307470335, + 0.82434015220157, + 0.8181187963300758, + 0.8135579418994486, + 0.809009026741244 + ], + "peak_layer": 1, + "peak_normalized": 1.3738985779819257, + "population_cv": 0.16161423472023242, + "rest_mean": 0.0001421073115866252, + "spike_contrast": 0.8576417505420824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187716347398237, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001908981939777732, + 0.0001847442181315273, + 0.00017900991952046752, + 0.00017368211410939693, + 0.00016855206922627985, + 0.00016333843814209104, + 0.00015942829486448318, + 0.00015560684551019222, + 0.0001518549834145233, + 0.00014945046859793365, + 0.00014607100456487387, + 0.0001432897406630218, + 0.00014164773165248334, + 0.00013921030040364712, + 0.000136215501697734, + 0.0001340555027127266, + 0.0001319019211223349, + 0.00012984243221580982, + 0.0001278161071240902, + 0.00012616344611160457, + 0.00012439746933523566, + 0.00012323897681199014, + 0.00012188227265141904, + 0.00012065946793882176, + 0.00011920763063244522, + 0.00011786540562752634, + 0.00011683493357850239, + 0.00011575440294109285, + 0.0001145390560850501, + 0.00011367462138878182, + 0.00011304090730845928, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552221000485588, + "first_to_last_ratio": 1.262697055715921, + "imbalance_abs_log_ratio": 0.2332499537229619, + "last_quartile_mean": 0.00011524712863320019, + "mean": 0.00012953460168318998, + "normalized": [ + 1.1676213631634658, + 1.1525055688027706, + 1.1391580058081985, + 1.1289307982409096, + 1.1166731577821778, + 1.1054001772932092, + 1.094813144753513, + 1.0822854389705017, + 1.0679308603491235, + 1.0564894822224282, + 1.0456838360716472, + 1.0337072702557388, + 1.0234945545465333, + 1.0158819477934218, + 1.0051772952892828, + 0.9951467950359763, + 0.9866643397779161, + 0.9787559418439278, + 0.970448736962688, + 0.9607908684400951, + 0.9509939230158972, + 0.9428061355388626, + 0.934924980051271, + 0.9261034265480621, + 0.9170582603044295, + 0.9058562786860701, + 0.8981778407016658, + 0.8907564927732574, + 0.8831168120837269, + 0.8778216107141017, + 0.8745192645108896, + 0.8703053916682415 + ], + "peak_layer": 1, + "peak_normalized": 1.1676213631634658, + "population_cv": 0.08967236486753827, + "rest_mean": 0.00013110875065817877, + "spike_contrast": 0.9231588022206524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103419721825049, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124736819416285, + 0.00014928934979252517, + 0.000147560378536582, + 0.00014623560127802193, + 0.00014464781270362437, + 0.00014318757166620344, + 0.00014181618462316692, + 0.00014019341324456036, + 0.0001383339986205101, + 0.00013685194426216185, + 0.00013545223919209093, + 0.00013390085950959474, + 0.00013257795944809914, + 0.00013159186346456409, + 0.00013020524056628346, + 0.00012890594371128827, + 0.00012780717224813998, + 0.00012678276107180864, + 0.0001257066905964166, + 0.0001244556624442339, + 0.00012318661902099848, + 0.0001221260172314942, + 0.00012110513489460573, + 0.0001199624384753406, + 0.00011879077646881342, + 0.00011733973224181682, + 0.00011634510883595794, + 0.00011538378748809919, + 0.0001143941844929941, + 0.00011370827269274741, + 0.00011328050459269434, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029741139769612346, + "first_to_last_ratio": 2.278072415841521, + "imbalance_abs_log_ratio": 0.8233296537390045, + "last_quartile_mean": 0.00013055396993877366, + "mean": 0.00019328556300024502, + "normalized": [ + 1.889398576482247, + 1.780233894901915, + 1.6772594716902607, + 1.566224958959574, + 1.4762232610465842, + 1.375571144634086, + 1.3083686481730985, + 1.2364401527267181, + 1.173148897934763, + 1.1508864184295666, + 1.080457040307468, + 1.0318582947152708, + 1.0081957004611959, + 0.9698994715426978, + 0.9066127339785177, + 0.8754550048258108, + 0.8446146862114313, + 0.8115724449181009, + 0.7806677558079268, + 0.7613480996239201, + 0.7403134026279838, + 0.7294052762839086, + 0.7160186976339193, + 0.7062575707090604, + 0.6942777320753531, + 0.6866005528973866, + 0.6829938476675228, + 0.6753403086482458, + 0.6715967314167978, + 0.6686860556999406, + 0.6640286432895989, + 0.6600445236791291 + ], + "peak_layer": 1, + "peak_normalized": 1.889398576482247, + "population_cv": 0.3579154260008552, + "rest_mean": 0.00020340604748559632, + "spike_contrast": 0.681567477924495, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013863494677934796, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003651934675872326, + 0.0003440935106482357, + 0.00032419004128314555, + 0.0003027286729775369, + 0.0002853326441254467, + 0.00026587804313749075, + 0.00025288877077400684, + 0.00023898603103589267, + 0.00022675274522043765, + 0.00022244972933549434, + 0.0002088367473334074, + 0.00019944331143051386, + 0.00019486967357806861, + 0.00018746756541077048, + 0.00017523515271022916, + 0.00016921281348913908, + 0.0001632518251426518, + 0.00015686523693148047, + 0.00015089180669747293, + 0.00014715759607497603, + 0.00014309189282357693, + 0.00014098350948188454, + 0.00013839607709087431, + 0.0001365093921776861, + 0.0001341938623227179, + 0.00013270997442305088, + 0.00013201285037212074, + 0.00013053353177383542, + 0.0001298099523410201, + 0.00012924736074637622, + 0.00012834715016651899, + 0.00012757707736454904 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0022265110164880753, + 0.0040087271481752396, + 0.005379301495850086, + 0.006410556845366955, + 0.0027253651060163975, + 0.004838948138058186, + 0.006175771821290255, + 0.00742313964292407, + 0.0030937574338167906, + 0.005150026176124811, + 0.007019025273621082, + 0.008143599145114422, + 0.004255065228790045, + 0.0066224560141563416, + 0.008129785768687725, + 0.010099578648805618, + 0.0041999719105660915, + 0.00719039561226964, + 0.009030314162373543, + 0.010736005380749702, + 0.005131533369421959, + 0.00787449348717928, + 0.009912566281855106, + 0.012113703414797783, + 0.0059690922498703, + 0.00876442901790142, + 0.010806876234710217, + 0.012353328987956047, + 0.006502739619463682, + 0.009470963850617409, + 0.012123330496251583, + 0.01384514570236206 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015962236011546338, + "first_to_last_ratio": 3.7698422272252903, + "imbalance_abs_log_ratio": 1.3270331510418396, + "last_quartile_mean": 4.2341920561739244e-05, + "mean": 8.61727822893954e-05, + "normalized": [ + 2.542734798705971, + 2.3221448837042376, + 2.101145461274622, + 1.8911858524935854, + 1.7004204056817789, + 1.5354142426015858, + 1.4229001770142031, + 1.3028733810328659, + 1.2049412416934622, + 1.1705258152816296, + 1.0773908232649334, + 1.0133220774376726, + 0.990609360100461, + 0.9272534949358001, + 0.8550385909364457, + 0.8185651575045715, + 0.7703972051466339, + 0.7173947009730917, + 0.6760256695667757, + 0.6514523109028573, + 0.618419077603279, + 0.6053620388084041, + 0.5838930841859182, + 0.5697043695032278, + 0.5457802700901755, + 0.5276149005446632, + 0.5104701332743111, + 0.49517988582787786, + 0.47527562953524777, + 0.46579059316028215, + 0.45896435764104826, + 0.45181000957238127 + ], + "peak_layer": 1, + "peak_normalized": 2.542734798705971, + "population_cv": 0.5705202801130907, + "rest_mean": 9.280119640472414e-05, + "spike_contrast": 0.542873885449774, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.03793460666202e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021911453222855926, + 0.00020010568550787866, + 0.00018106155039276928, + 0.00016296874673571438, + 0.00014652995741926134, + 0.00013231091725174338, + 0.0001226152671733871, + 0.00011227222421439365, + 0.00010383313929196447, + 0.00010086746624438092, + 9.284176485380158e-05, + 8.732078276807442e-05, + 8.536356472177431e-05, + 7.99040135461837e-05, + 7.368105434579775e-05, + 7.053803710732609e-05, + 6.638727063545957e-05, + 6.181989738252014e-05, + 5.825501284562051e-05, + 5.613745815935545e-05, + 5.329089253791608e-05, + 5.216573117650114e-05, + 5.0315691623836756e-05, + 4.9093010602518916e-05, + 4.703140439232811e-05, + 4.546604395727627e-05, + 4.398863165988587e-05, + 4.267102849553339e-05, + 4.095582335139625e-05, + 4.013847137684934e-05, + 3.955023566959426e-05, + 3.893372559105046e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021442477373057045, + "first_to_last_ratio": 1.8431254562763404, + "imbalance_abs_log_ratio": 0.6114627481432037, + "last_quartile_mean": 0.00011633759004325839, + "mean": 0.00015221248349917005, + "normalized": [ + 1.7740805953138137, + 1.6484457246979003, + 1.5301434809496548, + 1.4217241600875643, + 1.3264958046529274, + 1.2454577490392524, + 1.1897861330408546, + 1.1336265611690741, + 1.086914920172498, + 1.064301358569379, + 1.0227209980571423, + 0.9941753882589759, + 0.9788979908211966, + 0.9511107967643203, + 0.9228143274453928, + 0.9052447611165039, + 0.8846530997418666, + 0.8652930871753776, + 0.8497381552743479, + 0.8388656494639406, + 0.8256334792987466, + 0.8176309647342147, + 0.8076951735882674, + 0.8000661307359944, + 0.7892605467734257, + 0.78144613040264, + 0.7731219607566042, + 0.7664988488799842, + 0.7584935618388989, + 0.7529639051546407, + 0.7485195296370439, + 0.7441790263875565 + ], + "peak_layer": 1, + "peak_normalized": 1.7740805953138137, + "population_cv": 0.2736411578809574, + "rest_mean": 0.00015762286892384981, + "spike_contrast": 0.7803207938393821, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012299640220589936, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00027003721334040165, + 0.00025091401766985655, + 0.00023290693934541196, + 0.0002164041652576998, + 0.000201909220777452, + 0.0001895742170745507, + 0.00018110030214302242, + 0.00017255211423616856, + 0.00016544201935175806, + 0.00016199995297938585, + 0.00015567090304102749, + 0.00015132590488065034, + 0.0001490004942752421, + 0.00014477093645837158, + 0.00014046386058907956, + 0.00013778955326415598, + 0.0001346552453469485, + 0.00013170840975362808, + 0.00012934075493831187, + 0.00012768582382705063, + 0.00012567172234412283, + 0.00012445363972801715, + 0.00012294128828216344, + 0.00012178005272289738, + 0.00012013530795229599, + 0.00011894585622940212, + 0.00011767881369451061, + 0.00011667069338727742, + 0.00011545218876563013, + 0.00011461050598882139, + 0.00011393401655368507, + 0.00011327333777444437 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.190154030919075, + "first_to_last_ratio": 1.84312545379361, + "imbalance_abs_log_ratio": 0.6114627467961817, + "last_quartile_mean": 0.10316933691501617, + "mean": 0.13498355145566165, + "normalized": [ + 1.774080628234443, + 1.6484456620184311, + 1.5301434020504485, + 1.4217242444591272, + 1.326495766996536, + 1.2454576834375484, + 1.189786122599316, + 1.133626627255238, + 1.0869149569673349, + 1.0643014004698326, + 1.0227209907477024, + 0.9941753870542451, + 0.9788979589698585, + 0.9511107512164827, + 0.9228143032197887, + 0.9052447972745712, + 0.8846531331847195, + 0.8652931173486506, + 0.8497382179112226, + 0.8388656682549475, + 0.8256334805754935, + 0.8176309131099365, + 0.8076951535479161, + 0.8000661540388709, + 0.7892605573866218, + 0.7814461538045975, + 0.7731219580171443, + 0.7664988542586116, + 0.7584935821787124, + 0.7529638601652359, + 0.7485195164462104, + 0.7441789968002049 + ], + "peak_layer": 1, + "peak_normalized": 1.774080628234443, + "population_cv": 0.27364115345546325, + "rest_mean": 0.13978153549962574, + "spike_contrast": 0.7803207857775153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10907443761825561, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.2394717037677765, + 0.22251304984092712, + 0.2065441906452179, + 0.19190938770771027, + 0.1790551096200943, + 0.16811630129814148, + 0.16060155630111694, + 0.1530209481716156, + 0.14671564102172852, + 0.1436631828546524, + 0.13805051147937775, + 0.13419732451438904, + 0.13213512301445007, + 0.1283843070268631, + 0.1245647519826889, + 0.12219315767288208, + 0.11941362172365189, + 0.11680033802986145, + 0.11470068246126175, + 0.1132330670952797, + 0.11144693940877914, + 0.11036672443151474, + 0.10902556031942368, + 0.10799577087163925, + 0.10653719305992126, + 0.10548237711191177, + 0.1043587476015091, + 0.10346473753452301, + 0.10238415747880936, + 0.10163773596286774, + 0.10103782266378403, + 0.10045192390680313 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.883367768750759e-05, + "first_to_last_ratio": 2.249759357551583, + "imbalance_abs_log_ratio": 0.810823258297238, + "last_quartile_mean": 3.059601795030176e-05, + "mean": 4.691846504556452e-05, + "normalized": [ + 1.7359059710084281, + 1.6475789727938472, + 1.5536456441204052, + 1.4921194549590373, + 1.4365657321885223, + 1.3438513058553838, + 1.2787343947807137, + 1.2483299149152907, + 1.1912354282513808, + 1.1603083401306369, + 1.099547144002948, + 1.059972440022979, + 1.0583955432739494, + 1.0176800955770247, + 0.9754963047746087, + 0.9631595708780456, + 0.9242400656751482, + 0.8752301116137658, + 0.8454245377074081, + 0.8232853322610043, + 0.7863494898171388, + 0.7774642190168813, + 0.7497486129019063, + 0.7388483538626245, + 0.709315327139865, + 0.6908343794807076, + 0.6685469983108738, + 0.6592365073812129, + 0.6314341746588822, + 0.6235898169926835, + 0.6199474545603838, + 0.6139783610863121 + ], + "peak_layer": 1, + "peak_normalized": 1.7359059710084281, + "population_cv": 0.3220406003070692, + "rest_mean": 4.9070239684725595e-05, + "spike_contrast": 0.7193541792517962, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.52988819940947e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.144604362314567e-05, + 7.730187644483522e-05, + 7.28946688468568e-05, + 7.000795449130237e-05, + 6.740145909134299e-05, + 6.305144052021205e-05, + 5.999625500408001e-05, + 5.8569723478285596e-05, + 5.589093780145049e-05, + 5.4439886298496276e-05, + 5.158906424185261e-05, + 4.973227987647988e-05, + 4.9658294301480055e-05, + 4.774798799189739e-05, + 4.576878927764483e-05, + 4.51899686595425e-05, + 4.3363925215089694e-05, + 4.1064453398576006e-05, + 3.9666021621087566e-05, + 3.86272840842139e-05, + 3.689431105158292e-05, + 3.6477427784120664e-05, + 3.5177054087398574e-05, + 3.466563066467643e-05, + 3.3279986382694915e-05, + 3.2412888685939834e-05, + 3.136719897156581e-05, + 3.093036502832547e-05, + 2.9625922252307646e-05, + 2.9257877031341195e-05, + 2.9086982976878062e-05, + 2.8806922273361124e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017341253987979144, + "first_to_last_ratio": 1.4955148931299893, + "imbalance_abs_log_ratio": 0.4024705576674927, + "last_quartile_mean": 0.00011595507385209203, + "mean": 0.00013990553861731314, + "normalized": [ + 1.3768818258178377, + 1.3340879911950096, + 1.2910054180628048, + 1.2527236427111486, + 1.2147542166907819, + 1.1769889670731195, + 1.1474411155803916, + 1.1220953663134672, + 1.0944075691302455, + 1.0773007485615413, + 1.0508678613623372, + 1.0320596054419082, + 1.0208161736522385, + 1.000559648637946, + 0.9805197815181176, + 0.9663128275349431, + 0.948744191771126, + 0.9319305822463463, + 0.9182019814348666, + 0.9075038868226692, + 0.8943650364968407, + 0.886262676120959, + 0.8757907045344768, + 0.8679002174812178, + 0.856288373659771, + 0.8478178092769747, + 0.8387215581135022, + 0.8313975747674917, + 0.8224084044020752, + 0.8161864847722404, + 0.8112200992010112, + 0.8064376596145924 + ], + "peak_layer": 1, + "peak_normalized": 1.3768818258178377, + "population_cv": 0.16325888721442564, + "rest_mean": 0.0001431150389921472, + "spike_contrast": 0.8564734877369171, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001225742365932092, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001926333934534341, + 0.00018664629897102714, + 0.00018061880837194622, + 0.0001752629759721458, + 0.00016995084297377616, + 0.0001646672753849998, + 0.00016053336730692536, + 0.00015698735660407692, + 0.00015311368042603135, + 0.00015072034148033708, + 0.00014702223415952176, + 0.00014439085498452187, + 0.0001428178366040811, + 0.00013998383656144142, + 0.00013718014815822244, + 0.00013519251660909504, + 0.00013273456715978682, + 0.00013038225006312132, + 0.00012846154277212918, + 0.00012696482008323073, + 0.00012512662215158343, + 0.0001239930570591241, + 0.00012252797023393214, + 0.000121424047392793, + 0.00011979948612861335, + 0.00011861440725624561, + 0.00011734179133782163, + 0.0001163171255029738, + 0.00011505949078127742, + 0.00011418900976423174, + 0.00011349418491590768, + 0.00011282509512966499 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014675183047074825, + "first_to_last_ratio": 1.2676545313577883, + "imbalance_abs_log_ratio": 0.23716836728277432, + "last_quartile_mean": 0.00011576642282307148, + "mean": 0.00013041823922321782, + "normalized": [ + 1.1700698937843759, + 1.1560440994564782, + 1.1417398043557874, + 1.130463200949706, + 1.118403537277814, + 1.1065304334184665, + 1.0939881912841023, + 1.0846815122144253, + 1.0709468270677047, + 1.0579907390333845, + 1.0442838370166418, + 1.035550673160169, + 1.025959021745254, + 1.0140203095307772, + 1.0062047700154308, + 0.9976631870190885, + 0.9861765922450596, + 0.9770016877798172, + 0.9687702939454803, + 0.9602422119882037, + 0.9504995944091904, + 0.9425883202531964, + 0.9330014667281973, + 0.9259385264566372, + 0.9147443798098084, + 0.906202796813466, + 0.8958065504175365, + 0.8885655303249116, + 0.8806194435723184, + 0.8757905905296257, + 0.8713783728577705, + 0.8681336045391749 + ], + "peak_layer": 1, + "peak_normalized": 1.1700698937843759, + "population_cv": 0.09121429133712369, + "rest_mean": 0.0001320278307604086, + "spike_contrast": 0.921975648780332, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001217264449223876, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001525984553154558, + 0.0001507692359155044, + 0.000148903694935143, + 0.00014743302017450333, + 0.00014586022007279098, + 0.00014431175077334046, + 0.00014267601363826543, + 0.00014146225294098258, + 0.000139670999487862, + 0.00013798128929920495, + 0.00013619365927297622, + 0.00013505469541996717, + 0.00013380376913119107, + 0.00013224674330558628, + 0.00013122745440341532, + 0.00013011347618885338, + 0.00012861541472375393, + 0.00012741883983835578, + 0.0001263453159481287, + 0.0001252330985153094, + 0.0001239624834852293, + 0.00012293070903979242, + 0.00012168040848337114, + 0.00012075927224941552, + 0.00011929935135412961, + 0.00011818537313956767, + 0.00011682951299007982, + 0.00011588515189941972, + 0.00011484883725643158, + 0.00011421906674513593, + 0.00011364363308530301, + 0.00011322045611450449 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00030004180916876066, + "first_to_last_ratio": 2.2837506327398005, + "imbalance_abs_log_ratio": 0.8258191057718037, + "last_quartile_mean": 0.00013138116082700435, + "mean": 0.00019473525253488333, + "normalized": [ + 1.9000096511818072, + 1.783700741635724, + 1.6708066910504589, + 1.570366884717585, + 1.4783058799917985, + 1.376185321596543, + 1.3064973297630273, + 1.2402700043336092, + 1.1767953849206394, + 1.1505341943958838, + 1.0771542571420398, + 1.026098162637436, + 1.013898063759585, + 0.9669473025647412, + 0.9095398902231254, + 0.8755205777387188, + 0.842301214161826, + 0.8073124727024382, + 0.7791991818813049, + 0.760574607765185, + 0.7402055397447304, + 0.7308143413325056, + 0.7152667122155245, + 0.7043714155678553, + 0.6942467749765324, + 0.6868749151283189, + 0.680655564574882, + 0.676363713518914, + 0.6701061029149729, + 0.6672276319278244, + 0.6636177363814285, + 0.6582317375530352 + ], + "peak_layer": 1, + "peak_normalized": 1.9000096511818072, + "population_cv": 0.35930746026536914, + "rest_mean": 0.00020494150898971216, + "spike_contrast": 0.6812747128050888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013962146767880766, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0003699988592416048, + 0.0003473494143690914, + 0.0003253649629186839, + 0.0003058057918678969, + 0.0002878782688640058, + 0.0002679917961359024, + 0.0002544210874475539, + 0.00024152429250534624, + 0.00022916354646440595, + 0.000224049566895701, + 0.00020975990628357977, + 0.00019981748482678086, + 0.000197441695490852, + 0.0001882987271528691, + 0.00017711948021315038, + 0.00017049472080543637, + 0.00016402573965024203, + 0.00015721219824627042, + 0.00015173754945863038, + 0.00014811068831477314, + 0.00014414411270990968, + 0.0001423153153154999, + 0.0001392876438330859, + 0.00013716594548895955, + 0.00013519432104658335, + 0.00013375876005738974, + 0.00013254763325676322, + 0.0001317118585575372, + 0.00013049328117631376, + 0.00012993274140171707, + 0.00012922976748086512, + 0.00012818092363886535 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0034885562490671873, + 0.004831995815038681, + 0.005952076520770788, + 0.006925336085259914, + 0.00385080324485898, + 0.005542940925806761, + 0.006729288958013058, + 0.007927131839096546, + 0.004081308841705322, + 0.005811207927763462, + 0.007534808944910765, + 0.008581922389566898, + 0.00499920267611742, + 0.007102519739419222, + 0.008542952127754688, + 0.010447698645293713, + 0.005041283555328846, + 0.007636935915797949, + 0.009387043304741383, + 0.011020516976714134, + 0.005807018838822842, + 0.008367395959794521, + 0.01024632342159748, + 0.012378672137856483, + 0.006453148555010557, + 0.009127611294388771, + 0.011037957854568958, + 0.012611051090061665, + 0.007010402157902718, + 0.009794782847166061, + 0.012445803731679916, + 0.014087481424212456 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.00015858902861509705, + "first_to_last_ratio": 3.757869257587208, + "imbalance_abs_log_ratio": 1.3238521099622325, + "last_quartile_mean": 4.220184837322449e-05, + "mean": 8.57325238712292e-05, + "normalized": [ + 2.5462257183789148, + 2.3032637538343064, + 2.0924289533379263, + 1.8886492391629932, + 1.7031256747388024, + 1.5374176267930122, + 1.4224231993460965, + 1.3049597038569594, + 1.2055618292025094, + 1.169245405308266, + 1.0769812048488787, + 1.0144334047898542, + 0.9913512297333369, + 0.9317090592952362, + 0.8577499774884012, + 0.81555804918997, + 0.7682725064221458, + 0.7217807351250844, + 0.6783324192661732, + 0.6511800118936032, + 0.6192916242842709, + 0.6053728292385717, + 0.5848720876072855, + 0.5718124973482859, + 0.5462302773879167, + 0.5275173250132701, + 0.5112998770983732, + 0.49691833866078555, + 0.47721658788927634, + 0.46551989467389665, + 0.46012635628943105, + 0.45317260249616614 + ], + "peak_layer": 1, + "peak_normalized": 2.5462257183789148, + "population_cv": 0.5686111305901927, + "rest_mean": 9.231303704803361e-05, + "spike_contrast": 0.5437775023084314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.019775271648541e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00021829435718245804, + 0.00019746461475733668, + 0.0001793892151908949, + 0.0001619186659809202, + 0.00014601326256524771, + 0.00013180669338908046, + 0.00012194793089292943, + 0.00011187748896190897, + 0.00010335585830034688, + 0.00010024235962191597, + 9.233231685357168e-05, + 8.69699360919185e-05, + 8.499104296788573e-05, + 7.987776916706935e-05, + 7.353707042057067e-05, + 6.991984992055222e-05, + 6.586594099644572e-05, + 6.188008410390466e-05, + 5.8155150327365845e-05, + 5.582730591413565e-05, + 5.309343396220356e-05, + 5.190014053368941e-05, + 5.014256021240726e-05, + 4.90229285787791e-05, + 4.682970029534772e-05, + 4.522539165918715e-05, + 4.383502891869284e-05, + 4.2602063331287354e-05, + 4.0912982512963936e-05, + 3.991019548266195e-05, + 3.944779382436536e-05, + 3.885163096128963e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00021242368711682502, + "first_to_last_ratio": 1.8345658968465297, + "imbalance_abs_log_ratio": 0.6068078850437921, + "last_quartile_mean": 0.0001157896194854402, + "mean": 0.0001511171253696375, + "normalized": [ + 1.7717451736709768, + 1.636674793183293, + 1.5242755762025908, + 1.4186515254985965, + 1.3264266232459345, + 1.2453738409548603, + 1.1893512719369177, + 1.1330135273991528, + 1.085860457802012, + 1.0628542818802846, + 1.023252039669589, + 0.9942070699734702, + 0.9784308644775869, + 0.9531083940389613, + 0.9232390376562555, + 0.9040072643238835, + 0.8853775311228679, + 0.8680181295617843, + 0.8515367513400655, + 0.8395795332993111, + 0.8267439067367663, + 0.8185133286645813, + 0.8091730400239778, + 0.800791325997512, + 0.7909899739933552, + 0.7821072339266707, + 0.7753403490721149, + 0.7682422073314642, + 0.7604971517068638, + 0.7549863467424895, + 0.7508596466770359, + 0.7467718018887739 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "rest_mean": 0.00015645541295978345, + "spike_contrast": 0.7816308178118705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012229037238284946, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002677410375326872, + 0.0002473295899108052, + 0.00023034414334688336, + 0.00021438254043459892, + 0.00020044577831868082, + 0.00018819731485564262, + 0.00017973134526982903, + 0.000171217747265473, + 0.0001640921109355986, + 0.00016061548376455903, + 0.00015463090676348656, + 0.00015024171443656087, + 0.0001478576596127823, + 0.00014403100067283958, + 0.00013951722939964384, + 0.00013661097909789532, + 0.00013379570737015456, + 0.0001311724045081064, + 0.0001286817860091105, + 0.00012687484559137374, + 0.0001249351626029238, + 0.00012369138130452484, + 0.00012227990373503417, + 0.00012101328320568427, + 0.00011953213106608018, + 0.0001181897969217971, + 0.0001171672047348693, + 0.00011609455395955592, + 0.00011492414341773838, + 0.0001140913664130494, + 0.00011346775136189535, + 0.00011285000800853595 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.18837944976985455, + "first_to_last_ratio": 1.8345658800697515, + "imbalance_abs_log_ratio": 0.6068078758989703, + "last_quartile_mean": 0.10268339328467846, + "mean": 0.1340121782850474, + "normalized": [ + 1.7717451751532867, + 1.6366747548098712, + 1.5242756140248972, + 1.4186514377480737, + 1.3264266357207033, + 1.2453739091726768, + 1.1893513017871604, + 1.1330134634090196, + 1.0858604665419038, + 1.0628542749024097, + 1.0232520332438115, + 0.9942070832344971, + 0.9784308561813967, + 0.9531084116998004, + 0.9232390798282929, + 0.9040072094634526, + 0.8853775581660035, + 0.8680181154203164, + 0.8515367049614931, + 0.8395794986042229, + 0.8267439263823189, + 0.8185133396785964, + 0.809173050926764, + 0.8007913534932437, + 0.7909900046670136, + 0.782107217826699, + 0.7753403703776178, + 0.7682422246066585, + 0.7604971603471317, + 0.7549863443945886, + 0.7508596538022533, + 0.7467717694238248 + ], + "peak_layer": 1, + "peak_normalized": 1.7717451751532867, + "population_cv": 0.27141865758333467, + "rest_mean": 0.13874622461972413, + "spike_contrast": 0.7816308398662988, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10844832807779312, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.23743543028831482, + 0.21933434903621674, + 0.20427149534225464, + 0.19011656939983368, + 0.17775732278823853, + 0.16689527034759521, + 0.15938755869865417, + 0.15183760225772858, + 0.14551852643489838, + 0.14243541657924652, + 0.13712823390960693, + 0.1332358568906784, + 0.1311216503381729, + 0.12772813439369202, + 0.1237252801656723, + 0.12114797532558441, + 0.1186513751745224, + 0.1163249984383583, + 0.11411628872156143, + 0.11251387745141983, + 0.11079375445842743, + 0.10969075560569763, + 0.10843904316425323, + 0.107315793633461, + 0.10600229352712631, + 0.104811891913414, + 0.10390505194664001, + 0.10295381397008896, + 0.1019158810377121, + 0.10117736458778381, + 0.10062433779239655, + 0.10007651150226593 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.873207303215167e-05, + "first_to_last_ratio": 2.248757814140914, + "imbalance_abs_log_ratio": 0.8103779811585758, + "last_quartile_mean": 3.0564462122129044e-05, + "mean": 4.6857780716891284e-05, + "normalized": [ + 1.7400119939228862, + 1.6405409364325143, + 1.555121247708426, + 1.4944054279879166, + 1.4327475398601994, + 1.3414100714975594, + 1.2804101716466747, + 1.2499370459040815, + 1.192804641985012, + 1.1608858830017554, + 1.098191983597808, + 1.057744066133373, + 1.0542717513613309, + 1.021869458338179, + 0.9775749297615711, + 0.9570927445976433, + 0.9210091333285576, + 0.8815125232807631, + 0.847820728512489, + 0.8204970180441978, + 0.7871838673851995, + 0.7759041238100245, + 0.7506938949785397, + 0.7421070509174572, + 0.7072987335050381, + 0.6891811912847667, + 0.6695407652827761, + 0.6616333377893114, + 0.6345027924930273, + 0.6212629426015038, + 0.6200302724102036, + 0.614801730639213 + ], + "peak_layer": 1, + "peak_normalized": 1.7400119939228862, + "population_cv": 0.32175451551289735, + "rest_mean": 4.9004235565748616e-05, + "spike_contrast": 0.7196709452950106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.5266924533061685e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153310045599937e-05, + 7.687210745643824e-05, + 7.28695304132998e-05, + 7.002452184678987e-05, + 6.713537004543468e-05, + 6.28554989816621e-05, + 5.9997179050697014e-05, + 5.8569276006892323e-05, + 5.589217835222371e-05, + 5.439653614303097e-05, + 5.1458839152473956e-05, + 4.9563539505470544e-05, + 4.9400834541302174e-05, + 4.788253500009887e-05, + 4.58069916930981e-05, + 4.4847241952084005e-05, + 4.315644400776364e-05, + 4.130572051508352e-05, + 3.972699778387323e-05, + 3.8446669350378215e-05, + 3.688568904181011e-05, + 3.635714529082179e-05, + 3.5175849916413426e-05, + 3.477348946034908e-05, + 3.3142448955914006e-05, + 3.2293501135427505e-05, + 3.13731943606399e-05, + 3.1002669857116416e-05, + 2.9731392714893445e-05, + 2.9111002731951885e-05, + 2.905324254243169e-05, + 2.8808244678657502e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00017190765356644988, + "first_to_last_ratio": 1.4894621368061003, + "imbalance_abs_log_ratio": 0.3984150727783497, + "last_quartile_mean": 0.00011541592721187044, + "mean": 0.00013894661265112518, + "normalized": [ + 1.3739040544627474, + 1.3296102750467884, + 1.2883342981472405, + 1.2499918159027226, + 1.2130627732820631, + 1.175544413473833, + 1.1474083142393565, + 1.1199112790671786, + 1.092909618037032, + 1.0755929927697745, + 1.0512861461262244, + 1.0312571117882559, + 1.0194388253067554, + 1.0019061415077835, + 0.980344379776871, + 0.9647961249933756, + 0.9493075664595542, + 0.9344817410273759, + 0.9198929201794295, + 0.9079992279088673, + 0.8952920956888009, + 0.8869530532606651, + 0.8771902552778059, + 0.8683889858357474, + 0.8579407806493952, + 0.8482823463732029, + 0.8408658204630546, + 0.8330843095877946, + 0.8243382853592501, + 0.818115422616924, + 0.8135611224181157, + 0.8090075029660142 + ], + "peak_layer": 1, + "peak_normalized": 1.3739040544627474, + "population_cv": 0.16161403917150835, + "rest_mean": 0.00014210757001786045, + "spike_contrast": 0.8576421569620596, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012187744287075474, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00019089931447524577, + 0.00018474484386388212, + 0.00017900968668982387, + 0.00017368212866131216, + 0.0001685509632807225, + 0.00016333791427314281, + 0.00015942849859129637, + 0.00015560787869617343, + 0.00015185608936008066, + 0.00014945000293664634, + 0.00014607264893129468, + 0.0001432896824553609, + 0.00014164757158141583, + 0.0001392114645568654, + 0.00013621553080156446, + 0.0001340551534667611, + 0.00013190307072363794, + 0.00012984307250007987, + 0.0001278160052606836, + 0.00012616341700777411, + 0.0001243978040292859, + 0.00012323912233114243, + 0.00012188261462142691, + 0.00012065970804542303, + 0.00011920796532649547, + 0.00011786595860030502, + 0.00011683545744745061, + 0.00011575424287002534, + 0.00011453901242930442, + 0.00011367436673026532, + 0.00011304156214464456, + 0.00011240885214647278 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 0.00014552358334185556, + "first_to_last_ratio": 1.262638843036328, + "imbalance_abs_log_ratio": 0.2332038508028458, + "last_quartile_mean": 0.00011525352965691127, + "mean": 0.00012953498708156985, + "normalized": [ + 1.1676304712442283, + 1.1525836984023852, + 1.1391521450612143, + 1.1288261090295841, + 1.1167600441451297, + 1.1054423860149858, + 1.094731137320412, + 1.082319740349084, + 1.067957453000188, + 1.0563935463552665, + 1.04560972624621, + 1.0337959762176818, + 1.0235535208863231, + 1.0159040893770892, + 1.0050628637064216, + 0.9950792389307415, + 0.9865814183778941, + 0.9786230528286792, + 0.9704418054120744, + 0.9608381133357455, + 0.9508407831226554, + 0.9429749854457838, + 0.9346839821902475, + 0.9262276149685831, + 0.9170542960948607, + 0.9059488475652709, + 0.8984648923618161, + 0.8908037775288885, + 0.8831483920181569, + 0.8778579808382188, + 0.8744051093280881, + 0.8703028022960919 + ], + "peak_layer": 1, + "peak_normalized": 1.1676304712442283, + "population_cv": 0.0896642712448568, + "rest_mean": 0.00013110964479252557, + "spike_contrast": 0.9231344927669946, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012103183544240892, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00015124899800866842, + 0.00014929991448298097, + 0.00014756005839444697, + 0.00014622247545048594, + 0.00014465949789155275, + 0.00014319346519187093, + 0.00014180598373059183, + 0.00014019827358424664, + 0.00013833785487804562, + 0.00013683992438018322, + 0.0001354430423816666, + 0.00013391274842433631, + 0.0001325859921053052, + 0.00013159512309357524, + 0.00013019080506637692, + 0.00012889757636003196, + 0.00012779681128449738, + 0.00012676592450588942, + 0.00012570616672746837, + 0.00012446215259842575, + 0.00012316714855842292, + 0.00012214825255796313, + 0.00012107427755836397, + 0.00011997888213954866, + 0.00011879061639774591, + 0.00011735207226593047, + 0.0001163826382253319, + 0.0001153902558144182, + 0.00011439861555118114, + 0.00011371332220733166, + 0.00011326605454087257, + 0.0001127346622524783 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00029742828155576717, + "first_to_last_ratio": 2.2781890121807993, + "imbalance_abs_log_ratio": 0.8233808344454435, + "last_quartile_mean": 0.00013055469935352448, + "mean": 0.00019329176166138495, + "normalized": [ + 1.8891850069388627, + 1.7801603926271052, + 1.6773544463608039, + 1.566500262900107, + 1.476456430986117, + 1.3754925511314788, + 1.3084504582544147, + 1.236424592515421, + 1.172965600373877, + 1.150871569148191, + 1.0802294364746856, + 1.0319209662753672, + 1.0080063247818516, + 0.9698985570693867, + 0.9067931019504507, + 0.8756163462587222, + 0.8444435806568995, + 0.8113939671049883, + 0.7809677247391495, + 0.7613442367276676, + 0.7402582678385915, + 0.7293562882295931, + 0.7162426695419845, + 0.7062419232587185, + 0.6940340549797547, + 0.6866100033591334, + 0.6831688143117768, + 0.6753320518823956, + 0.6715919825156708, + 0.6686410475337363, + 0.6640239865195754, + 0.6600233567535221 + ], + "peak_layer": 1, + "peak_normalized": 1.8891850069388627, + "population_cv": 0.357915029863326, + "rest_mean": 0.00020341356920888993, + "spike_contrast": 0.6815376252628051, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001386340009048581, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00036516389809548855, + 0.0003440903383307159, + 0.0003242187958676368, + 0.0003027915954589844, + 0.00028538686456158757, + 0.00026587137836031616, + 0.0002529126941226423, + 0.00023899068764876574, + 0.00022672458726447076, + 0.00022245399304665625, + 0.0002087994507746771, + 0.00019946182146668434, + 0.00019483931828290224, + 0.00018747340072877705, + 0.00017527563613839447, + 0.00016924942610785365, + 0.00016322398732881993, + 0.00015683576930314302, + 0.0001509546273155138, + 0.00014716156874783337, + 0.0001430858246749267, + 0.0001409785618307069, + 0.00013844380737282336, + 0.00013651074550580233, + 0.00013415106514003128, + 0.00013271605712361634, + 0.00013205090363044292, + 0.00013053612201474607, + 0.00012981319741811603, + 0.00012924280599690974, + 0.00012835036613978446, + 0.00012757707736454904 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.02802751213312149, + 0.01413465291261673, + 0.014225807972252369, + 0.01433455292135477, + 0.01440478302538395, + 0.009703479707241058, + 0.00977881159633398, + 0.009858808480203152, + 0.009950309991836548, + 0.00752287870272994, + 0.007571155205368996, + 0.007693085819482803, + 0.007788818795233965, + 0.006353048607707024, + 0.006389216054230928, + 0.0064707002602517605, + 0.00656962301582098, + 0.005553483963012695, + 0.0056249531917274, + 0.005688921548426151, + 0.005768724251538515, + 0.004966245498508215, + 0.005057469941675663, + 0.005098348017781973, + 0.005216081161051989, + 0.0046758754178881645, + 0.004727107938379049, + 0.004777040798217058, + 0.004837613552808762, + 0.0044186655431985855, + 0.004510571248829365, + 0.0045849340967834 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 0.0001580166108396952, + "first_to_last_ratio": 1.9713740353643698, + "imbalance_abs_log_ratio": 0.6787307795042399, + "last_quartile_mean": 8.015557068574708e-05, + "mean": 0.00011843875881822896, + "normalized": [ + 0.8918679597819287, + 1.5814819593588525, + 1.5722239987443252, + 1.3246067564285042, + 1.1252650527356378, + 1.512743589128775, + 1.3788112428745087, + 1.2863035124436846, + 1.097095055843533, + 1.468154353173571, + 1.1523254606068964, + 1.0839888571909224, + 1.054518217028451, + 1.146335080549274, + 1.1665208527784754, + 0.8931136212023528, + 0.8203804970547371, + 1.073741223277434, + 0.919654742394771, + 0.8223971330190447, + 0.7711288002244273, + 0.8470775332584999, + 0.8298711629738247, + 0.7662487463744663, + 0.7210382462310316, + 0.7071376059209309, + 0.7189559392079661, + 0.727980518763353, + 0.6111916361855597, + 0.6986829946379066, + 0.6244678178312321, + 0.6046898327751223 + ], + "peak_layer": 2, + "peak_normalized": 1.5814819593588525, + "population_cv": 0.2904715201464359, + "rest_mean": 0.00012310891098523926, + "spike_contrast": 0.7572151875143335, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.321993711637333e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010563173418631777, + 0.0001873087603598833, + 0.00018621225899551064, + 0.00015688478015363216, + 0.0001332749961875379, + 0.00017916747310664505, + 0.00016330469225067645, + 0.00015234819147735834, + 0.00012993857671972364, + 0.00017388637934345752, + 0.0001364799973089248, + 0.0001283862948184833, + 0.00012489582877606153, + 0.00013577050413005054, + 0.00013816128193866462, + 0.00010577926877886057, + 9.71648478298448e-05, + 0.00012717257777694613, + 0.00010892276623053476, + 9.74036956904456e-05, + 9.133153798757121e-05, + 0.0001003268116619438, + 9.828891052166e-05, + 9.07535504666157e-05, + 8.539887494407594e-05, + 8.375250035896897e-05, + 8.515224908478558e-05, + 8.622110908618197e-05, + 7.238877878990024e-05, + 8.275114669231698e-05, + 7.396119326585904e-05, + 7.161871326388791e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001664285200604354, + "first_to_last_ratio": 1.9719337364651075, + "imbalance_abs_log_ratio": 0.6790146534176558, + "last_quartile_mean": 8.439863722742302e-05, + "mean": 0.0001245867656507471, + "normalized": [ + 0.8984334461398115, + 1.5927952483854981, + 1.5658595485525297, + 1.3198735865176374, + 1.131213894321221, + 1.5125315092450449, + 1.3796810696180641, + 1.286366052942558, + 1.0960606282761096, + 1.4536884557404346, + 1.1543342746982923, + 1.0797559630746076, + 1.0497530570909623, + 1.1457182987306291, + 1.1551962694199982, + 0.9023721663305979, + 0.8238710779144349, + 1.0632503993742801, + 0.9211624832492573, + 0.8274335806257342, + 0.7739032412019898, + 0.850990856837025, + 0.8281317613098801, + 0.7681943949939335, + 0.7208251570035091, + 0.7089546837555952, + 0.7203162530736348, + 0.7236174125342815, + 0.6145740290254166, + 0.6994991973458914, + 0.6274381906330746, + 0.6042038120380651 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952483854981, + "population_cv": 0.2889665774838781, + "rest_mean": 0.00012946851159169995, + "spike_contrast": 0.7586812913967177, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.822533756960183e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00011193291720701382, + 0.0001984412083402276, + 0.0001950853766174987, + 0.000164438781212084, + 0.00014093428035266697, + 0.00018844140868168324, + 0.00017189000209327787, + 0.00016026418597903103, + 0.00013655464863404632, + 0.00018111034296453, + 0.00014381477376446128, + 0.0001345233031315729, + 0.00013078533811494708, + 0.00014274133718572557, + 0.00014392216689884663, + 0.00011242362961638719, + 0.00010264343291055411, + 0.0001324669283349067, + 0.00011476465442683548, + 0.00010308727360097691, + 9.641810174798593e-05, + 0.00010602219845168293, + 0.00010317425767425448, + 9.570685506332666e-05, + 8.980527491075918e-05, + 8.832637104205787e-05, + 8.974187221610919e-05, + 9.015315299620852e-05, + 7.656779052922502e-05, + 8.714834257261828e-05, + 7.817049481673166e-05, + 7.527579873567447e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.14759047608822584, + "first_to_last_ratio": 1.9719337947467823, + "imbalance_abs_log_ratio": 0.6790146829732503, + "last_quartile_mean": 0.07484555337578058, + "mean": 0.1104847895912826, + "normalized": [ + 0.8984334974418968, + 1.5927952138130326, + 1.565859646322909, + 1.3198735575362357, + 1.1312139236939245, + 1.5125315007911495, + 1.3796810264894532, + 1.286366010451999, + 1.0960606168617568, + 1.4536884922319049, + 1.1543342013148132, + 1.0797558953237962, + 1.0497530983161607, + 1.1457182562234747, + 1.1551962948455465, + 0.9023721940039697, + 0.8238711061937815, + 1.0632504861804335, + 0.9211625151546727, + 0.8274335809310764, + 0.7739032642161865, + 0.8509909070635955, + 0.8281317391228538, + 0.7681943896830757, + 0.7208251689669645, + 0.7089546589522648, + 0.7203162343708721, + 0.7236173971219663, + 0.6145740294939214, + 0.6994991437375198, + 0.6274381319743458, + 0.6042038211744466 + ], + "peak_layer": 2, + "peak_normalized": 1.5927952138130326, + "population_cv": 0.28896658018674404, + "rest_mean": 0.11481397047086998, + "spike_contrast": 0.7586813040631772, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08710721284151077, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.09926323592662811, + 0.1759796440601349, + 0.1730036735534668, + 0.14582595229148865, + 0.12498193234205246, + 0.16711172461509705, + 0.15243376791477203, + 0.14212387800216675, + 0.12109802663326263, + 0.16061046719551086, + 0.1275363713502884, + 0.1192966029047966, + 0.11598175019025803, + 0.12658444046974182, + 0.12763161957263947, + 0.09969840198755264, + 0.0910252258181572, + 0.11747300624847412, + 0.10177444666624069, + 0.09141882508993149, + 0.08550453931093216, + 0.09402155131101608, + 0.09149596095085144, + 0.08487379550933838, + 0.0796402171254158, + 0.0783287063241005, + 0.07958398759365082, + 0.07994871586561203, + 0.06790108233690262, + 0.07728401571512222, + 0.069322369992733, + 0.06675533205270767 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.093040158499207e-05, + "first_to_last_ratio": 0.8860093039525244, + "imbalance_abs_log_ratio": 0.12102782735710801, + "last_quartile_mean": 5.748291960117058e-05, + "mean": 5.7093861016710434e-05, + "normalized": [ + 0.4337136137178416, + 0.9185569667519601, + 0.9444095926066538, + 0.8728787927927439, + 0.7849140301431445, + 1.1296292591692207, + 1.0726593944235638, + 0.9796135302182263, + 0.9386642177229508, + 1.3046353648934683, + 1.0461832118506482, + 1.0032917361864164, + 0.9662905330123907, + 1.0760080966276753, + 1.1828739682986207, + 0.9087955578115051, + 0.9194941496879511, + 1.198292373503509, + 1.0946132296674007, + 1.0673695485587213, + 0.9244260207272295, + 1.0525679458300632, + 1.125653431968099, + 0.9999504891362015, + 0.9629027708994021, + 1.0063234987710272, + 0.9975933861195996, + 1.0791525148711678, + 0.9412468232151083, + 1.085173475571988, + 1.0031556955576197, + 0.9789667796878816 + ], + "peak_layer": 10, + "peak_normalized": 1.3046353648934683, + "population_cv": 0.1441687540611157, + "rest_mean": 5.695535414647397e-05, + "spike_contrast": 1.0155638391297448, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.784179811598733e-05, + "top_five_layers": [ + 10, + 18, + 15, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.4762384782661684e-05, + 5.2443963795667514e-05, + 5.3919990023132414e-05, + 4.9836020480142906e-05, + 4.481377254705876e-05, + 6.449489592341706e-05, + 6.124226638348773e-05, + 5.592991874436848e-05, + 5.3591964388033375e-05, + 7.448667020071298e-05, + 5.973063889541663e-05, + 5.728179894504137e-05, + 5.516925739357248e-05, + 6.143345672171563e-05, + 6.75348419463262e-05, + 5.1886647270293906e-05, + 5.249747118796222e-05, + 6.84151382301934e-05, + 6.249569560168311e-05, + 6.094024865888059e-05, + 5.277905074763112e-05, + 6.009516800986603e-05, + 6.426790059776977e-05, + 5.7091034250333905e-05, + 5.4975836974335834e-05, + 5.74548939766828e-05, + 5.695645813830197e-05, + 6.161298369988799e-05, + 5.373941530706361e-05, + 6.19567435933277e-05, + 5.727403186028823e-05, + 5.58929932594765e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.801741053801379e-05, + "first_to_last_ratio": 0.954195857595535, + "imbalance_abs_log_ratio": 0.046886327142486896, + "last_quartile_mean": 6.08024129178375e-05, + "mean": 6.296396202287724e-05, + "normalized": [ + 0.4796364558374551, + 0.9568353419130547, + 0.9804638447804884, + 0.8949545354119591, + 0.8237377234766136, + 1.1597846916175047, + 1.085649359633547, + 0.9904450076993521, + 0.9595078978373608, + 1.2941748206155441, + 1.0680753760655664, + 1.0231559828609242, + 0.9904019047543832, + 1.096490963646882, + 1.1850351594596897, + 0.9422215946487361, + 0.9418631353047864, + 1.1705024179981143, + 1.0926353878857977, + 1.0382275689133567, + 0.9320833313178482, + 1.0784583323883348, + 1.0985080196909318, + 0.9917906747078287, + 0.9476388143731139, + 0.9855066582268529, + 0.978396290111904, + 1.028407839812513, + 0.8982055143685478, + 1.0282804954549911, + 0.9451124964564701, + 0.9138123627295472 + ], + "peak_layer": 10, + "peak_normalized": 1.2941748206155441, + "population_cv": 0.13431233496292436, + "rest_mean": 6.285090866084729e-05, + "spike_contrast": 1.0115120295379672, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.357445017783903e-05, + "top_five_layers": [ + 10, + 15, + 18, + 6, + 23 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 3.019981159013696e-05, + 6.0246144130360335e-05, + 6.173388828756288e-05, + 5.634988337988034e-05, + 5.186579073779285e-05, + 7.302463927771896e-05, + 6.835678505012766e-05, + 6.236234185053036e-05, + 6.041441884008236e-05, + 8.148637425620109e-05, + 6.725025741616264e-05, + 6.442195444833487e-05, + 6.235962791834027e-05, + 6.903941539349034e-05, + 7.461450877599418e-05, + 5.9326004702597857e-05, + 5.930343468207866e-05, + 7.369946979451925e-05, + 6.879665306769311e-05, + 6.537092122016475e-05, + 5.8687659475253895e-05, + 6.790400948375463e-05, + 6.916641723364592e-05, + 6.244707037694752e-05, + 5.966709431959316e-05, + 6.205140380188823e-05, + 6.160370685392991e-05, + 6.475263216998428e-05, + 5.655457789544016e-05, + 6.474461406469345e-05, + 5.950802733423188e-05, + 5.753724690293893e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7888660295284353e-05, + "first_to_last_ratio": 0.41855462748958633, + "imbalance_abs_log_ratio": 0.8709478659353102, + "last_quartile_mean": 4.2739129185065394e-05, + "mean": 3.1986895351110434e-05, + "normalized": [ + 0.2593955501951102, + 0.4997829486650695, + 0.48480994473028793, + 0.4968760363401999, + 0.5474125689219601, + 0.7788406177150305, + 0.7554473604850604, + 0.6514322393717875, + 0.7578135841604356, + 0.9674810188795085, + 0.9032772556875033, + 0.9019220651946442, + 0.8311356963419073, + 0.9587970177975769, + 1.1401394869948454, + 0.8953886492354074, + 1.0262272842074007, + 1.1413916918814533, + 1.174037958462901, + 1.194927715786966, + 1.0267140632187253, + 1.3434898267078448, + 1.3729067467361273, + 1.2011929483563188, + 1.1735959904166378, + 1.3266745689353452, + 1.2739952988364465, + 1.4049535244931144, + 1.2752889026856997, + 1.4680979746684197, + 1.4182292846928506, + 1.3483241791974143 + ], + "peak_layer": 30, + "peak_normalized": 1.4680979746684197, + "population_cv": 0.3184955307507873, + "rest_mean": 3.066251991657299e-05, + "spike_contrast": 1.276428773763581, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.913852269761264e-05, + "top_five_layers": [ + 30, + 31, + 28, + 23, + 32 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 8.297258318634704e-06, + 1.5986504877218977e-05, + 1.5507564967265353e-05, + 1.589352177688852e-05, + 1.7510028555989265e-05, + 2.491269333404489e-05, + 2.416441566310823e-05, + 2.083729486912489e-05, + 2.4240103812189773e-05, + 3.094671410508454e-05, + 2.889303505071439e-05, + 2.8849686714238487e-05, + 2.6585450541460887e-05, + 3.066893987124786e-05, + 3.6469522456172854e-05, + 2.8640703021665104e-05, + 3.282582474639639e-05, + 3.6509576602838933e-05, + 3.755382931558415e-05, + 3.822202779701911e-05, + 3.2841395295690745e-05, + 4.2974068492185324e-05, + 4.391502443468198e-05, + 3.842243313556537e-05, + 3.7539692129939795e-05, + 4.243620060151443e-05, + 4.0751154301688075e-05, + 4.494010136113502e-05, + 4.0792532672639936e-05, + 4.695989628089592e-05, + 4.5364751713350415e-05, + 4.3128704419359565e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00022091567916504573, + "first_to_last_ratio": 1.232986382535498, + "imbalance_abs_log_ratio": 0.20943917994887448, + "last_quartile_mean": 0.00017917122386279516, + "mean": 0.0002198539991695725, + "normalized": [ + 0.4672441326905094, + 1.0299890566632133, + 1.08986441571188, + 1.0084939820045402, + 0.8814237539069011, + 1.2693826047962031, + 1.1886514905905403, + 1.1035827467935222, + 1.0473307057033197, + 1.4274044616309502, + 1.1442998336482333, + 1.0911283607583675, + 1.1094038041384326, + 1.1946873974128753, + 1.2511466100570947, + 0.9923165311956169, + 0.9239703707694655, + 1.2473578196113966, + 1.1181336030529359, + 1.0160745414007064, + 0.9397394354798955, + 0.9905446518023159, + 1.0121913654482901, + 0.9359943959598221, + 0.8839446941554127, + 0.8510229499852975, + 0.8648513533654478, + 0.8786898836618169, + 0.7455559493639876, + 0.8399702482950041, + 0.7392957942269532, + 0.7163130557190521 + ], + "peak_layer": 10, + "peak_normalized": 1.4274044616309502, + "population_cv": 0.18952709251819305, + "rest_mean": 0.0002217885959436427, + "spike_contrast": 0.9441746799407325, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020940717658959329, + "top_five_layers": [ + 10, + 6, + 15, + 18, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010272549116052687, + 0.00022644721320830286, + 0.00023961105034686625, + 0.00022172143508214504, + 0.0001937845372594893, + 0.0002790788421407342, + 0.0002613297838252038, + 0.00024262708029709756, + 0.00023025984410196543, + 0.000313820579322055, + 0.00025157889467664063, + 0.00023988893372006714, + 0.00024390686303377151, + 0.000262656802078709, + 0.00027506958576850593, + 0.0002181647578254342, + 0.00020313858112785965, + 0.00027423660503700376, + 0.0002458261442370713, + 0.00022338805138133466, + 0.00020660547306761146, + 0.00021777520305477083, + 0.00022253431961871684, + 0.00020578211115207523, + 0.0001943387760547921, + 0.00018710079893935472, + 0.0001901410287246108, + 0.00019318348495289683, + 0.00016391345707233995, + 0.00018467081827111542, + 0.00016253713693004102, + 0.00015748428995721042 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014061377383768559, + 0.014165290631353855, + 0.014281988143920898, + 0.014350161887705326, + 0.00966146495193243, + 0.009741888381540775, + 0.009820535778999329, + 0.00991273857653141, + 0.0074927168898284435, + 0.007546746172010899, + 0.00766373984515667, + 0.007762390188872814, + 0.006325984839349985, + 0.006375366821885109, + 0.006445344537496567, + 0.006555322092026472, + 0.005531130358576775, + 0.005611324217170477, + 0.0056777093559503555, + 0.00574630219489336, + 0.004946626722812653, + 0.0050392793491482735, + 0.005084683652967215, + 0.005203457549214363, + 0.004667261149734259, + 0.004717131145298481, + 0.004768412560224533, + 0.004826710093766451, + 0.0044052014127373695, + 0.004504534415900707, + 0.004574235528707504, + 0.004644096828997135 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.729895206561196e-05, + "first_to_last_ratio": 1.2469721805352876, + "imbalance_abs_log_ratio": 0.22071835733540474, + "last_quartile_mean": 2.9911615229138988e-05, + "mean": 3.340667609563752e-05, + "normalized": [ + 1.308752778077389, + 1.2456464596610868, + 1.0236956931411572, + 0.9922285647386262, + 1.26903181062375, + 1.1170846373362797, + 1.0287316533628064, + 0.9469237624012169, + 1.1300261751959062, + 1.1132698789554223, + 1.0126240729566007, + 0.9379202574439188, + 1.1130503370514597, + 1.070713488608195, + 0.9897642722064988, + 0.922085688720865, + 1.036831181462568, + 0.9898105546019274, + 0.9395130252451869, + 0.9001967292772246, + 0.9975952535130206, + 0.9404117749145336, + 0.9224862220158919, + 0.8885787590271026, + 0.9461670180110204, + 0.8974372095121397, + 0.8904691498559152, + 0.8684437934704528, + 0.9244147277585869, + 0.9015689206268677, + 0.8740901912628669, + 0.8604359589635168 + ], + "peak_layer": 1, + "peak_normalized": 1.308752778077389, + "population_cv": 0.1164905978879517, + "rest_mean": 3.378375206213807e-05, + "spike_contrast": 0.9285666618330386, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.137046587653458e-05, + "top_five_layers": [ + 1, + 5, + 2, + 9, + 6 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.3721080146497115e-05, + 4.161290780757554e-05, + 3.419827044126578e-05, + 3.314705827506259e-05, + 4.239413465256803e-05, + 3.73180846509058e-05, + 3.4366505133220926e-05, + 3.1633575417799875e-05, + 3.7750418414361775e-05, + 3.719064625329338e-05, + 3.3828404411906376e-05, + 3.1332798243965954e-05, + 3.718331208801828e-05, + 3.5768978705164045e-05, + 3.306473445263691e-05, + 3.080381793552078e-05, + 3.463708344497718e-05, + 3.3066280593629926e-05, + 3.138600732199848e-05, + 3.007258055731654e-05, + 3.332634150865488e-05, + 3.1416031561093405e-05, + 3.0817198421573266e-05, + 2.9684462788281962e-05, + 3.1608295103069395e-05, + 2.9980394174344838e-05, + 2.9747614462394267e-05, + 2.9011820515734144e-05, + 3.0881623388268054e-05, + 3.01184209092753e-05, + 2.9200447897892445e-05, + 2.874430538213346e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.705871965095866e-05, + "first_to_last_ratio": 0.7180297289018083, + "imbalance_abs_log_ratio": 0.33124430563536084, + "last_quartile_mean": 9.33926785364747e-05, + "mean": 8.228966771639534e-05, + "normalized": [ + 0.8685826408392191, + 0.8249006353131848, + 0.7154905339803773, + 0.6884730939760333, + 0.9470487095509537, + 0.8638069620260335, + 0.8256155016985178, + 0.7853664463613442, + 0.9851686390305121, + 0.9692511243547932, + 0.9083804947955202, + 0.8729855457896263, + 1.0422548534804916, + 1.0255492616133721, + 0.9871528463964067, + 0.9398223255303129, + 1.0958375595009802, + 1.0726325633807163, + 1.0495259774381736, + 1.0230838788771643, + 1.1576441025653552, + 1.1142532603054807, + 1.1008567438213033, + 1.056908774400763, + 1.159063578749664, + 1.1192434432436902, + 1.1032241584740294, + 1.1060435702772593, + 1.1786235078649367, + 1.1628365878275773, + 1.141840825041133, + 1.1085318534950752 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235078649367, + "population_cv": 0.1341844256762087, + "rest_mean": 8.049536754023629e-05, + "spike_contrast": 1.1426606459269575, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.197888866765424e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.14753768988885e-05, + 6.788079917896539e-05, + 5.887747829547152e-05, + 5.66542221349664e-05, + 7.793232362018898e-05, + 7.108238787623122e-05, + 6.793962529627606e-05, + 6.462754390668124e-05, + 8.106919995043427e-05, + 7.975935295689851e-05, + 7.475032907677814e-05, + 7.183769048424438e-05, + 8.576680556870997e-05, + 8.4392107964959e-05, + 8.123247971525416e-05, + 7.733766688033938e-05, + 9.017610864248127e-05, + 8.826657722238451e-05, + 8.636514394311234e-05, + 8.418923243880272e-05, + 9.526214853394777e-05, + 9.169153054244816e-05, + 9.0589135652408e-05, + 8.697267185198143e-05, + 9.537895675748587e-05, + 9.210217103827745e-05, + 9.078394941752777e-05, + 9.101595787797123e-05, + 9.698853682493791e-05, + 9.56894364207983e-05, + 9.396170207764953e-05, + 9.122071787714958e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05946834245696664, + "first_to_last_ratio": 0.7180297222365345, + "imbalance_abs_log_ratio": 0.33124431491808765, + "last_quartile_mean": 0.08282156102359295, + "mean": 0.07297530013602227, + "normalized": [ + 0.8685826122359729, + 0.8249005917343738, + 0.71549052911611, + 0.6884731350635278, + 0.9470486699630107, + 0.8638070113577097, + 0.8256154769796886, + 0.7853664269052657, + 0.9851686341386658, + 0.9692511556795396, + 0.9083805454214465, + 0.8729855579949275, + 1.0422548040385637, + 1.0255492365750694, + 0.9871528950273681, + 0.9398223256645052, + 1.0958375053304956, + 1.07263252629438, + 1.0495259690486791, + 1.0230837911446407, + 1.1576441435704576, + 1.1142533045483842, + 1.1008568124470575, + 1.0569088284972943, + 1.1590636022344982, + 1.119243411839159, + 1.1032241423789504, + 1.1060435591240758, + 1.1786235037173105, + 1.1628366076958818, + 1.141840809884862, + 1.1085318743481278 + ], + "peak_layer": 29, + "peak_normalized": 1.1786235037173105, + "population_cv": 0.13418442668798974, + "rest_mean": 0.07138409617322462, + "spike_contrast": 1.1426607032635474, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08156780153512955, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.06338507682085037, + 0.0601973682641983, + 0.05221313610672951, + 0.050241533666849136, + 0.0691111609339714, + 0.06303657591342926, + 0.060249537229537964, + 0.05731235072016716, + 0.07189297676086426, + 0.07073139399290085, + 0.06628934293985367, + 0.06370638310909271, + 0.07605885714292526, + 0.0748397633433342, + 0.07203777879476547, + 0.06858381628990173, + 0.07996907085180283, + 0.07827568054199219, + 0.07658947259187698, + 0.07465984672307968, + 0.0844794288277626, + 0.08131296932697296, + 0.0803353562951088, + 0.07712823897600174, + 0.08458301424980164, + 0.08167712390422821, + 0.08050811290740967, + 0.08071386069059372, + 0.0860104039311409, + 0.08485835045576096, + 0.08332617580890656, + 0.08089544624090195 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.408341275549901e-06, + "first_to_last_ratio": 0.49124437919492936, + "imbalance_abs_log_ratio": 0.7108135577065074, + "last_quartile_mean": 1.5080765479069669e-05, + "mean": 1.1844782648040564e-05, + "normalized": [ + 0.5657990507718705, + 0.5382233871697658, + 0.49942145823009426, + 0.48906984415931193, + 0.8062921756393574, + 0.7122953827323945, + 0.7013508394120195, + 0.6911626993599347, + 0.8940675881372303, + 0.868722127857376, + 0.9735911331661544, + 0.854855860776803, + 0.989483587379368, + 1.0059088486455374, + 1.0303729006301217, + 0.9941025538400318, + 1.1620912657632405, + 1.127012997009919, + 1.113401270263767, + 1.066097770938846, + 1.1826034542812638, + 1.1643547935214302, + 1.2049778184223279, + 1.1791491538213539, + 1.2365172470752577, + 1.219921447624247, + 1.2337761202582145, + 1.2288278254433027, + 1.3039217543779833, + 1.3631114810840155, + 1.2856635723507386, + 1.3138525898567213 + ], + "peak_layer": 30, + "peak_normalized": 1.3631114810840155, + "population_cv": 0.2549101540724733, + "rest_mean": 1.1420299651271543e-05, + "spike_contrast": 1.237882653019464, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4136990830593277e-05, + "top_five_layers": [ + 30, + 32, + 29, + 31, + 25 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 6.701766778860474e-06, + 6.3751390371180605e-06, + 5.915538622502936e-06, + 5.7929260037781205e-06, + 9.550355571263935e-06, + 8.436983989668079e-06, + 8.307348252856173e-06, + 8.186671948351432e-06, + 1.0590036254143342e-05, + 1.0289824786013924e-05, + 1.1531975360412616e-05, + 1.0125581866304856e-05, + 1.1720218026312068e-05, + 1.1914771675947122e-05, + 1.220454305439489e-05, + 1.1774928680097219e-05, + 1.3764718460151926e-05, + 1.334922399109928e-05, + 1.3187996046326589e-05, + 1.2627696378331166e-05, + 1.4007680874783546e-05, + 1.379152945446549e-05, + 1.4272700354922563e-05, + 1.3966765436634887e-05, + 1.4646278032159898e-05, + 1.4449704394792207e-05, + 1.4613809980801307e-05, + 1.4555198504240252e-05, + 1.5444669770658948e-05, + 1.614575921848882e-05, + 1.5228405572997872e-05, + 1.5562298358418047e-05 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.6029601637419546e-05, + "first_to_last_ratio": 0.6070213107493427, + "imbalance_abs_log_ratio": 0.49919138022039955, + "last_quartile_mean": 9.230252817360451e-05, + "mean": 7.840498903988191e-05, + "normalized": [ + 0.6598133090619259, + 0.6419677920829977, + 0.6005256373772865, + 0.5913463573822231, + 0.8562376513989928, + 0.8062968559250673, + 0.7923156566726156, + 0.768439227921366, + 0.9790122729268586, + 0.9665730330981341, + 0.9163710065350301, + 0.8881686280668529, + 1.0631190324760253, + 1.0511030519715439, + 1.0188080221730096, + 0.9712812246823179, + 1.1361272291219966, + 1.1138561419734825, + 1.0918134348286874, + 1.0639217496772315, + 1.203414696914147, + 1.1592426077267617, + 1.1442664103435203, + 1.097952968622282, + 1.2042281788778202, + 1.1625445130141736, + 1.1452986211445626, + 1.1473932954411665, + 1.2233942816523278, + 1.205318760675343, + 1.1829469858778916, + 1.146901364356358 + ], + "peak_layer": 29, + "peak_normalized": 1.2233942816523278, + "population_cv": 0.1898597249088972, + "rest_mean": 7.605543874820728e-05, + "spike_contrast": 1.1977126437531995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.109256061492487e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 5.173265526536852e-05, + 5.033347770222463e-05, + 4.708420601673424e-05, + 4.636450466932729e-05, + 6.713330367347226e-05, + 6.321769615169615e-05, + 6.212150037754327e-05, + 6.024946924299002e-05, + 7.675944652874023e-05, + 7.578414806630462e-05, + 7.184805872384459e-05, + 6.963685154914856e-05, + 8.335383608937263e-05, + 8.241172326961532e-05, + 7.987963181221858e-05, + 7.615329377586022e-05, + 8.907804294722155e-05, + 8.733187860343605e-05, + 8.560362039133906e-05, + 8.341677312273532e-05, + 9.435371612198651e-05, + 9.089040395338088e-05, + 8.971619536168873e-05, + 8.608499047113582e-05, + 9.441749716643244e-05, + 9.114928980125114e-05, + 8.97971258382313e-05, + 8.996135875349864e-05, + 9.592021524440497e-05, + 9.450300422031432e-05, + 9.274894546251744e-05, + 8.992278890218586e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.8511469685763586e-05, + "first_to_last_ratio": 0.5271352332510719, + "imbalance_abs_log_ratio": 0.6402981537748061, + "last_quartile_mean": 9.202850924339145e-05, + "mean": 7.532146378252946e-05, + "normalized": [ + 0.548104082267993, + 0.5416830199005969, + 0.528884702689083, + 0.5247448675895723, + 0.7777813723908718, + 0.7487613756360456, + 0.7496059866168319, + 0.7329069161583202, + 0.9480676391902623, + 0.9399123881532756, + 0.9000936195631334, + 0.883441930780847, + 1.0560589316963704, + 1.0534552095740957, + 1.032095781768693, + 0.9835620691098637, + 1.1601754560133901, + 1.1467788545782247, + 1.1233340534265408, + 1.0967375255670042, + 1.2396083970871608, + 1.1951467072205837, + 1.1828066060767042, + 1.1317734013234224, + 1.244698956870482, + 1.2025862582796845, + 1.1881970083126314, + 1.1882256015362047, + 1.2689460104605979, + 1.2589162611011824, + 1.2307315535498626, + 1.1921774555104672 + ], + "peak_layer": 29, + "peak_normalized": 1.2689460104605979, + "population_cv": 0.23486334953575583, + "rest_mean": 7.254842300123225e-05, + "spike_contrast": 1.244629176846487, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.029588400153444e-05, + "top_five_layers": [ + 29, + 30, + 25, + 21, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 4.1284001781605184e-05, + 4.080035796505399e-05, + 3.9836369978729635e-05, + 3.952455153921619e-05, + 5.858363147126511e-05, + 5.639780283672735e-05, + 5.6461420172126964e-05, + 5.520362174138427e-05, + 7.140984234865755e-05, + 7.079557690303773e-05, + 6.77963689668104e-05, + 6.654213939327747e-05, + 7.954390457598493e-05, + 7.934778841445222e-05, + 7.773896504659206e-05, + 7.408333476632833e-05, + 8.738611359149218e-05, + 8.637706196168438e-05, + 8.461116522084922e-05, + 8.26078758109361e-05, + 9.336911898571998e-05, + 9.002019942272455e-05, + 8.909072494134307e-05, + 8.524682925781235e-05, + 9.375254740007222e-05, + 9.058055729838088e-05, + 8.949673792812973e-05, + 8.949889161158353e-05, + 9.557887096889317e-05, + 9.482341556577012e-05, + 9.270050213672221e-05, + 8.979655103757977e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.61669284151867e-05, + "first_to_last_ratio": 0.8297833624837898, + "imbalance_abs_log_ratio": 0.18659062131596024, + "last_quartile_mean": 0.00011589401856326731, + "mean": 0.00010716335873439675, + "normalized": [ + 0.9380742209455786, + 0.8898820858744699, + 0.7837417114644826, + 0.7583102664636657, + 1.0822070619439073, + 0.9520749081713696, + 0.9107982467806989, + 0.8640017807748708, + 1.0669538303069324, + 1.0266950649381863, + 0.9594029855154438, + 0.8901506143407661, + 1.0795289063552718, + 1.0379165674077444, + 0.9706971366471856, + 0.914979822300863, + 1.052939971267624, + 1.0225707935964177, + 0.9851238030379684, + 0.9636652893153618, + 1.0990044510799908, + 1.0523904893769196, + 1.0422234788441622, + 1.0049019599938585, + 1.0962247910521359, + 1.0599378706264646, + 1.05039930496957, + 1.0583562345651827, + 1.1199788667078585, + 1.1100529546792268, + 1.0850973923636869, + 1.0717171382921347 + ], + "peak_layer": 29, + "peak_normalized": 1.1199788667078585, + "population_cv": 0.09156717959150794, + "rest_mean": 0.00010599351123716958, + "spike_contrast": 1.0706366257223145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011348053521942347, + "top_five_layers": [ + 29, + 30, + 21, + 25, + 31 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010052718425868079, + 9.536275319987908e-05, + 8.398839418077841e-05, + 8.12630751170218e-05, + 0.00011597294360399246, + 0.00010202754492638633, + 9.760419925441965e-05, + 9.25893327803351e-05, + 0.00011433835607022047, + 0.00011002409155480564, + 0.00010281284630764276, + 9.539152961224318e-05, + 0.000115685943455901, + 0.0001112266254494898, + 0.0001040231654769741, + 9.805231093196198e-05, + 0.00011283658386673778, + 0.00010958212078548968, + 0.00010556917550275102, + 0.00010326960909878835, + 0.00011777300824178383, + 0.0001127776995417662, + 0.00011168816854478791, + 0.00010768866923172027, + 0.00011747513053705916, + 0.00011358650226611644, + 0.00011256431753281504, + 0.00011341700883349404, + 0.00012002069706795737, + 0.00011895700299646705, + 0.00011628268111962825, + 0.00011484880815260112 + ] + } + } + } + } + } + }, + "step": 0 + }, + { + "modes": { + "learned": { + "bits_per_byte": 5.150682269327325, + "forward": { + "activation_sha256": { + "attention_branch_output": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", + "mlp_branch_output": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", + "post_attention_state": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", + "post_mlp_state": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", + "pre_attention_input": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", + "pre_mlp_input": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f" + }, + "logits_sha256": "bdadcb0e52fa595647848016703b410d7bcf45d1b7a624e8a3f61c3e1956509d", + "loss_nats": 3.570180892944336, + "mixer_summary_sha256": "023101d1bfcf41624a0444a7f14bed4c44082764bd9047e90be0fb0a456542ad" + }, + "loss_nats": 3.570180892944336, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.6930912733078003, + "group": 1, + "latest_source_mass": 0.49764734506607056, + "layer": 1, + "max_source_mass": 0.5023525953292847, + "mixer_index": 1, + "normalized_entropy": 0.9999193428845807, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5023525953292847, + "median": 0.5031890273094177, + "p05": 0.4936259984970093, + "p95": 0.5090169310569763 + }, + { + "label": "current_group_partial", + "mean": 0.49764734506607056, + "median": 0.4968109726905823, + "p05": 0.4909830391407013, + "p95": 0.5063740015029907 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.692903995513916, + "group": 1, + "latest_source_mass": 0.48966121673583984, + "layer": 2, + "max_source_mass": 0.5103387832641602, + "mixer_index": 2, + "normalized_entropy": 0.9996491581400753, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5103387832641602, + "median": 0.5101123452186584, + "p05": 0.5045775175094604, + "p95": 0.5169917345046997 + }, + { + "label": "current_group_partial", + "mean": 0.48966121673583984, + "median": 0.48988765478134155, + "p05": 0.4830082654953003, + "p95": 0.49542248249053955 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6928629875183105, + "group": 1, + "latest_source_mass": 0.4946908950805664, + "layer": 2, + "max_source_mass": 0.5053091049194336, + "mixer_index": 3, + "normalized_entropy": 0.9995899961081784, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5053091049194336, + "median": 0.5084525346755981, + "p05": 0.4841057062149048, + "p95": 0.5181366801261902 + }, + { + "label": "current_group_partial", + "mean": 0.4946908950805664, + "median": 0.49154743552207947, + "p05": 0.4818633496761322, + "p95": 0.5158942937850952 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6927823424339294, + "group": 1, + "latest_source_mass": 0.4873165488243103, + "layer": 3, + "max_source_mass": 0.5126834511756897, + "mixer_index": 4, + "normalized_entropy": 0.9994736498448697, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5126834511756897, + "median": 0.5128336548805237, + "p05": 0.5050104260444641, + "p95": 0.5203934907913208 + }, + { + "label": "current_group_partial", + "mean": 0.4873165488243103, + "median": 0.4871663451194763, + "p05": 0.4796065092086792, + "p95": 0.4949895739555359 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6928155422210693, + "group": 1, + "latest_source_mass": 0.49704670906066895, + "layer": 3, + "max_source_mass": 0.502953290939331, + "mixer_index": 5, + "normalized_entropy": 0.999521547013135, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.502953290939331, + "median": 0.5012187957763672, + "p05": 0.4864824414253235, + "p95": 0.5226666927337646 + }, + { + "label": "current_group_partial", + "mean": 0.49704670906066895, + "median": 0.4987812042236328, + "p05": 0.4773333668708801, + "p95": 0.5135174989700317 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.692751407623291, + "group": 1, + "latest_source_mass": 0.48708435893058777, + "layer": 4, + "max_source_mass": 0.5129156112670898, + "mixer_index": 6, + "normalized_entropy": 0.9994290203469708, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5129156112670898, + "median": 0.512997567653656, + "p05": 0.504144549369812, + "p95": 0.5216691493988037 + }, + { + "label": "current_group_partial", + "mean": 0.48708435893058777, + "median": 0.4870024621486664, + "p05": 0.4783308207988739, + "p95": 0.4958554208278656 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6925709247589111, + "group": 1, + "latest_source_mass": 0.48873060941696167, + "layer": 4, + "max_source_mass": 0.5112694501876831, + "mixer_index": 7, + "normalized_entropy": 0.9991686386135645, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5112694501876831, + "median": 0.5127402544021606, + "p05": 0.4926953613758087, + "p95": 0.5298716425895691 + }, + { + "label": "current_group_partial", + "mean": 0.48873060941696167, + "median": 0.48725977540016174, + "p05": 0.4701283872127533, + "p95": 0.5073046088218689 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6929247379302979, + "group": 2, + "latest_source_mass": 0.49048691987991333, + "layer": 5, + "max_source_mass": 0.5095131397247314, + "mixer_index": 8, + "normalized_entropy": 0.9996790831213254, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5095131397247314, + "median": 0.5093363523483276, + "p05": 0.502471923828125, + "p95": 0.5177270770072937 + }, + { + "label": "completed_group_1", + "mean": 0.49048691987991333, + "median": 0.49066367745399475, + "p05": 0.4822728931903839, + "p95": 0.4975281059741974 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0983518362045288, + "group": 2, + "latest_source_mass": 0.33183109760284424, + "layer": 5, + "max_source_mass": 0.33983534574508667, + "mixer_index": 9, + "normalized_entropy": 0.999762925950977, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33983534574508667, + "median": 0.3411724865436554, + "p05": 0.32688987255096436, + "p95": 0.3497551679611206 + }, + { + "label": "completed_group_1", + "mean": 0.32833361625671387, + "median": 0.328115850687027, + "p05": 0.31867384910583496, + "p95": 0.33767229318618774 + }, + { + "label": "current_group_partial", + "mean": 0.33183109760284424, + "median": 0.331381618976593, + "p05": 0.3268507122993469, + "p95": 0.33714187145233154 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.098447322845459, + "group": 2, + "latest_source_mass": 0.3288182318210602, + "layer": 6, + "max_source_mass": 0.340813547372818, + "mixer_index": 10, + "normalized_entropy": 0.9998498416371705, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.340813547372818, + "median": 0.340879887342453, + "p05": 0.33481645584106445, + "p95": 0.34668514132499695 + }, + { + "label": "completed_group_1", + "mean": 0.3303682208061218, + "median": 0.33054590225219727, + "p05": 0.325665682554245, + "p95": 0.33449190855026245 + }, + { + "label": "current_group_partial", + "mean": 0.3288182318210602, + "median": 0.3287218511104584, + "p05": 0.3254890441894531, + "p95": 0.33243927359580994 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0980432033538818, + "group": 2, + "latest_source_mass": 0.32141047716140747, + "layer": 6, + "max_source_mass": 0.34726741909980774, + "mixer_index": 11, + "normalized_entropy": 0.9994819962236925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34726741909980774, + "median": 0.34668511152267456, + "p05": 0.3404209613800049, + "p95": 0.35565274953842163 + }, + { + "label": "completed_group_1", + "mean": 0.3313220739364624, + "median": 0.3316044509410858, + "p05": 0.3264642655849457, + "p95": 0.33527836203575134 + }, + { + "label": "current_group_partial", + "mean": 0.32141047716140747, + "median": 0.321965754032135, + "p05": 0.3149893283843994, + "p95": 0.326220840215683 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0985240936279297, + "group": 2, + "latest_source_mass": 0.32844245433807373, + "layer": 7, + "max_source_mass": 0.3373557925224304, + "mixer_index": 12, + "normalized_entropy": 0.9999197214148341, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3373557925224304, + "median": 0.33718767762184143, + "p05": 0.33250534534454346, + "p95": 0.34282490611076355 + }, + { + "label": "completed_group_1", + "mean": 0.33420175313949585, + "median": 0.3341767191886902, + "p05": 0.3307935893535614, + "p95": 0.3377305567264557 + }, + { + "label": "current_group_partial", + "mean": 0.32844245433807373, + "median": 0.3287346363067627, + "p05": 0.3250963091850281, + "p95": 0.33099210262298584 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.098268985748291, + "group": 2, + "latest_source_mass": 0.32483887672424316, + "layer": 7, + "max_source_mass": 0.3445510268211365, + "mixer_index": 13, + "normalized_entropy": 0.9996875122157655, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3445510268211365, + "median": 0.3445873260498047, + "p05": 0.3388661742210388, + "p95": 0.3502572774887085 + }, + { + "label": "completed_group_1", + "mean": 0.330610066652298, + "median": 0.330571711063385, + "p05": 0.3262380361557007, + "p95": 0.3352304697036743 + }, + { + "label": "current_group_partial", + "mean": 0.32483887672424316, + "median": 0.32478615641593933, + "p05": 0.3213251233100891, + "p95": 0.3284890055656433 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0985302925109863, + "group": 2, + "latest_source_mass": 0.3295760750770569, + "layer": 8, + "max_source_mass": 0.3371829390525818, + "mixer_index": 14, + "normalized_entropy": 0.9999253638813536, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3371829390525818, + "median": 0.33710986375808716, + "p05": 0.330502450466156, + "p95": 0.34345686435699463 + }, + { + "label": "completed_group_1", + "mean": 0.3332410156726837, + "median": 0.33316272497177124, + "p05": 0.32938438653945923, + "p95": 0.33757662773132324 + }, + { + "label": "current_group_partial", + "mean": 0.3295760750770569, + "median": 0.3295734226703644, + "p05": 0.3265466094017029, + "p95": 0.3326798677444458 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0983161926269531, + "group": 2, + "latest_source_mass": 0.32498204708099365, + "layer": 8, + "max_source_mass": 0.3423102796077728, + "mixer_index": 15, + "normalized_entropy": 0.9997304817684903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3423102796077728, + "median": 0.34171032905578613, + "p05": 0.3346346318721771, + "p95": 0.35120290517807007 + }, + { + "label": "completed_group_1", + "mean": 0.3327076733112335, + "median": 0.3330804705619812, + "p05": 0.32750606536865234, + "p95": 0.33743682503700256 + }, + { + "label": "current_group_partial", + "mean": 0.32498204708099365, + "median": 0.3252435028553009, + "p05": 0.31977349519729614, + "p95": 0.3294060230255127 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.098489761352539, + "group": 3, + "latest_source_mass": 0.327616423368454, + "layer": 9, + "max_source_mass": 0.33793729543685913, + "mixer_index": 16, + "normalized_entropy": 0.9998884708310343, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33793729543685913, + "median": 0.3376197814941406, + "p05": 0.3321118652820587, + "p95": 0.34503501653671265 + }, + { + "label": "completed_group_1", + "mean": 0.3344462513923645, + "median": 0.3347727060317993, + "p05": 0.33071762323379517, + "p95": 0.33719509840011597 + }, + { + "label": "completed_group_2", + "mean": 0.327616423368454, + "median": 0.3275512754917145, + "p05": 0.32320931553840637, + "p95": 0.33203738927841187 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3861088752746582, + "group": 3, + "latest_source_mass": 0.2466907799243927, + "layer": 9, + "max_source_mass": 0.2561652958393097, + "mixer_index": 17, + "normalized_entropy": 0.9998662002454641, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2561652958393097, + "median": 0.25643306970596313, + "p05": 0.25058528780937195, + "p95": 0.2609494626522064 + }, + { + "label": "completed_group_1", + "mean": 0.2513129711151123, + "median": 0.2511752247810364, + "p05": 0.24789966642856598, + "p95": 0.25490516424179077 + }, + { + "label": "completed_group_2", + "mean": 0.2458309531211853, + "median": 0.2456333041191101, + "p05": 0.2419084906578064, + "p95": 0.2499997466802597 + }, + { + "label": "current_group_partial", + "mean": 0.2466907799243927, + "median": 0.246699720621109, + "p05": 0.24314038455486298, + "p95": 0.25063008069992065 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3861844539642334, + "group": 3, + "latest_source_mass": 0.24695077538490295, + "layer": 10, + "max_source_mass": 0.2536832094192505, + "mixer_index": 18, + "normalized_entropy": 0.9999207187457876, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2536832094192505, + "median": 0.253880113363266, + "p05": 0.24814625084400177, + "p95": 0.2593393921852112 + }, + { + "label": "completed_group_1", + "mean": 0.25214672088623047, + "median": 0.2522420883178711, + "p05": 0.24960598349571228, + "p95": 0.2543295919895172 + }, + { + "label": "completed_group_2", + "mean": 0.24721932411193848, + "median": 0.24704834818840027, + "p05": 0.24384759366512299, + "p95": 0.25051552057266235 + }, + { + "label": "current_group_partial", + "mean": 0.24695077538490295, + "median": 0.24677348136901855, + "p05": 0.24500520527362823, + "p95": 0.2492748200893402 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3860987424850464, + "group": 3, + "latest_source_mass": 0.24684447050094604, + "layer": 10, + "max_source_mass": 0.25705933570861816, + "mixer_index": 19, + "normalized_entropy": 0.9998588909828025, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25705933570861816, + "median": 0.2568470239639282, + "p05": 0.2513251304626465, + "p95": 0.26331475377082825 + }, + { + "label": "completed_group_1", + "mean": 0.24983704090118408, + "median": 0.24969333410263062, + "p05": 0.2465076744556427, + "p95": 0.25355273485183716 + }, + { + "label": "completed_group_2", + "mean": 0.2462591528892517, + "median": 0.24622508883476257, + "p05": 0.24337750673294067, + "p95": 0.24924828112125397 + }, + { + "label": "current_group_partial", + "mean": 0.24684447050094604, + "median": 0.24683824181556702, + "p05": 0.24403956532478333, + "p95": 0.24949496984481812 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862221240997314, + "group": 3, + "latest_source_mass": 0.24620777368545532, + "layer": 11, + "max_source_mass": 0.25285202264785767, + "mixer_index": 20, + "normalized_entropy": 0.9999478920046239, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2514326572418213, + "median": 0.25176799297332764, + "p05": 0.24684108793735504, + "p95": 0.2554701566696167 + }, + { + "label": "completed_group_1", + "mean": 0.25285202264785767, + "median": 0.25268346071243286, + "p05": 0.2510001063346863, + "p95": 0.2550324499607086 + }, + { + "label": "completed_group_2", + "mean": 0.24950754642486572, + "median": 0.2494041621685028, + "p05": 0.2475442737340927, + "p95": 0.2515066862106323 + }, + { + "label": "current_group_partial", + "mean": 0.24620777368545532, + "median": 0.24642038345336914, + "p05": 0.2445429414510727, + "p95": 0.2473805695772171 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.386016845703125, + "group": 3, + "latest_source_mass": 0.24290117621421814, + "layer": 11, + "max_source_mass": 0.25482746958732605, + "mixer_index": 21, + "normalized_entropy": 0.999799814942231, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25482746958732605, + "median": 0.2545710802078247, + "p05": 0.2466055005788803, + "p95": 0.2635689973831177 + }, + { + "label": "completed_group_1", + "mean": 0.2536851167678833, + "median": 0.2534823715686798, + "p05": 0.2508774995803833, + "p95": 0.2568957209587097 + }, + { + "label": "completed_group_2", + "mean": 0.24858617782592773, + "median": 0.24861112236976624, + "p05": 0.24390698969364166, + "p95": 0.252436101436615 + }, + { + "label": "current_group_partial", + "mean": 0.24290117621421814, + "median": 0.24232372641563416, + "p05": 0.23918122053146362, + "p95": 0.24772994220256805 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862277269363403, + "group": 3, + "latest_source_mass": 0.24656552076339722, + "layer": 12, + "max_source_mass": 0.2530350983142853, + "mixer_index": 22, + "normalized_entropy": 0.9999519335969191, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25171470642089844, + "median": 0.2513781189918518, + "p05": 0.24833296239376068, + "p95": 0.25546184182167053 + }, + { + "label": "completed_group_1", + "mean": 0.2530350983142853, + "median": 0.25303736329078674, + "p05": 0.25148919224739075, + "p95": 0.2544969618320465 + }, + { + "label": "completed_group_2", + "mean": 0.24868467450141907, + "median": 0.24892255663871765, + "p05": 0.2467103898525238, + "p95": 0.25010940432548523 + }, + { + "label": "current_group_partial", + "mean": 0.24656552076339722, + "median": 0.24677836894989014, + "p05": 0.24485518038272858, + "p95": 0.2477675825357437 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3859888315200806, + "group": 3, + "latest_source_mass": 0.24197682738304138, + "layer": 12, + "max_source_mass": 0.2546530067920685, + "mixer_index": 23, + "normalized_entropy": 0.9997796069807546, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.25458407402038574, + "median": 0.2539653182029724, + "p05": 0.24643409252166748, + "p95": 0.26325899362564087 + }, + { + "label": "completed_group_1", + "mean": 0.2546530067920685, + "median": 0.2548711895942688, + "p05": 0.25055474042892456, + "p95": 0.25881263613700867 + }, + { + "label": "completed_group_2", + "mean": 0.248786062002182, + "median": 0.24910664558410645, + "p05": 0.2454056739807129, + "p95": 0.2517758011817932 + }, + { + "label": "current_group_partial", + "mean": 0.24197682738304138, + "median": 0.2416902482509613, + "p05": 0.23907747864723206, + "p95": 0.2456972599029541 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3862541913986206, + "group": 4, + "latest_source_mass": 0.24767187237739563, + "layer": 13, + "max_source_mass": 0.2521768808364868, + "mixer_index": 24, + "normalized_entropy": 0.999971023671165, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2501963973045349, + "median": 0.250087708234787, + "p05": 0.24644337594509125, + "p95": 0.25403982400894165 + }, + { + "label": "completed_group_1", + "mean": 0.2521768808364868, + "median": 0.2521744668483734, + "p05": 0.25072887539863586, + "p95": 0.25345146656036377 + }, + { + "label": "completed_group_2", + "mean": 0.24995486438274384, + "median": 0.2498224377632141, + "p05": 0.2482302337884903, + "p95": 0.2518054246902466 + }, + { + "label": "completed_group_3", + "mean": 0.24767187237739563, + "median": 0.2475992739200592, + "p05": 0.24603278934955597, + "p95": 0.2497682422399521 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6090729236602783, + "group": 4, + "latest_source_mass": 0.19462624192237854, + "layer": 13, + "max_source_mass": 0.20631682872772217, + "mixer_index": 25, + "normalized_entropy": 0.9997732197241024, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20360878109931946, + "median": 0.20399323105812073, + "p05": 0.19613273441791534, + "p95": 0.21039927005767822 + }, + { + "label": "completed_group_1", + "mean": 0.20631682872772217, + "median": 0.20582494139671326, + "p05": 0.20368342101573944, + "p95": 0.2099049985408783 + }, + { + "label": "completed_group_2", + "mean": 0.19986985623836517, + "median": 0.19948531687259674, + "p05": 0.19628161191940308, + "p95": 0.20406028628349304 + }, + { + "label": "completed_group_3", + "mean": 0.19557830691337585, + "median": 0.19561558961868286, + "p05": 0.1910029649734497, + "p95": 0.20039188861846924 + }, + { + "label": "current_group_partial", + "mean": 0.19462624192237854, + "median": 0.1954103410243988, + "p05": 0.19068367779254913, + "p95": 0.19781020283699036 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093083620071411, + "group": 4, + "latest_source_mass": 0.1976698487997055, + "layer": 14, + "max_source_mass": 0.20362424850463867, + "mixer_index": 26, + "normalized_entropy": 0.9999195057939432, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2017374038696289, + "median": 0.2026747167110443, + "p05": 0.19666710495948792, + "p95": 0.20696133375167847 + }, + { + "label": "completed_group_1", + "mean": 0.20362424850463867, + "median": 0.20354315638542175, + "p05": 0.20209211111068726, + "p95": 0.20522044599056244 + }, + { + "label": "completed_group_2", + "mean": 0.20011675357818604, + "median": 0.1997961699962616, + "p05": 0.19781088829040527, + "p95": 0.2023434340953827 + }, + { + "label": "completed_group_3", + "mean": 0.1968517303466797, + "median": 0.19661611318588257, + "p05": 0.19452345371246338, + "p95": 0.19933368265628815 + }, + { + "label": "current_group_partial", + "mean": 0.1976698487997055, + "median": 0.19780653715133667, + "p05": 0.19629782438278198, + "p95": 0.19887655973434448 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.608644723892212, + "group": 4, + "latest_source_mass": 0.18768835067749023, + "layer": 14, + "max_source_mass": 0.20831988751888275, + "mixer_index": 27, + "normalized_entropy": 0.9995071642492324, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20431089401245117, + "median": 0.20455676317214966, + "p05": 0.19753149151802063, + "p95": 0.21038773655891418 + }, + { + "label": "completed_group_1", + "mean": 0.20831988751888275, + "median": 0.20877230167388916, + "p05": 0.2030324637889862, + "p95": 0.2128065973520279 + }, + { + "label": "completed_group_2", + "mean": 0.20422273874282837, + "median": 0.2047901451587677, + "p05": 0.20030567049980164, + "p95": 0.20715001225471497 + }, + { + "label": "completed_group_3", + "mean": 0.19545812904834747, + "median": 0.19552549719810486, + "p05": 0.1923498660326004, + "p95": 0.19825047254562378 + }, + { + "label": "current_group_partial", + "mean": 0.18768835067749023, + "median": 0.18748146295547485, + "p05": 0.18511706590652466, + "p95": 0.19059684872627258 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.609345555305481, + "group": 4, + "latest_source_mass": 0.19806164503097534, + "layer": 15, + "max_source_mass": 0.20386312901973724, + "mixer_index": 28, + "normalized_entropy": 0.9999426152895332, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2002599537372589, + "median": 0.20108085870742798, + "p05": 0.19542044401168823, + "p95": 0.20405235886573792 + }, + { + "label": "completed_group_1", + "mean": 0.20386312901973724, + "median": 0.20386025309562683, + "p05": 0.20271247625350952, + "p95": 0.20498213171958923 + }, + { + "label": "completed_group_2", + "mean": 0.2001732587814331, + "median": 0.19999440014362335, + "p05": 0.1985137164592743, + "p95": 0.2019573152065277 + }, + { + "label": "completed_group_3", + "mean": 0.1976420283317566, + "median": 0.19760513305664062, + "p05": 0.1957571804523468, + "p95": 0.20000392198562622 + }, + { + "label": "current_group_partial", + "mean": 0.19806164503097534, + "median": 0.19796663522720337, + "p05": 0.1969158947467804, + "p95": 0.19929495453834534 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6086196899414062, + "group": 4, + "latest_source_mass": 0.18893863260746002, + "layer": 15, + "max_source_mass": 0.21025606989860535, + "mixer_index": 29, + "normalized_entropy": 0.9994916097810468, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20515945553779602, + "median": 0.2049580067396164, + "p05": 0.19868221879005432, + "p95": 0.21147878468036652 + }, + { + "label": "completed_group_1", + "mean": 0.21025606989860535, + "median": 0.2101147472858429, + "p05": 0.20689907670021057, + "p95": 0.21399712562561035 + }, + { + "label": "completed_group_2", + "mean": 0.20179016888141632, + "median": 0.20176216959953308, + "p05": 0.199407696723938, + "p95": 0.20402206480503082 + }, + { + "label": "completed_group_3", + "mean": 0.1938556730747223, + "median": 0.19379490613937378, + "p05": 0.19004228711128235, + "p95": 0.1977127194404602 + }, + { + "label": "current_group_partial", + "mean": 0.18893863260746002, + "median": 0.1887960582971573, + "p05": 0.18639500439167023, + "p95": 0.19145210087299347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.609370470046997, + "group": 4, + "latest_source_mass": 0.1981920301914215, + "layer": 16, + "max_source_mass": 0.20340535044670105, + "mixer_index": 30, + "normalized_entropy": 0.9999580956888227, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2002836912870407, + "median": 0.20032534003257751, + "p05": 0.19668522477149963, + "p95": 0.20378869771957397 + }, + { + "label": "completed_group_1", + "mean": 0.20340535044670105, + "median": 0.20332223176956177, + "p05": 0.2024117112159729, + "p95": 0.20457619428634644 + }, + { + "label": "completed_group_2", + "mean": 0.20010054111480713, + "median": 0.1998724788427353, + "p05": 0.19848622381687164, + "p95": 0.20170091092586517 + }, + { + "label": "completed_group_3", + "mean": 0.1980184018611908, + "median": 0.19809064269065857, + "p05": 0.19669371843338013, + "p95": 0.19908645749092102 + }, + { + "label": "current_group_partial", + "mean": 0.1981920301914215, + "median": 0.19810855388641357, + "p05": 0.1971922218799591, + "p95": 0.19958281517028809 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6083989143371582, + "group": 4, + "latest_source_mass": 0.18735264241695404, + "layer": 16, + "max_source_mass": 0.21095217764377594, + "mixer_index": 31, + "normalized_entropy": 0.9993544341854289, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20363345742225647, + "median": 0.20384372770786285, + "p05": 0.19405215978622437, + "p95": 0.21326100826263428 + }, + { + "label": "completed_group_1", + "mean": 0.21095217764377594, + "median": 0.21146853268146515, + "p05": 0.20567592978477478, + "p95": 0.21522513031959534 + }, + { + "label": "completed_group_2", + "mean": 0.20392385125160217, + "median": 0.20381325483322144, + "p05": 0.20115600526332855, + "p95": 0.2067122757434845 + }, + { + "label": "completed_group_3", + "mean": 0.19413790106773376, + "median": 0.19391843676567078, + "p05": 0.18873284757137299, + "p95": 0.1996975690126419 + }, + { + "label": "current_group_partial", + "mean": 0.18735264241695404, + "median": 0.18673613667488098, + "p05": 0.1829182654619217, + "p95": 0.1927962601184845 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.6093392372131348, + "group": 5, + "latest_source_mass": 0.19563516974449158, + "layer": 17, + "max_source_mass": 0.20331618189811707, + "mixer_index": 32, + "normalized_entropy": 0.9999386896380388, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20084109902381897, + "median": 0.20074692368507385, + "p05": 0.198189377784729, + "p95": 0.20358796417713165 + }, + { + "label": "completed_group_1", + "mean": 0.20331618189811707, + "median": 0.20337940752506256, + "p05": 0.20178094506263733, + "p95": 0.20489434897899628 + }, + { + "label": "completed_group_2", + "mean": 0.20119993388652802, + "median": 0.20122915506362915, + "p05": 0.20026232302188873, + "p95": 0.2019881308078766 + }, + { + "label": "completed_group_3", + "mean": 0.19900763034820557, + "median": 0.198960542678833, + "p05": 0.19746547937393188, + "p95": 0.20062875747680664 + }, + { + "label": "completed_group_4", + "mean": 0.19563516974449158, + "median": 0.19558461010456085, + "p05": 0.193980410695076, + "p95": 0.19730384647846222 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7908520698547363, + "group": 5, + "latest_source_mass": 0.1625179648399353, + "layer": 17, + "max_source_mass": 0.17730244994163513, + "mixer_index": 33, + "normalized_entropy": 0.9994935707672249, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17160072922706604, + "median": 0.1718936264514923, + "p05": 0.16487310826778412, + "p95": 0.17849747836589813 + }, + { + "label": "completed_group_1", + "mean": 0.17730244994163513, + "median": 0.1778835654258728, + "p05": 0.17289136350154877, + "p95": 0.18126235902309418 + }, + { + "label": "completed_group_2", + "mean": 0.1690303534269333, + "median": 0.16928547620773315, + "p05": 0.16646793484687805, + "p95": 0.17125293612480164 + }, + { + "label": "completed_group_3", + "mean": 0.16099892556667328, + "median": 0.16061408817768097, + "p05": 0.15715192258358002, + "p95": 0.1656636893749237 + }, + { + "label": "completed_group_4", + "mean": 0.15854957699775696, + "median": 0.15825147926807404, + "p05": 0.15517033636569977, + "p95": 0.1620505452156067 + }, + { + "label": "current_group_partial", + "mean": 0.1625179648399353, + "median": 0.16315963864326477, + "p05": 0.15939058363437653, + "p95": 0.16489268839359283 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7916775941848755, + "group": 5, + "latest_source_mass": 0.16463731229305267, + "layer": 18, + "max_source_mass": 0.16981256008148193, + "mixer_index": 34, + "normalized_entropy": 0.9999543046683522, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16706335544586182, + "median": 0.16679415106773376, + "p05": 0.16531850397586823, + "p95": 0.16906341910362244 + }, + { + "label": "completed_group_1", + "mean": 0.16981256008148193, + "median": 0.1699288785457611, + "p05": 0.1682884395122528, + "p95": 0.17106440663337708 + }, + { + "label": "completed_group_2", + "mean": 0.16858771443367004, + "median": 0.16860879957675934, + "p05": 0.16782470047473907, + "p95": 0.16934865713119507 + }, + { + "label": "completed_group_3", + "mean": 0.16527186334133148, + "median": 0.16520482301712036, + "p05": 0.1643052101135254, + "p95": 0.16638660430908203 + }, + { + "label": "completed_group_4", + "mean": 0.16462719440460205, + "median": 0.16459274291992188, + "p05": 0.1638425886631012, + "p95": 0.16539013385772705 + }, + { + "label": "current_group_partial", + "mean": 0.16463731229305267, + "median": 0.16451048851013184, + "p05": 0.1639435589313507, + "p95": 0.16552197933197021 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7898297309875488, + "group": 5, + "latest_source_mass": 0.153368279337883, + "layer": 18, + "max_source_mass": 0.18101517856121063, + "mixer_index": 35, + "normalized_entropy": 0.9989229925815112, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17284835875034332, + "median": 0.17269954085350037, + "p05": 0.1658562421798706, + "p95": 0.18014436960220337 + }, + { + "label": "completed_group_1", + "mean": 0.18101517856121063, + "median": 0.1814284771680832, + "p05": 0.17416825890541077, + "p95": 0.18707150220870972 + }, + { + "label": "completed_group_2", + "mean": 0.1735592633485794, + "median": 0.17370939254760742, + "p05": 0.17075185477733612, + "p95": 0.17625606060028076 + }, + { + "label": "completed_group_3", + "mean": 0.16231875121593475, + "median": 0.1621963381767273, + "p05": 0.15821577608585358, + "p95": 0.16650474071502686 + }, + { + "label": "completed_group_4", + "mean": 0.15689018368721008, + "median": 0.15674838423728943, + "p05": 0.15215492248535156, + "p95": 0.1616627275943756 + }, + { + "label": "current_group_partial", + "mean": 0.153368279337883, + "median": 0.1528519243001938, + "p05": 0.15056025981903076, + "p95": 0.15731072425842285 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917213439941406, + "group": 5, + "latest_source_mass": 0.16463561356067657, + "layer": 19, + "max_source_mass": 0.16885727643966675, + "mixer_index": 36, + "normalized_entropy": 0.9999787219018127, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1661592274904251, + "median": 0.1661478579044342, + "p05": 0.16439414024353027, + "p95": 0.1678619235754013 + }, + { + "label": "completed_group_1", + "mean": 0.16885727643966675, + "median": 0.16877895593643188, + "p05": 0.16819745302200317, + "p95": 0.1697411835193634 + }, + { + "label": "completed_group_2", + "mean": 0.16764025390148163, + "median": 0.16765299439430237, + "p05": 0.1669837087392807, + "p95": 0.1682416945695877 + }, + { + "label": "completed_group_3", + "mean": 0.16661542654037476, + "median": 0.1665019690990448, + "p05": 0.1658560335636139, + "p95": 0.16755254566669464 + }, + { + "label": "completed_group_4", + "mean": 0.16609220206737518, + "median": 0.16612619161605835, + "p05": 0.16555683314800262, + "p95": 0.16653065383434296 + }, + { + "label": "current_group_partial", + "mean": 0.16463561356067657, + "median": 0.16466060280799866, + "p05": 0.16384081542491913, + "p95": 0.16551899909973145 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7890949249267578, + "group": 5, + "latest_source_mass": 0.15094393491744995, + "layer": 19, + "max_source_mass": 0.18413972854614258, + "mixer_index": 37, + "normalized_entropy": 0.9985128895105295, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17241930961608887, + "median": 0.17162901163101196, + "p05": 0.1626318097114563, + "p95": 0.18299198150634766 + }, + { + "label": "completed_group_1", + "mean": 0.18413972854614258, + "median": 0.1848709136247635, + "p05": 0.17655718326568604, + "p95": 0.19053134322166443 + }, + { + "label": "completed_group_2", + "mean": 0.17484498023986816, + "median": 0.17485198378562927, + "p05": 0.1718076914548874, + "p95": 0.17815767228603363 + }, + { + "label": "completed_group_3", + "mean": 0.1621202528476715, + "median": 0.16197144985198975, + "p05": 0.15698590874671936, + "p95": 0.16731801629066467 + }, + { + "label": "completed_group_4", + "mean": 0.15553176403045654, + "median": 0.15512266755104065, + "p05": 0.1502593755722046, + "p95": 0.16059254109859467 + }, + { + "label": "current_group_partial", + "mean": 0.15094393491744995, + "median": 0.15083211660385132, + "p05": 0.1470862179994583, + "p95": 0.1549530029296875 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917208671569824, + "group": 5, + "latest_source_mass": 0.16669265925884247, + "layer": 20, + "max_source_mass": 0.168935626745224, + "mixer_index": 38, + "normalized_entropy": 0.9999784557739275, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16663745045661926, + "median": 0.16669239103794098, + "p05": 0.16406413912773132, + "p95": 0.16939817368984222 + }, + { + "label": "completed_group_1", + "mean": 0.168935626745224, + "median": 0.16887718439102173, + "p05": 0.16814815998077393, + "p95": 0.1698606014251709 + }, + { + "label": "completed_group_2", + "mean": 0.16597259044647217, + "median": 0.16582076251506805, + "p05": 0.16482511162757874, + "p95": 0.1670236438512802 + }, + { + "label": "completed_group_3", + "mean": 0.16514870524406433, + "median": 0.165022611618042, + "p05": 0.16397784650325775, + "p95": 0.1662973016500473 + }, + { + "label": "completed_group_4", + "mean": 0.16661295294761658, + "median": 0.16667577624320984, + "p05": 0.16593852639198303, + "p95": 0.16715332865715027 + }, + { + "label": "current_group_partial", + "mean": 0.16669265925884247, + "median": 0.1666145920753479, + "p05": 0.16592761874198914, + "p95": 0.1674128621816635 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.789101004600525, + "group": 5, + "latest_source_mass": 0.14993149042129517, + "layer": 20, + "max_source_mass": 0.18365857005119324, + "mixer_index": 39, + "normalized_entropy": 0.9985162826410648, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17229264974594116, + "median": 0.1720556616783142, + "p05": 0.16247707605361938, + "p95": 0.18216519057750702 + }, + { + "label": "completed_group_1", + "mean": 0.18365857005119324, + "median": 0.18395906686782837, + "p05": 0.17734748125076294, + "p95": 0.18972161412239075 + }, + { + "label": "completed_group_2", + "mean": 0.1747075319290161, + "median": 0.1746184229850769, + "p05": 0.17210862040519714, + "p95": 0.1774953156709671 + }, + { + "label": "completed_group_3", + "mean": 0.1640234887599945, + "median": 0.1635349988937378, + "p05": 0.15958356857299805, + "p95": 0.16904467344284058 + }, + { + "label": "completed_group_4", + "mean": 0.15538626909255981, + "median": 0.15498605370521545, + "p05": 0.14995241165161133, + "p95": 0.16101743280887604 + }, + { + "label": "current_group_partial", + "mean": 0.14993149042129517, + "median": 0.15014703571796417, + "p05": 0.14564283192157745, + "p95": 0.1527552753686905 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7917447090148926, + "group": 6, + "latest_source_mass": 0.1661863625049591, + "layer": 21, + "max_source_mass": 0.16768339276313782, + "mixer_index": 40, + "normalized_entropy": 0.9999917621681839, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1658121943473816, + "median": 0.1657177209854126, + "p05": 0.16437731683254242, + "p95": 0.16778677701950073 + }, + { + "label": "completed_group_1", + "mean": 0.16768339276313782, + "median": 0.16760629415512085, + "p05": 0.1666821986436844, + "p95": 0.16875657439231873 + }, + { + "label": "completed_group_2", + "mean": 0.16629494726657867, + "median": 0.16622786223888397, + "p05": 0.1655496209859848, + "p95": 0.16718709468841553 + }, + { + "label": "completed_group_3", + "mean": 0.1669958382844925, + "median": 0.16695073246955872, + "p05": 0.1664678305387497, + "p95": 0.16760030388832092 + }, + { + "label": "completed_group_4", + "mean": 0.16702723503112793, + "median": 0.16713355481624603, + "p05": 0.16618871688842773, + "p95": 0.16781742870807648 + }, + { + "label": "completed_group_5", + "mean": 0.1661863625049591, + "median": 0.16608929634094238, + "p05": 0.16534176468849182, + "p95": 0.16705378890037537 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.9436898231506348, + "group": 6, + "latest_source_mass": 0.13651785254478455, + "layer": 21, + "max_source_mass": 0.15856821835041046, + "mixer_index": 41, + "normalized_entropy": 0.9988589781980651, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14778707921504974, + "median": 0.14752492308616638, + "p05": 0.13982102274894714, + "p95": 0.1569063663482666 + }, + { + "label": "completed_group_1", + "mean": 0.15856821835041046, + "median": 0.1585446149110794, + "p05": 0.15396656095981598, + "p95": 0.16390106081962585 + }, + { + "label": "completed_group_2", + "mean": 0.1500256508588791, + "median": 0.14993107318878174, + "p05": 0.14748509228229523, + "p95": 0.15272916853427887 + }, + { + "label": "completed_group_3", + "mean": 0.1408001333475113, + "median": 0.14052921533584595, + "p05": 0.13647492229938507, + "p95": 0.14521925151348114 + }, + { + "label": "completed_group_4", + "mean": 0.13533657789230347, + "median": 0.13547882437705994, + "p05": 0.13068656623363495, + "p95": 0.13946673274040222 + }, + { + "label": "completed_group_5", + "mean": 0.130964457988739, + "median": 0.13209164142608643, + "p05": 0.12633010745048523, + "p95": 0.13486123085021973 + }, + { + "label": "current_group_partial", + "mean": 0.13651785254478455, + "median": 0.1365797072649002, + "p05": 0.13271403312683105, + "p95": 0.14000433683395386 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458942413330078, + "group": 6, + "latest_source_mass": 0.14246684312820435, + "layer": 22, + "max_source_mass": 0.14393000304698944, + "mixer_index": 42, + "normalized_entropy": 0.9999918250478763, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.142832413315773, + "median": 0.14290162920951843, + "p05": 0.14148348569869995, + "p95": 0.1440550833940506 + }, + { + "label": "completed_group_1", + "mean": 0.1429906189441681, + "median": 0.14299973845481873, + "p05": 0.14231431484222412, + "p95": 0.14357265830039978 + }, + { + "label": "completed_group_2", + "mean": 0.14157448709011078, + "median": 0.14162302017211914, + "p05": 0.1408008486032486, + "p95": 0.1421196460723877 + }, + { + "label": "completed_group_3", + "mean": 0.14285019040107727, + "median": 0.14287817478179932, + "p05": 0.1423148512840271, + "p95": 0.14345195889472961 + }, + { + "label": "completed_group_4", + "mean": 0.14393000304698944, + "median": 0.14395871758460999, + "p05": 0.14325706660747528, + "p95": 0.14449745416641235 + }, + { + "label": "completed_group_5", + "mean": 0.14335544407367706, + "median": 0.14326418936252594, + "p05": 0.14292818307876587, + "p95": 0.14392802119255066 + }, + { + "label": "current_group_partial", + "mean": 0.14246684312820435, + "median": 0.14247724413871765, + "p05": 0.14211606979370117, + "p95": 0.14281725883483887 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9425597190856934, + "group": 6, + "latest_source_mass": 0.12947803735733032, + "layer": 22, + "max_source_mass": 0.1615946888923645, + "mixer_index": 43, + "normalized_entropy": 0.9982782195923864, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1483290195465088, + "median": 0.14828777313232422, + "p05": 0.13995377719402313, + "p95": 0.15759113430976868 + }, + { + "label": "completed_group_1", + "mean": 0.1615946888923645, + "median": 0.16178560256958008, + "p05": 0.15419340133666992, + "p95": 0.16922280192375183 + }, + { + "label": "completed_group_2", + "mean": 0.15280696749687195, + "median": 0.1528012454509735, + "p05": 0.15010300278663635, + "p95": 0.1558753252029419 + }, + { + "label": "completed_group_3", + "mean": 0.14239415526390076, + "median": 0.14164817333221436, + "p05": 0.1380041539669037, + "p95": 0.14742238819599152 + }, + { + "label": "completed_group_4", + "mean": 0.13462956249713898, + "median": 0.1342291682958603, + "p05": 0.12952053546905518, + "p95": 0.14025796949863434 + }, + { + "label": "completed_group_5", + "mean": 0.1307675838470459, + "median": 0.13075318932533264, + "p05": 0.12602169811725616, + "p95": 0.13501127064228058 + }, + { + "label": "current_group_partial", + "mean": 0.12947803735733032, + "median": 0.1295393407344818, + "p05": 0.1264311969280243, + "p95": 0.13163118064403534 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458987712860107, + "group": 6, + "latest_source_mass": 0.14346569776535034, + "layer": 23, + "max_source_mass": 0.14346569776535034, + "mixer_index": 44, + "normalized_entropy": 0.9999941529832156, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1426449418067932, + "median": 0.14265665411949158, + "p05": 0.14160317182540894, + "p95": 0.1436314880847931 + }, + { + "label": "completed_group_1", + "mean": 0.14291533827781677, + "median": 0.14293473958969116, + "p05": 0.1422625482082367, + "p95": 0.14348529279232025 + }, + { + "label": "completed_group_2", + "mean": 0.1416938751935959, + "median": 0.14168253540992737, + "p05": 0.14088502526283264, + "p95": 0.14245855808258057 + }, + { + "label": "completed_group_3", + "mean": 0.14274153113365173, + "median": 0.14274577796459198, + "p05": 0.14230182766914368, + "p95": 0.14324283599853516 + }, + { + "label": "completed_group_4", + "mean": 0.14325544238090515, + "median": 0.1432678997516632, + "p05": 0.14277629554271698, + "p95": 0.14376230537891388 + }, + { + "label": "completed_group_5", + "mean": 0.1432831883430481, + "median": 0.14325812458992004, + "p05": 0.14287351071834564, + "p95": 0.1437322348356247 + }, + { + "label": "current_group_partial", + "mean": 0.14346569776535034, + "median": 0.1435171663761139, + "p05": 0.142959862947464, + "p95": 0.1437678039073944 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.941068410873413, + "group": 6, + "latest_source_mass": 0.1254161298274994, + "layer": 23, + "max_source_mass": 0.16456490755081177, + "mixer_index": 45, + "normalized_entropy": 0.9975118387741332, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14912807941436768, + "median": 0.14865416288375854, + "p05": 0.13610364496707916, + "p95": 0.16207551956176758 + }, + { + "label": "completed_group_1", + "mean": 0.16456490755081177, + "median": 0.16586321592330933, + "p05": 0.15482422709465027, + "p95": 0.17167139053344727 + }, + { + "label": "completed_group_2", + "mean": 0.15460123121738434, + "median": 0.1543833166360855, + "p05": 0.15155401825904846, + "p95": 0.15836837887763977 + }, + { + "label": "completed_group_3", + "mean": 0.1425517499446869, + "median": 0.14188045263290405, + "p05": 0.13683314621448517, + "p95": 0.1494884341955185 + }, + { + "label": "completed_group_4", + "mean": 0.13539987802505493, + "median": 0.13422535359859467, + "p05": 0.12956365942955017, + "p95": 0.14296601712703705 + }, + { + "label": "completed_group_5", + "mean": 0.1283380389213562, + "median": 0.12796106934547424, + "p05": 0.12316946685314178, + "p95": 0.13433398306369781 + }, + { + "label": "current_group_partial", + "mean": 0.1254161298274994, + "median": 0.12523823976516724, + "p05": 0.12156305462121964, + "p95": 0.12899920344352722 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458861351013184, + "group": 6, + "latest_source_mass": 0.14452049136161804, + "layer": 24, + "max_source_mass": 0.14452049136161804, + "mixer_index": 46, + "normalized_entropy": 0.9999876592688483, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14264336228370667, + "median": 0.1426892876625061, + "p05": 0.14129234850406647, + "p95": 0.14382319152355194 + }, + { + "label": "completed_group_1", + "mean": 0.1430070698261261, + "median": 0.14298146963119507, + "p05": 0.14245536923408508, + "p95": 0.1435846984386444 + }, + { + "label": "completed_group_2", + "mean": 0.14314037561416626, + "median": 0.14310884475708008, + "p05": 0.14277565479278564, + "p95": 0.14360389113426208 + }, + { + "label": "completed_group_3", + "mean": 0.14221979677677155, + "median": 0.14216911792755127, + "p05": 0.14183637499809265, + "p95": 0.14285607635974884 + }, + { + "label": "completed_group_4", + "mean": 0.14132915437221527, + "median": 0.1413242518901825, + "p05": 0.140913724899292, + "p95": 0.14174436032772064 + }, + { + "label": "completed_group_5", + "mean": 0.1431397795677185, + "median": 0.14310240745544434, + "p05": 0.14276552200317383, + "p95": 0.14377698302268982 + }, + { + "label": "current_group_partial", + "mean": 0.14452049136161804, + "median": 0.1444905698299408, + "p05": 0.14409083127975464, + "p95": 0.14507406949996948 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9415628910064697, + "group": 6, + "latest_source_mass": 0.1248510330915451, + "layer": 24, + "max_source_mass": 0.16342055797576904, + "mixer_index": 47, + "normalized_entropy": 0.9977659512948458, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14895881712436676, + "median": 0.14996516704559326, + "p05": 0.13763123750686646, + "p95": 0.15872029960155487 + }, + { + "label": "completed_group_1", + "mean": 0.16342055797576904, + "median": 0.1639186143875122, + "p05": 0.15447522699832916, + "p95": 0.17102839052677155 + }, + { + "label": "completed_group_2", + "mean": 0.15406624972820282, + "median": 0.1540084183216095, + "p05": 0.1514483094215393, + "p95": 0.1571783721446991 + }, + { + "label": "completed_group_3", + "mean": 0.14293310046195984, + "median": 0.14186778664588928, + "p05": 0.13832569122314453, + "p95": 0.1491597443819046 + }, + { + "label": "completed_group_4", + "mean": 0.1363803744316101, + "median": 0.13557225465774536, + "p05": 0.13139937818050385, + "p95": 0.14291447401046753 + }, + { + "label": "completed_group_5", + "mean": 0.12938985228538513, + "median": 0.12914851307868958, + "p05": 0.12471500039100647, + "p95": 0.13491052389144897 + }, + { + "label": "current_group_partial", + "mean": 0.1248510330915451, + "median": 0.12450477480888367, + "p05": 0.120968759059906, + "p95": 0.12821169197559357 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9458723068237305, + "group": 7, + "latest_source_mass": 0.14437910914421082, + "layer": 25, + "max_source_mass": 0.14437910914421082, + "mixer_index": 48, + "normalized_entropy": 0.999980552939918, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14191991090774536, + "median": 0.14168427884578705, + "p05": 0.14056351780891418, + "p95": 0.14397425949573517 + }, + { + "label": "completed_group_1", + "mean": 0.1418565809726715, + "median": 0.1417136788368225, + "p05": 0.14059004187583923, + "p95": 0.14328429102897644 + }, + { + "label": "completed_group_2", + "mean": 0.14164230227470398, + "median": 0.14142388105392456, + "p05": 0.14061963558197021, + "p95": 0.14320334792137146 + }, + { + "label": "completed_group_3", + "mean": 0.14289972186088562, + "median": 0.14285793900489807, + "p05": 0.1423202008008957, + "p95": 0.14347271621227264 + }, + { + "label": "completed_group_4", + "mean": 0.14397987723350525, + "median": 0.1439221203327179, + "p05": 0.14329849183559418, + "p95": 0.14462903141975403 + }, + { + "label": "completed_group_5", + "mean": 0.14332249760627747, + "median": 0.14335353672504425, + "p05": 0.14246267080307007, + "p95": 0.14414797723293304 + }, + { + "label": "completed_group_6", + "mean": 0.14437910914421082, + "median": 0.14459103345870972, + "p05": 0.14338219165802002, + "p95": 0.14524540305137634 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.073223114013672, + "group": 7, + "latest_source_mass": 0.11086538434028625, + "layer": 25, + "max_source_mass": 0.14827430248260498, + "mixer_index": 49, + "normalized_entropy": 0.9970095684146328, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13280925154685974, + "median": 0.13328123092651367, + "p05": 0.11957645416259766, + "p95": 0.1447097212076187 + }, + { + "label": "completed_group_1", + "mean": 0.14827430248260498, + "median": 0.14952056109905243, + "p05": 0.13636183738708496, + "p95": 0.15699218213558197 + }, + { + "label": "completed_group_2", + "mean": 0.13796664774417877, + "median": 0.1381954699754715, + "p05": 0.13450786471366882, + "p95": 0.1413417011499405 + }, + { + "label": "completed_group_3", + "mean": 0.12756124138832092, + "median": 0.12679177522659302, + "p05": 0.12232367694377899, + "p95": 0.13387270271778107 + }, + { + "label": "completed_group_4", + "mean": 0.12019774317741394, + "median": 0.11888951063156128, + "p05": 0.11402077227830887, + "p95": 0.12811852991580963 + }, + { + "label": "completed_group_5", + "mean": 0.11310221254825592, + "median": 0.11254298686981201, + "p05": 0.10767743736505508, + "p95": 0.12006338685750961 + }, + { + "label": "completed_group_6", + "mean": 0.10922321677207947, + "median": 0.10910944640636444, + "p05": 0.10472994297742844, + "p95": 0.11491495370864868 + }, + { + "label": "current_group_partial", + "mean": 0.11086538434028625, + "median": 0.11154524236917496, + "p05": 0.10694204270839691, + "p95": 0.11396989226341248 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079413890838623, + "group": 7, + "latest_source_mass": 0.1263430416584015, + "layer": 26, + "max_source_mass": 0.1263430416584015, + "mixer_index": 50, + "normalized_entropy": 0.9999867027561686, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12475188821554184, + "median": 0.12463723123073578, + "p05": 0.12364111095666885, + "p95": 0.1259622424840927 + }, + { + "label": "completed_group_1", + "mean": 0.12460128962993622, + "median": 0.12450964748859406, + "p05": 0.12383783608675003, + "p95": 0.125508114695549 + }, + { + "label": "completed_group_2", + "mean": 0.12424702197313309, + "median": 0.12409250438213348, + "p05": 0.1235705241560936, + "p95": 0.12524327635765076 + }, + { + "label": "completed_group_3", + "mean": 0.12420950829982758, + "median": 0.12421596050262451, + "p05": 0.12374235689640045, + "p95": 0.12457412481307983 + }, + { + "label": "completed_group_4", + "mean": 0.12484367191791534, + "median": 0.12490126490592957, + "p05": 0.12441762536764145, + "p95": 0.1252550482749939 + }, + { + "label": "completed_group_5", + "mean": 0.12469653785228729, + "median": 0.12477756291627884, + "p05": 0.12382431328296661, + "p95": 0.12524525821208954 + }, + { + "label": "completed_group_6", + "mean": 0.12630701065063477, + "median": 0.12644168734550476, + "p05": 0.12546542286872864, + "p95": 0.12699897587299347 + }, + { + "label": "current_group_partial", + "mean": 0.1263430416584015, + "median": 0.12645362317562103, + "p05": 0.12567421793937683, + "p95": 0.1267925351858139 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.072925567626953, + "group": 7, + "latest_source_mass": 0.10599855333566666, + "layer": 26, + "max_source_mass": 0.14843900501728058, + "mixer_index": 51, + "normalized_entropy": 0.9968664788491151, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13228744268417358, + "median": 0.13281425833702087, + "p05": 0.12057555466890335, + "p95": 0.14273115992546082 + }, + { + "label": "completed_group_1", + "mean": 0.14843900501728058, + "median": 0.14951951801776886, + "p05": 0.13698264956474304, + "p95": 0.1572267711162567 + }, + { + "label": "completed_group_2", + "mean": 0.13848665356636047, + "median": 0.1387266218662262, + "p05": 0.13503478467464447, + "p95": 0.1420348733663559 + }, + { + "label": "completed_group_3", + "mean": 0.12843601405620575, + "median": 0.12770286202430725, + "p05": 0.12394566833972931, + "p95": 0.13404949009418488 + }, + { + "label": "completed_group_4", + "mean": 0.12169650197029114, + "median": 0.12072620540857315, + "p05": 0.11642728000879288, + "p95": 0.12845680117607117 + }, + { + "label": "completed_group_5", + "mean": 0.11442162096500397, + "median": 0.11409753561019897, + "p05": 0.10971812158823013, + "p95": 0.12062239646911621 + }, + { + "label": "completed_group_6", + "mean": 0.11023421585559845, + "median": 0.11024647206068039, + "p05": 0.10620385408401489, + "p95": 0.11548472940921783 + }, + { + "label": "current_group_partial", + "mean": 0.10599855333566666, + "median": 0.10623474419116974, + "p05": 0.1023266613483429, + "p95": 0.10840564221143723 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0793263912200928, + "group": 7, + "latest_source_mass": 0.12782885134220123, + "layer": 27, + "max_source_mass": 0.12782885134220123, + "mixer_index": 52, + "normalized_entropy": 0.9999446243342576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12420415878295898, + "median": 0.12419730424880981, + "p05": 0.12226121127605438, + "p95": 0.12590435147285461 + }, + { + "label": "completed_group_1", + "mean": 0.12273682653903961, + "median": 0.12260904908180237, + "p05": 0.12171939015388489, + "p95": 0.12380002439022064 + }, + { + "label": "completed_group_2", + "mean": 0.12360995262861252, + "median": 0.12356993556022644, + "p05": 0.12293750047683716, + "p95": 0.12423694133758545 + }, + { + "label": "completed_group_3", + "mean": 0.1238352432847023, + "median": 0.12381016463041306, + "p05": 0.12339816987514496, + "p95": 0.12445184588432312 + }, + { + "label": "completed_group_4", + "mean": 0.1241140365600586, + "median": 0.12404628843069077, + "p05": 0.12358136475086212, + "p95": 0.1247347816824913 + }, + { + "label": "completed_group_5", + "mean": 0.12607695162296295, + "median": 0.1261017620563507, + "p05": 0.12524889409542084, + "p95": 0.12690351903438568 + }, + { + "label": "completed_group_6", + "mean": 0.127593994140625, + "median": 0.12773489952087402, + "p05": 0.1267755776643753, + "p95": 0.1284082531929016 + }, + { + "label": "current_group_partial", + "mean": 0.12782885134220123, + "median": 0.1278689205646515, + "p05": 0.12723052501678467, + "p95": 0.12838256359100342 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.073427200317383, + "group": 7, + "latest_source_mass": 0.10513068735599518, + "layer": 27, + "max_source_mass": 0.1465209424495697, + "mixer_index": 53, + "normalized_entropy": 0.9971077131807252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13135746121406555, + "median": 0.13279247283935547, + "p05": 0.11876361072063446, + "p95": 0.14146237075328827 + }, + { + "label": "completed_group_1", + "mean": 0.1465209424495697, + "median": 0.1469956636428833, + "p05": 0.13593894243240356, + "p95": 0.1559162735939026 + }, + { + "label": "completed_group_2", + "mean": 0.13772746920585632, + "median": 0.137643963098526, + "p05": 0.13452906906604767, + "p95": 0.14171628654003143 + }, + { + "label": "completed_group_3", + "mean": 0.12966199219226837, + "median": 0.1287793666124344, + "p05": 0.12537093460559845, + "p95": 0.13508695363998413 + }, + { + "label": "completed_group_4", + "mean": 0.12215200066566467, + "median": 0.12143252789974213, + "p05": 0.11671329289674759, + "p95": 0.129044771194458 + }, + { + "label": "completed_group_5", + "mean": 0.11654195189476013, + "median": 0.11616532504558563, + "p05": 0.11162350326776505, + "p95": 0.12263412773609161 + }, + { + "label": "completed_group_6", + "mean": 0.11090746521949768, + "median": 0.11034701764583588, + "p05": 0.10647446662187576, + "p95": 0.11587852239608765 + }, + { + "label": "current_group_partial", + "mean": 0.10513068735599518, + "median": 0.10488355159759521, + "p05": 0.10140107572078705, + "p95": 0.10828588157892227 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.079422950744629, + "group": 7, + "latest_source_mass": 0.12521177530288696, + "layer": 28, + "max_source_mass": 0.12591606378555298, + "mixer_index": 54, + "normalized_entropy": 0.9999910596499906, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12477636337280273, + "median": 0.12485313415527344, + "p05": 0.12292245030403137, + "p95": 0.12664391100406647 + }, + { + "label": "completed_group_1", + "mean": 0.12493627518415451, + "median": 0.12481105327606201, + "p05": 0.12430030107498169, + "p95": 0.12584592401981354 + }, + { + "label": "completed_group_2", + "mean": 0.1242394968867302, + "median": 0.12431945651769638, + "p05": 0.12357322871685028, + "p95": 0.12491215020418167 + }, + { + "label": "completed_group_3", + "mean": 0.12470392882823944, + "median": 0.12473497539758682, + "p05": 0.12409180402755737, + "p95": 0.12546151876449585 + }, + { + "label": "completed_group_4", + "mean": 0.12560302019119263, + "median": 0.12554584443569183, + "p05": 0.12500521540641785, + "p95": 0.12630382180213928 + }, + { + "label": "completed_group_5", + "mean": 0.12461308389902115, + "median": 0.12459056079387665, + "p05": 0.12410678714513779, + "p95": 0.12512892484664917 + }, + { + "label": "completed_group_6", + "mean": 0.12591606378555298, + "median": 0.12592178583145142, + "p05": 0.12557773292064667, + "p95": 0.12638571858406067 + }, + { + "label": "current_group_partial", + "mean": 0.12521177530288696, + "median": 0.12524773180484772, + "p05": 0.12447575479745865, + "p95": 0.1259688436985016 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0725789070129395, + "group": 7, + "latest_source_mass": 0.10425455868244171, + "layer": 28, + "max_source_mass": 0.14827941358089447, + "mixer_index": 55, + "normalized_entropy": 0.9966997703328787, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13186156749725342, + "median": 0.13315711915493011, + "p05": 0.11918869614601135, + "p95": 0.14256390929222107 + }, + { + "label": "completed_group_1", + "mean": 0.14827941358089447, + "median": 0.14931634068489075, + "p05": 0.13603198528289795, + "p95": 0.15790212154388428 + }, + { + "label": "completed_group_2", + "mean": 0.13866659998893738, + "median": 0.13880926370620728, + "p05": 0.1348804384469986, + "p95": 0.14264559745788574 + }, + { + "label": "completed_group_3", + "mean": 0.12988534569740295, + "median": 0.12922212481498718, + "p05": 0.12565752863883972, + "p95": 0.13522912561893463 + }, + { + "label": "completed_group_4", + "mean": 0.12183371186256409, + "median": 0.12081405520439148, + "p05": 0.1163119301199913, + "p95": 0.12904007732868195 + }, + { + "label": "completed_group_5", + "mean": 0.11524046957492828, + "median": 0.1147729679942131, + "p05": 0.11042807996273041, + "p95": 0.12173305451869965 + }, + { + "label": "completed_group_6", + "mean": 0.1099783331155777, + "median": 0.10993096977472305, + "p05": 0.10556004196405411, + "p95": 0.11569196730852127 + }, + { + "label": "current_group_partial", + "mean": 0.10425455868244171, + "median": 0.10401678085327148, + "p05": 0.10061369091272354, + "p95": 0.10750719904899597 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0793380737304688, + "group": 8, + "latest_source_mass": 0.12683480978012085, + "layer": 29, + "max_source_mass": 0.12747105956077576, + "mixer_index": 56, + "normalized_entropy": 0.9999502424341857, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12417487800121307, + "median": 0.1240256130695343, + "p05": 0.12294038385152817, + "p95": 0.1256686896085739 + }, + { + "label": "completed_group_1", + "mean": 0.1225123256444931, + "median": 0.12230424582958221, + "p05": 0.12168098986148834, + "p95": 0.12367479503154755 + }, + { + "label": "completed_group_2", + "mean": 0.12277349829673767, + "median": 0.1226874440908432, + "p05": 0.12208077311515808, + "p95": 0.12351377308368683 + }, + { + "label": "completed_group_3", + "mean": 0.1243998259305954, + "median": 0.12447299808263779, + "p05": 0.12367013841867447, + "p95": 0.12505075335502625 + }, + { + "label": "completed_group_4", + "mean": 0.12567630410194397, + "median": 0.12574473023414612, + "p05": 0.12491010874509811, + "p95": 0.1263245940208435 + }, + { + "label": "completed_group_5", + "mean": 0.12615728378295898, + "median": 0.12627410888671875, + "p05": 0.1253039538860321, + "p95": 0.1267564296722412 + }, + { + "label": "completed_group_6", + "mean": 0.12747105956077576, + "median": 0.12754422426223755, + "p05": 0.1268254816532135, + "p95": 0.1280263066291809 + }, + { + "label": "completed_group_7", + "mean": 0.12683480978012085, + "median": 0.12676092982292175, + "p05": 0.1260336935520172, + "p95": 0.12750548124313354 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1906280517578125, + "group": 8, + "latest_source_mass": 0.09856453537940979, + "layer": 29, + "max_source_mass": 0.13291704654693604, + "mixer_index": 57, + "normalized_entropy": 0.9969977918295433, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11854428797960281, + "median": 0.11933885514736176, + "p05": 0.10912582278251648, + "p95": 0.12757083773612976 + }, + { + "label": "completed_group_1", + "mean": 0.13291704654693604, + "median": 0.13307878375053406, + "p05": 0.12339349836111069, + "p95": 0.14226418733596802 + }, + { + "label": "completed_group_2", + "mean": 0.12378417700529099, + "median": 0.1235300600528717, + "p05": 0.12091188877820969, + "p95": 0.1279762238264084 + }, + { + "label": "completed_group_3", + "mean": 0.11745279282331467, + "median": 0.11693152785301208, + "p05": 0.11398620158433914, + "p95": 0.12128151953220367 + }, + { + "label": "completed_group_4", + "mean": 0.11140252649784088, + "median": 0.11079759150743484, + "p05": 0.10681547224521637, + "p95": 0.11677644401788712 + }, + { + "label": "completed_group_5", + "mean": 0.10409101843833923, + "median": 0.1040785163640976, + "p05": 0.09952308982610703, + "p95": 0.10878291726112366 + }, + { + "label": "completed_group_6", + "mean": 0.09838292002677917, + "median": 0.09840121120214462, + "p05": 0.09399936348199844, + "p95": 0.10255295783281326 + }, + { + "label": "completed_group_7", + "mean": 0.09486069530248642, + "median": 0.09467358142137527, + "p05": 0.09144656360149384, + "p95": 0.09743223339319229 + }, + { + "label": "current_group_partial", + "mean": 0.09856453537940979, + "median": 0.09877555072307587, + "p05": 0.09461960196495056, + "p95": 0.10164139419794083 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.197185516357422, + "group": 8, + "latest_source_mass": 0.11096570640802383, + "layer": 30, + "max_source_mass": 0.11249847710132599, + "mixer_index": 58, + "normalized_entropy": 0.999982222582434, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11084984987974167, + "median": 0.11077362298965454, + "p05": 0.108895443379879, + "p95": 0.11311701685190201 + }, + { + "label": "completed_group_1", + "mean": 0.11021924018859863, + "median": 0.11018644273281097, + "p05": 0.10941747575998306, + "p95": 0.11104057729244232 + }, + { + "label": "completed_group_2", + "mean": 0.11018813401460648, + "median": 0.11026144027709961, + "p05": 0.10942888259887695, + "p95": 0.1108226552605629 + }, + { + "label": "completed_group_3", + "mean": 0.11046157777309418, + "median": 0.11042240262031555, + "p05": 0.10973355174064636, + "p95": 0.11123604327440262 + }, + { + "label": "completed_group_4", + "mean": 0.11131839454174042, + "median": 0.11132219433784485, + "p05": 0.1105581521987915, + "p95": 0.11199404299259186 + }, + { + "label": "completed_group_5", + "mean": 0.11167056858539581, + "median": 0.1118055060505867, + "p05": 0.11075222492218018, + "p95": 0.1122528463602066 + }, + { + "label": "completed_group_6", + "mean": 0.11249847710132599, + "median": 0.11257164180278778, + "p05": 0.11187897622585297, + "p95": 0.11291356384754181 + }, + { + "label": "completed_group_7", + "mean": 0.1118280440568924, + "median": 0.11168843507766724, + "p05": 0.11146453768014908, + "p95": 0.11255130171775818 + }, + { + "label": "current_group_partial", + "mean": 0.11096570640802383, + "median": 0.11086957156658173, + "p05": 0.11044884473085403, + "p95": 0.11181740462779999 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1899101734161377, + "group": 8, + "latest_source_mass": 0.09411308169364929, + "layer": 30, + "max_source_mass": 0.13352498412132263, + "mixer_index": 59, + "normalized_entropy": 0.9966710713162742, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1190595030784607, + "median": 0.11995811760425568, + "p05": 0.10872812569141388, + "p95": 0.1278010904788971 + }, + { + "label": "completed_group_1", + "mean": 0.13352498412132263, + "median": 0.13314831256866455, + "p05": 0.12346142530441284, + "p95": 0.14397428929805756 + }, + { + "label": "completed_group_2", + "mean": 0.1239522397518158, + "median": 0.12362635135650635, + "p05": 0.12061451375484467, + "p95": 0.12857040762901306 + }, + { + "label": "completed_group_3", + "mean": 0.11779220402240753, + "median": 0.11750936508178711, + "p05": 0.11429227143526077, + "p95": 0.12138796597719193 + }, + { + "label": "completed_group_4", + "mean": 0.11234382539987564, + "median": 0.11192750930786133, + "p05": 0.1077679693698883, + "p95": 0.11798053979873657 + }, + { + "label": "completed_group_5", + "mean": 0.10513705015182495, + "median": 0.1050291359424591, + "p05": 0.10075211524963379, + "p95": 0.11021918058395386 + }, + { + "label": "completed_group_6", + "mean": 0.09944089502096176, + "median": 0.09912091493606567, + "p05": 0.09499282389879227, + "p95": 0.10404862463474274 + }, + { + "label": "completed_group_7", + "mean": 0.09463620185852051, + "median": 0.09429345279932022, + "p05": 0.090972401201725, + "p95": 0.0975509062409401 + }, + { + "label": "current_group_partial", + "mean": 0.09411308169364929, + "median": 0.09450676292181015, + "p05": 0.09012747555971146, + "p95": 0.0978383868932724 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.197190284729004, + "group": 8, + "latest_source_mass": 0.11225657165050507, + "layer": 31, + "max_source_mass": 0.11242865771055222, + "mixer_index": 60, + "normalized_entropy": 0.9999843927618645, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11067462712526321, + "median": 0.11063650250434875, + "p05": 0.10940495133399963, + "p95": 0.11191637068986893 + }, + { + "label": "completed_group_1", + "mean": 0.11021950840950012, + "median": 0.11016013473272324, + "p05": 0.10925821959972382, + "p95": 0.11120235174894333 + }, + { + "label": "completed_group_2", + "mean": 0.11038416624069214, + "median": 0.1102733239531517, + "p05": 0.10960154980421066, + "p95": 0.11125709116458893 + }, + { + "label": "completed_group_3", + "mean": 0.11103622615337372, + "median": 0.11098648607730865, + "p05": 0.11065587401390076, + "p95": 0.11149194091558456 + }, + { + "label": "completed_group_4", + "mean": 0.11016052961349487, + "median": 0.11016929149627686, + "p05": 0.10976231843233109, + "p95": 0.11053308844566345 + }, + { + "label": "completed_group_5", + "mean": 0.11144963651895523, + "median": 0.11151091754436493, + "p05": 0.11074185371398926, + "p95": 0.11193440854549408 + }, + { + "label": "completed_group_6", + "mean": 0.11242865771055222, + "median": 0.11247988045215607, + "p05": 0.11164618283510208, + "p95": 0.11297892034053802 + }, + { + "label": "completed_group_7", + "mean": 0.11139006912708282, + "median": 0.11143708229064941, + "p05": 0.11081651598215103, + "p95": 0.1119559034705162 + }, + { + "label": "current_group_partial", + "mean": 0.11225657165050507, + "median": 0.11227837204933167, + "p05": 0.1117941215634346, + "p95": 0.11280770599842072 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1896841526031494, + "group": 8, + "latest_source_mass": 0.09325435757637024, + "layer": 31, + "max_source_mass": 0.13428261876106262, + "mixer_index": 61, + "normalized_entropy": 0.9965682048112662, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11968795955181122, + "median": 0.1202392578125, + "p05": 0.11006385833024979, + "p95": 0.12905627489089966 + }, + { + "label": "completed_group_1", + "mean": 0.13428261876106262, + "median": 0.13435032963752747, + "p05": 0.12363448739051819, + "p95": 0.14427264034748077 + }, + { + "label": "completed_group_2", + "mean": 0.12373493611812592, + "median": 0.12346962094306946, + "p05": 0.12061072885990143, + "p95": 0.1282179057598114 + }, + { + "label": "completed_group_3", + "mean": 0.1179073229432106, + "median": 0.11775766313076019, + "p05": 0.11464057117700577, + "p95": 0.12117566168308258 + }, + { + "label": "completed_group_4", + "mean": 0.11132999509572983, + "median": 0.11077313870191574, + "p05": 0.10667262226343155, + "p95": 0.1171092763543129 + }, + { + "label": "completed_group_5", + "mean": 0.10504785180091858, + "median": 0.1049019992351532, + "p05": 0.10071787983179092, + "p95": 0.10987957566976547 + }, + { + "label": "completed_group_6", + "mean": 0.09994535893201828, + "median": 0.09992853552103043, + "p05": 0.09573008865118027, + "p95": 0.1044701635837555 + }, + { + "label": "completed_group_7", + "mean": 0.09480959922075272, + "median": 0.09453058242797852, + "p05": 0.09107209742069244, + "p95": 0.09781570732593536 + }, + { + "label": "current_group_partial", + "mean": 0.09325435757637024, + "median": 0.09382589161396027, + "p05": 0.08936256170272827, + "p95": 0.09653929620981216 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1971938610076904, + "group": 8, + "latest_source_mass": 0.10965746641159058, + "layer": 32, + "max_source_mass": 0.11217275261878967, + "mixer_index": 62, + "normalized_entropy": 0.9999860203964374, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11116788536310196, + "median": 0.11119179427623749, + "p05": 0.11027558147907257, + "p95": 0.11194087564945221 + }, + { + "label": "completed_group_1", + "mean": 0.1117495447397232, + "median": 0.11171998083591461, + "p05": 0.11123906075954437, + "p95": 0.1124686524271965 + }, + { + "label": "completed_group_2", + "mean": 0.11143069714307785, + "median": 0.11137667298316956, + "p05": 0.11108753085136414, + "p95": 0.1119472086429596 + }, + { + "label": "completed_group_3", + "mean": 0.11117202788591385, + "median": 0.11118572950363159, + "p05": 0.11080538481473923, + "p95": 0.11158231645822525 + }, + { + "label": "completed_group_4", + "mean": 0.11217275261878967, + "median": 0.11215600371360779, + "p05": 0.1119328960776329, + "p95": 0.11246690899133682 + }, + { + "label": "completed_group_5", + "mean": 0.11135637760162354, + "median": 0.11132287979125977, + "p05": 0.11110222339630127, + "p95": 0.1116970106959343 + }, + { + "label": "completed_group_6", + "mean": 0.11160033196210861, + "median": 0.11164811253547668, + "p05": 0.11120022088289261, + "p95": 0.111964151263237 + }, + { + "label": "completed_group_7", + "mean": 0.10969290882349014, + "median": 0.10973770171403885, + "p05": 0.10930771380662918, + "p95": 0.10996787250041962 + }, + { + "label": "current_group_partial", + "mean": 0.10965746641159058, + "median": 0.10969755053520203, + "p05": 0.10922328382730484, + "p95": 0.1099214255809784 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.188610792160034, + "group": 8, + "latest_source_mass": 0.09157620370388031, + "layer": 32, + "max_source_mass": 0.1360917091369629, + "mixer_index": 63, + "normalized_entropy": 0.9960796974214496, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11987313628196716, + "median": 0.12083137035369873, + "p05": 0.10883265733718872, + "p95": 0.12953230738639832 + }, + { + "label": "completed_group_1", + "mean": 0.1360917091369629, + "median": 0.13644108176231384, + "p05": 0.12369051575660706, + "p95": 0.14713600277900696 + }, + { + "label": "completed_group_2", + "mean": 0.12462760508060455, + "median": 0.12453563511371613, + "p05": 0.12107435613870621, + "p95": 0.12926824390888214 + }, + { + "label": "completed_group_3", + "mean": 0.11755925416946411, + "median": 0.11717041581869125, + "p05": 0.11395573616027832, + "p95": 0.12139421701431274 + }, + { + "label": "completed_group_4", + "mean": 0.11237476021051407, + "median": 0.11172428727149963, + "p05": 0.10730767995119095, + "p95": 0.11879656463861465 + }, + { + "label": "completed_group_5", + "mean": 0.10438287258148193, + "median": 0.10406170785427094, + "p05": 0.09989674389362335, + "p95": 0.10985209792852402 + }, + { + "label": "completed_group_6", + "mean": 0.09920359402894974, + "median": 0.09911085665225983, + "p05": 0.0946069061756134, + "p95": 0.10452912747859955 + }, + { + "label": "completed_group_7", + "mean": 0.09431086480617523, + "median": 0.09389802813529968, + "p05": 0.09041668474674225, + "p95": 0.09788760542869568 + }, + { + "label": "current_group_partial", + "mean": 0.09157620370388031, + "median": 0.09197241812944412, + "p05": 0.0875491350889206, + "p95": 0.09489993005990982 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1930694580078125, + "group": null, + "latest_source_mass": 0.11946354806423187, + "layer": null, + "max_source_mass": 0.12094368040561676, + "mixer_index": null, + "normalized_entropy": 0.9981089236979843, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10071488469839096, + "median": 0.09996568411588669, + "p05": 0.09106086939573288, + "p95": 0.1130409762263298 + }, + { + "label": "completed_group_1", + "mean": 0.09411877393722534, + "median": 0.09266924858093262, + "p05": 0.08850140869617462, + "p95": 0.10204548388719559 + }, + { + "label": "completed_group_2", + "mean": 0.10181896388530731, + "median": 0.10170605033636093, + "p05": 0.09935513883829117, + "p95": 0.10529708862304688 + }, + { + "label": "completed_group_3", + "mean": 0.10989432036876678, + "median": 0.11005622148513794, + "p05": 0.1059446707367897, + "p95": 0.11359670758247375 + }, + { + "label": "completed_group_4", + "mean": 0.1145806610584259, + "median": 0.11529573798179626, + "p05": 0.10914603620767593, + "p95": 0.11935751885175705 + }, + { + "label": "completed_group_5", + "mean": 0.11861918121576309, + "median": 0.11936421692371368, + "p05": 0.11333125829696655, + "p95": 0.12279194593429565 + }, + { + "label": "completed_group_6", + "mean": 0.11984598636627197, + "median": 0.12033014744520187, + "p05": 0.11644290387630463, + "p95": 0.12263244390487671 + }, + { + "label": "completed_group_7", + "mean": 0.12094368040561676, + "median": 0.12095880508422852, + "p05": 0.118732750415802, + "p95": 0.12314030528068542 + }, + { + "label": "completed_group_8", + "mean": 0.11946354806423187, + "median": 0.118886798620224, + "p05": 0.11734164506196976, + "p95": 0.12209592759609222 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", + "optimizer_state_before": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.002919268561527133, + 0.0026904908008873463, + 0.0028405722696334124, + 0.0029144894797354937, + 0.0030634324066340923, + 0.0032732728868722916, + 0.003848863998427987, + 0.003392934100702405, + 0.0037359180860221386, + 0.004302032291889191, + 0.004344160668551922, + 0.004835603758692741, + 0.005531937349587679, + 0.005483300890773535, + 0.005397035740315914, + 0.005739198997616768, + 0.005659298039972782, + 0.007230010814964771, + 0.006938732694834471, + 0.007343437522649765, + 0.007882392033934593, + 0.00830139685422182, + 0.008992243558168411, + 0.009393857792019844, + 0.012831038795411587, + 0.00905293133109808, + 0.012243949808180332, + 0.012258118949830532, + 0.013431218452751637, + 0.01483316533267498, + 0.015388047322630882, + 0.014634119346737862 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.570841959619429e-05, + "first_to_last_ratio": 3.472594525431938, + "imbalance_abs_log_ratio": 1.2449020164225795, + "last_quartile_mean": 4.523539814726973e-06, + "mean": 8.683279858701098e-06, + "normalized": [ + 2.229894338065868, + 2.1043034837063286, + 1.976946278509285, + 1.8612974320657731, + 1.7187480804487953, + 1.6201790212455152, + 1.5254325865776683, + 1.4355362848487128, + 1.3017373037261373, + 1.2100830304331005, + 1.133220479031311, + 1.0535486714736453, + 0.9690659040803394, + 0.9036010498699347, + 0.8438110062145913, + 0.7926660684312774, + 0.749751881449241, + 0.7082035160351038, + 0.6752208628773162, + 0.6475764877057508, + 0.6244377002240272, + 0.6007972561165787, + 0.5813749906205735, + 0.5649799471303525, + 0.5550363681005585, + 0.5437815898914894, + 0.5335487688172357, + 0.5248093961192007, + 0.5123845047471113, + 0.5051729354463279, + 0.4992577444849533, + 0.49359503150589623 + ], + "peak_layer": 1, + "peak_normalized": 2.229894338065868, + "population_cv": 0.5276859177486317, + "rest_mean": 9.350082985259144e-06, + "spike_contrast": 0.543582659458801, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.082542975287651e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.936279659275897e-05, + 1.8272256056661718e-05, + 1.7166377801913768e-05, + 1.6162166502908804e-05, + 1.49243705891422e-05, + 1.4068467862671241e-05, + 1.3245758054836188e-05, + 1.2465163308661431e-05, + 1.1303349310765043e-05, + 1.050748960551573e-05, + 9.840070561040193e-06, + 9.148257959168404e-06, + 8.414670446654782e-06, + 7.846220796636771e-06, + 7.3270471148134675e-06, + 6.8829413066850975e-06, + 6.510305411211448e-06, + 6.149529326648917e-06, + 5.863131718797376e-06, + 5.623087872663746e-06, + 5.42216730536893e-06, + 5.216890713199973e-06, + 5.048241746408166e-06, + 4.905878995487001e-06, + 4.819536115974188e-06, + 4.7218077270372305e-06, + 4.632953277905472e-06, + 4.557066858978942e-06, + 4.4491780499811284e-06, + 4.38655797552201e-06, + 4.335194716986734e-06, + 4.286023795430083e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.106548153184121e-05, + "first_to_last_ratio": 3.396664379322294, + "imbalance_abs_log_ratio": 1.2227938851577476, + "last_quartile_mean": 1.7978073401536676e-05, + "mean": 3.421872185072061e-05, + "normalized": [ + 2.125690241222297, + 2.035753111677548, + 1.9424653444151043, + 1.8443604241196392, + 1.713002582679694, + 1.6251047960941019, + 1.5375111780105508, + 1.4526202049778152, + 1.3158411261187606, + 1.226794175835664, + 1.149073116741286, + 1.0695301297827082, + 0.9835511244361359, + 0.9163244633971447, + 0.8552224227935168, + 0.8029396208596036, + 0.7578907347648619, + 0.7156154592837152, + 0.6817576032295484, + 0.6538669675872317, + 0.6299332335154186, + 0.605919603383038, + 0.5863501185278558, + 0.569785799336057, + 0.5595323652477712, + 0.5483698809054829, + 0.5380511691309848, + 0.5293393622233508, + 0.5166837305455455, + 0.5094821346031305, + 0.5036034749610113, + 0.498034299593426 + ], + "peak_layer": 1, + "peak_normalized": 2.125690241222297, + "population_cv": 0.5121606369233174, + "rest_mean": 3.681488218447797e-05, + "spike_contrast": 0.5486763735168884, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.0199456048430876e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.273840310517699e-05, + 6.966086948523298e-05, + 6.646868132520467e-05, + 6.311165634542704e-05, + 5.8616758906282485e-05, + 5.560900899581611e-05, + 5.261166734271683e-05, + 4.970680674887262e-05, + 4.502640149439685e-05, + 4.197932867100462e-05, + 3.931981336791068e-05, + 3.659795402199961e-05, + 3.365586235304363e-05, + 3.1355451937997714e-05, + 2.9264618206070736e-05, + 2.747556754911784e-05, + 2.593405224615708e-05, + 2.448744635330513e-05, + 2.3328873794525862e-05, + 2.2374491891241632e-05, + 2.1555510102189146e-05, + 2.073379437206313e-05, + 2.006415161304176e-05, + 1.9497341781971045e-05, + 1.9146482372889295e-05, + 1.8764516426017508e-05, + 1.8411423297948204e-05, + 1.8113316400558688e-05, + 1.76802568603307e-05, + 1.7433827451895922e-05, + 1.723266723274719e-05, + 1.70420971699059e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.054153479635715485, + "first_to_last_ratio": 3.3966644557946295, + "imbalance_abs_log_ratio": 1.2227939076716985, + "last_quartile_mean": 0.01594313490204513, + "mean": 0.030345504521392286, + "normalized": [ + 2.1256903113360446, + 2.035753146214189, + 1.9424653011478115, + 1.8443605007189683, + 1.71300252819031, + 1.6251048166450035, + 1.5375111878366567, + 1.452620175683203, + 1.3158411334809852, + 1.2267941193182312, + 1.1490731743165223, + 1.0695301881432524, + 0.9835511318647784, + 0.9163244786384582, + 0.8552223856485347, + 0.8029396127492687, + 0.7578907435068827, + 0.7156154476500971, + 0.6817576088078939, + 0.6538669486644281, + 0.6299332311205004, + 0.6059195951826442, + 0.586350084864764, + 0.569785800788618, + 0.5595323690576035, + 0.5483698809409665, + 0.5380511395546987, + 0.5293393594226915, + 0.5166837108984478, + 0.5094821244722924, + 0.5036034882338023, + 0.4980342749014515 + ], + "peak_layer": 1, + "peak_normalized": 2.1256903113360446, + "population_cv": 0.5121606472523333, + "rest_mean": 0.0326478054953946, + "spike_contrast": 0.5486763655311122, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017913079261779784, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06450514495372772, + 0.06177595630288124, + 0.05894508957862854, + 0.05596804991364479, + 0.05198192596435547, + 0.049314625561237335, + 0.04665655270218849, + 0.04408049210906029, + 0.039929863065481186, + 0.03722768649458885, + 0.034869205206632614, + 0.032455433160066605, + 0.029846355319023132, + 0.027806328609585762, + 0.025952154770493507, + 0.024365607649087906, + 0.02299857698380947, + 0.021715711802244186, + 0.02068827860057354, + 0.01984192244708538, + 0.019115641713142395, + 0.01838693581521511, + 0.017793089151382446, + 0.01729043759405613, + 0.016979292035102844, + 0.01664056070148945, + 0.01632743328809738, + 0.01606306992471218, + 0.015679027885198593, + 0.015460492111742496, + 0.015282101929187775, + 0.015113101340830326 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.484954840540013e-06, + "first_to_last_ratio": 4.317118296735341, + "imbalance_abs_log_ratio": 1.462588118730986, + "last_quartile_mean": 1.2705129819323702e-06, + "mean": 2.775344690064685e-06, + "normalized": [ + 2.642311887415898, + 2.416230245995366, + 2.21143682159549, + 2.0333492354809493, + 1.8430856012839139, + 1.6939904160227164, + 1.5446558421616008, + 1.4254585070372157, + 1.2827451334693212, + 1.1755524566426843, + 1.0923958043799127, + 1.0016640021807137, + 0.9226630606403423, + 0.8576190807297013, + 0.7936412075441582, + 0.7370849544400642, + 0.6937134258189417, + 0.6483319177623661, + 0.6136592809430983, + 0.5846488972705621, + 0.5617626261670844, + 0.5391223387293338, + 0.5197976899046192, + 0.5027943403920482, + 0.49023336166728243, + 0.4787835081603097, + 0.4696409445710295, + 0.46085508801940983, + 0.450168541694312, + 0.44355065941556593, + 0.43718887869853934, + 0.4318642437654502 + ], + "peak_layer": 1, + "peak_normalized": 2.642311887415898, + "population_cv": 0.6359571845984164, + "rest_mean": 3.020632700807053e-06, + "spike_contrast": 0.48029322852403633, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.450789432055899e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.333326266234508e-06, + 6.705871783196926e-06, + 6.1374994402285665e-06, + 5.643245003739139e-06, + 5.1151978368579876e-06, + 4.701407306129113e-06, + 4.286952389520593e-06, + 3.95613869841327e-06, + 3.5600598948803963e-06, + 3.2625632684357697e-06, + 3.031774895134731e-06, + 2.779962869681185e-06, + 2.5607080260670045e-06, + 2.380188561801333e-06, + 2.202627911174204e-06, + 2.0456648144318024e-06, + 1.9252938727731816e-06, + 1.7993445453612367e-06, + 1.7031160268743406e-06, + 1.622602212592028e-06, + 1.5590849216096103e-06, + 1.4962503200877109e-06, + 1.4426177585846744e-06, + 1.3954276028016466e-06, + 1.3605665571958525e-06, + 1.3287892670632573e-06, + 1.3034155017521698e-06, + 1.2790317214239622e-06, + 1.2493728718254715e-06, + 1.2310059673836804e-06, + 1.2133498330513248e-06, + 1.1985721357632428e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.000926375942072e-05, + "first_to_last_ratio": 2.861076547442411, + "imbalance_abs_log_ratio": 1.0511979691949909, + "last_quartile_mean": 1.7479177131463075e-05, + "mean": 3.001758557275025e-05, + "normalized": [ + 1.966517821002492, + 1.8844494706669952, + 1.8016591622079403, + 1.7146543004927615, + 1.602713342111759, + 1.525842314618408, + 1.4518503935852287, + 1.3803041859872354, + 1.2718526716709437, + 1.1969100977048808, + 1.1317671023781757, + 1.06503487809827, + 0.9973441232158428, + 0.941363281859895, + 0.8896494105618908, + 0.8445592656875696, + 0.8090600018046681, + 0.7713325693634966, + 0.7401533589124044, + 0.7139326567185207, + 0.692297727813069, + 0.6685034690624881, + 0.6485427267474059, + 0.6313224439068734, + 0.6206897714046083, + 0.6084823527417841, + 0.5970360381608709, + 0.5872159169829965, + 0.572597446631392, + 0.564250570870669, + 0.5573663965692749, + 0.5507447304591896 + ], + "peak_layer": 1, + "peak_normalized": 1.966517821002492, + "population_cv": 0.43784911874358545, + "rest_mean": 3.1950544494234406e-05, + "spike_contrast": 0.6128098192589182, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.957960739673581e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.9030116972280666e-05, + 5.656662324327044e-05, + 5.408145807450637e-05, + 5.146978219272569e-05, + 4.810958489542827e-05, + 4.580210224958137e-05, + 4.3581043428275734e-05, + 4.143339901929721e-05, + 3.817794640781358e-05, + 3.592835128074512e-05, + 3.397291584406048e-05, + 3.196977559127845e-05, + 2.993786256411113e-05, + 2.8257452868274413e-05, + 2.670512731128838e-05, + 2.5351630029035732e-05, + 2.4286027837661095e-05, + 2.3153541405918077e-05, + 2.2217616788111627e-05, + 2.1430534616229124e-05, + 2.0781106286449358e-05, + 2.006686008826364e-05, + 1.9467686797725037e-05, + 1.8950775483972393e-05, + 1.8631608327268623e-05, + 1.8265171092934906e-05, + 1.7921580365509726e-05, + 1.7626804037718102e-05, + 1.7187992852996103e-05, + 1.6937439795583487e-05, + 1.673079350439366e-05, + 1.6532027075299993e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.89006590921781e-05, + "first_to_last_ratio": 2.1665975586085744, + "imbalance_abs_log_ratio": 0.7731579916979808, + "last_quartile_mean": 1.7954723034563358e-05, + "mean": 2.587040432899812e-05, + "normalized": [ + 1.7831448102947671, + 1.6974226005892297, + 1.620036533641332, + 1.533717800359919, + 1.4465980773830347, + 1.3745334732258696, + 1.316859965180907, + 1.257080202982369, + 1.1876570648434144, + 1.1203272435244702, + 1.0718495078032313, + 1.0193313865605265, + 0.978296344153428, + 0.937610047259807, + 0.9066038289594328, + 0.877320807240572, + 0.8649781691504772, + 0.8401000985755385, + 0.8145599740177561, + 0.792555397726043, + 0.7795027531450014, + 0.7587311609912605, + 0.7416782788097599, + 0.7272996278499536, + 0.7222997702968516, + 0.7135070943498951, + 0.7058159198574021, + 0.6988176661381593, + 0.6867067750666979, + 0.6814848031835403, + 0.675082580859005, + 0.668490235980348 + ], + "peak_layer": 1, + "peak_normalized": 1.7831448102947671, + "population_cv": 0.3297379118959793, + "rest_mean": 2.708773947098827e-05, + "spike_contrast": 0.7123811340152144, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9296794562251307e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.6130677219480276e-05, + 4.391300899442285e-05, + 4.191100015304983e-05, + 3.967789962189272e-05, + 3.742407716345042e-05, + 3.555973671609536e-05, + 3.406769974390045e-05, + 3.252117312513292e-05, + 3.072516847169027e-05, + 2.8983318770769984e-05, + 2.7729180146707222e-05, + 2.63705151155591e-05, + 2.530892197682988e-05, + 2.4256351025542244e-05, + 2.3454207621398382e-05, + 2.269664400955662e-05, + 2.2377334971679375e-05, + 2.173372922698036e-05, + 2.1072995878057554e-05, + 2.050372859230265e-05, + 2.0166051399428397e-05, + 1.9628681911854073e-05, + 1.9187516954843886e-05, + 1.8815535440808162e-05, + 1.8686187104322016e-05, + 1.8458717022440396e-05, + 1.8259743228554726e-05, + 1.8078695575241e-05, + 1.776538192643784e-05, + 1.7630287402425893e-05, + 1.7464659322286025e-05, + 1.729411269479897e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011786452796513913, + "first_to_last_ratio": 5.272419649365271, + "imbalance_abs_log_ratio": 1.6624893936843081, + "last_quartile_mean": 2.2354921611622558e-05, + "mean": 5.981176059322024e-05, + "normalized": [ + 2.342046121512161, + 2.2441163908348334, + 2.142933559820905, + 2.032574241487723, + 1.8909432158278896, + 1.7811012647720825, + 1.7046454997674345, + 1.6263691566595195, + 1.4513535263674326, + 1.3484633711734058, + 1.2565665103260477, + 1.1592512217173825, + 1.0440274588303153, + 0.9501757209203151, + 0.8639054268917737, + 0.7923577171085253, + 0.7123005746652016, + 0.6458221031841243, + 0.5963034371094506, + 0.5569297557861542, + 0.5166112374905155, + 0.4756772431215453, + 0.44740588281370386, + 0.42808243396944073, + 0.41207290926584317, + 0.3977052623291704, + 0.38540407372369656, + 0.3748332006962577, + 0.3630559483389979, + 0.3565727390637327, + 0.35157439090519904, + 0.34881840351921967 + ], + "peak_layer": 1, + "peak_normalized": 2.342046121512161, + "population_cv": 0.6458102290367006, + "rest_mean": 6.583757459776718e-05, + "spike_contrast": 0.41423708475420845, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.7272364968666805e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014008190191816539, + 0.00013422455231193453, + 0.00012817262904718518, + 0.00012157184391980991, + 0.00011310064292047173, + 0.00010653080244082958, + 0.00010195784852840006, + 9.727600263431668e-05, + 8.680800965521485e-05, + 8.065396832535043e-05, + 7.515745528507978e-05, + 6.933685654075816e-05, + 6.244512042030692e-05, + 5.683168274117634e-05, + 5.167170456843451e-05, + 4.739231007988565e-05, + 4.2603951442288235e-05, + 3.862775702145882e-05, + 3.566595842130482e-05, + 3.331094922032207e-05, + 3.089942765654996e-05, + 2.8451093385228887e-05, + 2.6760133550851606e-05, + 2.56043640547432e-05, + 2.464680619596038e-05, + 2.3787451937096193e-05, + 2.3051696189213544e-05, + 2.241943366243504e-05, + 2.171501546399668e-05, + 2.1327243302948773e-05, + 2.1028283299528994e-05, + 2.086344284180086e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.005079356953501701, + 0.0052208867855370045, + 0.005689430516213179, + 0.005868426524102688, + 0.00590060418471694, + 0.0059102908708155155, + 0.0058776624500751495, + 0.006663382984697819, + 0.006277414970099926, + 0.006413500756025314, + 0.006856496445834637, + 0.006499431096017361, + 0.006718362681567669, + 0.0070298356004059315, + 0.006969383917748928, + 0.0068367342464625835, + 0.007033169735223055, + 0.007157569285482168, + 0.0075136395171284676, + 0.007708695251494646, + 0.007887008599936962, + 0.008205465972423553, + 0.00814912747591734, + 0.008287571370601654, + 0.008026733994483948, + 0.008120592683553696, + 0.008479002863168716, + 0.00907627958804369, + 0.00841403380036354, + 0.008773921057581902, + 0.008821850642561913, + 0.009164483286440372 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5008465311439068e-05, + "first_to_last_ratio": 3.3668968034019833, + "imbalance_abs_log_ratio": 1.213991490288974, + "last_quartile_mean": 4.457655279566097e-06, + "mean": 8.299250268350988e-06, + "normalized": [ + 2.2568151540450523, + 2.1152935419859507, + 1.9767562187382288, + 1.86698957878165, + 1.7143188557206628, + 1.6128346254213421, + 1.5135326124803135, + 1.410756892623042, + 1.2736883716277778, + 1.1902867485409472, + 1.1070374521817832, + 1.0322313304507316, + 0.9480382659054776, + 0.8848529200942086, + 0.8311995014573598, + 0.7835740569083032, + 0.742040914352895, + 0.7071972712748903, + 0.6782348645232779, + 0.6514608668278806, + 0.6287747596064291, + 0.6086256047198574, + 0.5913261103101772, + 0.5772101349211559, + 0.569144818247716, + 0.5582752103529628, + 0.549190400479895, + 0.5409965375823161, + 0.5285413070813697, + 0.5224600186388942, + 0.5164682084504763, + 0.5118468456669756 + ], + "peak_layer": 1, + "peak_normalized": 2.2568151540450523, + "population_cv": 0.524209711679407, + "rest_mean": 8.921669009227858e-06, + "spike_contrast": 0.5535050742757021, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.9381890676158944e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8729873772826977e-05, + 1.7555350495968014e-05, + 1.640559457882773e-05, + 1.5494613762712106e-05, + 1.422756122337887e-05, + 1.338531819783384e-05, + 1.2561185940285213e-05, + 1.1708224519679789e-05, + 1.0570658560027368e-05, + 9.878487617243081e-06, + 9.187580872094259e-06, + 8.566746146243531e-06, + 7.86800683272304e-06, + 7.343615834543016e-06, + 6.898332685523201e-06, + 6.5030772020691074e-06, + 6.1583832575706765e-06, + 5.8692071434052195e-06, + 5.62884088139981e-06, + 5.406636773841456e-06, + 5.2183590923959855e-06, + 5.051136213296559e-06, + 4.907563379674684e-06, + 4.790411367139313e-06, + 4.723475285572931e-06, + 4.633265689335531e-06, + 4.557868578558555e-06, + 4.489865659706993e-06, + 4.38649658462964e-06, + 4.336026449891506e-06, + 4.286298917577369e-06, + 4.247945071256254e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.820503929498955e-05, + "first_to_last_ratio": 3.2848952430521337, + "imbalance_abs_log_ratio": 1.1893347619462469, + "last_quartile_mean": 1.7718994058668613e-05, + "mean": 3.2650613661644456e-05, + "normalized": [ + 2.1450013507239705, + 2.0461366500203595, + 1.9398189683428497, + 1.8522773613526717, + 1.7094590145500226, + 1.6168974254280155, + 1.5255888819635426, + 1.4261239538940118, + 1.2879114356833505, + 1.2060871976977126, + 1.1221510774968064, + 1.0481371490840103, + 0.9613461884209538, + 0.8968067603991513, + 0.8421359861574166, + 0.7933538356930859, + 0.7503585255090798, + 0.7148355279210235, + 0.6855522988186872, + 0.6585248527378349, + 0.6351822812313331, + 0.6146906504361358, + 0.5972743074473386, + 0.5828693520617939, + 0.574802048908203, + 0.5639397943233103, + 0.554845965325609, + 0.5465864044553121, + 0.5340118249846835, + 0.5278506086231465, + 0.5219704528761507, + 0.5174718674324269 + ], + "peak_layer": 1, + "peak_normalized": 2.1450013507239705, + "population_cv": 0.5072019698199706, + "rest_mean": 3.5063350541922434e-05, + "spike_contrast": 0.5596109386261622, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9621834508143365e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.003561040619388e-05, + 6.680761725874618e-05, + 6.333627970889211e-05, + 6.047799251973629e-05, + 5.581488585448824e-05, + 5.279269316815771e-05, + 4.981141319149174e-05, + 4.656382225221023e-05, + 4.205109871691093e-05, + 3.937948713428341e-05, + 3.663892130134627e-05, + 3.422232111915946e-05, + 3.138854299322702e-05, + 2.9281291062943637e-05, + 2.749625673459377e-05, + 2.5903489586198702e-05, + 2.4499666324118152e-05, + 2.3339818653766997e-05, + 2.238370325358119e-05, + 2.1501240553334355e-05, + 2.0739091269206256e-05, + 2.007002694881521e-05, + 1.9501372662489302e-05, + 1.903104202938266e-05, + 1.8767639630823396e-05, + 1.841298035287764e-05, + 1.811606125556864e-05, + 1.7846381524577737e-05, + 1.7435813788324594e-05, + 1.7234646293218248e-05, + 1.704265559965279e-05, + 1.6895774024305865e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05161681119352579, + "first_to_last_ratio": 3.284895287464527, + "imbalance_abs_log_ratio": 1.1893347754664323, + "last_quartile_mean": 0.015713380998931825, + "mean": 0.028954890876775607, + "normalized": [ + 2.145001289725201, + 2.046136691673054, + 1.939818911776489, + 1.8522774119626542, + 1.7094589830823987, + 1.6168973606656405, + 1.5255889998464391, + 1.4261239529824867, + 1.287911366616444, + 1.2060871893436458, + 1.1221510842273104, + 1.0481372445524282, + 0.961346209259541, + 0.8968067750917011, + 0.8421359558072424, + 0.7933538963986676, + 0.7503585064155442, + 0.7148355327095294, + 0.6855522990290889, + 0.6585248332975209, + 0.6351822731624658, + 0.6146906547931704, + 0.5972742941320782, + 0.5828693766065183, + 0.5748020438293477, + 0.5639398000251236, + 0.5548459661026797, + 0.5465864176570202, + 0.534011826874325, + 0.527850568480463, + 0.5219704287182262, + 0.5174718551555549 + ], + "peak_layer": 1, + "peak_normalized": 2.145001289725201, + "population_cv": 0.5072019680935582, + "rest_mean": 0.03109453007992771, + "spike_contrast": 0.5596109391274233, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017400839179754258, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06210827827453613, + 0.05924566462635994, + 0.05616724491119385, + 0.05363249033689499, + 0.049497198313474655, + 0.04681708663702011, + 0.044173263013362885, + 0.04129326343536377, + 0.037291333079338074, + 0.034922122955322266, + 0.032491762191057205, + 0.030348699539899826, + 0.027835674583911896, + 0.025966942310333252, + 0.02438395470380783, + 0.02297147549688816, + 0.021726548671722412, + 0.020697984844446182, + 0.019850092008709908, + 0.019067514687776566, + 0.01839163340628147, + 0.017798300832509995, + 0.017294012010097504, + 0.01687691919505596, + 0.016643330454826355, + 0.01632881537079811, + 0.016065504401922226, + 0.015826350077986717, + 0.015462254174053669, + 0.015283855609595776, + 0.015113596804440022, + 0.014983341097831726 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.355004418561293e-06, + "first_to_last_ratio": 4.2803271552883535, + "imbalance_abs_log_ratio": 1.4540294448239433, + "last_quartile_mean": 1.251073626917787e-06, + "mean": 2.683941971781678e-06, + "normalized": [ + 2.7030485685097614, + 2.4621902558382964, + 2.2386850638453395, + 2.05248673466923, + 1.8575399210018237, + 1.6957758104696088, + 1.5437340831941542, + 1.4081496862748333, + 1.2505669189373825, + 1.154757255828014, + 1.0675003825206393, + 0.9762526930125442, + 0.897424076523062, + 0.8318154411009656, + 0.7746171026378299, + 0.7211559397261628, + 0.6781980327113627, + 0.6399672113971655, + 0.60846523760761, + 0.5811474026512373, + 0.5581318010721549, + 0.5402081583654614, + 0.5217772343602641, + 0.5073418706694675, + 0.49649585255921586, + 0.4853123246471542, + 0.4772675785905054, + 0.4696153655827239, + 0.458545908189773, + 0.453049428909843, + 0.4466290334907211, + 0.4421476251056929 + ], + "peak_layer": 1, + "peak_normalized": 2.7030485685097614, + "population_cv": 0.6457571562979046, + "rest_mean": 2.9201333467830814e-06, + "spike_contrast": 0.48234391361810997, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4085085467741009e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.254825504787732e-06, + 6.608375770156272e-06, + 6.008500804455252e-06, + 5.508755293703871e-06, + 4.985529358236818e-06, + 4.551363872451475e-06, + 4.143292699154699e-06, + 3.7793920455442276e-06, + 3.3564490422577364e-06, + 3.0993014661362395e-06, + 2.8651090815401403e-06, + 2.6202055778412614e-06, + 2.4086341454676585e-06, + 2.232544375146972e-06, + 2.079027353829588e-06, + 1.9355406948307063e-06, + 1.82024416517379e-06, + 1.7176348592329305e-06, + 1.633085389585176e-06, + 1.5597659057675628e-06, + 1.4979933666836587e-06, + 1.4498873497359455e-06, + 1.4004198192196782e-06, + 1.3616761407320155e-06, + 1.332566057499207e-06, + 1.3025501175434329e-06, + 1.280958485949668e-06, + 1.2604203902810696e-06, + 1.2307106089792796e-06, + 1.2159583775428473e-06, + 1.198726408802031e-06, + 1.1866985687447595e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.78605675198196e-05, + "first_to_last_ratio": 2.7793570801926766, + "imbalance_abs_log_ratio": 1.0222196348386712, + "last_quartile_mean": 1.7220013887708774e-05, + "mean": 2.881263026210945e-05, + "normalized": [ + 1.9752570579468534, + 1.888104295535499, + 1.7945647548145947, + 1.7154982752112686, + 1.5967511433618535, + 1.5186186336526764, + 1.441625524284827, + 1.3583546018661343, + 1.2494560102010674, + 1.1811084039550475, + 1.1110423537101946, + 1.0496234480105349, + 0.9817226877095458, + 0.9275068527847158, + 0.8805127756083984, + 0.8381970075463177, + 0.803976865779107, + 0.7714567335637176, + 0.7440882717080923, + 0.7183088339322109, + 0.696644002608348, + 0.6758702747761651, + 0.6578620296540808, + 0.6426087873852322, + 0.6340400523715064, + 0.6220565885755293, + 0.611876165213302, + 0.6024355818665901, + 0.5879093007530156, + 0.5807453084805843, + 0.5738005724905904, + 0.5683768046423998 + ], + "peak_layer": 1, + "peak_normalized": 1.9752570579468534, + "population_cv": 0.4317079414337261, + "rest_mean": 3.0619262057977416e-05, + "spike_contrast": 0.622380073312624, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.905681856442243e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.691235128324479e-05, + 5.440125096356496e-05, + 5.1706130761886016e-05, + 4.9428017518948764e-05, + 4.6006600314285606e-05, + 4.375539720058441e-05, + 4.153702320763841e-05, + 3.9137768908403814e-05, + 3.600011405069381e-05, + 3.4030839742626995e-05, + 3.201205254299566e-05, + 3.0242412321968004e-05, + 2.8286012820899487e-05, + 2.6723912014858797e-05, + 2.5369889044668525e-05, + 2.4150660465238616e-05, + 2.3164688172983006e-05, + 2.2227697627386078e-05, + 2.14391402550973e-05, + 2.069636684609577e-05, + 2.0072146071470343e-05, + 1.9473600332275964e-05, + 1.8954735423903912e-05, + 1.8515249394113198e-05, + 1.8268361600348726e-05, + 1.7923086488735862e-05, + 1.7629761714488268e-05, + 1.7357753677060828e-05, + 1.6939213310251944e-05, + 1.673279984970577e-05, + 1.6532703739358112e-05, + 1.637643072172068e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.7207858895271784e-05, + "first_to_last_ratio": 2.0918563583707814, + "imbalance_abs_log_ratio": 0.7380518814986469, + "last_quartile_mean": 1.778700470822514e-05, + "mean": 2.5014859602379147e-05, + "normalized": [ + 1.766896017685199, + 1.6852067556838466, + 1.5917675487924012, + 1.5166567899677899, + 1.4315811273281913, + 1.3662079609937838, + 1.303579546183775, + 1.2375462607814267, + 1.160212708426433, + 1.1098915366489155, + 1.0567965268051207, + 1.0112629998208633, + 0.9697402160962814, + 0.9378324250878841, + 0.907941204027948, + 0.8814449641361121, + 0.8690589692914835, + 0.8427704073086626, + 0.8201725639084954, + 0.8014260695185076, + 0.784571291095966, + 0.7669834591017225, + 0.7521317253856684, + 0.7398605496651066, + 0.7380371143547211, + 0.7300178093260107, + 0.722929928084616, + 0.7169054508143122, + 0.7047914878526057, + 0.6980364293666598, + 0.6913059490086015, + 0.6864362074508885 + ], + "peak_layer": 1, + "peak_normalized": 1.766896017685199, + "population_cv": 0.3175074310788083, + "rest_mean": 2.614369299812501e-05, + "spike_contrast": 0.7236605504321191, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.8919159265351483e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.419865581439808e-05, + 4.215521039441228e-05, + 3.981784175266512e-05, + 3.79389566660393e-05, + 3.581080090953037e-05, + 3.417550033191219e-05, + 3.260885932832025e-05, + 3.095704596489668e-05, + 2.9022558010183275e-05, + 2.776378096314147e-05, + 2.6435616746312007e-05, + 2.5296601961599663e-05, + 2.4257915356429294e-05, + 2.345974644413218e-05, + 2.2712021745974198e-05, + 2.2049222025088966e-05, + 2.173938810301479e-05, + 2.1081783415866084e-05, + 2.051650153589435e-05, + 2.0047560610692017e-05, + 1.962594069482293e-05, + 1.9185983546776697e-05, + 1.8814469513017684e-05, + 1.8507507775211707e-05, + 1.846189479692839e-05, + 1.8261293007526547e-05, + 1.808399065339472e-05, + 1.793328920030035e-05, + 1.763026011758484e-05, + 1.7461283277953044e-05, + 1.7292921256739646e-05, + 1.7171105355373584e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011250984425714705, + "first_to_last_ratio": 5.136972637567696, + "imbalance_abs_log_ratio": 1.6364639249316602, + "last_quartile_mean": 2.1901974605498253e-05, + "mean": 5.6501408323583746e-05, + "normalized": [ + 2.382164866319327, + 2.279581056980086, + 2.1640865436402454, + 2.0688186882965165, + 1.8923495642236086, + 1.8092596842626791, + 1.7252927191320069, + 1.6086472166206554, + 1.4275667956980447, + 1.3299589734449933, + 1.2272752348601612, + 1.1370747967113455, + 1.007765000781145, + 0.9145523912616853, + 0.8390743254016101, + 0.7615077898186029, + 0.6843792173338624, + 0.6320158419323604, + 0.5900143308948815, + 0.550751795487684, + 0.5037684703042699, + 0.4736161081951874, + 0.4535521893538639, + 0.4358391510862496, + 0.4210578321323425, + 0.40810041507267314, + 0.3968839045479846, + 0.3877734415065673, + 0.37747110329431527, + 0.3721870546903874, + 0.36935648798288095, + 0.3682570087317776 + ], + "peak_layer": 1, + "peak_normalized": 2.382164866319327, + "population_cv": 0.6517494715357345, + "rest_mean": 6.217700879292183e-05, + "spike_contrast": 0.41579944566424976, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5853165789158082e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00013459566980600357, + 0.00012879954010713845, + 0.00012227393744979054, + 0.00011689116945490241, + 0.00010692041541915387, + 0.00010222572018392384, + 9.748146840138361e-05, + 9.089083323488012e-05, + 8.065953443292528e-05, + 7.514455501222983e-05, + 6.934277917025611e-05, + 6.424632738344371e-05, + 5.694014180335216e-05, + 5.16734980919864e-05, + 4.740888107335195e-05, + 4.302626257413067e-05, + 3.866838960675523e-05, + 3.5709785151993856e-05, + 3.3336640626657754e-05, + 3.111825208179653e-05, + 2.8463628041208722e-05, + 2.67599771177629e-05, + 2.5626337446738034e-05, + 2.4625525838928297e-05, + 2.3790360501152463e-05, + 2.3058248189045116e-05, + 2.2424499547923915e-05, + 2.1909745555603877e-05, + 2.1327648937585764e-05, + 2.102909274981357e-05, + 2.086916174448561e-05, + 2.080703961837571e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.002919268561527133, + 0.0065401275642216206, + 0.010618876665830612, + 0.015120964497327805, + 0.0030634324066340923, + 0.007598300464451313, + 0.012669830583035946, + 0.017725100740790367, + 0.0037359180860221386, + 0.009108707308769226, + 0.015153576619923115, + 0.021781234070658684, + 0.005531937349587679, + 0.0119093032553792, + 0.01904802769422531, + 0.02626260370016098, + 0.005659298039972782, + 0.013823030516505241, + 0.02223494090139866, + 0.0319429449737072, + 0.007882392033934593, + 0.01784195750951767, + 0.028768405318260193, + 0.040818072855472565, + 0.012831038795411587, + 0.023826664313673973, + 0.03824474662542343, + 0.053081437945365906, + 0.013431218452751637, + 0.03125866502523422, + 0.04969283565878868, + 0.06860727071762085 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.57084181182654e-05, + "first_to_last_ratio": 3.4725850785839505, + "imbalance_abs_log_ratio": 1.2448992960183491, + "last_quartile_mean": 4.523551695001515e-06, + "mean": 8.68330698722275e-06, + "normalized": [ + 2.2299043393712443, + 2.1042761707719917, + 1.976933189224246, + 1.8613268098100455, + 1.7187475287766918, + 1.6201752163443084, + 1.525403311499459, + 1.435524363342176, + 1.3017427682121105, + 1.2100915045221226, + 1.1332349539879822, + 1.0535572156473645, + 0.9690912612097308, + 0.9035997979344527, + 0.843793758654846, + 0.7926704001101033, + 0.7497625268960744, + 0.7082004131597335, + 0.6752277610326668, + 0.6475699083216085, + 0.6244433954092409, + 0.6007951696153412, + 0.5813694559888941, + 0.5649745160906161, + 0.5550358385641657, + 0.5437760679691019, + 0.5335476779691889, + 0.5248102179054668, + 0.5123899215671036, + 0.50517146191784, + 0.49925770343476866, + 0.49359537473931375 + ], + "peak_layer": 1, + "peak_normalized": 2.2299043393712443, + "population_cv": 0.5276847791721199, + "rest_mean": 9.350114733361235e-06, + "spike_contrast": 0.543581047186555, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.082545158074936e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.9362943930900656e-05, + 1.8272075976710767e-05, + 1.716631777526345e-05, + 1.61624720931286e-05, + 1.4924412425898481e-05, + 1.4068478776607662e-05, + 1.3245545233075973e-05, + 1.2465098734537605e-05, + 1.1303432074782904e-05, + 1.0507596016395837e-05, + 9.840226994128898e-06, + 9.148360732069705e-06, + 8.414916919718962e-06, + 7.846234439057298e-06, + 7.32692024030257e-06, + 6.883000423840713e-06, + 6.510418188554468e-06, + 6.149521595943952e-06, + 5.863209935341729e-06, + 5.6230483096442185e-06, + 5.4222336984821595e-06, + 5.2168888942105696e-06, + 5.0482094593462534e-06, + 4.905847163172439e-06, + 4.819546575163258e-06, + 4.721774530480616e-06, + 4.632958280126331e-06, + 4.557088232104434e-06, + 4.449238986126147e-06, + 4.3865588850167114e-06, + 4.33520790465991e-06, + 4.286040166334715e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.106540149630746e-05, + "first_to_last_ratio": 3.3966554168443395, + "imbalance_abs_log_ratio": 1.2227912465426896, + "last_quartile_mean": 1.7978097275772598e-05, + "mean": 3.421876306219929e-05, + "normalized": [ + 2.125681514851671, + 2.035741091532698, + 1.9424547124055425, + 1.8443747880483625, + 1.7129915891324252, + 1.6251025199478857, + 1.537504861060443, + 1.452620900762345, + 1.3158482592376062, + 1.2267970572702998, + 1.1490758791488886, + 1.0695328816728442, + 0.9835696082276737, + 0.9163259113865182, + 0.85522266858747, + 0.8029399296205194, + 0.7578942872402048, + 0.7156133216460457, + 0.6817566226795772, + 0.6538643195822892, + 0.6299348137903584, + 0.6059197773220268, + 0.5863493060398601, + 0.5697864420541977, + 0.5595343492561913, + 0.5483692204744319, + 0.5380517969111698, + 0.5293376615585488, + 0.5166827361721865, + 0.509484072574387, + 0.5036056858005552, + 0.49803141400477713 + ], + "peak_layer": 1, + "peak_normalized": 2.125681514851671, + "population_cv": 0.512158847941706, + "rest_mean": 3.6814922000134914e-05, + "spike_contrast": 0.5486771042805106, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.019950479734689e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.273819210240617e-05, + 6.966054206714034e-05, + 6.646839756285772e-05, + 6.311222387012094e-05, + 5.861645331606269e-05, + 5.560899808187969e-05, + 5.261151454760693e-05, + 4.9706890422385186e-05, + 4.502669980865903e-05, + 4.1979477828135714e-05, + 3.931995524908416e-05, + 3.659809226519428e-05, + 3.365653537912294e-05, + 3.1355539249489084e-05, + 2.926466186181642e-05, + 2.7475611204863526e-05, + 2.5934205041266978e-05, + 2.4487402697559446e-05, + 2.332886833755765e-05, + 2.2374428226612508e-05, + 2.1555590137722902e-05, + 2.073382529488299e-05, + 2.006414797506295e-05, + 1.9497387256706133e-05, + 1.9146573322359473e-05, + 1.8764516426017508e-05, + 1.841146695369389e-05, + 1.8113280020770617e-05, + 1.7680244127404876e-05, + 1.7433914763387293e-05, + 1.7232763639185578e-05, + 1.704201895336155e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05415340978652239, + "first_to_last_ratio": 3.3966554118601855, + "imbalance_abs_log_ratio": 1.2227912450753184, + "last_quartile_mean": 0.015943156788125634, + "mean": 0.030345541599672288, + "normalized": [ + 2.1256815759027154, + 2.0357410833262275, + 1.9424548253937048, + 1.8443746973080621, + 1.712991596235519, + 1.6251025854549026, + 1.5375048897542698, + 1.4526208560186746, + 1.3158482418228976, + 1.2267970397813441, + 1.1490758214573777, + 1.0695328097128658, + 0.9835695720716091, + 0.9163258756382576, + 0.8552226910646754, + 0.8029399206669349, + 0.7578942982897275, + 0.715613345638086, + 0.6817566530269693, + 0.6538643082892013, + 0.6299347939093796, + 0.6059197755459048, + 0.5863493070405259, + 0.5697864549709054, + 0.5595343861542013, + 0.5483692109047464, + 0.5380518325125886, + 0.5293376691592189, + 0.5166827726726805, + 0.5094840492707883, + 0.5036056657399419, + 0.49803139526509604 + ], + "peak_layer": 1, + "peak_normalized": 2.1256815759027154, + "population_cv": 0.51215885272173, + "rest_mean": 0.03264784129957358, + "spike_contrast": 0.5486771102516746, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01791312322020531, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0645049586892128, + 0.06177566573023796, + 0.05894484370946884, + 0.055968549102544785, + 0.05198165774345398, + 0.04931461811065674, + 0.04665641859173775, + 0.04408056661486626, + 0.03993012756109238, + 0.0372278206050396, + 0.03486932814121246, + 0.032455552369356155, + 0.029846951365470886, + 0.02780640497803688, + 0.02595219574868679, + 0.02436564676463604, + 0.022998712956905365, + 0.021715674549341202, + 0.02068827487528324, + 0.019841866567730904, + 0.019115712493658066, + 0.01838696375489235, + 0.017793087288737297, + 0.017290478572249413, + 0.01697937399148941, + 0.01664056070148945, + 0.016327474266290665, + 0.016063038259744644, + 0.015679018571972847, + 0.01546056941151619, + 0.015282186679542065, + 0.015113032422959805 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.484899475050042e-06, + "first_to_last_ratio": 4.317013540640096, + "imbalance_abs_log_ratio": 1.4625638531540355, + "last_quartile_mean": 1.2705309870852943e-06, + "mean": 2.7753340354763623e-06, + "normalized": [ + 2.6423318625200567, + 2.4162339509651107, + 2.211337495963179, + 2.033398824132887, + 1.8430633472212699, + 1.6940067504930827, + 1.5445742745339275, + 1.4254731551929667, + 1.2827132729228703, + 1.1755637695304801, + 1.092360755278899, + 1.0016920978710038, + 0.9227083034090567, + 0.8576286815024052, + 0.7936258208713879, + 0.7371254703657203, + 0.6937452548738572, + 0.6483007348957919, + 0.6136555742307378, + 0.5846345106553689, + 0.5617683465972004, + 0.5391257192612848, + 0.5197876012499301, + 0.5027932393500675, + 0.49022455226643835, + 0.4787926786546725, + 0.46965294739774904, + 0.460864066796148, + 0.45018919494858056, + 0.44356620781452427, + 0.43718650171280415, + 0.43187503652054116 + ], + "peak_layer": 1, + "peak_normalized": 2.6423318625200567, + "population_cv": 0.6359510157609364, + "rest_mean": 3.020622224775488e-06, + "spike_contrast": 0.48029104777209103, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4507778132610838e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.333353551075561e-06, + 6.705856321786996e-06, + 6.137200216471683e-06, + 5.643360964313615e-06, + 5.115116437082179e-06, + 4.701434590970166e-06, + 4.28670955443522e-06, + 3.956164164264919e-06, + 3.5599578041001223e-06, + 3.2625821404508315e-06, + 3.031665983144194e-06, + 2.7800301722891163e-06, + 2.560823759267805e-06, + 2.380206069574342e-06, + 2.2025767520972295e-06, + 2.0457694063225063e-06, + 1.9253748178016394e-06, + 1.7992510947806295e-06, + 1.7030992012223578e-06, + 1.622556055735913e-06, + 1.559094812364492e-06, + 1.496253958066518e-06, + 1.4425842209675466e-06, + 1.3954191899756552e-06, + 1.3605368849312072e-06, + 1.3288096170072095e-06, + 1.3034438097747625e-06, + 1.2790517303074012e-06, + 1.249425395144499e-06, + 1.2310443935348303e-06, + 1.2133385780543904e-06, + 1.1985974879280548e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.00092546644737e-05, + "first_to_last_ratio": 2.8610718587446584, + "imbalance_abs_log_ratio": 1.0511963304057041, + "last_quartile_mean": 1.7479202597314725e-05, + "mean": 3.0017626613698667e-05, + "normalized": [ + 1.9665203436980145, + 1.8844342899401736, + 1.80165076038572, + 1.7146572887379672, + 1.6027118780052416, + 1.5258449550389437, + 1.4518445303422347, + 1.3803062982231078, + 1.2718602647538741, + 1.1969127030718432, + 1.1317702815894917, + 1.0650354822632313, + 0.9973554850676779, + 0.9413624795803887, + 0.8896480124159175, + 0.8445590199428843, + 0.809061501322106, + 0.7713306058151453, + 0.7401513167984001, + 0.7139298020909397, + 0.6922990839843121, + 0.6685043123895319, + 0.6485424460151992, + 0.631323216874115, + 0.6206922556356658, + 0.6084815208074952, + 0.5970366156159529, + 0.5872145081510212, + 0.5725964819678343, + 0.5642525868906152, + 0.5573685431962108, + 0.5507411293887441 + ], + "peak_layer": 1, + "peak_normalized": 1.9665203436980145, + "population_cv": 0.4378481821603461, + "rest_mean": 3.195058242353197e-05, + "spike_contrast": 0.6128109021943274, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9579665240598844e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.903027340536937e-05, + 5.6566244893474504e-05, + 5.4081279813544825e-05, + 5.14699422637932e-05, + 4.810960672330111e-05, + 4.580224413075484e-05, + 4.358092701295391e-05, + 4.1433519072597846e-05, + 3.8178226532181725e-05, + 3.592847861000337e-05, + 3.397305772523396e-05, + 3.196983743691817e-05, + 2.9938244551885873e-05, + 2.825746742018964e-05, + 2.670512185432017e-05, + 2.5351657313876785e-05, + 2.4286106054205447e-05, + 2.3153514121077023e-05, + 2.2217585865291767e-05, + 2.1430478227557614e-05, + 2.0781175408046693e-05, + 2.006691283895634e-05, + 1.9467704987619072e-05, + 1.895082459668629e-05, + 1.8631708371685818e-05, + 1.8265171092934906e-05, + 1.7921622202266008e-05, + 1.7626785847824067e-05, + 1.7187987396027893e-05, + 1.693752346909605e-05, + 1.673088081588503e-05, + 1.6531941582798027e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.890059224431752e-05, + "first_to_last_ratio": 2.166537206678912, + "imbalance_abs_log_ratio": 0.7731301356847547, + "last_quartile_mean": 1.7955192333829473e-05, + "mean": 2.5870607089473197e-05, + "normalized": [ + 1.7831451783784935, + 1.6977459463469209, + 1.6200768511580206, + 1.5334592693492033, + 1.446421368117004, + 1.3744004997462946, + 1.3169700168196983, + 1.2570593821131775, + 1.1875487586493927, + 1.1204242107881341, + 1.071956417312422, + 1.0193758496059768, + 0.9782769348268451, + 0.9375232472756838, + 0.9065756301529028, + 0.8773601959197916, + 0.8648597359592212, + 0.8401947622081907, + 0.814617151100937, + 0.7925804041893227, + 0.77941283303955, + 0.7586878792831788, + 0.7416534116249156, + 0.7273676136251142, + 0.7223613266422326, + 0.7135679461862469, + 0.7058096849258172, + 0.6989156870230683, + 0.6866714405082665, + 0.6814500720405863, + 0.6750012133598903, + 0.6685290817235002 + ], + "peak_layer": 1, + "peak_normalized": 1.7831451783784935, + "population_cv": 0.3297373433822509, + "rest_mean": 2.7087979105999693e-05, + "spike_contrast": 0.7123749662061754, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9296798200230114e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.61310482933186e-05, + 4.392171831568703e-05, + 4.19123716710601e-05, + 3.967152224504389e-05, + 3.741979890037328e-05, + 3.5556575312511995e-05, + 3.407081385375932e-05, + 3.2520889362785965e-05, + 3.0722607334610075e-05, + 2.8986054530832916e-05, + 2.7732163289329037e-05, + 2.6371872081654146e-05, + 2.5308618205599487e-05, + 2.4254295567516237e-05, + 2.3453661924577318e-05, + 2.2697840904584154e-05, + 2.2374446416506544e-05, + 2.1736348571721464e-05, + 2.107464024447836e-05, + 2.0504536223597825e-05, + 2.016388316405937e-05, + 1.962771602848079e-05, + 1.9187024008715525e-05, + 1.8817441741703078e-05, + 1.8687926058191806e-05, + 1.8460435967426747e-05, + 1.825972503866069e-05, + 1.808137312764302e-05, + 1.776460703695193e-05, + 1.762952706485521e-05, + 1.7462691175751388e-05, + 1.7295253201154992e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011785902643168811, + "first_to_last_ratio": 5.2723364637525565, + "imbalance_abs_log_ratio": 1.6624736160576234, + "last_quartile_mean": 2.235423085039656e-05, + "mean": 5.981010565392353e-05, + "normalized": [ + 2.341513132793844, + 2.2440750819699944, + 2.142476081227823, + 2.0324729445905114, + 1.8907305822050442, + 1.7814901971753774, + 1.70505397052119, + 1.6266178018526998, + 1.4513520805328384, + 1.3486545714944895, + 1.25655955310435, + 1.1590271011245943, + 1.0440203382713282, + 0.9503717152908809, + 0.8638113904441559, + 0.7921778834221871, + 0.7124380420896897, + 0.6459278658421956, + 0.5962877600798272, + 0.5569469907381962, + 0.5166814306914113, + 0.4757859010752184, + 0.4474223682075307, + 0.42807794732183224, + 0.41212661540589873, + 0.3977379814905876, + 0.3854116965392957, + 0.3748299777949146, + 0.36298713381472003, + 0.3565353744030076, + 0.3515636207422689, + 0.3488348677420969 + ], + "peak_layer": 1, + "peak_normalized": 2.341513132793844, + "population_cv": 0.6457650964234652, + "rest_mean": 6.583521119157215e-05, + "spike_contrast": 0.4142849283380522, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.7274535750620998e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001400461478624493, + 0.00013421836774796247, + 0.00012814172077924013, + 0.00012156242155469954, + 0.00011308479588478804, + 0.00010655111691448838, + 0.00010197945812251419, + 9.728818258736283e-05, + 8.68055212777108e-05, + 8.066317241173238e-05, + 7.515495963161811e-05, + 6.932153337402269e-05, + 6.244296673685312e-05, + 5.684183270204812e-05, + 5.1664650527527556e-05, + 4.7380242904182523e-05, + 4.2610994569258764e-05, + 3.863301390083507e-05, + 3.566403393051587e-05, + 3.3311058359686285e-05, + 3.090277095907368e-05, + 2.845680501195602e-05, + 2.6760379114421085e-05, + 2.5603387257433496e-05, + 2.4649336410220712e-05, + 2.3788750695530325e-05, + 2.305151429027319e-05, + 2.2418620574171655e-05, + 2.1710298824473284e-05, + 2.1324418412405066e-05, + 2.1027057300671004e-05, + 2.0863850295427255e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.005866481456905603, + 0.010142817161977291, + 0.01482025533914566, + 0.01956234686076641, + 0.0067119961604475975, + 0.011908270418643951, + 0.017094247043132782, + 0.02295343205332756, + 0.007494525518268347, + 0.013766046613454819, + 0.02038806490600109, + 0.02682213857769966, + 0.009193498641252518, + 0.016781754791736603, + 0.024297412484884262, + 0.03140362724661827, + 0.009884056635200977, + 0.018740201368927956, + 0.02799132652580738, + 0.03797563910484314, + 0.012623735703527927, + 0.02351362258195877, + 0.034899674355983734, + 0.047332651913166046, + 0.01744839735329151, + 0.029514454305171967, + 0.044660378247499466, + 0.0601838044822216, + 0.018972838297486305, + 0.0376126803457737, + 0.0565216988325119, + 0.07583927363157272 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5008519199000148e-05, + "first_to_last_ratio": 3.366911210627585, + "imbalance_abs_log_ratio": 1.2139957693612529, + "last_quartile_mean": 4.457652210021479e-06, + "mean": 8.299272167278104e-06, + "normalized": [ + 2.256857417490706, + 2.1152794126450303, + 1.976744865869958, + 1.8670032822761609, + 1.7142943873365317, + 1.61284428729019, + 1.5135327831047092, + 1.4107548139296946, + 1.2736784355647999, + 1.19029358022618, + 1.1070446131205842, + 1.032219511002358, + 0.948055380482258, + 0.88484028406668, + 0.831196979445672, + 0.7835724276747893, + 0.7420372577590195, + 0.7071991859856492, + 0.678238718639715, + 0.6514579423864029, + 0.6287823606114806, + 0.6086317246696183, + 0.5913247143847147, + 0.5772065844972074, + 0.5691335632017953, + 0.558269353765152, + 0.5491862116730458, + 0.5409941785864528, + 0.5285468164414351, + 0.5224629687441996, + 0.516466078557371, + 0.5118498785704394 + ], + "peak_layer": 1, + "peak_normalized": 2.256857417490706, + "population_cv": 0.5242114468317668, + "rest_mean": 8.921693312947389e-06, + "spike_contrast": 0.5535045654951547, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.938197980663972e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8730273950495757e-05, + 1.7555279555381276e-05, + 1.6405543647124432e-05, + 1.5494768376811408e-05, + 1.4227395695343148e-05, + 1.3385433703660965e-05, + 1.256122050108388e-05, + 1.1708238162100315e-05, + 1.0570603990345262e-05, + 9.878570381260943e-06, + 9.187664545606822e-06, + 8.566670658183284e-06, + 7.868169632274657e-06, + 7.343530342041049e-06, + 6.8983299570390955e-06, + 6.5030808400479145e-06, + 6.158369160402799e-06, + 5.869238520972431e-06, + 5.628887720376952e-06, + 5.406626769399736e-06, + 5.218435944698285e-06, + 5.051200332673034e-06, + 4.907564743916737e-06, + 4.7903945414873306e-06, + 4.723394340544473e-06, + 4.63322930954746e-06, + 4.5578458411910105e-06, + 4.489857929002028e-06, + 4.386553882795852e-06, + 4.336062374932226e-06, + 4.286292551114457e-06, + 4.247981451044325e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.8205007462674985e-05, + "first_to_last_ratio": 3.284886533563498, + "imbalance_abs_log_ratio": 1.189332110567881, + "last_quartile_mean": 1.7719031347951386e-05, + "mean": 3.265063452317918e-05, + "normalized": [ + 2.145023601546889, + 2.046116178200582, + 1.9398041355219169, + 1.8522906626530224, + 1.7094551367874427, + 1.616905863157966, + 1.5255793277696308, + 1.4261117891382666, + 1.2879069358899493, + 1.2060838643982497, + 1.1221569343801026, + 1.0481379278736214, + 0.9613520366262555, + 0.8968015077026028, + 0.8421366180143057, + 0.7933562257498653, + 0.7503570432885839, + 0.714837243906681, + 0.6855582118161445, + 0.6585243205638621, + 0.6351846609275962, + 0.6146934889106834, + 0.5972735915655352, + 0.5828673640378894, + 0.5748017373594797, + 0.5639401582431183, + 0.5548451094205582, + 0.5465856652489405, + 0.5340146592968225, + 0.5278533354505021, + 0.5219685037625007, + 0.5174761607904339 + ], + "peak_layer": 1, + "peak_normalized": 2.145023601546889, + "population_cv": 0.5072011380661324, + "rest_mean": 3.5063370281325966e-05, + "spike_contrast": 0.5596113913680666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.962186142918654e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.003638165770099e-05, + 6.680699152639136e-05, + 6.333583587547764e-05, + 6.047846545698121e-05, + 5.581479490501806e-05, + 5.2793002396356314e-05, + 4.981113306712359e-05, + 4.656345481635071e-05, + 4.2050978663610294e-05, + 3.9379403460770845e-05, + 3.663913594209589e-05, + 3.422236841288395e-05, + 3.138875399599783e-05, + 2.928113826783374e-05, + 2.7496294933371246e-05, + 2.5903584173647687e-05, + 2.449963358230889e-05, + 2.3339889594353735e-05, + 2.2383910618373193e-05, + 2.1501236915355548e-05, + 2.0739182218676433e-05, + 2.0070132450200617e-05, + 1.950136174855288e-05, + 1.9030989278689958e-05, + 1.87676414498128e-05, + 1.8413003999739885e-05, + 1.8116044884664007e-05, + 1.7846368791651912e-05, + 1.7435917470720597e-05, + 1.7234746337635443e-05, + 1.7042602848960087e-05, + 1.689592500042636e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05161678325384855, + "first_to_last_ratio": 3.2848865977727097, + "imbalance_abs_log_ratio": 1.189332130114739, + "last_quartile_mean": 0.015713414060883224, + "mean": 0.028954909328604117, + "normalized": [ + 2.1450237245911468, + 2.046116217659094, + 1.9398041664812302, + 1.8522906413119131, + 1.7094550632279355, + 1.6169058509968128, + 1.5255792788818974, + 1.4261118508948372, + 1.2879069434477655, + 1.206083847584953, + 1.1221569306992827, + 1.0481379275281273, + 0.9613519652186596, + 0.8968015075628347, + 0.8421366414011558, + 0.7933562213086081, + 0.7503569989748414, + 0.7148372643644912, + 0.6855582307408538, + 0.6585243493162974, + 0.6351846345398151, + 0.614693479533182, + 0.5972735918667607, + 0.5828673969375834, + 0.5748016775305409, + 0.5639401482692175, + 0.5548451622169414, + 0.5465856833639338, + 0.5340146386986252, + 0.5278533199019263, + 0.5219685200223859, + 0.5174761249263505 + ], + "peak_layer": 1, + "peak_normalized": 2.1450237245911468, + "population_cv": 0.507201147039954, + "rest_mean": 0.031094547602589482, + "spike_contrast": 0.5596113785438716, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.017400862649083138, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.06210896745324135, + 0.05924510955810547, + 0.05616685375571251, + 0.05363290756940842, + 0.04949711635708809, + 0.0468173623085022, + 0.04417300969362259, + 0.0412929393351078, + 0.03729122877120972, + 0.034922048449516296, + 0.03249195218086243, + 0.03034873865544796, + 0.02783585898578167, + 0.025966806337237358, + 0.024383990094065666, + 0.022971557453274727, + 0.021726518869400024, + 0.020698048174381256, + 0.01985027641057968, + 0.019067512825131416, + 0.018391713500022888, + 0.017798393964767456, + 0.017294002696871758, + 0.01687687262892723, + 0.016643330454826355, + 0.016328835859894753, + 0.01606549136340618, + 0.015826338902115822, + 0.015462345443665981, + 0.015283945016562939, + 0.015113551169633865, + 0.014983474276959896 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 5.35499910370163e-06, + "first_to_last_ratio": 4.280329422122994, + "imbalance_abs_log_ratio": 1.4540299744175837, + "last_quartile_mean": 1.251071722663255e-06, + "mean": 2.683941740855289e-06, + "normalized": [ + 2.703190107906874, + 2.4622030057009887, + 2.2387169403666407, + 2.052519103467962, + 1.857495181173395, + 1.6956946287041017, + 1.543681691896796, + 1.4080949959710705, + 1.2505201784095583, + 1.154791072279902, + 1.0675304639467325, + 0.9762128756212061, + 0.8974710865803879, + 0.8318006025975303, + 0.7746066644618449, + 0.7211640498251165, + 0.6781926692188598, + 0.6399504079116765, + 0.6084710506699328, + 0.581158338700675, + 0.5581318067356622, + 0.5402181590129327, + 0.5217735093776118, + 0.5073536475319059, + 0.4964690402033343, + 0.48529694803264734, + 0.47725139647875997, + 0.4696126527079396, + 0.4585674232309517, + 0.45305527095834874, + 0.4466379671326055, + 0.4421670631860494 + ], + "peak_layer": 1, + "peak_normalized": 2.703190107906874, + "population_cv": 0.645765617676911, + "rest_mean": 2.9201339657447534e-06, + "spike_contrast": 0.48234216065938457, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4085037264521817e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.255204764078371e-06, + 6.608409421460237e-06, + 6.008585842209868e-06, + 5.508841695700539e-06, + 4.985408850188833e-06, + 4.551145593723049e-06, + 4.143151727475924e-06, + 3.779244934776216e-06, + 3.3563233046152163e-06, + 3.099391960859066e-06, + 2.8651895718212472e-06, + 2.6200984848401276e-06, + 2.408760110483854e-06, + 2.232504357380094e-06, + 2.078999159493833e-06, + 1.9355622953298735e-06, + 1.8202296132585616e-06, + 1.7175896118715173e-06, + 1.6331008509951062e-06, + 1.5597951232848573e-06, + 1.497993252996821e-06, + 1.44991406614281e-06, + 1.400409701091121e-06, + 1.3617076319860644e-06, + 1.3324939800440916e-06, + 1.3025087355345022e-06, + 1.2809149438908207e-06, + 1.2604130006366177e-06, + 1.2307682482060045e-06, + 1.2159739526396152e-06, + 1.1987502830379526e-06, + 1.186750637316436e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.7860596623650054e-05, + "first_to_last_ratio": 2.779352348042467, + "imbalance_abs_log_ratio": 1.022217932231161, + "last_quartile_mean": 1.7220053678101976e-05, + "mean": 2.88126707914671e-05, + "normalized": [ + 1.9752636229242324, + 1.8880994931589683, + 1.7945585688576082, + 1.7155011651574092, + 1.5967516750805444, + 1.5186230631712434, + 1.44161945600051, + 1.358346630504372, + 1.2494545051794677, + 1.1811062374956933, + 1.1110457151250825, + 1.0496228553992981, + 0.981726104768848, + 0.9275032753698961, + 0.8805136203767746, + 0.8381981012329272, + 0.8039749772857018, + 0.7714588681042523, + 0.7440936013254396, + 0.7183075709964174, + 0.6966458004631945, + 0.6758729225629136, + 0.6578613567984469, + 0.6426061157750497, + 0.6340386554461201, + 0.6220565974014273, + 0.6118747994647586, + 0.6024343556608059, + 0.5879125773227442, + 0.5807482794696509, + 0.5737976188800432, + 0.5683818132401579 + ], + "peak_layer": 1, + "peak_normalized": 1.9752636229242324, + "population_cv": 0.4317074018350235, + "rest_mean": 3.0619305444243194e-05, + "spike_contrast": 0.62238001123762, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.9056843666476197e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.6912620493676513e-05, + 5.440118911792524e-05, + 5.170602526050061e-05, + 4.942817031405866e-05, + 4.600668034981936e-05, + 4.375558637548238e-05, + 4.1536906792316586e-05, + 3.9137594285421073e-05, + 3.6000121326651424e-05, + 3.4030825190711766e-05, + 3.201219442416914e-05, + 3.0242437787819654e-05, + 2.8286151064094156e-05, + 2.672384653124027e-05, + 2.5369949071318842e-05, + 2.4150725948857144e-05, + 2.3164666345110163e-05, + 2.222779039584566e-05, + 2.1439323973027058e-05, + 2.0696359570138156e-05, + 2.00722261070041e-05, + 1.9473704014671966e-05, + 1.8954742699861526e-05, + 1.85151984624099e-05, + 1.8268347048433498e-05, + 1.7923111954587512e-05, + 1.762974716257304e-05, + 1.7357742763124406e-05, + 1.6939331544563174e-05, + 1.6732908989069983e-05, + 1.653264189371839e-05, + 1.6376598068745807e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.7207921195658855e-05, + "first_to_last_ratio": 2.0918054721335677, + "imbalance_abs_log_ratio": 0.7380275553273271, + "last_quartile_mean": 1.7787467186280992e-05, + "mean": 2.50158799417477e-05, + "normalized": [ + 1.7669834832946238, + 1.6851855745889466, + 1.5913246601136481, + 1.5163874050648884, + 1.4314556947301431, + 1.3660671618942286, + 1.3038144666622677, + 1.2377581340639199, + 1.1604947777085126, + 1.1096820799312197, + 1.0568639468917405, + 1.0113831764859333, + 0.9697099699695068, + 0.9379529791327526, + 0.9077702331627978, + 0.8815756711399201, + 0.8691427723616427, + 0.8426692090660671, + 0.8202682499619266, + 0.8014514792168752, + 0.7847031862420706, + 0.7670164542718427, + 0.7519889963555914, + 0.7399739814225563, + 0.7380541298189486, + 0.7300354427325324, + 0.7228094770104511, + 0.7167792102532495, + 0.7048467977298535, + 0.6982014484170743, + 0.6912475034823692, + 0.6864022468218989 + ], + "peak_layer": 1, + "peak_normalized": 1.7669834832946238, + "population_cv": 0.3174743127828502, + "rest_mean": 2.6144617987921703e-05, + "spike_contrast": 0.7236936680868339, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.8920694492408076e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.420264667714946e-05, + 4.21564000134822e-05, + 3.9808386645745486e-05, + 3.793376527028158e-05, + 3.58091238013003e-05, + 3.417337211431004e-05, + 3.26160661643371e-05, + 3.096360887866467e-05, + 2.9030798032181337e-05, + 2.7759673685068265e-05, + 2.6438381610205397e-05, + 2.5300640118075535e-05, + 2.4258148187072948e-05, + 2.3463719116989523e-05, + 2.2708671167492867e-05, + 2.2053391148801893e-05, + 2.1742371245636605e-05, + 2.1080111764604226e-05, + 2.0519732061075047e-05, + 2.004901398322545e-05, + 1.9630040696938522e-05, + 1.918759153340943e-05, + 1.881166645034682e-05, + 1.851110027928371e-05, + 1.846307350206189e-05, + 1.826247898861766e-05, + 1.8081715097650886e-05, + 1.793086266843602e-05, + 1.763236286933534e-05, + 1.746612360875588e-05, + 1.729216455714777e-05, + 1.7170956198242493e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011250466286583105, + "first_to_last_ratio": 5.136757236447842, + "imbalance_abs_log_ratio": 1.6364219925229777, + "last_quartile_mean": 2.1901884338149102e-05, + "mean": 5.649989418543555e-05, + "normalized": [ + 2.3823389400332498, + 2.2792913554127336, + 2.1636479699592535, + 2.0682892196556057, + 1.89270947379848, + 1.8094109355775225, + 1.7253047001597102, + 1.6089010079061294, + 1.4276863120740728, + 1.3301882974473787, + 1.227270134970172, + 1.1372581289133685, + 1.0077700511385481, + 0.9144159274954295, + 0.8389896038222593, + 0.761607782391717, + 0.6842191357138161, + 0.6319663297187056, + 0.5899873238938539, + 0.550774281755648, + 0.5037238917813971, + 0.4736966989455727, + 0.453500437871209, + 0.43589448694769534, + 0.4210071093140732, + 0.4081032387076438, + 0.39686588746098733, + 0.38782343274226294, + 0.37750089000360326, + 0.37216747430584624, + 0.36939626290542954, + 0.36829327717662536 + ], + "peak_layer": 1, + "peak_normalized": 2.3823389400332498, + "population_cv": 0.6517280428580817, + "rest_mean": 6.217536584127056e-05, + "spike_contrast": 0.4157972678427941, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5852347243926487e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001346018980257213, + 0.00012877972039859742, + 0.00012224588135723025, + 0.00011685812205541879, + 0.00010693788499338552, + 0.00010223152639809996, + 9.747953299665824e-05, + 9.090273670153692e-05, + 8.066412556217983e-05, + 7.515549805248156e-05, + 6.934063276275992e-05, + 6.425496394513175e-05, + 5.6938901252578944e-05, + 5.1664403144968674e-05, + 4.740282383863814e-05, + 4.3030759115936235e-05, + 3.8658308767480776e-05, + 3.570603075786494e-05, + 3.333422137075104e-05, + 3.111868863925338e-05, + 2.8460346584324725e-05, + 2.676381336641498e-05, + 2.5622726752771996e-05, + 2.4627992388559505e-05, + 2.3786857127561234e-05, + 2.3057789803715423e-05, + 2.242288064735476e-05, + 2.1911982912570238e-05, + 2.132876034011133e-05, + 2.1027422917541116e-05, + 2.08708497666521e-05, + 2.080853118968662e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.028571506962180138, + 0.01502692885696888, + 0.015751885250210762, + 0.016715344041585922, + 0.01783851347863674, + 0.012985615991055965, + 0.014028912410140038, + 0.015184907242655754, + 0.016659293323755264, + 0.013609340414404869, + 0.014725575223565102, + 0.016002535820007324, + 0.017315881326794624, + 0.014967667870223522, + 0.016154561191797256, + 0.01737469807267189, + 0.018479224294424057, + 0.01647813245654106, + 0.017671043053269386, + 0.01892261393368244, + 0.020303834229707718, + 0.01861351728439331, + 0.01984807848930359, + 0.021230308338999748, + 0.022779423743486404, + 0.02138628251850605, + 0.02279111184179783, + 0.024287279695272446, + 0.02611272968351841, + 0.024725768715143204, + 0.026407435536384583, + 0.028016721829771996 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.961159703498197e-06, + "first_to_last_ratio": 10.14195644845838, + "imbalance_abs_log_ratio": 2.3166809231867567, + "last_quartile_mean": 4.891718603516892e-07, + "mean": 2.1833543613070105e-06, + "normalized": [ + 2.1149003512951285, + 3.2694277026187555, + 2.6224151681895895, + 2.367649688300867, + 1.872384189095874, + 2.4215723461348544, + 1.997727875442672, + 1.5120426998182623, + 1.407451434597839, + 1.4959940566742542, + 1.146826747734668, + 1.0540980256857684, + 0.9488948361068527, + 0.9447470595114825, + 0.8146834014726949, + 0.6963182667169648, + 0.5482907339349459, + 0.5718154556650352, + 0.5280062120840487, + 0.46712124843842245, + 0.37653152797943845, + 0.3855401511265584, + 0.3587208946748597, + 0.2844717467264502, + 0.29151483884188434, + 0.260094980728135, + 0.263913337696718, + 0.20680766564782654, + 0.1887856679838017, + 0.2117583846844945, + 0.19386962063621954, + 0.1756236837546329 + ], + "peak_layer": 2, + "peak_normalized": 3.2694277026187555, + "population_cv": 0.8462629641613505, + "rest_mean": 2.4504692364475717e-06, + "spike_contrast": 0.3023641450084502, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.409340355479799e-07, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.617576905729948e-06, + 7.13831923349062e-06, + 5.725661594624398e-06, + 5.1694182729988825e-06, + 4.0880781853047665e-06, + 5.2871505431539845e-06, + 4.361747869552346e-06, + 3.30132502313063e-06, + 3.0729652280570008e-06, + 3.2662851481291e-06, + 2.5039291813300224e-06, + 2.3014695216261316e-06, + 2.071773678835598e-06, + 2.0627176127163693e-06, + 1.7787425576898386e-06, + 1.5203095244942233e-06, + 1.197112965201086e-06, + 1.2484757689890102e-06, + 1.1528246659509023e-06, + 1.0198912150372053e-06, + 8.221017537834996e-07, + 8.417707704211352e-07, + 7.832148298803077e-07, + 6.211026288838184e-07, + 6.364801947711385e-07, + 5.678795105268364e-07, + 5.762163368672191e-07, + 4.515344187439041e-07, + 4.121860115446907e-07, + 4.6234359274421877e-07, + 4.2328608174102555e-07, + 3.834487358744809e-07 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.940424550019088e-06, + "first_to_last_ratio": 7.362808193451028, + "imbalance_abs_log_ratio": 1.9964414079930852, + "last_quartile_mean": 1.214268294802423e-06, + "mean": 4.16189458363192e-06, + "normalized": [ + 1.8755411782360838, + 2.990879153140948, + 2.507234682429089, + 2.1623895678054548, + 1.7817542090336904, + 2.3421906478647685, + 1.984613814607898, + 1.5406942028635409, + 1.3930431107090302, + 1.467345705614427, + 1.208878006326314, + 1.0663129583432038, + 0.9225426136071778, + 0.9347168647367088, + 0.8446162613986855, + 0.7123769822844362, + 0.5995323166764556, + 0.6272780709200748, + 0.55800311830824, + 0.5090119191108281, + 0.4207840547535497, + 0.45050976377329044, + 0.4129238940687079, + 0.3527585690707129, + 0.3423379312955225, + 0.33392765114492584, + 0.32776335752361546, + 0.2883552290395252, + 0.26514548144992983, + 0.28609378158280374, + 0.25631521585884554, + 0.23412968642151585 + ], + "peak_layer": 2, + "peak_normalized": 2.990879153140948, + "population_cv": 0.7649777321045976, + "rest_mean": 4.627515910147782e-06, + "spike_contrast": 0.3560310655730732, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.6475394204462646e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.805804671079386e-06, + 1.2447723747754935e-05, + 1.0434846444695722e-05, + 8.99963742995169e-06, + 7.415473191940691e-06, + 9.747950571181718e-06, + 8.259753485617694e-06, + 6.412206857930869e-06, + 5.797698577225674e-06, + 6.106938144512242e-06, + 5.0312228268012404e-06, + 4.437882125785109e-06, + 3.839525106741348e-06, + 3.890193056577118e-06, + 3.515203843562631e-06, + 2.9648379040736472e-06, + 2.4951903014880372e-06, + 2.610665205793339e-06, + 2.3223501557367854e-06, + 2.1184539491514442e-06, + 1.7512588783574756e-06, + 1.8749741457213531e-06, + 1.718545718176756e-06, + 1.4681439779451466e-06, + 1.4247743820305914e-06, + 1.3897716826249962e-06, + 1.3641165423905477e-06, + 1.2001040659015416e-06, + 1.1035075431209407e-06, + 1.1906921599802445e-06, + 1.0667569085853756e-06, + 9.744230737851467e-07 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.00792845810065046, + "first_to_last_ratio": 7.362808269910068, + "imbalance_abs_log_ratio": 1.9964414183775794, + "last_quartile_mean": 0.001076825283234939, + "mean": 0.003690809797262773, + "normalized": [ + 1.8755411129192268, + 2.990879196958467, + 2.5072346853780427, + 2.162389622214153, + 1.7817542918869094, + 2.3421908242130387, + 1.9846136654580344, + 1.5406942156700558, + 1.3930430879772915, + 1.4673456922088532, + 1.2088779947214396, + 1.0663129426719855, + 0.9225426097974535, + 0.9347168545387744, + 0.8446162145203677, + 0.7123769861164736, + 0.5995323323039541, + 0.6272780785788562, + 0.5580030712234716, + 0.5090119005316229, + 0.4207840635329223, + 0.45050976081671684, + 0.4129238737745274, + 0.35275859035228563, + 0.3423379200763214, + 0.3339276703286944, + 0.3277633648171629, + 0.2883552364940563, + 0.2651454743312869, + 0.2860937736207281, + 0.25631521203611624, + 0.2341296799307093 + ], + "peak_layer": 2, + "peak_normalized": 2.990879196958467, + "population_cv": 0.7649777429861212, + "rest_mean": 0.004103727453319287, + "spike_contrast": 0.3560310647277095, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0014610544545575976, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.006922265514731407, + 0.011038766242563725, + 0.009253726340830326, + 0.007980968803167343, + 0.006576116196811199, + 0.008644580841064453, + 0.007324831560254097, + 0.005686409305781126, + 0.005141457077115774, + 0.005415693856775761, + 0.004461738746613264, + 0.003935558255761862, + 0.0034049293026328087, + 0.0034498621243983507, + 0.0031173177994787693, + 0.002629247959703207, + 0.002212759805843234, + 0.00231516407802701, + 0.002059483202174306, + 0.001878666109405458, + 0.0015530339442193508, + 0.001662745838984847, + 0.0015240234788507223, + 0.0013019648613408208, + 0.0012635041493922472, + 0.0012324635172262788, + 0.0012097122380509973, + 0.0010642643319442868, + 0.0009786015143617988, + 0.001055917702615261, + 0.0009460106957703829, + 0.000864128116518259 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.5133993492308946e-06, + "first_to_last_ratio": 6.5659297300766655, + "imbalance_abs_log_ratio": 1.881894116932073, + "last_quartile_mean": 5.350954843663658e-07, + "mean": 1.7443735549349526e-06, + "normalized": [ + 1.589774202228725, + 2.5970667163618275, + 2.31809881449688, + 2.0431360816541173, + 1.739210008196666, + 2.257659305838541, + 2.011325443479608, + 1.5567888778424819, + 1.467448725425628, + 1.5902279397678998, + 1.2912623477212033, + 1.1632146422568665, + 1.0012751149296444, + 1.0140542592766522, + 0.8991815616774139, + 0.774122319459461, + 0.6409300827695702, + 0.6678856249973933, + 0.6105498714134762, + 0.5542747505087504, + 0.4535313629747497, + 0.4812253529456957, + 0.4453558428285356, + 0.3783599505586694, + 0.3693486135392654, + 0.34798394288991763, + 0.34559725866794794, + 0.2988109680432934, + 0.27684960189841046, + 0.29841376846953566, + 0.272740676748336, + 0.2442959701328381 + ], + "peak_layer": 2, + "peak_normalized": 2.5970667163618275, + "population_cv": 0.6989714208275406, + "rest_mean": 1.9299347726515677e-06, + "spike_contrast": 0.38464666774479367, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.423429792652314e-07, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.773160076685599e-06, + 4.530254500423325e-06, + 4.043630269734422e-06, + 3.5639925499708625e-06, + 3.033831944776466e-06, + 3.938201189157553e-06, + 3.5085029139736434e-06, + 2.7156213491252856e-06, + 2.559778749855468e-06, + 2.773951564449817e-06, + 2.2524438918480882e-06, + 2.0290808606659994e-06, + 1.746597831697727e-06, + 1.7688894331513438e-06, + 1.5685085372751928e-06, + 1.3503585023499909e-06, + 1.1180214869455085e-06, + 1.1650420219666557e-06, + 1.0650270496626035e-06, + 9.66862216955633e-07, + 7.911281159067585e-07, + 8.394367796427105e-07, + 7.768669547658646e-07, + 6.600010920010391e-07, + 6.442819540097844e-07, + 6.070139875191671e-07, + 6.028507186783827e-07, + 5.212379505792342e-07, + 4.829291242458567e-07, + 5.205450861467398e-07, + 4.7576162387485965e-07, + 4.261434298769018e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.804085307341666e-06, + "first_to_last_ratio": 6.304609762674019, + "imbalance_abs_log_ratio": 1.8412810741903929, + "last_quartile_mean": 7.619956647886283e-07, + "mean": 2.3993721356418973e-06, + "normalized": [ + 1.4988203443996868, + 2.5234715181763763, + 2.2553217408275312, + 1.9958915618525483, + 1.7576503840860649, + 2.319660501024385, + 2.0260662582488056, + 1.6409258036937564, + 1.4769401358690522, + 1.5639642180334559, + 1.318106777079787, + 1.1580760773918135, + 0.96371832012835, + 1.0214962965942758, + 0.8943161393616675, + 0.7727506438550246, + 0.6438659223631747, + 0.6719055424540622, + 0.6043445279969172, + 0.5577548422477306, + 0.4603517050297865, + 0.4900258115330936, + 0.45428577798054076, + 0.38963894059637383, + 0.37546196475112736, + 0.3630494912829247, + 0.35422949063900455, + 0.315438725345089, + 0.2890783675728055, + 0.3094773464831404, + 0.27956031292056627, + 0.2543545101810816 + ], + "peak_layer": 2, + "peak_normalized": 2.5234715181763763, + "population_cv": 0.6891965904370858, + "rest_mean": 2.650882836237374e-06, + "spike_contrast": 0.3927802233252255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.041214352426323e-06, + "top_five_layers": [ + 2, + 6, + 3, + 7, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5962277706858004e-06, + 6.054747245798353e-06, + 5.411356141848955e-06, + 4.788886599271791e-06, + 4.217257355776383e-06, + 5.565728770307032e-06, + 4.861286925006425e-06, + 3.937191650038585e-06, + 3.5437290080153616e-06, + 3.752532165890443e-06, + 3.1626286727259867e-06, + 2.7786554710473865e-06, + 2.312318883923581e-06, + 2.4509497507096967e-06, + 2.145797225239221e-06, + 1.8541163626650814e-06, + 1.5448739532075706e-06, + 1.6121514363476308e-06, + 1.4500474208034575e-06, + 1.3382614270085469e-06, + 1.1045550536437077e-06, + 1.1757542779378127e-06, + 1.090000637304911e-06, + 9.348888170279679e-07, + 9.008729762172152e-07, + 8.710908332432155e-07, + 8.499283694618498e-07, + 7.56854888095404e-07, + 6.936065801710356e-07, + 7.4255132176404e-07, + 6.707692250529362e-07, + 6.102911243033304e-07 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.828203619742453e-06, + "first_to_last_ratio": 5.846579859482356, + "imbalance_abs_log_ratio": 1.765856850870783, + "last_quartile_mean": 4.837364215859452e-07, + "mean": 1.4834222348270032e-06, + "normalized": [ + 1.2646232648739053, + 2.247789759160063, + 2.020880547315863, + 1.842393507943919, + 1.7522315558445254, + 2.3420641180772317, + 2.0534732489830905, + 1.7288628769088255, + 1.548314882558432, + 1.6499586193135367, + 1.4182480230650574, + 1.2414487105462828, + 0.9740236314282787, + 1.1015223442115594, + 0.9385100939682994, + 0.8083610783044318, + 0.6755087261126067, + 0.7006206946100645, + 0.6299107486392663, + 0.5832268076063307, + 0.47664186001684955, + 0.5068198063687983, + 0.4772500226016884, + 0.4085558916985, + 0.3939522492022734, + 0.36559718120224766, + 0.3572742323275962, + 0.3258561263199403, + 0.2984295494995466, + 0.3178990099824609, + 0.2887716969712665, + 0.26097913433726316 + ], + "peak_layer": 6, + "peak_normalized": 2.3420641180772317, + "population_cv": 0.6540082332746273, + "rest_mean": 1.6337852183891368e-06, + "spike_contrast": 0.41098555430286204, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.714621235914819e-07, + "top_five_layers": [ + 6, + 2, + 7, + 3, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.87597026979347e-06, + 3.3344213079544716e-06, + 2.9978191378177144e-06, + 2.7330474949849304e-06, + 2.5992992505052825e-06, + 3.4742699881462613e-06, + 3.0461678761639632e-06, + 2.564633632573532e-06, + 2.2968047233007383e-06, + 2.447585302434163e-06, + 2.1038606519141467e-06, + 1.8415926206216682e-06, + 1.4448883121076506e-06, + 1.6340227375621907e-06, + 1.3922067410021555e-06, + 1.1991407973255264e-06, + 1.002064664135105e-06, + 1.0393163165645092e-06, + 9.34423610488011e-07, + 8.651716143504018e-07, + 7.070611331982946e-07, + 7.518277698181919e-07, + 7.079632950990344e-07, + 6.06060893915128e-07, + 5.843975259267609e-07, + 5.42334987585491e-07, + 5.299885401655047e-07, + 4.833822231375962e-07, + 4.4269702925703314e-07, + 4.7157845983747393e-07, + 4.2837035607590224e-07, + 3.8714225070179964e-07 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 1.4976229522289941e-05, + "first_to_last_ratio": 6.400592033457801, + "imbalance_abs_log_ratio": 1.8563904913150628, + "last_quartile_mean": 2.339819417329636e-06, + "mean": 7.376071508247151e-06, + "normalized": [ + 1.5647793422352725, + 2.5744682287007357, + 2.3807087714141812, + 2.046233253632905, + 1.770279995956188, + 2.2790875337951744, + 2.0129055774210696, + 1.6145788731579245, + 1.4481155829279344, + 1.5649856288248702, + 1.2953241737825878, + 1.1378217116015181, + 0.9671807788478195, + 0.9743671476553158, + 0.8907704628221471, + 0.7647048808528004, + 0.6263127195536261, + 0.6555825424956646, + 0.6004321371903539, + 0.5418622203213719, + 0.44846797056121285, + 0.4882635266638759, + 0.4317412168950189, + 0.383285230653721, + 0.3707675618784486, + 0.36766574152703, + 0.3569460829427414, + 0.31124413981080545, + 0.2846577996070008, + 0.3121781014204342, + 0.2794134285368139, + 0.25486763631343645 + ], + "peak_layer": 2, + "peak_normalized": 2.5744682287007357, + "population_cv": 0.7019462950147708, + "rest_mean": 8.162162531435671e-06, + "spike_contrast": 0.3836213712934221, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.131179983029142e-06, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.154192432295531e-05, + 1.8989461750607006e-05, + 1.756027813826222e-05, + 1.5093162801349536e-05, + 1.3057711839792319e-05, + 1.681071262282785e-05, + 1.484733547840733e-05, + 1.1909249224117957e-05, + 1.068140409188345e-05, + 1.1543445907591376e-05, + 9.554403732181527e-06, + 8.392654308408964e-06, + 7.1339945861836895e-06, + 7.187001756392419e-06, + 6.5703866312105674e-06, + 5.640517883875873e-06, + 4.61972740595229e-06, + 4.835623713006498e-06, + 4.428830379765714e-06, + 3.996814484708011e-06, + 3.307931820017984e-06, + 3.601466687541688e-06, + 3.184554088875302e-06, + 2.8271392693568487e-06, + 2.7348080493538873e-06, + 2.7119288006360875e-06, + 2.632859832374379e-06, + 2.295759031767375e-06, + 2.099656285281526e-06, + 2.302647999385954e-06, + 2.0609734292520443e-06, + 1.8799219105858356e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.014360599219799042, + 0.014914286322891712, + 0.015590262599289417, + 0.016720104962587357, + 0.01203431747853756, + 0.01319553330540657, + 0.014193638227880001, + 0.015358141623437405, + 0.012656479142606258, + 0.013699201866984367, + 0.01495310477912426, + 0.01618308201432228, + 0.01409733947366476, + 0.01521329116076231, + 0.016374915838241577, + 0.017517324537038803, + 0.015613679774105549, + 0.016780734062194824, + 0.017834074795246124, + 0.01904122717678547, + 0.017550688236951828, + 0.018688000738620758, + 0.019847052171826363, + 0.021104788407683372, + 0.019968591630458832, + 0.021149026229977608, + 0.02249406836926937, + 0.02368815243244171, + 0.02278727851808071, + 0.024227207526564598, + 0.02562388963997364, + 0.0269329734146595 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.831441280359286e-06, + "first_to_last_ratio": 3.949798443260095, + "imbalance_abs_log_ratio": 1.373664550588347, + "last_quartile_mean": 1.982744535666825e-06, + "mean": 4.823673624088087e-06, + "normalized": [ + 1.646132756330255, + 1.5986846162740493, + 1.5570334831325985, + 1.4112719107036644, + 1.9597644907314193, + 1.739595316367215, + 1.586190473612319, + 1.4896705111709025, + 1.6742871420623147, + 1.4631400905900616, + 1.3644640801393357, + 1.1849574810055368, + 1.2519074382611204, + 1.1640272230786968, + 1.006304156490551, + 0.9127806011294972, + 0.9624471043561035, + 0.8599074596436582, + 0.7423282252089072, + 0.6792100847159988, + 0.7315895600277779, + 0.6384373973703829, + 0.5798636442076341, + 0.5076487157583509, + 0.5180109917100456, + 0.4991175722294294, + 0.4463413444888274, + 0.40238605877214556, + 0.38468977858519854, + 0.36602261687556353, + 0.35225030635593113, + 0.31953736861450793 + ], + "peak_layer": 5, + "peak_normalized": 1.9597644907314193, + "population_cv": 0.4957051427675951, + "rest_mean": 5.185350830658906e-06, + "spike_contrast": 0.5536012513623684, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.870616708605667e-06, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.940407158457674e-06, + 7.711532816756517e-06, + 7.510621344408719e-06, + 6.807515092077665e-06, + 9.45326428336557e-06, + 8.391240044147708e-06, + 7.651265150343534e-06, + 7.185684353316901e-06, + 8.076214726315811e-06, + 7.057710263325134e-06, + 6.5817293943837285e-06, + 5.715848146792268e-06, + 6.038792889739852e-06, + 5.61488741368521e-06, + 4.854082817473682e-06, + 4.402955710247625e-06, + 4.6425307118624914e-06, + 4.147912932239706e-06, + 3.580749080356327e-06, + 3.2762877708591986e-06, + 3.5289492643642006e-06, + 3.079613634326961e-06, + 2.7970729661319638e-06, + 2.448731720505748e-06, + 2.49871595769946e-06, + 2.407580268481979e-06, + 2.1530049707507715e-06, + 1.9409790183999576e-06, + 1.8556179384177085e-06, + 1.7655736428423552e-06, + 1.6991405118460534e-06, + 1.5413439768963144e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.103702101725503e-05, + "first_to_last_ratio": 3.8954992147916703, + "imbalance_abs_log_ratio": 1.3598218392207078, + "last_quartile_mean": 7.967405281306128e-06, + "mean": 1.931944555622067e-05, + "normalized": [ + 1.6032842527864015, + 1.5688395889631517, + 1.533796016047905, + 1.3882386982091592, + 1.9400717090010198, + 1.7314283899901333, + 1.585633146781874, + 1.5008455649726484, + 1.675406616182152, + 1.4714366899853428, + 1.3716626344955323, + 1.1913524891080052, + 1.2645012175425632, + 1.1767240803978434, + 1.0170067664778755, + 0.9256058574210715, + 0.9711818883737663, + 0.8683475684259824, + 0.7494840741626005, + 0.6894104604993242, + 0.7371611971197432, + 0.6427392384511877, + 0.5847748939175224, + 0.5118396580731392, + 0.5189878230973387, + 0.5013130375398106, + 0.4483794446409543, + 0.404418519263501, + 0.386302743223401, + 0.36748312424709273, + 0.35304323396633686, + 0.31929937663562014 + ], + "peak_layer": 5, + "peak_normalized": 1.9400717090010198, + "population_cv": 0.4898474013710311, + "rest_mean": 2.0753733493890008e-05, + "spike_contrast": 0.5576967968781986, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1574290692806243e-05, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.0974562832852826e-05, + 3.0309111025417224e-05, + 2.9632088626385666e-05, + 2.682000194909051e-05, + 3.74811097572092e-05, + 3.345023651490919e-05, + 3.063355325139128e-05, + 2.8995504180784337e-05, + 3.236792690586299e-05, + 2.8427341021597385e-05, + 2.649976158863865e-05, + 2.3016269551590085e-05, + 2.44294624280883e-05, + 2.2733656805939972e-05, + 1.9648006855277345e-05, + 1.7882191968965344e-05, + 1.876269561762456e-05, + 1.677599357208237e-05, + 1.4479616766038816e-05, + 1.3319027857505716e-05, + 1.4241545613913331e-05, + 1.2417365724104457e-05, + 1.1297526725684293e-05, + 9.888458407658618e-06, + 1.0026556992670521e-05, + 9.68508993537398e-06, + 8.662442269269377e-06, + 7.813141564838588e-06, + 7.46315481592319e-06, + 7.099570211721584e-06, + 6.820599537604721e-06, + 6.168686923047062e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.027523940661922097, + "first_to_last_ratio": 3.8954991154967806, + "imbalance_abs_log_ratio": 1.3598218137310631, + "last_quartile_mean": 0.007065574871376157, + "mean": 0.017132677690824494, + "normalized": [ + 1.6032842798207116, + 1.5688395364239456, + 1.5337959695585364, + 1.3882387052917204, + 1.940071709225884, + 1.7314283707937437, + 1.5856331209472507, + 1.5008455668619336, + 1.6754066305691406, + 1.4714366809574604, + 1.3716626672028296, + 1.1913525502963036, + 1.2645012070621602, + 1.176724101825654, + 1.0170067499274977, + 0.9256058277235238, + 0.9711818600927834, + 0.8683475735699788, + 0.7494840504534797, + 0.6894104671183356, + 0.7371612027997355, + 0.6427392553352459, + 0.5847748722004306, + 0.5118396849118217, + 0.5189878410918901, + 0.5013130358668625, + 0.44837945479745345, + 0.40441852259963446, + 0.38630275368745787, + 0.36748313889002426, + 0.3530432360984768, + 0.31929937599809327 + ], + "peak_layer": 5, + "peak_normalized": 1.940071709225884, + "population_cv": 0.4898473968405949, + "rest_mean": 0.018404618560991907, + "spike_contrast": 0.5576968062611827, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01026419699192047, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 7 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.027468552812933922, + 0.026878422126173973, + 0.02627803198993206, + 0.023784246295690536, + 0.033238623291254044, + 0.02966400422155857, + 0.02716614119708538, + 0.02571350336074829, + 0.02870420180261135, + 0.02520965039730072, + 0.02350025437772274, + 0.020411059260368347, + 0.021664291620254517, + 0.020160434767603874, + 0.017424048855900764, + 0.015858106315135956, + 0.016638945788145065, + 0.014877119101583958, + 0.01284066867083311, + 0.011811447329819202, + 0.012629545293748379, + 0.011011844500899315, + 0.010018759407103062, + 0.008769184350967407, + 0.008891651406884193, + 0.008588834665715694, + 0.00768194068223238, + 0.006928772199898958, + 0.0066184005700051785, + 0.0062959701754152775, + 0.006048575975000858, + 0.005470453295856714 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0978330894649844e-06, + "first_to_last_ratio": 3.7123595485838843, + "imbalance_abs_log_ratio": 1.3116676713818918, + "last_quartile_mean": 5.650942647150714e-07, + "mean": 1.3341314888393185e-06, + "normalized": [ + 1.5241148267186329, + 1.5504517836489524, + 1.525800361966804, + 1.340018587688921, + 1.9164903896693077, + 1.732090277095866, + 1.5197216980945576, + 1.4707803330175575, + 1.6808225666055663, + 1.5059009907724967, + 1.3766280212942543, + 1.196783029382642, + 1.2615435432700803, + 1.196050358398629, + 1.046816748775451, + 0.9373627935785216, + 0.9843932320660254, + 0.8775573102072941, + 0.7634158116096246, + 0.6842432896373823, + 0.7549093975240115, + 0.6636832533885216, + 0.5908781007193045, + 0.5110061412689536, + 0.5365346292575633, + 0.5196779133499698, + 0.4612664477302504, + 0.4092178350653177, + 0.3933915708760512, + 0.3701864458066216, + 0.3616807560410091, + 0.33658155547385904 + ], + "peak_layer": 5, + "peak_normalized": 1.9164903896693077, + "population_cv": 0.4775581328679472, + "rest_mean": 1.4301389744223006e-06, + "spike_contrast": 0.570357903168614, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.156910666912154e-07, + "top_five_layers": [ + 5, + 6, + 9, + 2, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.0333695829322096e-06, + 2.068506546493154e-06, + 2.035618308582343e-06, + 1.7877609934657812e-06, + 2.5568501769157592e-06, + 2.3108361801860156e-06, + 2.0275085717003094e-06, + 1.9622143554443028e-06, + 2.242438313260209e-06, + 2.009069930863916e-06, + 1.8366027916272287e-06, + 1.596665924807894e-06, + 1.6830649656185415e-06, + 1.5956884453771636e-06, + 1.3965911875857273e-06, + 1.2505652193794958e-06, + 1.3133100082995952e-06, + 1.170776840808685e-06, + 1.018497073346225e-06, + 9.128705187322339e-07, + 1.0071483984575025e-06, + 8.85440726960951e-07, + 7.883090802351944e-07, + 6.817493840571842e-07, + 7.158077437452448e-07, + 6.933186682545056e-07, + 6.153900926619826e-07, + 5.45950399555295e-07, + 5.248360821497045e-07, + 4.938773940921237e-07, + 4.825296855415218e-07, + 4.4904405172019324e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.398965625616256e-05, + "first_to_last_ratio": 3.369027618649736, + "imbalance_abs_log_ratio": 1.2146241622684435, + "last_quartile_mean": 7.120646955627308e-06, + "mean": 1.5399188356468585e-05, + "normalized": [ + 1.5913426512839013, + 1.5418831400755482, + 1.5021141560062705, + 1.3562195006410476, + 1.8805635081013792, + 1.6560198739348433, + 1.512688238851242, + 1.4219848670303201, + 1.6092469390600324, + 1.4165297374033572, + 1.3242582904493114, + 1.1412781561654184, + 1.2305966607275267, + 1.1486603349616307, + 1.006300256787253, + 0.9179301571399356, + 0.9843528747997042, + 0.8966403626677242, + 0.7866772482945813, + 0.7196405302926673, + 0.7789613145990362, + 0.6869940328779693, + 0.6281068759775288, + 0.5617779475196961, + 0.5746692969502841, + 0.5501909649183, + 0.500671447520295, + 0.45822743358408635, + 0.4365896209302579, + 0.4181501204665046, + 0.398078906415543, + 0.36265455356680404 + ], + "peak_layer": 5, + "peak_normalized": 1.8805635081013792, + "population_cv": 0.4466412974933768, + "rest_mean": 1.640840013418778e-05, + "spike_contrast": 0.6063628797090791, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.949444756784941e-06, + "top_five_layers": [ + 5, + 6, + 9, + 1, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.45053852268029e-05, + 2.37437488976866e-05, + 2.3131338821258396e-05, + 2.088467954308726e-05, + 2.8959151677554473e-05, + 2.5501361960778013e-05, + 2.3294171114685014e-05, + 2.1897412807447836e-05, + 2.478109672665596e-05, + 2.181340823881328e-05, + 2.0392502847244032e-05, + 1.7574757293914445e-05, + 1.895018976938445e-05, + 1.7688436855678447e-05, + 1.5496207197429612e-05, + 1.4135379387880675e-05, + 1.5158235328271985e-05, + 1.3807533832732588e-05, + 1.2114191122236662e-05, + 1.108188007492572e-05, + 1.1995372005912941e-05, + 1.0579150512057822e-05, + 9.672336091171019e-06, + 8.650924428366125e-06, + 8.849440746416803e-06, + 8.472494300804101e-06, + 7.7099339250708e-06, + 7.056330559862545e-06, + 6.72312580718426e-06, + 6.439172466343734e-06, + 6.130092060629977e-06, + 5.584585778706241e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7287268860854965e-05, + "first_to_last_ratio": 2.7170527496823964, + "imbalance_abs_log_ratio": 0.9995477449345839, + "last_quartile_mean": 6.362507633639325e-06, + "mean": 1.1818911630712137e-05, + "normalized": [ + 1.5531683121051287, + 1.4809645584609277, + 1.4249957798471882, + 1.2911504637898432, + 1.7654831615088196, + 1.5114625239638928, + 1.4072245252680236, + 1.266979301315116, + 1.5685685065032353, + 1.3590910493816417, + 1.257358469917724, + 1.0915032257961577, + 1.1980939699800082, + 1.1009846962306815, + 1.0217422520554849, + 0.917572741077425, + 1.0166318370670364, + 0.924735094262937, + 0.841915821310778, + 0.7538749370193452, + 0.8515117201007877, + 0.7457322090867645, + 0.703238735238229, + 0.6393538153043691, + 0.6469780758226346, + 0.6196156550317132, + 0.5947215623716378, + 0.5572499311933502, + 0.511086746910104, + 0.5033639872021801, + 0.45209119593165065, + 0.4215551389451849 + ], + "peak_layer": 5, + "peak_normalized": 1.7654831615088196, + "population_cv": 0.3798574342343211, + "rest_mean": 1.2437515848702148e-05, + "spike_contrast": 0.6816834612878747, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.478448853566078e-06, + "top_five_layers": [ + 5, + 9, + 1, + 6, + 2 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.8356759028392844e-05, + 1.7503389244666323e-05, + 1.6841899196151644e-05, + 1.5259993233485147e-05, + 2.086608947138302e-05, + 1.7863842003862374e-05, + 1.6631862308713607e-05, + 1.4974316400184762e-05, + 1.8538772565079853e-05, + 1.6062977010733448e-05, + 1.4860608644085005e-05, + 1.2900380170322023e-05, + 1.4160166756482795e-05, + 1.301244083151687e-05, + 1.2075881386408582e-05, + 1.0844711141544394e-05, + 1.2015481843263842e-05, + 1.092936236091191e-05, + 9.950528692570515e-06, + 8.909981261240318e-06, + 1.0063941772386897e-05, + 8.813743079372216e-06, + 8.311516467074398e-06, + 7.556466243840987e-06, + 7.646576705155894e-06, + 7.323182671825634e-06, + 7.028961590549443e-06, + 6.586087692994624e-06, + 6.040489097358659e-06, + 5.949214482825482e-06, + 5.343225893739145e-06, + 4.982322934665717e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.156662675493862e-05, + "first_to_last_ratio": 4.432028161582752, + "imbalance_abs_log_ratio": 1.488857303572528, + "last_quartile_mean": 1.3891298635826388e-05, + "mean": 3.828891439638937e-05, + "normalized": [ + 1.5694774864844574, + 1.533915591681154, + 1.4779364465001996, + 1.3758197901662568, + 1.9142257050168354, + 1.758664969175181, + 1.6344745902568742, + 1.5990784946964705, + 1.7383903346505918, + 1.5307661661772007, + 1.4323492558141884, + 1.2429043804879258, + 1.3172537017266024, + 1.2162602046192026, + 1.0552102311500136, + 0.9826538220680813, + 1.0107614019133901, + 0.8826720328909949, + 0.739013117314438, + 0.7006180982562433, + 0.737790003472691, + 0.6192970376209981, + 0.5556544994109884, + 0.47239582046029627, + 0.4740970917174548, + 0.4544292159121539, + 0.39665333426527116, + 0.35034805233110755, + 0.3379110425614577, + 0.30972196454961737, + 0.31015209244222897, + 0.26910402420943297 + ], + "peak_layer": 5, + "peak_normalized": 1.9142257050168354, + "population_cv": 0.5160505486771831, + "rest_mean": 4.132475101801834e-05, + "spike_contrast": 0.5298373517131695, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1895396639592945e-05, + "top_five_layers": [ + 5, + 6, + 9, + 7, + 8 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.009358912706375e-05, + 5.873196278116666e-05, + 5.658858208335005e-05, + 5.2678646170534194e-05, + 7.329362415475771e-05, + 6.733737245667726e-05, + 6.258225766941905e-05, + 6.122697959654033e-05, + 6.656107871094719e-05, + 5.861137469764799e-05, + 5.484309804160148e-05, + 4.758945942739956e-05, + 5.04362142237369e-05, + 4.6569282858399674e-05, + 4.0402854210697114e-05, + 3.76247480744496e-05, + 3.870095679303631e-05, + 3.379655390745029e-05, + 2.8296009986661375e-05, + 2.682590638869442e-05, + 2.8249178285477683e-05, + 2.3712211259407923e-05, + 2.1275407561915927e-05, + 1.8087523130816408e-05, + 1.8152662960346788e-05, + 1.7399601347278804e-05, + 1.5187425560725387e-05, + 1.3414446584647521e-05, + 1.293824698223034e-05, + 1.1858917787321843e-05, + 1.1875386917381547e-05, + 1.0303700946678873e-05 + ] + } + } + } + } + } + }, + "step": 100 + }, + { + "modes": { + "learned": { + "bits_per_byte": 3.0609520355704922, + "forward": { + "activation_sha256": { + "attention_branch_output": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", + "mlp_branch_output": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", + "post_attention_state": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", + "post_mlp_state": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", + "pre_attention_input": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", + "pre_mlp_input": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d" + }, + "logits_sha256": "b2134e9ad4fc330ab7158864584c7289a039b641b1626767dcba8e1a49e62a3d", + "loss_nats": 2.121690273284912, + "mixer_summary_sha256": "24302def7a1bb63aa3c7a0b6a721bf1efbd647790e9d6d8427706d94cca54b03" + }, + "loss_nats": 2.121690273284912, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.658259391784668, + "group": 1, + "latest_source_mass": 0.6112743616104126, + "layer": 1, + "max_source_mass": 0.6112743616104126, + "mixer_index": 1, + "normalized_entropy": 0.9496675601463258, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3887256383895874, + "median": 0.36817848682403564, + "p05": 0.2958398759365082, + "p95": 0.519643247127533 + }, + { + "label": "current_group_partial", + "mean": 0.6112743616104126, + "median": 0.6318215131759644, + "p05": 0.48035678267478943, + "p95": 0.7041600942611694 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6694332361221313, + "group": 1, + "latest_source_mass": 0.5721787214279175, + "layer": 2, + "max_source_mass": 0.5721787214279175, + "mixer_index": 2, + "normalized_entropy": 0.9657880099596494, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42782124876976013, + "median": 0.40344932675361633, + "p05": 0.3339918255805969, + "p95": 0.5918412208557129 + }, + { + "label": "current_group_partial", + "mean": 0.5721787214279175, + "median": 0.596550703048706, + "p05": 0.4081588089466095, + "p95": 0.6660082340240479 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6890689730644226, + "group": 1, + "latest_source_mass": 0.4954703748226166, + "layer": 2, + "max_source_mass": 0.5045296549797058, + "mixer_index": 3, + "normalized_entropy": 0.9941163902704933, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5045296549797058, + "median": 0.50140380859375, + "p05": 0.4251278042793274, + "p95": 0.583143949508667 + }, + { + "label": "current_group_partial", + "mean": 0.4954703748226166, + "median": 0.49859619140625, + "p05": 0.41685599088668823, + "p95": 0.5748721957206726 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6740086078643799, + "group": 1, + "latest_source_mass": 0.5477453470230103, + "layer": 3, + "max_source_mass": 0.5477453470230103, + "mixer_index": 4, + "normalized_entropy": 0.9723888760824149, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.45225459337234497, + "median": 0.42421072721481323, + "p05": 0.3554007411003113, + "p95": 0.6163875460624695 + }, + { + "label": "current_group_partial", + "mean": 0.5477453470230103, + "median": 0.575789213180542, + "p05": 0.3836124539375305, + "p95": 0.644599199295044 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.679557204246521, + "group": 1, + "latest_source_mass": 0.5538821220397949, + "layer": 3, + "max_source_mass": 0.5538821220397949, + "mixer_index": 5, + "normalized_entropy": 0.9803938085668243, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4461178183555603, + "median": 0.43929627537727356, + "p05": 0.3631950318813324, + "p95": 0.562574565410614 + }, + { + "label": "current_group_partial", + "mean": 0.5538821220397949, + "median": 0.5607037544250488, + "p05": 0.4374254047870636, + "p95": 0.63680499792099 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6740646362304688, + "group": 1, + "latest_source_mass": 0.5581084489822388, + "layer": 4, + "max_source_mass": 0.5581084489822388, + "mixer_index": 6, + "normalized_entropy": 0.9724697079283204, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.44189155101776123, + "median": 0.42476987838745117, + "p05": 0.33982762694358826, + "p95": 0.5899009108543396 + }, + { + "label": "current_group_partial", + "mean": 0.5581084489822388, + "median": 0.5752301216125488, + "p05": 0.410099059343338, + "p95": 0.6601723432540894 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6792672276496887, + "group": 1, + "latest_source_mass": 0.5544062852859497, + "layer": 4, + "max_source_mass": 0.5544062852859497, + "mixer_index": 7, + "normalized_entropy": 0.9799754607686005, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4455936849117279, + "median": 0.44227513670921326, + "p05": 0.348201721906662, + "p95": 0.5532940626144409 + }, + { + "label": "current_group_partial", + "mean": 0.5544062852859497, + "median": 0.5577248334884644, + "p05": 0.4467058777809143, + "p95": 0.6517983078956604 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674970269203186, + "group": 2, + "latest_source_mass": 0.5532590746879578, + "layer": 5, + "max_source_mass": 0.5532590746879578, + "mixer_index": 8, + "normalized_entropy": 0.9737762601269252, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.44674086570739746, + "median": 0.43155425786972046, + "p05": 0.3413114547729492, + "p95": 0.5883275270462036 + }, + { + "label": "completed_group_1", + "mean": 0.5532590746879578, + "median": 0.5684457421302795, + "p05": 0.4116724729537964, + "p95": 0.6586886048316956 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.093062162399292, + "group": 2, + "latest_source_mass": 0.35190510749816895, + "layer": 5, + "max_source_mass": 0.35190510749816895, + "mixer_index": 9, + "normalized_entropy": 0.99494805735739, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3082374334335327, + "median": 0.30379241704940796, + "p05": 0.26716136932373047, + "p95": 0.3615886867046356 + }, + { + "label": "completed_group_1", + "mean": 0.33985745906829834, + "median": 0.34154799580574036, + "p05": 0.2781605124473572, + "p95": 0.3903803527355194 + }, + { + "label": "current_group_partial", + "mean": 0.35190510749816895, + "median": 0.35370177030563354, + "p05": 0.3136623501777649, + "p95": 0.3873046040534973 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0853323936462402, + "group": 2, + "latest_source_mass": 0.37136080861091614, + "layer": 6, + "max_source_mass": 0.37136080861091614, + "mixer_index": 10, + "normalized_entropy": 0.9879121186256079, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30180174112319946, + "median": 0.28923720121383667, + "p05": 0.2442891150712967, + "p95": 0.3929156959056854 + }, + { + "label": "completed_group_1", + "mean": 0.3268374800682068, + "median": 0.3262294828891754, + "p05": 0.2930828928947449, + "p95": 0.3670434355735779 + }, + { + "label": "current_group_partial", + "mean": 0.37136080861091614, + "median": 0.38435930013656616, + "p05": 0.2483101636171341, + "p95": 0.4514022171497345 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0921990871429443, + "group": 2, + "latest_source_mass": 0.3605259656906128, + "layer": 6, + "max_source_mass": 0.3605259656906128, + "mixer_index": 11, + "normalized_entropy": 0.9941624524035314, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3034760355949402, + "median": 0.30043965578079224, + "p05": 0.2575032711029053, + "p95": 0.3658854067325592 + }, + { + "label": "completed_group_1", + "mean": 0.3359980285167694, + "median": 0.33590996265411377, + "p05": 0.29249265789985657, + "p95": 0.3839111328125 + }, + { + "label": "current_group_partial", + "mean": 0.3605259656906128, + "median": 0.3592720031738281, + "p05": 0.3142351508140564, + "p95": 0.40569764375686646 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.076724648475647, + "group": 2, + "latest_source_mass": 0.3958718776702881, + "layer": 7, + "max_source_mass": 0.3958718776702881, + "mixer_index": 12, + "normalized_entropy": 0.9800770113185262, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29931002855300903, + "median": 0.2845022678375244, + "p05": 0.24265936017036438, + "p95": 0.3946967124938965 + }, + { + "label": "completed_group_1", + "mean": 0.30481812357902527, + "median": 0.2940869927406311, + "p05": 0.26745280623435974, + "p95": 0.36952510476112366 + }, + { + "label": "current_group_partial", + "mean": 0.3958718776702881, + "median": 0.4253297448158264, + "p05": 0.24267785251140594, + "p95": 0.4795311689376831 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0856893062591553, + "group": 2, + "latest_source_mass": 0.3888435363769531, + "layer": 7, + "max_source_mass": 0.3888435363769531, + "mixer_index": 13, + "normalized_entropy": 0.988236994486361, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2801036834716797, + "median": 0.2685231566429138, + "p05": 0.24713139235973358, + "p95": 0.3503561317920685 + }, + { + "label": "completed_group_1", + "mean": 0.3310527801513672, + "median": 0.330751895904541, + "p05": 0.2913045883178711, + "p95": 0.37040621042251587 + }, + { + "label": "current_group_partial", + "mean": 0.3888435363769531, + "median": 0.3877270221710205, + "p05": 0.3418702185153961, + "p95": 0.44118839502334595 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0752873420715332, + "group": 2, + "latest_source_mass": 0.3948463201522827, + "layer": 8, + "max_source_mass": 0.3948463201522827, + "mixer_index": 14, + "normalized_entropy": 0.9787687186488199, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29463040828704834, + "median": 0.28890830278396606, + "p05": 0.23444202542304993, + "p95": 0.37455716729164124 + }, + { + "label": "completed_group_1", + "mean": 0.3105233311653137, + "median": 0.2965858280658722, + "p05": 0.2587319016456604, + "p95": 0.39421546459198 + }, + { + "label": "current_group_partial", + "mean": 0.3948463201522827, + "median": 0.4135509133338928, + "p05": 0.24363072216510773, + "p95": 0.5001816153526306 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.088913917541504, + "group": 2, + "latest_source_mass": 0.3843393623828888, + "layer": 8, + "max_source_mass": 0.3843393623828888, + "mixer_index": 15, + "normalized_entropy": 0.9911721621661782, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29400017857551575, + "median": 0.2848791778087616, + "p05": 0.2631606459617615, + "p95": 0.35448360443115234 + }, + { + "label": "completed_group_1", + "mean": 0.32166045904159546, + "median": 0.32034605741500854, + "p05": 0.2860022187232971, + "p95": 0.3604046106338501 + }, + { + "label": "current_group_partial", + "mean": 0.3843393623828888, + "median": 0.38500386476516724, + "p05": 0.336969792842865, + "p95": 0.43113112449645996 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.054490327835083, + "group": 3, + "latest_source_mass": 0.42586958408355713, + "layer": 9, + "max_source_mass": 0.42586958408355713, + "mixer_index": 16, + "normalized_entropy": 0.959838460494086, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2850497364997864, + "median": 0.2810911536216736, + "p05": 0.21189084649085999, + "p95": 0.37678587436676025 + }, + { + "label": "completed_group_1", + "mean": 0.2890806794166565, + "median": 0.26693958044052124, + "p05": 0.21384978294372559, + "p95": 0.4104633927345276 + }, + { + "label": "completed_group_2", + "mean": 0.42586958408355713, + "median": 0.45180952548980713, + "p05": 0.2289215475320816, + "p95": 0.5656958818435669 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3700100183486938, + "group": 3, + "latest_source_mass": 0.28284209966659546, + "layer": 9, + "max_source_mass": 0.28284209966659546, + "mixer_index": 17, + "normalized_entropy": 0.9882533297199292, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21610885858535767, + "median": 0.21061542630195618, + "p05": 0.18654318153858185, + "p95": 0.26685822010040283 + }, + { + "label": "completed_group_1", + "mean": 0.22824729979038239, + "median": 0.22523541748523712, + "p05": 0.19356240332126617, + "p95": 0.2682129740715027 + }, + { + "label": "completed_group_2", + "mean": 0.2728016972541809, + "median": 0.27193963527679443, + "p05": 0.21500371396541595, + "p95": 0.3283920884132385 + }, + { + "label": "current_group_partial", + "mean": 0.28284209966659546, + "median": 0.2946581244468689, + "p05": 0.19060052931308746, + "p95": 0.3525015413761139 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3620216846466064, + "group": 3, + "latest_source_mass": 0.2944645285606384, + "layer": 10, + "max_source_mass": 0.2944645285606384, + "mixer_index": 18, + "normalized_entropy": 0.9824909650114454, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22791308164596558, + "median": 0.22349298000335693, + "p05": 0.18910717964172363, + "p95": 0.2777109742164612 + }, + { + "label": "completed_group_1", + "mean": 0.2214205265045166, + "median": 0.19864881038665771, + "p05": 0.15984180569648743, + "p95": 0.34694117307662964 + }, + { + "label": "completed_group_2", + "mean": 0.2562018632888794, + "median": 0.2503240704536438, + "p05": 0.20511329174041748, + "p95": 0.32819920778274536 + }, + { + "label": "current_group_partial", + "mean": 0.2944645285606384, + "median": 0.30793535709381104, + "p05": 0.16727104783058167, + "p95": 0.3644053339958191 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3784711360931396, + "group": 3, + "latest_source_mass": 0.2763957977294922, + "layer": 10, + "max_source_mass": 0.2763957977294922, + "mixer_index": 19, + "normalized_entropy": 0.994356736025074, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23347607254981995, + "median": 0.23056401312351227, + "p05": 0.20095914602279663, + "p95": 0.2848108112812042 + }, + { + "label": "completed_group_1", + "mean": 0.23307976126670837, + "median": 0.2289559543132782, + "p05": 0.205013245344162, + "p95": 0.2760506868362427 + }, + { + "label": "completed_group_2", + "mean": 0.2570483684539795, + "median": 0.2558031678199768, + "p05": 0.21930555999279022, + "p95": 0.29454779624938965 + }, + { + "label": "current_group_partial", + "mean": 0.2763957977294922, + "median": 0.2743210792541504, + "p05": 0.2263108193874359, + "p95": 0.3325941264629364 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3619427680969238, + "group": 3, + "latest_source_mass": 0.302674800157547, + "layer": 11, + "max_source_mass": 0.302674800157547, + "mixer_index": 20, + "normalized_entropy": 0.9824340387540098, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24715131521224976, + "median": 0.24610307812690735, + "p05": 0.21518638730049133, + "p95": 0.2835894525051117 + }, + { + "label": "completed_group_1", + "mean": 0.22003203630447388, + "median": 0.19626209139823914, + "p05": 0.16581544280052185, + "p95": 0.3317301273345947 + }, + { + "label": "completed_group_2", + "mean": 0.23014184832572937, + "median": 0.2278168797492981, + "p05": 0.19933417439460754, + "p95": 0.272026926279068 + }, + { + "label": "current_group_partial", + "mean": 0.302674800157547, + "median": 0.3237387239933014, + "p05": 0.16210556030273438, + "p95": 0.38430726528167725 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3804662227630615, + "group": 3, + "latest_source_mass": 0.28360050916671753, + "layer": 11, + "max_source_mass": 0.28360050916671753, + "mixer_index": 21, + "normalized_entropy": 0.995795886847494, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24302780628204346, + "median": 0.24201302230358124, + "p05": 0.21055130660533905, + "p95": 0.276764452457428 + }, + { + "label": "completed_group_1", + "mean": 0.23108476400375366, + "median": 0.22807812690734863, + "p05": 0.20644722878932953, + "p95": 0.2614594101905823 + }, + { + "label": "completed_group_2", + "mean": 0.24228686094284058, + "median": 0.2414286881685257, + "p05": 0.21957996487617493, + "p95": 0.2676010727882385 + }, + { + "label": "current_group_partial", + "mean": 0.28360050916671753, + "median": 0.2854132354259491, + "p05": 0.246682807803154, + "p95": 0.3173881471157074 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3502743244171143, + "group": 3, + "latest_source_mass": 0.31918564438819885, + "layer": 12, + "max_source_mass": 0.31918564438819885, + "mixer_index": 22, + "normalized_entropy": 0.974017035838133, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2455797642469406, + "median": 0.2419765591621399, + "p05": 0.2023906707763672, + "p95": 0.29628193378448486 + }, + { + "label": "completed_group_1", + "mean": 0.21441738307476044, + "median": 0.19096502661705017, + "p05": 0.1489589512348175, + "p95": 0.3387751877307892 + }, + { + "label": "completed_group_2", + "mean": 0.2208172082901001, + "median": 0.21868902444839478, + "p05": 0.19425739347934723, + "p95": 0.25476256012916565 + }, + { + "label": "current_group_partial", + "mean": 0.31918564438819885, + "median": 0.34166574478149414, + "p05": 0.153193861246109, + "p95": 0.42244112491607666 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3820215463638306, + "group": 3, + "latest_source_mass": 0.27603140473365784, + "layer": 12, + "max_source_mass": 0.27603140473365784, + "mixer_index": 23, + "normalized_entropy": 0.9969178156703975, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24522465467453003, + "median": 0.24368876218795776, + "p05": 0.22978872060775757, + "p95": 0.2647436261177063 + }, + { + "label": "completed_group_1", + "mean": 0.22846508026123047, + "median": 0.22824203968048096, + "p05": 0.21080705523490906, + "p95": 0.24792201817035675 + }, + { + "label": "completed_group_2", + "mean": 0.25027889013290405, + "median": 0.25114861130714417, + "p05": 0.2239076942205429, + "p95": 0.27430635690689087 + }, + { + "label": "current_group_partial", + "mean": 0.27603140473365784, + "median": 0.2767183184623718, + "p05": 0.23810210824012756, + "p95": 0.31143108010292053 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3491160869598389, + "group": 4, + "latest_source_mass": 0.3227139413356781, + "layer": 13, + "max_source_mass": 0.3227139413356781, + "mixer_index": 24, + "normalized_entropy": 0.9731815441202416, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24498310685157776, + "median": 0.24184581637382507, + "p05": 0.20811651647090912, + "p95": 0.2852743864059448 + }, + { + "label": "completed_group_1", + "mean": 0.21178749203681946, + "median": 0.1895410120487213, + "p05": 0.14949846267700195, + "p95": 0.3285062313079834 + }, + { + "label": "completed_group_2", + "mean": 0.2205154001712799, + "median": 0.2162817418575287, + "p05": 0.18882519006729126, + "p95": 0.26559314131736755 + }, + { + "label": "completed_group_3", + "mean": 0.3227139413356781, + "median": 0.34892457723617554, + "p05": 0.15726026892662048, + "p95": 0.42967018485069275 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.6032840013504028, + "group": 4, + "latest_source_mass": 0.22103822231292725, + "layer": 13, + "max_source_mass": 0.22103822231292725, + "mixer_index": 25, + "normalized_entropy": 0.9961763600595251, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20074918866157532, + "median": 0.20095838606357574, + "p05": 0.18337497115135193, + "p95": 0.2181507796049118 + }, + { + "label": "completed_group_1", + "mean": 0.17959120869636536, + "median": 0.17876555025577545, + "p05": 0.1576874852180481, + "p95": 0.20805387198925018 + }, + { + "label": "completed_group_2", + "mean": 0.18619459867477417, + "median": 0.18535691499710083, + "p05": 0.1644289195537567, + "p95": 0.21231110394001007 + }, + { + "label": "completed_group_3", + "mean": 0.21242676675319672, + "median": 0.2123909741640091, + "p05": 0.19269250333309174, + "p95": 0.23113366961479187 + }, + { + "label": "current_group_partial", + "mean": 0.22103822231292725, + "median": 0.2211262434720993, + "p05": 0.1813979148864746, + "p95": 0.26087528467178345 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5699318647384644, + "group": 4, + "latest_source_mass": 0.2694075107574463, + "layer": 14, + "max_source_mass": 0.2694075107574463, + "mixer_index": 26, + "normalized_entropy": 0.9754535124403232, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1938496232032776, + "median": 0.19102387130260468, + "p05": 0.16569040715694427, + "p95": 0.23268038034439087 + }, + { + "label": "completed_group_1", + "mean": 0.17019829154014587, + "median": 0.14846302568912506, + "p05": 0.11795832216739655, + "p95": 0.2854715883731842 + }, + { + "label": "completed_group_2", + "mean": 0.1625727415084839, + "median": 0.15400215983390808, + "p05": 0.1330171525478363, + "p95": 0.22402943670749664 + }, + { + "label": "completed_group_3", + "mean": 0.20397180318832397, + "median": 0.2051275074481964, + "p05": 0.14910343289375305, + "p95": 0.25531816482543945 + }, + { + "label": "current_group_partial", + "mean": 0.2694075107574463, + "median": 0.2924001216888428, + "p05": 0.12448020279407501, + "p95": 0.3373839557170868 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6068798303604126, + "group": 4, + "latest_source_mass": 0.21404394507408142, + "layer": 14, + "max_source_mass": 0.21404394507408142, + "mixer_index": 27, + "normalized_entropy": 0.9984105742421472, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19717517495155334, + "median": 0.19712674617767334, + "p05": 0.18285326659679413, + "p95": 0.21078822016716003 + }, + { + "label": "completed_group_1", + "mean": 0.1942940056324005, + "median": 0.19466334581375122, + "p05": 0.18043261766433716, + "p95": 0.20777744054794312 + }, + { + "label": "completed_group_2", + "mean": 0.19104912877082825, + "median": 0.18969203531742096, + "p05": 0.17138653993606567, + "p95": 0.2142438441514969 + }, + { + "label": "completed_group_3", + "mean": 0.20343773066997528, + "median": 0.2029801905155182, + "p05": 0.18992920219898224, + "p95": 0.2184751033782959 + }, + { + "label": "current_group_partial", + "mean": 0.21404394507408142, + "median": 0.21432773768901825, + "p05": 0.1852298229932785, + "p95": 0.24286989867687225 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5761134624481201, + "group": 4, + "latest_source_mass": 0.26560431718826294, + "layer": 15, + "max_source_mass": 0.26560431718826294, + "mixer_index": 28, + "normalized_entropy": 0.979294355048726, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1940101534128189, + "median": 0.19362978637218475, + "p05": 0.1631820648908615, + "p95": 0.2295525223016739 + }, + { + "label": "completed_group_1", + "mean": 0.18250693380832672, + "median": 0.16429191827774048, + "p05": 0.13507941365242004, + "p95": 0.2775976359844208 + }, + { + "label": "completed_group_2", + "mean": 0.16849923133850098, + "median": 0.16119162738323212, + "p05": 0.1375115066766739, + "p95": 0.2266949564218521 + }, + { + "label": "completed_group_3", + "mean": 0.18937936425209045, + "median": 0.18853023648262024, + "p05": 0.1678447425365448, + "p95": 0.21501585841178894 + }, + { + "label": "current_group_partial", + "mean": 0.26560431718826294, + "median": 0.28881219029426575, + "p05": 0.117766834795475, + "p95": 0.34617751836776733 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6027262210845947, + "group": 4, + "latest_source_mass": 0.22770702838897705, + "layer": 15, + "max_source_mass": 0.22770702838897705, + "mixer_index": 29, + "normalized_entropy": 0.9958297916945706, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20452985167503357, + "median": 0.20441509783267975, + "p05": 0.17931760847568512, + "p95": 0.2289508432149887 + }, + { + "label": "completed_group_1", + "mean": 0.18702909350395203, + "median": 0.186691552400589, + "p05": 0.1675943285226822, + "p95": 0.20997506380081177 + }, + { + "label": "completed_group_2", + "mean": 0.17977532744407654, + "median": 0.177423357963562, + "p05": 0.1569271981716156, + "p95": 0.20937731862068176 + }, + { + "label": "completed_group_3", + "mean": 0.20095868408679962, + "median": 0.2005838304758072, + "p05": 0.1845097541809082, + "p95": 0.21870870888233185 + }, + { + "label": "current_group_partial", + "mean": 0.22770702838897705, + "median": 0.22757767140865326, + "p05": 0.18680104613304138, + "p95": 0.2673209607601166 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.577937126159668, + "group": 4, + "latest_source_mass": 0.2687118649482727, + "layer": 16, + "max_source_mass": 0.2687118649482727, + "mixer_index": 30, + "normalized_entropy": 0.9804274610215993, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1962222456932068, + "median": 0.19565686583518982, + "p05": 0.16995516419410706, + "p95": 0.22565382719039917 + }, + { + "label": "completed_group_1", + "mean": 0.1818992793560028, + "median": 0.16701740026474, + "p05": 0.138178750872612, + "p95": 0.2653764486312866 + }, + { + "label": "completed_group_2", + "mean": 0.17167484760284424, + "median": 0.16360890865325928, + "p05": 0.14424335956573486, + "p95": 0.22631293535232544 + }, + { + "label": "completed_group_3", + "mean": 0.18149182200431824, + "median": 0.18099229037761688, + "p05": 0.16264864802360535, + "p95": 0.20300358533859253 + }, + { + "label": "current_group_partial", + "mean": 0.2687118649482727, + "median": 0.29608893394470215, + "p05": 0.1254507452249527, + "p95": 0.339657723903656 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5977532863616943, + "group": 4, + "latest_source_mass": 0.23852142691612244, + "layer": 16, + "max_source_mass": 0.23852142691612244, + "mixer_index": 31, + "normalized_entropy": 0.9927399336239481, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21533392369747162, + "median": 0.21377520263195038, + "p05": 0.1841924786567688, + "p95": 0.24801591038703918 + }, + { + "label": "completed_group_1", + "mean": 0.18402470648288727, + "median": 0.18508511781692505, + "p05": 0.15855011343955994, + "p95": 0.21014195680618286 + }, + { + "label": "completed_group_2", + "mean": 0.17019468545913696, + "median": 0.16691477596759796, + "p05": 0.1446647047996521, + "p95": 0.20511509478092194 + }, + { + "label": "completed_group_3", + "mean": 0.1919252574443817, + "median": 0.191789448261261, + "p05": 0.17424410581588745, + "p95": 0.20975348353385925 + }, + { + "label": "current_group_partial", + "mean": 0.23852142691612244, + "median": 0.24018850922584534, + "p05": 0.18989041447639465, + "p95": 0.27930569648742676 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.578121542930603, + "group": 5, + "latest_source_mass": 0.26854145526885986, + "layer": 17, + "max_source_mass": 0.26854145526885986, + "mixer_index": 32, + "normalized_entropy": 0.9805420456038999, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20422279834747314, + "median": 0.2033933699131012, + "p05": 0.17994040250778198, + "p95": 0.23156529664993286 + }, + { + "label": "completed_group_1", + "mean": 0.1835813671350479, + "median": 0.17072421312332153, + "p05": 0.14088554680347443, + "p95": 0.26194027066230774 + }, + { + "label": "completed_group_2", + "mean": 0.16642534732818604, + "median": 0.1602073311805725, + "p05": 0.13873112201690674, + "p95": 0.22039896249771118 + }, + { + "label": "completed_group_3", + "mean": 0.17722904682159424, + "median": 0.17722415924072266, + "p05": 0.1577582061290741, + "p95": 0.19813896715641022 + }, + { + "label": "completed_group_4", + "mean": 0.26854145526885986, + "median": 0.2872956693172455, + "p05": 0.12619024515151978, + "p95": 0.3464823067188263 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.782798409461975, + "group": 5, + "latest_source_mass": 0.19473549723625183, + "layer": 17, + "max_source_mass": 0.19473549723625183, + "mixer_index": 33, + "normalized_entropy": 0.99499873731939, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15961387753486633, + "median": 0.15747272968292236, + "p05": 0.12699076533317566, + "p95": 0.19599634408950806 + }, + { + "label": "completed_group_1", + "mean": 0.16475997865200043, + "median": 0.16349202394485474, + "p05": 0.1474122852087021, + "p95": 0.1848660409450531 + }, + { + "label": "completed_group_2", + "mean": 0.14714640378952026, + "median": 0.14494061470031738, + "p05": 0.12754470109939575, + "p95": 0.1751968413591385 + }, + { + "label": "completed_group_3", + "mean": 0.15768736600875854, + "median": 0.15663693845272064, + "p05": 0.13646987080574036, + "p95": 0.1821092814207077 + }, + { + "label": "completed_group_4", + "mean": 0.1760568618774414, + "median": 0.17550913989543915, + "p05": 0.15776243805885315, + "p95": 0.19570523500442505 + }, + { + "label": "current_group_partial", + "mean": 0.19473549723625183, + "median": 0.1956680715084076, + "p05": 0.14822278916835785, + "p95": 0.22800904512405396 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7707297801971436, + "group": 5, + "latest_source_mass": 0.20674791932106018, + "layer": 18, + "max_source_mass": 0.20674791932106018, + "mixer_index": 34, + "normalized_entropy": 0.9882631070787802, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16306114196777344, + "median": 0.16283683478832245, + "p05": 0.13633866608142853, + "p95": 0.19143974781036377 + }, + { + "label": "completed_group_1", + "mean": 0.1578725278377533, + "median": 0.14938931167125702, + "p05": 0.13056670129299164, + "p95": 0.21463912725448608 + }, + { + "label": "completed_group_2", + "mean": 0.139655202627182, + "median": 0.1338472068309784, + "p05": 0.11858384311199188, + "p95": 0.18817788362503052 + }, + { + "label": "completed_group_3", + "mean": 0.14299750328063965, + "median": 0.14077699184417725, + "p05": 0.12273441255092621, + "p95": 0.17249858379364014 + }, + { + "label": "completed_group_4", + "mean": 0.18966571986675262, + "median": 0.19467708468437195, + "p05": 0.13032010197639465, + "p95": 0.23041488230228424 + }, + { + "label": "current_group_partial", + "mean": 0.20674791932106018, + "median": 0.21543893218040466, + "p05": 0.11787950992584229, + "p95": 0.24730797111988068 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.781312108039856, + "group": 5, + "latest_source_mass": 0.20407646894454956, + "layer": 18, + "max_source_mass": 0.20407646894454956, + "mixer_index": 35, + "normalized_entropy": 0.9941692167014471, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15919369459152222, + "median": 0.15759795904159546, + "p05": 0.13659372925758362, + "p95": 0.19095730781555176 + }, + { + "label": "completed_group_1", + "mean": 0.1684478521347046, + "median": 0.16807237267494202, + "p05": 0.15075388550758362, + "p95": 0.1875023990869522 + }, + { + "label": "completed_group_2", + "mean": 0.14742717146873474, + "median": 0.14584535360336304, + "p05": 0.12703484296798706, + "p95": 0.1739335060119629 + }, + { + "label": "completed_group_3", + "mean": 0.1539241522550583, + "median": 0.1541770100593567, + "p05": 0.13218382000923157, + "p95": 0.1755600869655609 + }, + { + "label": "completed_group_4", + "mean": 0.1669306606054306, + "median": 0.16647151112556458, + "p05": 0.15041440725326538, + "p95": 0.18558931350708008 + }, + { + "label": "current_group_partial", + "mean": 0.20407646894454956, + "median": 0.20521500706672668, + "p05": 0.1562502235174179, + "p95": 0.24481025338172913 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7736209630966187, + "group": 5, + "latest_source_mass": 0.2125461846590042, + "layer": 19, + "max_source_mass": 0.2125461846590042, + "mixer_index": 36, + "normalized_entropy": 0.9898767069782805, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15695160627365112, + "median": 0.15683278441429138, + "p05": 0.12300770729780197, + "p95": 0.196682408452034 + }, + { + "label": "completed_group_1", + "mean": 0.16567477583885193, + "median": 0.16040010750293732, + "p05": 0.14332951605319977, + "p95": 0.20587366819381714 + }, + { + "label": "completed_group_2", + "mean": 0.1478763222694397, + "median": 0.14480885863304138, + "p05": 0.12841497361660004, + "p95": 0.18284933269023895 + }, + { + "label": "completed_group_3", + "mean": 0.14320014417171478, + "median": 0.14095112681388855, + "p05": 0.12106460332870483, + "p95": 0.17675133049488068 + }, + { + "label": "completed_group_4", + "mean": 0.17375096678733826, + "median": 0.17515000700950623, + "p05": 0.1435358226299286, + "p95": 0.19906878471374512 + }, + { + "label": "current_group_partial", + "mean": 0.2125461846590042, + "median": 0.2186175137758255, + "p05": 0.12126382440328598, + "p95": 0.2633163630962372 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.781569480895996, + "group": 5, + "latest_source_mass": 0.20039017498493195, + "layer": 19, + "max_source_mass": 0.20039017498493195, + "mixer_index": 37, + "normalized_entropy": 0.9943128592274447, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15368077158927917, + "median": 0.1525704562664032, + "p05": 0.12359238415956497, + "p95": 0.20425409078598022 + }, + { + "label": "completed_group_1", + "mean": 0.1725173443555832, + "median": 0.17324355244636536, + "p05": 0.1533699631690979, + "p95": 0.1897408664226532 + }, + { + "label": "completed_group_2", + "mean": 0.14978237450122833, + "median": 0.14767895638942719, + "p05": 0.1299348920583725, + "p95": 0.17630663514137268 + }, + { + "label": "completed_group_3", + "mean": 0.15674246847629547, + "median": 0.15763753652572632, + "p05": 0.13473035395145416, + "p95": 0.17680563032627106 + }, + { + "label": "completed_group_4", + "mean": 0.16688686609268188, + "median": 0.16692788898944855, + "p05": 0.14797358214855194, + "p95": 0.18507987260818481 + }, + { + "label": "current_group_partial", + "mean": 0.20039017498493195, + "median": 0.20044702291488647, + "p05": 0.1619955599308014, + "p95": 0.23957164585590363 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7793617248535156, + "group": 5, + "latest_source_mass": 0.20417512953281403, + "layer": 20, + "max_source_mass": 0.20417512953281403, + "mixer_index": 38, + "normalized_entropy": 0.9930806871193036, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1678246557712555, + "median": 0.17157375812530518, + "p05": 0.13025718927383423, + "p95": 0.19556939601898193 + }, + { + "label": "completed_group_1", + "mean": 0.1683025360107422, + "median": 0.16609728336334229, + "p05": 0.15072931349277496, + "p95": 0.19358061254024506 + }, + { + "label": "completed_group_2", + "mean": 0.14878366887569427, + "median": 0.1468905806541443, + "p05": 0.13322587311267853, + "p95": 0.1712985783815384 + }, + { + "label": "completed_group_3", + "mean": 0.14351718127727509, + "median": 0.1410684883594513, + "p05": 0.12425544857978821, + "p95": 0.1736954152584076 + }, + { + "label": "completed_group_4", + "mean": 0.16739682853221893, + "median": 0.16813713312149048, + "p05": 0.14868372678756714, + "p95": 0.1844603419303894 + }, + { + "label": "current_group_partial", + "mean": 0.20417512953281403, + "median": 0.20674002170562744, + "p05": 0.14016404747962952, + "p95": 0.2501453757286072 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7772380113601685, + "group": 5, + "latest_source_mass": 0.19795814156532288, + "layer": 20, + "max_source_mass": 0.19795814156532288, + "mixer_index": 39, + "normalized_entropy": 0.9918954200509164, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15367703139781952, + "median": 0.15482746064662933, + "p05": 0.11167150735855103, + "p95": 0.2207922786474228 + }, + { + "label": "completed_group_1", + "mean": 0.18459689617156982, + "median": 0.184708833694458, + "p05": 0.160706028342247, + "p95": 0.20910656452178955 + }, + { + "label": "completed_group_2", + "mean": 0.14832255244255066, + "median": 0.1467352658510208, + "p05": 0.12785059213638306, + "p95": 0.17438356578350067 + }, + { + "label": "completed_group_3", + "mean": 0.15016385912895203, + "median": 0.14991271495819092, + "p05": 0.12576226890087128, + "p95": 0.17565840482711792 + }, + { + "label": "completed_group_4", + "mean": 0.1652815043926239, + "median": 0.16595271229743958, + "p05": 0.14006288349628448, + "p95": 0.18747010827064514 + }, + { + "label": "current_group_partial", + "mean": 0.19795814156532288, + "median": 0.19790686666965485, + "p05": 0.1500549614429474, + "p95": 0.24641191959381104 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.780207633972168, + "group": 6, + "latest_source_mass": 0.20078188180923462, + "layer": 21, + "max_source_mass": 0.20078188180923462, + "mixer_index": 40, + "normalized_entropy": 0.9935527979875202, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16951784491539001, + "median": 0.1744978278875351, + "p05": 0.12857238948345184, + "p95": 0.2016112506389618 + }, + { + "label": "completed_group_1", + "mean": 0.17200809717178345, + "median": 0.17267316579818726, + "p05": 0.15072743594646454, + "p95": 0.19161207973957062 + }, + { + "label": "completed_group_2", + "mean": 0.15038041770458221, + "median": 0.14867597818374634, + "p05": 0.13418853282928467, + "p95": 0.17207789421081543 + }, + { + "label": "completed_group_3", + "mean": 0.14669056236743927, + "median": 0.14433687925338745, + "p05": 0.12725189328193665, + "p95": 0.1738971769809723 + }, + { + "label": "completed_group_4", + "mean": 0.16062124073505402, + "median": 0.16014984250068665, + "p05": 0.14366847276687622, + "p95": 0.17978575825691223 + }, + { + "label": "completed_group_5", + "mean": 0.20078188180923462, + "median": 0.20047783851623535, + "p05": 0.14466112852096558, + "p95": 0.25333014130592346 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.937079668045044, + "group": 6, + "latest_source_mass": 0.16695083677768707, + "layer": 21, + "max_source_mass": 0.16695083677768707, + "mixer_index": 41, + "normalized_entropy": 0.9954620304464951, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13008031249046326, + "median": 0.12692952156066895, + "p05": 0.10487981885671616, + "p95": 0.16793549060821533 + }, + { + "label": "completed_group_1", + "mean": 0.15163564682006836, + "median": 0.15097364783287048, + "p05": 0.13223227858543396, + "p95": 0.17222966253757477 + }, + { + "label": "completed_group_2", + "mean": 0.1323973685503006, + "median": 0.131024569272995, + "p05": 0.11479418724775314, + "p95": 0.15471340715885162 + }, + { + "label": "completed_group_3", + "mean": 0.13641127943992615, + "median": 0.13619393110275269, + "p05": 0.11877292394638062, + "p95": 0.15510697662830353 + }, + { + "label": "completed_group_4", + "mean": 0.13806137442588806, + "median": 0.1375710368156433, + "p05": 0.11978909373283386, + "p95": 0.15855638682842255 + }, + { + "label": "completed_group_5", + "mean": 0.1444631814956665, + "median": 0.1438722014427185, + "p05": 0.12040875852108002, + "p95": 0.17076769471168518 + }, + { + "label": "current_group_partial", + "mean": 0.16695083677768707, + "median": 0.17087092995643616, + "p05": 0.13015522062778473, + "p95": 0.195466548204422 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.939245343208313, + "group": 6, + "latest_source_mass": 0.16357660293579102, + "layer": 22, + "max_source_mass": 0.16357660293579102, + "mixer_index": 42, + "normalized_entropy": 0.9965749673230103, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14692580699920654, + "median": 0.1518530696630478, + "p05": 0.11718790233135223, + "p95": 0.16568630933761597 + }, + { + "label": "completed_group_1", + "mean": 0.14671877026557922, + "median": 0.14763988554477692, + "p05": 0.12195808440446854, + "p95": 0.1661134660243988 + }, + { + "label": "completed_group_2", + "mean": 0.13336658477783203, + "median": 0.13267168402671814, + "p05": 0.12229838222265244, + "p95": 0.14625124633312225 + }, + { + "label": "completed_group_3", + "mean": 0.1307726800441742, + "median": 0.12948819994926453, + "p05": 0.11789381504058838, + "p95": 0.1479169726371765 + }, + { + "label": "completed_group_4", + "mean": 0.131200909614563, + "median": 0.1309259533882141, + "p05": 0.11869271099567413, + "p95": 0.14432916045188904 + }, + { + "label": "completed_group_5", + "mean": 0.147438645362854, + "median": 0.14618173241615295, + "p05": 0.13021595776081085, + "p95": 0.16755904257297516 + }, + { + "label": "current_group_partial", + "mean": 0.16357660293579102, + "median": 0.1623307466506958, + "p05": 0.13464656472206116, + "p95": 0.1969548761844635 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9312366247177124, + "group": 6, + "latest_source_mass": 0.1666574329137802, + "layer": 22, + "max_source_mass": 0.16950437426567078, + "mixer_index": 43, + "normalized_entropy": 0.9924593001661848, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12334204465150833, + "median": 0.12106236815452576, + "p05": 0.08947738260030746, + "p95": 0.17906470596790314 + }, + { + "label": "completed_group_1", + "mean": 0.16950437426567078, + "median": 0.17181579768657684, + "p05": 0.1329171359539032, + "p95": 0.19668330252170563 + }, + { + "label": "completed_group_2", + "mean": 0.13870283961296082, + "median": 0.13818645477294922, + "p05": 0.12228117883205414, + "p95": 0.1556926816701889 + }, + { + "label": "completed_group_3", + "mean": 0.1300538182258606, + "median": 0.13022829592227936, + "p05": 0.1119839996099472, + "p95": 0.14633683860301971 + }, + { + "label": "completed_group_4", + "mean": 0.12885478138923645, + "median": 0.1284654438495636, + "p05": 0.11224459111690521, + "p95": 0.14672259986400604 + }, + { + "label": "completed_group_5", + "mean": 0.14288470149040222, + "median": 0.14101536571979523, + "p05": 0.11797811090946198, + "p95": 0.17301785945892334 + }, + { + "label": "current_group_partial", + "mean": 0.1666574329137802, + "median": 0.1637348234653473, + "p05": 0.12865713238716125, + "p95": 0.20903055369853973 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9345850944519043, + "group": 6, + "latest_source_mass": 0.16773909330368042, + "layer": 23, + "max_source_mass": 0.16773909330368042, + "mixer_index": 44, + "normalized_entropy": 0.9941800732120613, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1465124636888504, + "median": 0.14716467261314392, + "p05": 0.12261509895324707, + "p95": 0.17156414687633514 + }, + { + "label": "completed_group_1", + "mean": 0.153631791472435, + "median": 0.159563809633255, + "p05": 0.10715117305517197, + "p95": 0.18623968958854675 + }, + { + "label": "completed_group_2", + "mean": 0.13159149885177612, + "median": 0.13108083605766296, + "p05": 0.11123912036418915, + "p95": 0.1546429693698883 + }, + { + "label": "completed_group_3", + "mean": 0.13120220601558685, + "median": 0.13034585118293762, + "p05": 0.11503913998603821, + "p95": 0.1493607610464096 + }, + { + "label": "completed_group_4", + "mean": 0.12539167702198029, + "median": 0.12449093163013458, + "p05": 0.10983049124479294, + "p95": 0.14342987537384033 + }, + { + "label": "completed_group_5", + "mean": 0.14393125474452972, + "median": 0.1405339539051056, + "p05": 0.1222267672419548, + "p95": 0.17515066266059875 + }, + { + "label": "current_group_partial", + "mean": 0.16773909330368042, + "median": 0.16424956917762756, + "p05": 0.13410812616348267, + "p95": 0.21529987454414368 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9284589290618896, + "group": 6, + "latest_source_mass": 0.16532829403877258, + "layer": 23, + "max_source_mass": 0.169878289103508, + "mixer_index": 45, + "normalized_entropy": 0.9910318469730498, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12412484735250473, + "median": 0.12159939110279083, + "p05": 0.08492134511470795, + "p95": 0.18361014127731323 + }, + { + "label": "completed_group_1", + "mean": 0.169878289103508, + "median": 0.17313054203987122, + "p05": 0.1298009753227234, + "p95": 0.1993248164653778 + }, + { + "label": "completed_group_2", + "mean": 0.13807371258735657, + "median": 0.13714420795440674, + "p05": 0.12058109790086746, + "p95": 0.15811002254486084 + }, + { + "label": "completed_group_3", + "mean": 0.13034150004386902, + "median": 0.13021467626094818, + "p05": 0.11215148121118546, + "p95": 0.14708518981933594 + }, + { + "label": "completed_group_4", + "mean": 0.13028821349143982, + "median": 0.13050158321857452, + "p05": 0.1127624660730362, + "p95": 0.14778561890125275 + }, + { + "label": "completed_group_5", + "mean": 0.1419651210308075, + "median": 0.1401965618133545, + "p05": 0.11490530520677567, + "p95": 0.17368361353874207 + }, + { + "label": "current_group_partial", + "mean": 0.16532829403877258, + "median": 0.16024862229824066, + "p05": 0.1169055625796318, + "p95": 0.22413846850395203 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9329564571380615, + "group": 6, + "latest_source_mass": 0.17254900932312012, + "layer": 24, + "max_source_mass": 0.17254900932312012, + "mixer_index": 46, + "normalized_entropy": 0.9933431191961559, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1454196274280548, + "median": 0.145169198513031, + "p05": 0.12175656855106354, + "p95": 0.17354263365268707 + }, + { + "label": "completed_group_1", + "mean": 0.1566689908504486, + "median": 0.16300347447395325, + "p05": 0.11168468743562698, + "p95": 0.18563294410705566 + }, + { + "label": "completed_group_2", + "mean": 0.1286500096321106, + "median": 0.12701794505119324, + "p05": 0.11111085116863251, + "p95": 0.15331649780273438 + }, + { + "label": "completed_group_3", + "mean": 0.12666445970535278, + "median": 0.12615132331848145, + "p05": 0.11176232993602753, + "p95": 0.14301280677318573 + }, + { + "label": "completed_group_4", + "mean": 0.12782414257526398, + "median": 0.1281316727399826, + "p05": 0.11212349683046341, + "p95": 0.14346082508563995 + }, + { + "label": "completed_group_5", + "mean": 0.14222374558448792, + "median": 0.14010348916053772, + "p05": 0.12181462347507477, + "p95": 0.16952435672283173 + }, + { + "label": "current_group_partial", + "mean": 0.17254900932312012, + "median": 0.1675749123096466, + "p05": 0.1353597342967987, + "p95": 0.2285366952419281 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.9259672164916992, + "group": 6, + "latest_source_mass": 0.16209754347801208, + "layer": 24, + "max_source_mass": 0.17791953682899475, + "mixer_index": 47, + "normalized_entropy": 0.989751360013567, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12082186341285706, + "median": 0.11626945436000824, + "p05": 0.08509357273578644, + "p95": 0.18624860048294067 + }, + { + "label": "completed_group_1", + "mean": 0.17791953682899475, + "median": 0.1807878017425537, + "p05": 0.1341632455587387, + "p95": 0.2101013958454132 + }, + { + "label": "completed_group_2", + "mean": 0.1424642950296402, + "median": 0.14065685868263245, + "p05": 0.12273216992616653, + "p95": 0.16812381148338318 + }, + { + "label": "completed_group_3", + "mean": 0.13298648595809937, + "median": 0.13262662291526794, + "p05": 0.11336477845907211, + "p95": 0.15321806073188782 + }, + { + "label": "completed_group_4", + "mean": 0.12929362058639526, + "median": 0.12950879335403442, + "p05": 0.1104152500629425, + "p95": 0.1479472517967224 + }, + { + "label": "completed_group_5", + "mean": 0.1344166398048401, + "median": 0.13280367851257324, + "p05": 0.1057823970913887, + "p95": 0.16791269183158875 + }, + { + "label": "current_group_partial", + "mean": 0.16209754347801208, + "median": 0.15705879032611847, + "p05": 0.1103038564324379, + "p95": 0.22623871266841888 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.9201786518096924, + "group": 7, + "latest_source_mass": 0.15931037068367004, + "layer": 25, + "max_source_mass": 0.17355048656463623, + "mixer_index": 48, + "normalized_entropy": 0.9867766262187836, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13756656646728516, + "median": 0.14409756660461426, + "p05": 0.07605770230293274, + "p95": 0.1807018518447876 + }, + { + "label": "completed_group_1", + "mean": 0.17355048656463623, + "median": 0.1850908100605011, + "p05": 0.09468921273946762, + "p95": 0.2254364788532257 + }, + { + "label": "completed_group_2", + "mean": 0.14306488633155823, + "median": 0.1434939205646515, + "p05": 0.12189669907093048, + "p95": 0.16250990331172943 + }, + { + "label": "completed_group_3", + "mean": 0.13006147742271423, + "median": 0.12742848694324493, + "p05": 0.10484378784894943, + "p95": 0.15863747894763947 + }, + { + "label": "completed_group_4", + "mean": 0.11658850312232971, + "median": 0.11437725275754929, + "p05": 0.09685036540031433, + "p95": 0.14346592128276825 + }, + { + "label": "completed_group_5", + "mean": 0.139857679605484, + "median": 0.13671082258224487, + "p05": 0.10990621894598007, + "p95": 0.17913517355918884 + }, + { + "label": "completed_group_6", + "mean": 0.15931037068367004, + "median": 0.15141257643699646, + "p05": 0.10549820214509964, + "p95": 0.2391991913318634 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 2.043663501739502, + "group": 7, + "latest_source_mass": 0.08876943588256836, + "layer": 25, + "max_source_mass": 0.17510183155536652, + "mixer_index": 49, + "normalized_entropy": 0.9827943997351177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10298450291156769, + "median": 0.09445159137248993, + "p05": 0.06719416379928589, + "p95": 0.17596006393432617 + }, + { + "label": "completed_group_1", + "mean": 0.17510183155536652, + "median": 0.1808546781539917, + "p05": 0.12213055044412613, + "p95": 0.21415141224861145 + }, + { + "label": "completed_group_2", + "mean": 0.1414714753627777, + "median": 0.14097586274147034, + "p05": 0.11696422845125198, + "p95": 0.16771599650382996 + }, + { + "label": "completed_group_3", + "mean": 0.12628035247325897, + "median": 0.12579765915870667, + "p05": 0.10614679008722305, + "p95": 0.14692939817905426 + }, + { + "label": "completed_group_4", + "mean": 0.11767096817493439, + "median": 0.11719106882810593, + "p05": 0.10052644461393356, + "p95": 0.13653282821178436 + }, + { + "label": "completed_group_5", + "mean": 0.12069195508956909, + "median": 0.11935321986675262, + "p05": 0.09329675883054733, + "p95": 0.15252181887626648 + }, + { + "label": "completed_group_6", + "mean": 0.12702946364879608, + "median": 0.12134957313537598, + "p05": 0.08377101272344589, + "p95": 0.18232685327529907 + }, + { + "label": "current_group_partial", + "mean": 0.08876943588256836, + "median": 0.07879889756441116, + "p05": 0.06865477561950684, + "p95": 0.15027400851249695 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0607244968414307, + "group": 7, + "latest_source_mass": 0.11456632614135742, + "layer": 26, + "max_source_mass": 0.1498526930809021, + "mixer_index": 50, + "normalized_entropy": 0.9909990040771789, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1220984011888504, + "median": 0.12646546959877014, + "p05": 0.07202506065368652, + "p95": 0.15947024524211884 + }, + { + "label": "completed_group_1", + "mean": 0.1498526930809021, + "median": 0.15748712420463562, + "p05": 0.08880092203617096, + "p95": 0.19019576907157898 + }, + { + "label": "completed_group_2", + "mean": 0.12913095951080322, + "median": 0.12894359230995178, + "p05": 0.1061355471611023, + "p95": 0.15167440474033356 + }, + { + "label": "completed_group_3", + "mean": 0.1179434061050415, + "median": 0.118401437997818, + "p05": 0.09755907952785492, + "p95": 0.13774089515209198 + }, + { + "label": "completed_group_4", + "mean": 0.10775718837976456, + "median": 0.10657531023025513, + "p05": 0.09242597222328186, + "p95": 0.12714339792728424 + }, + { + "label": "completed_group_5", + "mean": 0.12020602822303772, + "median": 0.11893786489963531, + "p05": 0.10092492401599884, + "p95": 0.14495709538459778 + }, + { + "label": "completed_group_6", + "mean": 0.13844498991966248, + "median": 0.13359181582927704, + "p05": 0.10005055367946625, + "p95": 0.1863655149936676 + }, + { + "label": "current_group_partial", + "mean": 0.11456632614135742, + "median": 0.10762587189674377, + "p05": 0.08914398401975632, + "p95": 0.16366823017597198 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0388383865356445, + "group": 7, + "latest_source_mass": 0.08920245617628098, + "layer": 26, + "max_source_mass": 0.18260905146598816, + "mixer_index": 51, + "normalized_entropy": 0.9804740098096767, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10253133624792099, + "median": 0.09456855058670044, + "p05": 0.07405152916908264, + "p95": 0.16842687129974365 + }, + { + "label": "completed_group_1", + "mean": 0.18260905146598816, + "median": 0.19173294305801392, + "p05": 0.11063728481531143, + "p95": 0.22958147525787354 + }, + { + "label": "completed_group_2", + "mean": 0.14208021759986877, + "median": 0.14284399151802063, + "p05": 0.10792264342308044, + "p95": 0.17174816131591797 + }, + { + "label": "completed_group_3", + "mean": 0.12606358528137207, + "median": 0.12572120130062103, + "p05": 0.10405182838439941, + "p95": 0.14868314564228058 + }, + { + "label": "completed_group_4", + "mean": 0.11357533931732178, + "median": 0.11289471387863159, + "p05": 0.09577979147434235, + "p95": 0.13378244638442993 + }, + { + "label": "completed_group_5", + "mean": 0.11596501618623734, + "median": 0.1148330420255661, + "p05": 0.09200432896614075, + "p95": 0.1449994146823883 + }, + { + "label": "completed_group_6", + "mean": 0.12797299027442932, + "median": 0.12260732054710388, + "p05": 0.08455555140972137, + "p95": 0.1881599724292755 + }, + { + "label": "current_group_partial", + "mean": 0.08920245617628098, + "median": 0.07755134999752045, + "p05": 0.066501684486866, + "p95": 0.1560288667678833 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0634398460388184, + "group": 7, + "latest_source_mass": 0.12262405455112457, + "layer": 27, + "max_source_mass": 0.15090109407901764, + "mixer_index": 52, + "normalized_entropy": 0.9923048110176299, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12216806411743164, + "median": 0.12567293643951416, + "p05": 0.08283672481775284, + "p95": 0.1497604101896286 + }, + { + "label": "completed_group_1", + "mean": 0.15090109407901764, + "median": 0.15809711813926697, + "p05": 0.09385120868682861, + "p95": 0.1825941503047943 + }, + { + "label": "completed_group_2", + "mean": 0.12766355276107788, + "median": 0.12779396772384644, + "p05": 0.10599278658628464, + "p95": 0.14822138845920563 + }, + { + "label": "completed_group_3", + "mean": 0.1134132444858551, + "median": 0.11286742240190506, + "p05": 0.09987462311983109, + "p95": 0.12846460938453674 + }, + { + "label": "completed_group_4", + "mean": 0.10988184064626694, + "median": 0.10929364711046219, + "p05": 0.09590980410575867, + "p95": 0.12525902688503265 + }, + { + "label": "completed_group_5", + "mean": 0.11335283517837524, + "median": 0.11079756915569305, + "p05": 0.09715084731578827, + "p95": 0.14056096971035004 + }, + { + "label": "completed_group_6", + "mean": 0.13999530673027039, + "median": 0.1355634331703186, + "p05": 0.10303047299385071, + "p95": 0.18968813121318817 + }, + { + "label": "current_group_partial", + "mean": 0.12262405455112457, + "median": 0.11672497540712357, + "p05": 0.09879282861948013, + "p95": 0.16369464993476868 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.0386834144592285, + "group": 7, + "latest_source_mass": 0.08390261977910995, + "layer": 27, + "max_source_mass": 0.17931805551052094, + "mixer_index": 53, + "normalized_entropy": 0.9803994839943029, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10029368102550507, + "median": 0.09128708392381668, + "p05": 0.07227826118469238, + "p95": 0.17348244786262512 + }, + { + "label": "completed_group_1", + "mean": 0.17931805551052094, + "median": 0.1862885057926178, + "p05": 0.11921355128288269, + "p95": 0.22246700525283813 + }, + { + "label": "completed_group_2", + "mean": 0.14185777306556702, + "median": 0.14203041791915894, + "p05": 0.10853314399719238, + "p95": 0.1728457659482956 + }, + { + "label": "completed_group_3", + "mean": 0.13114282488822937, + "median": 0.1317460834980011, + "p05": 0.10662781447172165, + "p95": 0.15405671298503876 + }, + { + "label": "completed_group_4", + "mean": 0.11872631311416626, + "median": 0.11834143102169037, + "p05": 0.09892343729734421, + "p95": 0.1395292729139328 + }, + { + "label": "completed_group_5", + "mean": 0.11815309524536133, + "median": 0.11647512018680573, + "p05": 0.09220993518829346, + "p95": 0.14916184544563293 + }, + { + "label": "completed_group_6", + "mean": 0.12660562992095947, + "median": 0.12275825440883636, + "p05": 0.08448901027441025, + "p95": 0.1805192083120346 + }, + { + "label": "current_group_partial", + "mean": 0.08390261977910995, + "median": 0.07283803820610046, + "p05": 0.06168072670698166, + "p95": 0.14687296748161316 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0626261234283447, + "group": 7, + "latest_source_mass": 0.1115579903125763, + "layer": 28, + "max_source_mass": 0.15398627519607544, + "mixer_index": 54, + "normalized_entropy": 0.9919134931593667, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11644256114959717, + "median": 0.12025526911020279, + "p05": 0.07829730212688446, + "p95": 0.1473471075296402 + }, + { + "label": "completed_group_1", + "mean": 0.15398627519607544, + "median": 0.1615961492061615, + "p05": 0.09488844126462936, + "p95": 0.18759915232658386 + }, + { + "label": "completed_group_2", + "mean": 0.13596117496490479, + "median": 0.13718228042125702, + "p05": 0.11204125732183456, + "p95": 0.15343619883060455 + }, + { + "label": "completed_group_3", + "mean": 0.12036904692649841, + "median": 0.1201494038105011, + "p05": 0.10638508945703506, + "p95": 0.13539429008960724 + }, + { + "label": "completed_group_4", + "mean": 0.10964121669530869, + "median": 0.10774455964565277, + "p05": 0.09591315686702728, + "p95": 0.1285344362258911 + }, + { + "label": "completed_group_5", + "mean": 0.11886264383792877, + "median": 0.11775418370962143, + "p05": 0.0997074693441391, + "p95": 0.14473959803581238 + }, + { + "label": "completed_group_6", + "mean": 0.13317911326885223, + "median": 0.12975312769412994, + "p05": 0.09947147965431213, + "p95": 0.1780128926038742 + }, + { + "label": "current_group_partial", + "mean": 0.1115579903125763, + "median": 0.1049065813422203, + "p05": 0.0882733166217804, + "p95": 0.15690873563289642 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.020051956176758, + "group": 7, + "latest_source_mass": 0.07486395537853241, + "layer": 28, + "max_source_mass": 0.1913527548313141, + "mixer_index": 55, + "normalized_entropy": 0.9714396465047528, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09590239822864532, + "median": 0.08437805622816086, + "p05": 0.062284864485263824, + "p95": 0.19059056043624878 + }, + { + "label": "completed_group_1", + "mean": 0.1913527548313141, + "median": 0.19941553473472595, + "p05": 0.11809638142585754, + "p95": 0.24551606178283691 + }, + { + "label": "completed_group_2", + "mean": 0.14906750619411469, + "median": 0.14890097081661224, + "p05": 0.1085248589515686, + "p95": 0.18817238509655 + }, + { + "label": "completed_group_3", + "mean": 0.1341754049062729, + "median": 0.13468942046165466, + "p05": 0.10614670813083649, + "p95": 0.16129878163337708 + }, + { + "label": "completed_group_4", + "mean": 0.11847006529569626, + "median": 0.11653131991624832, + "p05": 0.09301862120628357, + "p95": 0.14944709837436676 + }, + { + "label": "completed_group_5", + "mean": 0.11603337526321411, + "median": 0.11465917527675629, + "p05": 0.08672616630792618, + "p95": 0.1505887657403946 + }, + { + "label": "completed_group_6", + "mean": 0.12013453245162964, + "median": 0.11430312693119049, + "p05": 0.07354772090911865, + "p95": 0.18300247192382812 + }, + { + "label": "current_group_partial", + "mean": 0.07486395537853241, + "median": 0.06415845453739166, + "p05": 0.052880410104990005, + "p95": 0.13596120476722717 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 2.0452136993408203, + "group": 8, + "latest_source_mass": 0.09500804543495178, + "layer": 29, + "max_source_mass": 0.15673108398914337, + "mixer_index": 56, + "normalized_entropy": 0.9835398871990577, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11420649290084839, + "median": 0.11760550737380981, + "p05": 0.0455118790268898, + "p95": 0.1930246502161026 + }, + { + "label": "completed_group_1", + "mean": 0.15673108398914337, + "median": 0.1630343794822693, + "p05": 0.0941019207239151, + "p95": 0.2044450342655182 + }, + { + "label": "completed_group_2", + "mean": 0.13583534955978394, + "median": 0.1358843892812729, + "p05": 0.10828851163387299, + "p95": 0.16276802122592926 + }, + { + "label": "completed_group_3", + "mean": 0.12758757174015045, + "median": 0.12776434421539307, + "p05": 0.10106872022151947, + "p95": 0.1536240577697754 + }, + { + "label": "completed_group_4", + "mean": 0.1112617775797844, + "median": 0.10795636475086212, + "p05": 0.0855436623096466, + "p95": 0.1458887755870819 + }, + { + "label": "completed_group_5", + "mean": 0.13417255878448486, + "median": 0.13457609713077545, + "p05": 0.10089332610368729, + "p95": 0.16806170344352722 + }, + { + "label": "completed_group_6", + "mean": 0.12519711256027222, + "median": 0.11858057975769043, + "p05": 0.08283591270446777, + "p95": 0.18291912972927094 + }, + { + "label": "completed_group_7", + "mean": 0.09500804543495178, + "median": 0.086635060608387, + "p05": 0.07067855447530746, + "p95": 0.14996609091758728 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 2.1403071880340576, + "group": 8, + "latest_source_mass": 0.10776971280574799, + "layer": 29, + "max_source_mass": 0.1673685610294342, + "mixer_index": 57, + "normalized_entropy": 0.9740957797899907, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08488726615905762, + "median": 0.07340645045042038, + "p05": 0.057354122400283813, + "p95": 0.1699385792016983 + }, + { + "label": "completed_group_1", + "mean": 0.1673685610294342, + "median": 0.17044895887374878, + "p05": 0.10701967775821686, + "p95": 0.213656947016716 + }, + { + "label": "completed_group_2", + "mean": 0.13726133108139038, + "median": 0.13688170909881592, + "p05": 0.09961593151092529, + "p95": 0.17454354465007782 + }, + { + "label": "completed_group_3", + "mean": 0.12220048904418945, + "median": 0.12267022579908371, + "p05": 0.09608018398284912, + "p95": 0.14777429401874542 + }, + { + "label": "completed_group_4", + "mean": 0.10616929084062576, + "median": 0.10591423511505127, + "p05": 0.08670090138912201, + "p95": 0.12526394426822662 + }, + { + "label": "completed_group_5", + "mean": 0.1037440150976181, + "median": 0.10274936258792877, + "p05": 0.08146549016237259, + "p95": 0.13024258613586426 + }, + { + "label": "completed_group_6", + "mean": 0.10401669144630432, + "median": 0.10373582690954208, + "p05": 0.06632979214191437, + "p95": 0.14398667216300964 + }, + { + "label": "completed_group_7", + "mean": 0.06658264249563217, + "median": 0.05958016216754913, + "p05": 0.04998533055186272, + "p95": 0.10914435982704163 + }, + { + "label": "current_group_partial", + "mean": 0.10776971280574799, + "median": 0.09594395756721497, + "p05": 0.05985837057232857, + "p95": 0.1909799873828888 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.18704891204834, + "group": 8, + "latest_source_mass": 0.1059001013636589, + "layer": 30, + "max_source_mass": 0.12684623897075653, + "mixer_index": 58, + "normalized_entropy": 0.9953688551489734, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10885682702064514, + "median": 0.11496149003505707, + "p05": 0.06692696362733841, + "p95": 0.13488291203975677 + }, + { + "label": "completed_group_1", + "mean": 0.12684623897075653, + "median": 0.12977978587150574, + "p05": 0.0880269706249237, + "p95": 0.15352922677993774 + }, + { + "label": "completed_group_2", + "mean": 0.11236055195331573, + "median": 0.1118544340133667, + "p05": 0.0995602011680603, + "p95": 0.12739220261573792 + }, + { + "label": "completed_group_3", + "mean": 0.10673344880342484, + "median": 0.10557802021503448, + "p05": 0.09419171512126923, + "p95": 0.12314252555370331 + }, + { + "label": "completed_group_4", + "mean": 0.10181289911270142, + "median": 0.10104846954345703, + "p05": 0.0905531719326973, + "p95": 0.1154235228896141 + }, + { + "label": "completed_group_5", + "mean": 0.11441421508789062, + "median": 0.11405807733535767, + "p05": 0.1015545204281807, + "p95": 0.12890149652957916 + }, + { + "label": "completed_group_6", + "mean": 0.11840946972370148, + "median": 0.11774120479822159, + "p05": 0.09769279509782791, + "p95": 0.1386050581932068 + }, + { + "label": "completed_group_7", + "mean": 0.10466624796390533, + "median": 0.10123022645711899, + "p05": 0.08714763075113297, + "p95": 0.1335311383008957 + }, + { + "label": "current_group_partial", + "mean": 0.1059001013636589, + "median": 0.10151755809783936, + "p05": 0.08478552103042603, + "p95": 0.14096680283546448 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1283018589019775, + "group": 8, + "latest_source_mass": 0.07073106616735458, + "layer": 30, + "max_source_mass": 0.1812124252319336, + "mixer_index": 59, + "normalized_entropy": 0.9686319190376982, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08667369186878204, + "median": 0.07598486542701721, + "p05": 0.0599454902112484, + "p95": 0.17076340317726135 + }, + { + "label": "completed_group_1", + "mean": 0.1812124252319336, + "median": 0.18718186020851135, + "p05": 0.11695106327533722, + "p95": 0.22937621176242828 + }, + { + "label": "completed_group_2", + "mean": 0.14178472757339478, + "median": 0.1410643458366394, + "p05": 0.10069074481725693, + "p95": 0.18480442464351654 + }, + { + "label": "completed_group_3", + "mean": 0.12440058588981628, + "median": 0.12452012300491333, + "p05": 0.09520045667886734, + "p95": 0.15427032113075256 + }, + { + "label": "completed_group_4", + "mean": 0.11160320043563843, + "median": 0.11176156997680664, + "p05": 0.08930732309818268, + "p95": 0.1340266317129135 + }, + { + "label": "completed_group_5", + "mean": 0.10330478101968765, + "median": 0.10138857364654541, + "p05": 0.07821988314390182, + "p95": 0.13451263308525085 + }, + { + "label": "completed_group_6", + "mean": 0.11197599023580551, + "median": 0.11038671433925629, + "p05": 0.06996595859527588, + "p95": 0.1621071994304657 + }, + { + "label": "completed_group_7", + "mean": 0.06831350922584534, + "median": 0.06078982725739479, + "p05": 0.049278710037469864, + "p95": 0.11596570909023285 + }, + { + "label": "current_group_partial", + "mean": 0.07073106616735458, + "median": 0.06323730200529099, + "p05": 0.051058635115623474, + "p95": 0.11510380357503891 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.173035144805908, + "group": 8, + "latest_source_mass": 0.097323939204216, + "layer": 31, + "max_source_mass": 0.13639645278453827, + "mixer_index": 60, + "normalized_entropy": 0.9889909148205337, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10752685368061066, + "median": 0.11246659606695175, + "p05": 0.04835075885057449, + "p95": 0.15864257514476776 + }, + { + "label": "completed_group_1", + "mean": 0.13639645278453827, + "median": 0.1424846053123474, + "p05": 0.0804441049695015, + "p95": 0.17550095915794373 + }, + { + "label": "completed_group_2", + "mean": 0.11839058995246887, + "median": 0.11810846626758575, + "p05": 0.0967480018734932, + "p95": 0.13973747193813324 + }, + { + "label": "completed_group_3", + "mean": 0.11235916614532471, + "median": 0.112240731716156, + "p05": 0.09452164173126221, + "p95": 0.12995943427085876 + }, + { + "label": "completed_group_4", + "mean": 0.10138258337974548, + "median": 0.09962283074855804, + "p05": 0.08511637151241302, + "p95": 0.12233416736125946 + }, + { + "label": "completed_group_5", + "mean": 0.11953261494636536, + "median": 0.12015771865844727, + "p05": 0.09630562365055084, + "p95": 0.14168336987495422 + }, + { + "label": "completed_group_6", + "mean": 0.11256720125675201, + "median": 0.11007197946310043, + "p05": 0.0816735252737999, + "p95": 0.14915722608566284 + }, + { + "label": "completed_group_7", + "mean": 0.09452059864997864, + "median": 0.0890384241938591, + "p05": 0.07612448185682297, + "p95": 0.13178984820842743 + }, + { + "label": "current_group_partial", + "mean": 0.097323939204216, + "median": 0.0898042768239975, + "p05": 0.07185541093349457, + "p95": 0.14817608892917633 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.1028335094451904, + "group": 8, + "latest_source_mass": 0.06014348566532135, + "layer": 31, + "max_source_mass": 0.19083696603775024, + "mixer_index": 61, + "normalized_entropy": 0.9570407736811942, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08200174570083618, + "median": 0.06842087209224701, + "p05": 0.05465565621852875, + "p95": 0.1860819309949875 + }, + { + "label": "completed_group_1", + "mean": 0.19083696603775024, + "median": 0.19397340714931488, + "p05": 0.1252523958683014, + "p95": 0.24377956986427307 + }, + { + "label": "completed_group_2", + "mean": 0.15512439608573914, + "median": 0.15396007895469666, + "p05": 0.10948964953422546, + "p95": 0.2031564563512802 + }, + { + "label": "completed_group_3", + "mean": 0.1322997808456421, + "median": 0.131678968667984, + "p05": 0.1001092940568924, + "p95": 0.16698354482650757 + }, + { + "label": "completed_group_4", + "mean": 0.11209174990653992, + "median": 0.11124030500650406, + "p05": 0.08621159940958023, + "p95": 0.1389508843421936 + }, + { + "label": "completed_group_5", + "mean": 0.10435720533132553, + "median": 0.10206517577171326, + "p05": 0.07588012516498566, + "p95": 0.13983221352100372 + }, + { + "label": "completed_group_6", + "mean": 0.1024213582277298, + "median": 0.10071991384029388, + "p05": 0.06045639514923096, + "p95": 0.15181156992912292 + }, + { + "label": "completed_group_7", + "mean": 0.06072331964969635, + "median": 0.05212578922510147, + "p05": 0.04197835922241211, + "p95": 0.10935051739215851 + }, + { + "label": "current_group_partial", + "mean": 0.06014348566532135, + "median": 0.054720573127269745, + "p05": 0.043509967625141144, + "p95": 0.09508825838565826 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.147671937942505, + "group": 8, + "latest_source_mass": 0.07905417680740356, + "layer": 32, + "max_source_mass": 0.14984291791915894, + "mixer_index": 62, + "normalized_entropy": 0.9774476219204733, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09926988929510117, + "median": 0.09770013391971588, + "p05": 0.04949922859668732, + "p95": 0.1754719465970993 + }, + { + "label": "completed_group_1", + "mean": 0.14984291791915894, + "median": 0.1559048891067505, + "p05": 0.08648592233657837, + "p95": 0.2032528817653656 + }, + { + "label": "completed_group_2", + "mean": 0.12505242228507996, + "median": 0.12578052282333374, + "p05": 0.09032565355300903, + "p95": 0.15844053030014038 + }, + { + "label": "completed_group_3", + "mean": 0.13004425168037415, + "median": 0.12868037819862366, + "p05": 0.10237257927656174, + "p95": 0.1618403047323227 + }, + { + "label": "completed_group_4", + "mean": 0.1122550517320633, + "median": 0.10850167274475098, + "p05": 0.08462762087583542, + "p95": 0.15187090635299683 + }, + { + "label": "completed_group_5", + "mean": 0.12226949632167816, + "median": 0.12273561954498291, + "p05": 0.08938803523778915, + "p95": 0.15480002760887146 + }, + { + "label": "completed_group_6", + "mean": 0.10689134150743484, + "median": 0.10258689522743225, + "p05": 0.07286856323480606, + "p95": 0.15216295421123505 + }, + { + "label": "completed_group_7", + "mean": 0.07532045990228653, + "median": 0.06964509189128876, + "p05": 0.054946303367614746, + "p95": 0.11525978893041611 + }, + { + "label": "current_group_partial", + "mean": 0.07905417680740356, + "median": 0.072466179728508, + "p05": 0.05414725840091705, + "p95": 0.11476384103298187 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.109609603881836, + "group": 8, + "latest_source_mass": 0.06053228676319122, + "layer": 32, + "max_source_mass": 0.18956288695335388, + "mixer_index": 63, + "normalized_entropy": 0.9601247071609755, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08386656641960144, + "median": 0.07061353325843811, + "p05": 0.055651482194662094, + "p95": 0.18354055285453796 + }, + { + "label": "completed_group_1", + "mean": 0.18956288695335388, + "median": 0.19494932889938354, + "p05": 0.12378176301717758, + "p95": 0.24272672832012177 + }, + { + "label": "completed_group_2", + "mean": 0.1487923562526703, + "median": 0.14935529232025146, + "p05": 0.10487234592437744, + "p95": 0.19193747639656067 + }, + { + "label": "completed_group_3", + "mean": 0.12990039587020874, + "median": 0.1300441324710846, + "p05": 0.09967920184135437, + "p95": 0.160600483417511 + }, + { + "label": "completed_group_4", + "mean": 0.11159682273864746, + "median": 0.11003521829843521, + "p05": 0.08588480204343796, + "p95": 0.14072611927986145 + }, + { + "label": "completed_group_5", + "mean": 0.10738209635019302, + "median": 0.10594583302736282, + "p05": 0.0788765475153923, + "p95": 0.14147546887397766 + }, + { + "label": "completed_group_6", + "mean": 0.10535432398319244, + "median": 0.10358455777168274, + "p05": 0.06328895688056946, + "p95": 0.15448220074176788 + }, + { + "label": "completed_group_7", + "mean": 0.0630122721195221, + "median": 0.05402897298336029, + "p05": 0.044587839394807816, + "p95": 0.11368448287248611 + }, + { + "label": "current_group_partial", + "mean": 0.06053228676319122, + "median": 0.055282287299633026, + "p05": 0.04416987672448158, + "p95": 0.09472700953483582 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 2.1622862815856934, + "group": null, + "latest_source_mass": 0.12579965591430664, + "layer": null, + "max_source_mass": 0.14956718683242798, + "mixer_index": null, + "normalized_entropy": 0.9840988963481907, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12356588989496231, + "median": 0.12339343130588531, + "p05": 0.0884157195687294, + "p95": 0.16069066524505615 + }, + { + "label": "completed_group_1", + "mean": 0.09235234558582306, + "median": 0.09365912526845932, + "p05": 0.06739562749862671, + "p95": 0.11328895390033722 + }, + { + "label": "completed_group_2", + "mean": 0.08736803382635117, + "median": 0.08676370978355408, + "p05": 0.07290066033601761, + "p95": 0.10374386608600616 + }, + { + "label": "completed_group_3", + "mean": 0.08735492825508118, + "median": 0.08669611811637878, + "p05": 0.07205022871494293, + "p95": 0.10462935268878937 + }, + { + "label": "completed_group_4", + "mean": 0.08889827132225037, + "median": 0.08936247229576111, + "p05": 0.07074585556983948, + "p95": 0.10606974363327026 + }, + { + "label": "completed_group_5", + "mean": 0.10504580289125443, + "median": 0.10402922332286835, + "p05": 0.0790465772151947, + "p95": 0.13349474966526031 + }, + { + "label": "completed_group_6", + "mean": 0.14004790782928467, + "median": 0.13968253135681152, + "p05": 0.10069862753152847, + "p95": 0.1819821298122406 + }, + { + "label": "completed_group_7", + "mean": 0.14956718683242798, + "median": 0.14930106699466705, + "p05": 0.12316055595874786, + "p95": 0.17985624074935913 + }, + { + "label": "completed_group_8", + "mean": 0.12579965591430664, + "median": 0.12568724155426025, + "p05": 0.07226289063692093, + "p95": 0.17831499874591827 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", + "optimizer_state_before": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.010549778118729591, + 0.013795096427202225, + 0.012347182258963585, + 0.012820021249353886, + 0.01913420483469963, + 0.018582463264465332, + 0.013614931143820286, + 0.013612493872642517, + 0.019129201769828796, + 0.02525762841105461, + 0.027986859902739525, + 0.014458909630775452, + 0.018231205642223358, + 0.016914349049329758, + 0.022478405386209488, + 0.03932488337159157, + 0.03611915931105614, + 0.0266753900796175, + 0.018954744562506676, + 0.014813725836575031, + 0.013802599161863327, + 0.016533223912119865, + 0.016679473221302032, + 0.01826235093176365, + 0.015529165975749493, + 0.011229476891458035, + 0.008122270926833153, + 0.008894859813153744, + 0.009738311171531677, + 0.007465044502168894, + 0.008605006150901318, + 0.009693782776594162 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.478636267573165e-05, + "first_to_last_ratio": 5.2633491541899735, + "imbalance_abs_log_ratio": 1.6607675454574897, + "last_quartile_mean": 6.60916873584938e-06, + "mean": 1.8399187894146962e-05, + "normalized": [ + 2.32572864627557, + 2.093980749525082, + 2.097253493167976, + 1.9748047942119253, + 1.9001199222002307, + 1.7176970887283356, + 1.5731226694511573, + 1.442465715104865, + 1.388173701809276, + 1.2578149167122274, + 1.163711179168628, + 1.0739337627510765, + 1.0625631931116428, + 0.9721718406735991, + 0.8978727300712377, + 0.830314261296444, + 0.8465768431906301, + 0.7713093251343669, + 0.7043450693146728, + 0.6424476146199775, + 0.6810764492568463, + 0.62147201099682, + 0.5661400138095133, + 0.5212255519238714, + 0.43050775556639687, + 0.41135344603945795, + 0.3923546966297886, + 0.3784948438109706, + 0.3251037643245192, + 0.3141780758593552, + 0.3117257919642976, + 0.30996008329924213 + ], + "peak_layer": 1, + "peak_normalized": 2.32572864627557, + "population_cv": 0.5996620573317034, + "rest_mean": 1.9884464530116895e-05, + "spike_contrast": 0.5219498892811426, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0378694059909322e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.279151835362427e-05, + 3.852754525723867e-05, + 3.858776108245365e-05, + 3.633480446296744e-05, + 3.496066346997395e-05, + 3.160423148074187e-05, + 2.8944179575773887e-05, + 2.6540197723079473e-05, + 2.5541268769302405e-05, + 2.3142772988649085e-05, + 2.141134064004291e-05, + 1.9759509086725302e-05, + 1.955029983946588e-05, + 1.7887172361952253e-05, + 1.65201290656114e-05, + 1.527710810478311e-05, + 1.5576326404698193e-05, + 1.4191465197654907e-05, + 1.295937727263663e-05, + 1.1820514373539481e-05, + 1.2531253560155164e-05, + 1.1434580301283859e-05, + 1.041651648847619e-05, + 9.590126865077764e-06, + 7.92099308455363e-06, + 7.56856934458483e-06, + 7.2190077844425105e-06, + 6.963997748243855e-06, + 5.981645244901301e-06, + 5.7806214499578346e-06, + 5.73550141780288e-06, + 5.7030138123082e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013640912038681563, + "first_to_last_ratio": 5.121983284590993, + "imbalance_abs_log_ratio": 1.6335417243184733, + "last_quartile_mean": 2.663209011188883e-05, + "mean": 7.298096016938871e-05, + "normalized": [ + 2.2789977416441083, + 2.0684974265127174, + 2.076163899895869, + 1.9536947342428674, + 1.8791834481613054, + 1.70558006750568, + 1.5594206835801008, + 1.4313066933158733, + 1.382104599488904, + 1.2645011290875958, + 1.1686177713576213, + 1.080869560986379, + 1.070406696124593, + 0.9820093608320563, + 0.9088928309552108, + 0.8405354206057669, + 0.8564067293095593, + 0.7807789902359659, + 0.7130468773072498, + 0.650906500520924, + 0.6919540061414183, + 0.6322816699001862, + 0.5753535339386485, + 0.5291429877926532, + 0.43715672246183995, + 0.41718055352967753, + 0.3982548633280403, + 0.38497320016438824, + 0.33031538135673594, + 0.31952371692646125, + 0.31683848721825286, + 0.31510371557135003 + ], + "peak_layer": 1, + "peak_normalized": 2.2789977416441083, + "population_cv": 0.5877415594416108, + "rest_mean": 7.874945926973683e-05, + "spike_contrast": 0.5311917747171685, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.183106502750888e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016632344340905547, + 0.00015096092829480767, + 0.00015152043488342315, + 0.00014258251758292317, + 0.00013714461238123477, + 0.00012447487097233534, + 0.00011380801879568025, + 0.00010445813677506521, + 0.00010086732072522864, + 9.228450653608888e-05, + 8.528684702469036e-05, + 7.888289837865159e-05, + 7.811930845491588e-05, + 7.166798604885116e-05, + 6.633187149418518e-05, + 6.134308205218986e-05, + 6.25013854005374e-05, + 5.698200038750656e-05, + 5.2038845751667395e-05, + 4.7503781388513744e-05, + 5.04994677612558e-05, + 4.614452336682007e-05, + 4.198985334369354e-05, + 3.861736331600696e-05, + 3.190411734976806e-05, + 3.044623736059293e-05, + 2.9065022317809053e-05, + 2.8095713787479326e-05, + 2.4106733690132387e-05, + 2.3319147658185102e-05, + 2.3123177015804686e-05, + 2.2996571715339087e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12096897326409817, + "first_to_last_ratio": 5.121983389958884, + "imbalance_abs_log_ratio": 1.6335417448901706, + "last_quartile_mean": 0.02361760358326137, + "mean": 0.06472024595132098, + "normalized": [ + 2.2789977721075467, + 2.0684973368733783, + 2.0761638538672402, + 1.953694825115509, + 1.8791834311760194, + 1.7055800372311585, + 1.5594207388065806, + 1.4313066964307424, + 1.3821046167764135, + 1.264501125605702, + 1.1686178592805807, + 1.080869532341778, + 1.0704067558461303, + 0.9820093835771068, + 0.9088927809148353, + 0.8405354027150204, + 0.8564067368028002, + 0.7807789687785609, + 0.7130468636740094, + 0.6509064717345558, + 0.6919539840797562, + 0.6322817027038486, + 0.5753535315685364, + 0.529143012125947, + 0.4371566943170802, + 0.41718053416692213, + 0.39825484202638917, + 0.3849732137717626, + 0.3303153774147322, + 0.31952373248220817, + 0.3168384772300972, + 0.31510370845705143 + ], + "peak_layer": 1, + "peak_normalized": 2.2789977721075467, + "population_cv": 0.5877415617202747, + "rest_mean": 0.06983580868001338, + "spike_contrast": 0.5311917756455903, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03709620721638203, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.147497296333313, + 0.13387365639209747, + 0.1343698352575302, + 0.12644360959529877, + 0.12162121385335922, + 0.11038555949926376, + 0.10092609375715256, + 0.09263452142477036, + 0.08945015072822571, + 0.08183882385492325, + 0.07563323527574539, + 0.06995414197444916, + 0.06927698850631714, + 0.06355588883161545, + 0.05882376432418823, + 0.05439965799450874, + 0.05542685464024544, + 0.050532206892967224, + 0.04614856839179993, + 0.04212682694196701, + 0.04478343203663826, + 0.04092142730951309, + 0.0372370220720768, + 0.03424626588821411, + 0.028292888775467873, + 0.027000026777386665, + 0.025775151327252388, + 0.024915561079978943, + 0.02137809246778488, + 0.0206796545535326, + 0.020505864173173904, + 0.02039358951151371 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.711734099302703e-06, + "first_to_last_ratio": 4.6783638296609995, + "imbalance_abs_log_ratio": 1.54294843975527, + "last_quartile_mean": 1.648382720986774e-06, + "mean": 4.007230394620365e-06, + "normalized": [ + 3.358984576738256, + 2.372641967350387, + 2.13819011970407, + 1.8393394452465484, + 1.59553806661366, + 1.4487556048903885, + 1.2955565469325476, + 1.3466327346665505, + 1.2739273872501113, + 1.0660089247937763, + 1.0478478795998651, + 1.0914936255170047, + 1.0411021864111147, + 0.9154927446436142, + 0.8065881507886284, + 0.7702055746502616, + 0.8215637644294851, + 0.7364762575936972, + 0.6730206907422001, + 0.6266437238096296, + 0.6693956877918972, + 0.6236755529871887, + 0.5897829924931797, + 0.560318828703654, + 0.45883664783466677, + 0.44720692888245495, + 0.4388849180240322, + 0.4318078866173612, + 0.37877623882612776, + 0.3751519735069292, + 0.3795957753533195, + 0.38055659660739233 + ], + "peak_layer": 1, + "peak_normalized": 3.358984576738256, + "population_cv": 0.6648021914060557, + "rest_mean": 4.31860559678575e-06, + "spike_contrast": 0.5385544594899636, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.3258043029272812e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.346022509096656e-05, + 9.50772300711833e-06, + 8.568220437155105e-06, + 7.3706569310161285e-06, + 6.39368863630807e-06, + 5.805497494293377e-06, + 5.19159357281751e-06, + 5.3962676247465424e-06, + 5.1049205467279535e-06, + 4.271743364370195e-06, + 4.19896787207108e-06, + 4.373866431706119e-06, + 4.171936325292336e-06, + 3.6685903523903107e-06, + 3.232184553780826e-06, + 3.0863911888445728e-06, + 3.292195287940558e-06, + 2.9512300443457207e-06, + 2.6969489681505365e-06, + 2.511105776648037e-06, + 2.6824227461474948e-06, + 2.4992116323119262e-06, + 2.363396333748824e-06, + 2.245326641059364e-06, + 1.838664161368797e-06, + 1.7920611981026013e-06, + 1.7587129832463688e-06, + 1.730353687889874e-06, + 1.5178436569840414e-06, + 1.5033203908387804e-06, + 1.5211277286653058e-06, + 1.5249779607984237e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.0001145691167039331, + "first_to_last_ratio": 4.836936726468446, + "imbalance_abs_log_ratio": 1.5762816126034256, + "last_quartile_mean": 2.368629634474928e-05, + "mean": 6.211562191538178e-05, + "normalized": [ + 2.348876988858354, + 2.071301897195793, + 2.0294561010843566, + 1.9098237582754058, + 1.8088773728255583, + 1.6555764779765272, + 1.5234729301569887, + 1.4082090466002533, + 1.3570526012658433, + 1.2477509254620376, + 1.1584754738367655, + 1.0770687308168136, + 1.0629673480990054, + 0.9794260409949351, + 0.9103157365029115, + 0.8467425542641047, + 0.8606040430432489, + 0.7903153532861079, + 0.7260518902950809, + 0.6672536282299563, + 0.7127906049054458, + 0.6531196817841458, + 0.5952239503818174, + 0.548639546167493, + 0.4562347554219186, + 0.43716465149452993, + 0.4212539905654889, + 0.40928696694430294, + 0.34099585105581787, + 0.33116538282585856, + 0.32832835587544684, + 0.32617736350768683 + ], + "peak_layer": 1, + "peak_normalized": 2.348876988858354, + "population_cv": 0.5738572576504661, + "rest_mean": 6.679497912248028e-05, + "spike_contrast": 0.5516446517557004, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6847092997049916e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014590195496566594, + 0.00012866020551882684, + 0.00012606092786882073, + 0.0001186298904940486, + 0.00011235954298172146, + 0.00010283716255798936, + 9.463146852795035e-05, + 8.747178071644157e-05, + 8.429416629951447e-05, + 7.750482473056763e-05, + 7.195942453108728e-05, + 6.690279406029731e-05, + 6.602687790291384e-05, + 6.0837657656520605e-05, + 5.6544828112237155e-05, + 5.259594036033377e-05, + 5.3456955356523395e-05, + 4.909092967864126e-05, + 4.509916470851749e-05, + 4.144687409279868e-05, + 4.4275431719142944e-05, + 4.0568935219198465e-05, + 3.697270585689694e-05, + 3.4079086617566645e-05, + 2.8339305572444573e-05, + 2.715475420700386e-05, + 2.6166453608311713e-05, + 2.542311449360568e-05, + 2.1181169358897023e-05, + 2.0570543711073697e-05, + 2.0394320017658174e-05, + 2.026070978899952e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.517959642835194e-05, + "first_to_last_ratio": 4.053573868997912, + "imbalance_abs_log_ratio": 1.39959892878961, + "last_quartile_mean": 2.3480414938603644e-05, + "mean": 5.345439518578132e-05, + "normalized": [ + 2.3487685774234603, + 2.032002289203053, + 1.963516724917696, + 1.8356141378280442, + 1.6872738480579543, + 1.5647260343512426, + 1.4530976571042717, + 1.3596062267041897, + 1.314646683688125, + 1.2118086283799, + 1.12751130576606, + 1.057258217084054, + 1.0421641943948246, + 0.9643812108591268, + 0.9028111172523068, + 0.8481148430618181, + 0.8638417335406758, + 0.7968091313416481, + 0.7398800228203574, + 0.6885877184509301, + 0.7528185246328142, + 0.6965652282580993, + 0.6386689529276455, + 0.5954414082316423, + 0.5183367890031948, + 0.5027841940839927, + 0.4926901944439643, + 0.48501928420581064, + 0.3867176094242744, + 0.3783808574805794, + 0.37618160958425234, + 0.373975045493992 + ], + "peak_layer": 1, + "peak_normalized": 2.3487685774234603, + "population_cv": 0.531886342087635, + "rest_mean": 5.701439672352367e-05, + "spike_contrast": 0.6003814623868411, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.423038688197266e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012555200373753905, + 0.0001086194533854723, + 0.00010495859896764159, + 9.812164353206754e-05, + 9.019220306072384e-05, + 8.364148379769176e-05, + 7.76744564063847e-05, + 7.267692853929475e-05, + 7.027364335954189e-05, + 6.47764973109588e-05, + 6.0270434914855286e-05, + 5.65150985494256e-05, + 5.570825669565238e-05, + 5.155041435500607e-05, + 4.8259222239721566e-05, + 4.533546598395333e-05, + 4.6176137402653694e-05, + 4.259295019437559e-05, + 3.9549839129904285e-05, + 3.6808040022151545e-05, + 4.02414589188993e-05, + 3.7234472983982414e-05, + 3.413966260268353e-05, + 3.182896034559235e-05, + 2.7707379558705725e-05, + 2.687602500373032e-05, + 2.633645635796711e-05, + 2.5926412490662187e-05, + 2.0671755919465795e-05, + 2.0226119886501692e-05, + 2.0108560420339927e-05, + 1.99906098714564e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002512736391508952, + "first_to_last_ratio": 5.815826874804025, + "imbalance_abs_log_ratio": 1.7605829727324667, + "last_quartile_mean": 4.320514426581212e-05, + "mean": 0.00013164091467388062, + "normalized": [ + 2.22465467951502, + 2.0950685860525193, + 2.148742715865945, + 2.011602148255569, + 1.9594409030366386, + 1.7609558829295935, + 1.610634891401338, + 1.4591459078647937, + 1.4125272633741999, + 1.2911637629028558, + 1.1922164013125294, + 1.0885645721953474, + 1.0786559850616189, + 0.9886788153600501, + 0.9103044070794555, + 0.8363940916690796, + 0.853516847878044, + 0.7731887115305843, + 0.7048080081053143, + 0.6396644863918306, + 0.6683488269421659, + 0.6076388883701213, + 0.5570727160911497, + 0.5013742768274315, + 0.394539708177536, + 0.3676631795999013, + 0.3470701351915981, + 0.3326099884380236, + 0.30382715452412723, + 0.2945209687200569, + 0.29298846283259594, + 0.29241662650296524 + ], + "peak_layer": 1, + "peak_normalized": 2.22465467951502, + "population_cv": 0.6145561150438986, + "rest_mean": 0.0001427135030098725, + "spike_contrast": 0.503448770748442, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.184893765952439e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00029285557684488595, + 0.0002757967449724674, + 0.0002828624565154314, + 0.0002648091467563063, + 0.00025794259272515774, + 0.00023181384312920272, + 0.00021202545030973852, + 0.00019208330195397139, + 0.00018594638095237315, + 0.00016996997874230146, + 0.0001569444575579837, + 0.0001432996359653771, + 0.0001419952604919672, + 0.00013015058357268572, + 0.00011983330477960408, + 0.00011010368325514719, + 0.00011235773854423314, + 0.00010178326920140535, + 9.278157085645944e-05, + 8.420601807301864e-05, + 8.798205089988187e-05, + 7.99901390564628e-05, + 7.333356188610196e-05, + 6.600136839551851e-05, + 5.193756805965677e-05, + 4.839951725443825e-05, + 4.568863005260937e-05, + 4.378508310765028e-05, + 3.9996084524318576e-05, + 3.877100971294567e-05, + 3.8569269236177206e-05, + 3.8493992178700864e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.017611408606171608, + 0.014558257535099983, + 0.01080919150263071, + 0.010327119380235672, + 0.008616750128567219, + 0.007770510856062174, + 0.007606723811477423, + 0.008487755432724953, + 0.00865182839334011, + 0.009020541794598103, + 0.008682658895850182, + 0.008706990629434586, + 0.009003226645290852, + 0.009712434373795986, + 0.00981286820024252, + 0.011337134055793285, + 0.010840619914233685, + 0.010956712998449802, + 0.012231985107064247, + 0.014082341454923153, + 0.014436324127018452, + 0.016183899715542793, + 0.017385432496666908, + 0.02254617027938366, + 0.022119678556919098, + 0.022372635081410408, + 0.024847008287906647, + 0.026897024363279343, + 0.026162855327129364, + 0.024674534797668457, + 0.030845578759908676, + 0.029862264171242714 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.2954689231701195e-05, + "first_to_last_ratio": 5.105739778813491, + "imbalance_abs_log_ratio": 1.63036535371595, + "last_quartile_mean": 6.454439642311627e-06, + "mean": 1.73502516958024e-05, + "normalized": [ + 2.118076439435723, + 2.157395185923538, + 2.1393573631676532, + 2.0293387218518246, + 1.9547446735792444, + 1.7470380730869022, + 1.5859544826144767, + 1.463120381926734, + 1.4022787424481573, + 1.2651844959124576, + 1.1596399794189511, + 1.0756457189909705, + 1.0548334272291358, + 0.9617847148778298, + 0.8861896603299768, + 0.8166507566511427, + 0.822632314310604, + 0.7462779082282098, + 0.6809665471807215, + 0.6235405742761265, + 0.6619400891948598, + 0.6046031773324564, + 0.5544108586463486, + 0.5123283891876923, + 0.43840604673891487, + 0.41933885865900805, + 0.40328491124319266, + 0.3940604884962617, + 0.3330148316096749, + 0.3301665032697355, + 0.32877203485044354, + 0.3290236493310315 + ], + "peak_layer": 2, + "peak_normalized": 2.157395185923538, + "population_cv": 0.5996242881005125, + "rest_mean": 1.878216889289669e-05, + "spike_contrast": 0.5120760486362497, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.617898831493221e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.6749159335158765e-05, + 3.7431349483085796e-05, + 3.7118388718226925e-05, + 3.5209537600167096e-05, + 3.391531208762899e-05, + 3.031155029020738e-05, + 2.751670945144724e-05, + 2.538550688768737e-05, + 2.4329889129148796e-05, + 2.195126944570802e-05, + 2.0120045519433916e-05, + 1.8662723960005678e-05, + 1.830162545957137e-05, + 1.6687206880305894e-05, + 1.5375613656942733e-05, + 1.4169096175464801e-05, + 1.4272877706389409e-05, + 1.2948109542776365e-05, + 1.1814940990007017e-05, + 1.0818585906235967e-05, + 1.1484827155072708e-05, + 1.049001730279997e-05, + 9.619167940400075e-06, + 8.88902650331147e-06, + 7.606455255881883e-06, + 7.2756347435642965e-06, + 6.997094715188723e-06, + 6.8370486587809864e-06, + 5.777891146863112e-06, + 5.728471933252877e-06, + 5.704277555196313e-06, + 5.708643129764823e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.0001289480715058744, + "first_to_last_ratio": 4.956339714424028, + "imbalance_abs_log_ratio": 1.6006675075081214, + "last_quartile_mean": 2.6016794436145574e-05, + "mean": 6.873570305288013e-05, + "normalized": [ + 2.073004605358973, + 2.134365700374101, + 2.1192095069414485, + 2.0040524994244264, + 1.9289936690931089, + 1.7292296834320953, + 1.569549055379467, + 1.4495827774599748, + 1.392480193841659, + 1.2691570699511074, + 1.1657818381111424, + 1.084357575904709, + 1.0620446910222163, + 0.9734770749855297, + 0.8984711717160666, + 0.8270404385478125, + 0.8348284439914535, + 0.7567459964705125, + 0.691239653970941, + 0.6326487613732772, + 0.6743242179293075, + 0.6159548780125667, + 0.5640484616188776, + 0.521373530426787, + 0.4457155128282148, + 0.4257578292711852, + 0.4106916383072167, + 0.4012558017197981, + 0.3390531928847147, + 0.336176289510552, + 0.3343946322908752, + 0.33499360784988247 + ], + "peak_layer": 2, + "peak_normalized": 2.134365700374101, + "population_cv": 0.5868619217766903, + "rest_mean": 7.428186829740837e-05, + "spike_contrast": 0.5221517931823592, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.878641073242761e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001424894289812073, + 0.0001467071269871667, + 0.00014566535537596792, + 0.0001377499575028196, + 0.00013259073602966964, + 0.0001188598180306144, + 0.00010788405779749155, + 9.963809134205803e-05, + 9.571310511091724e-05, + 8.723640348762274e-05, + 8.013083424884826e-05, + 7.4534080340527e-05, + 7.300038851099089e-05, + 6.691263115499169e-05, + 6.175704766064882e-05, + 5.68472059967462e-05, + 5.7382520026294515e-05, + 5.2015468099853024e-05, + 4.751284359372221e-05, + 4.3485557398526e-05, + 4.6350149204954505e-05, + 4.233809158904478e-05, + 3.877026756526902e-05, + 3.5836976167047396e-05, + 3.0636569135822356e-05, + 2.926476372522302e-05, + 2.8229178496985696e-05, + 2.7580599635257386e-05, + 2.330505958525464e-05, + 2.3107313609216362e-05, + 2.2984850147622637e-05, + 2.3026021153782494e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11435244046151638, + "first_to_last_ratio": 4.956339696350975, + "imbalance_abs_log_ratio": 1.6006675038616696, + "last_quartile_mean": 0.023071953793987632, + "mean": 0.0609555093687959, + "normalized": [ + 2.073004695896629, + 2.1343657739438058, + 2.1192095214175937, + 2.004052535226019, + 1.9289935285682593, + 1.7292296759967316, + 1.5695490131514764, + 1.4495827726824428, + 1.3924801813690295, + 1.269157021277261, + 1.165781889125704, + 1.0843576426531487, + 1.0620447122638885, + 0.9734770498631476, + 0.8984712131746528, + 0.8270404152126933, + 0.8348284100671275, + 0.7567460489501358, + 0.691239607862424, + 0.6326487463288526, + 0.6743241627324319, + 0.6159549038523712, + 0.5640484241789988, + 0.5213735345830548, + 0.44571554071981845, + 0.4257578260668156, + 0.4106916263563641, + 0.40125582081652367, + 0.3390531599397356, + 0.3361762979130643, + 0.33439464558088794, + 0.33499360222891106 + ], + "peak_layer": 2, + "peak_normalized": 2.1343657739438058, + "population_cv": 0.5868619269179132, + "rest_mean": 0.06587390429167836, + "spike_contrast": 0.522151786129607, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03439617678523064, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.12636105716228485, + 0.13010135293006897, + 0.1291774958372116, + 0.12215804308652878, + 0.1175827831029892, + 0.10540607571601868, + 0.09567265957593918, + 0.08836005628108978, + 0.08487933874130249, + 0.07736211270093918, + 0.07106082886457443, + 0.06609757244586945, + 0.0647374764084816, + 0.05933878943324089, + 0.05476677045226097, + 0.05041266977787018, + 0.05088739097118378, + 0.046127840876579285, + 0.04213486239314079, + 0.038563426584005356, + 0.041103772819042206, + 0.03754584491252899, + 0.03438185900449753, + 0.03178058937191963, + 0.027168817818164825, + 0.025952285155653954, + 0.025033917278051376, + 0.0244587529450655, + 0.02066715806722641, + 0.020491797477006912, + 0.020383195951581, + 0.020419705659151077 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.461751238224679e-06, + "first_to_last_ratio": 4.607278415178689, + "imbalance_abs_log_ratio": 1.527637317203245, + "last_quartile_mean": 1.6195572669630565e-06, + "mean": 3.872976176211296e-06, + "normalized": [ + 3.025668897657687, + 2.420035163018765, + 2.225735559561749, + 1.8932958237601154, + 1.6740889922677953, + 1.49074225689019, + 1.302073782365782, + 1.3813151404849735, + 1.2950916709377387, + 1.0715944778616184, + 1.0585199120860045, + 1.1104678074006644, + 1.039542517960571, + 0.9298369507481301, + 0.7931357450923319, + 0.7672068262710003, + 0.8115868243354613, + 0.7239540167999846, + 0.6595190440563047, + 0.6140523657384757, + 0.6490685381015874, + 0.6059744156497426, + 0.5735242411534173, + 0.5386197309103437, + 0.45666220052738465, + 0.4487521252190139, + 0.4444189656933061, + 0.44116761367860935, + 0.38204877349263455, + 0.3857109035588071, + 0.3910260087310755, + 0.39556270798873516 + ], + "peak_layer": 1, + "peak_normalized": 3.025668897657687, + "population_cv": 0.6453832806768665, + "rest_mean": 4.185131009156369e-06, + "spike_contrast": 0.5226455452702347, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.187340078307898e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1718343557731714e-05, + 9.372738531965297e-06, + 8.620220796728972e-06, + 7.332689619943267e-06, + 6.483706783910748e-06, + 5.773609245807165e-06, + 5.042900738772005e-06, + 5.349800630938262e-06, + 5.015859187551541e-06, + 4.150259883317631e-06, + 4.09962240155437e-06, + 4.300815362512367e-06, + 4.026123406219995e-06, + 3.601236358008464e-06, + 3.0717958452441962e-06, + 2.971373760374263e-06, + 3.1432564355782233e-06, + 2.8038566597388126e-06, + 2.554301545387716e-06, + 2.3782101834513014e-06, + 2.5138269847957417e-06, + 2.3469244752050145e-06, + 2.221245722466847e-06, + 2.0860613858530996e-06, + 1.7686418232187862e-06, + 1.7380062899974291e-06, + 1.7212240663866396e-06, + 1.7086316574932425e-06, + 1.4796657978877192e-06, + 1.4938491403881926e-06, + 1.5144344160944456e-06, + 1.5320049442379968e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00010837215813808143, + "first_to_last_ratio": 4.663749544487092, + "imbalance_abs_log_ratio": 1.5398197478822726, + "last_quartile_mean": 2.3237130790221272e-05, + "mean": 5.872993017419503e-05, + "normalized": [ + 2.116908005707516, + 2.1257566114370654, + 2.0713368933145166, + 1.958807069139986, + 1.853064595344867, + 1.6774795353818415, + 1.534246942340664, + 1.4245031207093415, + 1.3659761301608329, + 1.2527187093598229, + 1.1583387848911404, + 1.08159561433793, + 1.0590461894491214, + 0.974994469979534, + 0.9039318927032919, + 0.837094157953219, + 0.8447810620709018, + 0.7717232000223526, + 0.7087798354672495, + 0.6523243210626272, + 0.6962576906319006, + 0.6381512392879863, + 0.585489683522781, + 0.5414078956394395, + 0.4665886811070132, + 0.44848929689996414, + 0.43561388426614667, + 0.42649458236277493, + 0.3502420718536938, + 0.3476010806933041, + 0.3452465505067908, + 0.3450102023943839 + ], + "peak_layer": 2, + "peak_normalized": 2.1257566114370654, + "population_cv": 0.5682342302969126, + "rest_mean": 6.323713649989158e-05, + "spike_contrast": 0.5438420826580702, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4391016015433706e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012432585936039686, + 0.0001248455373570323, + 0.00012164947111159563, + 0.000115040602395311, + 0.00010883035429287702, + 9.851825598161668e-05, + 9.010621579363942e-05, + 8.366096881218255e-05, + 8.022368274396285e-05, + 7.357208232861012e-05, + 6.802915595471859e-05, + 6.352203490678221e-05, + 6.219770875759423e-05, + 5.7261357142124325e-05, + 5.308785694069229e-05, + 4.916248144581914e-05, + 4.961393278790638e-05, + 4.532324965111911e-05, + 4.162659024586901e-05, + 3.831096182693727e-05, + 4.089116555405781e-05, + 3.747857772395946e-05, + 3.4385768231004477e-05, + 3.179684790666215e-05, + 2.7402720661484636e-05, + 2.633974509080872e-05, + 2.558357300586067e-05, + 2.5047997041838244e-05, + 2.0569692424032837e-05, + 2.0414587197592482e-05, + 2.0276305804145522e-05, + 2.0262425096007064e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.030398450704524e-05, + "first_to_last_ratio": 3.89627084299362, + "imbalance_abs_log_ratio": 1.360019901586191, + "last_quartile_mean": 2.3177029561338713e-05, + "mean": 5.084823260403937e-05, + "normalized": [ + 2.109723573221063, + 2.0821310668378783, + 1.9942546197542355, + 1.874953814933449, + 1.728017146958865, + 1.5814957309589324, + 1.4629098120324804, + 1.3741245898657877, + 1.3101737824863438, + 1.2101890628861518, + 1.1282931314057325, + 1.0632895987411368, + 1.0440174426577644, + 0.9649796238381606, + 0.9017142254727002, + 0.8410904420982939, + 0.8516872374927745, + 0.7851358108806284, + 0.7291665843144397, + 0.6770579705243002, + 0.7378064582735888, + 0.6786248956615917, + 0.6313013382664657, + 0.5913982993163636, + 0.5312183147012203, + 0.5201489591253866, + 0.5114845807208003, + 0.5059300847390553, + 0.39756423665978435, + 0.3954241221367383, + 0.3931178424325881, + 0.39157560060529917 + ], + "peak_layer": 1, + "peak_normalized": 2.109723573221063, + "population_cv": 0.5213662772063792, + "rest_mean": 5.429395497668776e-05, + "spike_contrast": 0.5938291252788923, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.224133179173805e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010727571498136967, + 0.00010587268479866907, + 0.00010140432277694345, + 9.5338087703567e-05, + 8.786661783233285e-05, + 8.041626279009506e-05, + 7.438637840095907e-05, + 6.987180677242577e-05, + 6.662002124357969e-05, + 6.153597496449947e-05, + 5.7371711591258645e-05, + 5.406639684224501e-05, + 5.308644176693633e-05, + 4.90675083710812e-05, + 4.585057467920706e-05, + 4.276796244084835e-05, + 4.330679075792432e-05, + 3.9922768337419257e-05, + 3.7076832086313516e-05, + 3.4427201171638444e-05, + 3.751615440705791e-05, + 3.450687654549256e-05, + 3.210055729141459e-05, + 3.007155828527175e-05, + 2.7011512429453433e-05, + 2.6448655262356624e-05, + 2.60080869338708e-05, + 2.572565063019283e-05, + 2.021543878072407e-05, + 2.010661773965694e-05, + 1.998934749281034e-05, + 1.991092722164467e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00023694485207670368, + "first_to_last_ratio": 5.643445556811249, + "imbalance_abs_log_ratio": 1.7304947933060113, + "last_quartile_mean": 4.198584883852163e-05, + "mean": 0.00012326974319876172, + "normalized": [ + 2.0534243627746784, + 2.17548929765909, + 2.1923221930833754, + 2.045443753327701, + 2.026125682616779, + 1.7904266625829455, + 1.6133287652163053, + 1.4807635751829418, + 1.4348095534638061, + 1.3014197885977827, + 1.1747669830911103, + 1.0855875327354585, + 1.0626766295217445, + 0.9759316522493466, + 0.8962992674239908, + 0.8231145619826911, + 0.8258276905981559, + 0.7455975627999764, + 0.6776587351559191, + 0.6226196341065754, + 0.6492096639107944, + 0.5955964221479619, + 0.5398643713108281, + 0.4868843463557821, + 0.3972339380481051, + 0.3723774041823533, + 0.3563689360290182, + 0.3460491488507791, + 0.3146328182140966, + 0.31212742697325935, + 0.31138985452451795, + 0.31463178528213015 + ], + "peak_layer": 3, + "peak_normalized": 2.1923221930833754, + "population_cv": 0.61857583014248, + "rest_mean": 0.00013391299553741528, + "spike_contrast": 0.4913352905442922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.579618057003244e-05, + "top_five_layers": [ + 3, + 2, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0002531250938773155, + 0.0002681720070540905, + 0.0002702469937503338, + 0.000252141326200217, + 0.00024975999258458614, + 0.0002207054349128157, + 0.00019887462258338928, + 0.00018253334565088153, + 0.00017686860519461334, + 0.00016042568313423544, + 0.00014481322432402521, + 0.0001338200963800773, + 0.0001309958752244711, + 0.00012030284415232018, + 0.00011048658052459359, + 0.00010146512067876756, + 0.00010179956734646112, + 9.19096200959757e-05, + 8.353481825906783e-05, + 7.675016240682453e-05, + 8.002790855243802e-05, + 7.341901800828055e-05, + 6.654894241364673e-05, + 6.001810834277421e-05, + 4.896692553302273e-05, + 4.590286698658019e-05, + 4.3929507228313014e-05, + 4.2657389712985605e-05, + 3.878470670315437e-05, + 3.8475867768283933e-05, + 3.83849474019371e-05, + 3.878457937389612e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.010549778118729591, + 0.02910016104578972, + 0.04255657270550728, + 0.05177471786737442, + 0.01913420483469963, + 0.02246284671127796, + 0.029843268916010857, + 0.036163609474897385, + 0.019129201769828796, + 0.0314205139875412, + 0.03302976116538048, + 0.03871368616819382, + 0.018231205642223358, + 0.02580643631517887, + 0.0349617674946785, + 0.04659247398376465, + 0.03611915931105614, + 0.03664228320121765, + 0.04177439957857132, + 0.04776441678404808, + 0.013802599161863327, + 0.02693362347781658, + 0.03932841867208481, + 0.0514267161488533, + 0.015529165975749493, + 0.03558400645852089, + 0.05707564577460289, + 0.08332383632659912, + 0.009738311171531677, + 0.03187238425016403, + 0.05786185711622238, + 0.09122034162282944 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.478662188172166e-05, + "first_to_last_ratio": 5.263389233448399, + "imbalance_abs_log_ratio": 1.6607751602108825, + "last_quartile_mean": 6.609167655824422e-06, + "mean": 1.839926902391653e-05, + "normalized": [ + 2.3257405363021224, + 2.093986543353821, + 2.0972432569142194, + 1.9748429471145472, + 1.9001738268978305, + 1.7176742899564605, + 1.5731126681977952, + 1.4424450196971237, + 1.3881676796557327, + 1.2578123363623133, + 1.1637254248597446, + 1.0739089583500412, + 1.0625597930502229, + 0.9721654778743629, + 0.897852656475923, + 0.8303134671066331, + 0.8465754829823255, + 0.7713090877071682, + 0.7043469066766334, + 0.642452591912055, + 0.6810739404273755, + 0.621476487606478, + 0.5661443389519891, + 0.5212309648706807, + 0.4305042754919776, + 0.4113531398630616, + 0.3923510140548581, + 0.378496313745133, + 0.3251027262595914, + 0.314172044005768, + 0.31172491175027217, + 0.3099608915257388 + ], + "peak_layer": 1, + "peak_normalized": 2.3257405363021224, + "population_cv": 0.5996654929960216, + "rest_mean": 1.9884546603592485e-05, + "spike_contrast": 0.5219515586938898, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.037877009366639e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.2791925807250664e-05, + 3.852782174362801e-05, + 3.858774289255962e-05, + 3.633566666394472e-05, + 3.4961809433298185e-05, + 3.160395135637373e-05, + 2.8944123187102377e-05, + 2.653993396961596e-05, + 2.554127058829181e-05, + 2.314282755833119e-05, + 2.1411697161966003e-05, + 1.9759139831876382e-05, + 1.9550323486328125e-05, + 1.7887134163174778e-05, + 1.6519832570338622e-05, + 1.5277160855475813e-05, + 1.5576370060443878e-05, + 1.419152340531582e-05, + 1.2959468222106807e-05, + 1.1820658073702361e-05, + 1.2531262655102182e-05, + 1.1434713087510318e-05, + 1.0416641998745035e-05, + 9.59026874625124e-06, + 7.920963980723172e-06, + 7.568597084173234e-06, + 7.2189718594017904e-06, + 6.964055501157418e-06, + 5.9816525208589155e-06, + 5.780535957455868e-06, + 5.735510512749897e-06, + 5.703053830075078e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00013640988436236512, + "first_to_last_ratio": 5.122011664762131, + "imbalance_abs_log_ratio": 1.6335472651589908, + "last_quartile_mean": 2.6632091703504557e-05, + "mean": 7.298115508547198e-05, + "normalized": [ + 2.2789966397783896, + 2.0685142340171683, + 2.076140409583827, + 1.9537250082825501, + 1.8792316671609142, + 1.7055529809049683, + 1.5594088420998298, + 1.4313187223467239, + 1.382107687555985, + 1.2644854892365605, + 1.1686221274742359, + 1.0808639824268664, + 1.0704039370079022, + 0.9820006566284564, + 0.9088834247617501, + 0.8405321787627472, + 0.856406336273156, + 0.7807790484250425, + 0.7130462191259114, + 0.650909996157966, + 0.6919497653761542, + 0.6322822243882543, + 0.5753541906206389, + 0.5291452134901508, + 0.4371545081034775, + 0.4171814083388854, + 0.3982526780954131, + 0.3849724710776064, + 0.3303152718064163, + 0.31952201613190695, + 0.31683873767491905, + 0.31510192688522626 + ], + "peak_layer": 1, + "peak_normalized": 2.2789966397783896, + "population_cv": 0.5877439115216546, + "rest_mean": 7.874967775057519e-05, + "spike_contrast": 0.5311911602534674, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1831132693914694e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00016632380720693618, + 0.00015096255810931325, + 0.0001515191252110526, + 0.00014258510782383382, + 0.00013714849774260074, + 0.00012447322660591453, + 0.00011380745854694396, + 0.00010445929365232587, + 0.0001008678154903464, + 9.228361159330234e-05, + 8.528739272151142e-05, + 7.8882701927796e-05, + 7.81193157308735e-05, + 7.16675422154367e-05, + 6.633136217715219e-05, + 6.134300929261371e-05, + 6.250152364373207e-05, + 5.6982156820595264e-05, + 5.203893670113757e-05, + 4.750416337628849e-05, + 5.049929313827306e-05, + 4.6144687075866386e-05, + 4.199001341476105e-05, + 3.8617628888459876e-05, + 3.190404095221311e-05, + 3.044638106075581e-05, + 2.9064940463285893e-05, + 2.809573561535217e-05, + 2.4106790078803897e-05, + 2.331908581254538e-05, + 2.312325705133844e-05, + 2.2996502593741752e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.12096964661031961, + "first_to_last_ratio": 5.122011546850009, + "imbalance_abs_log_ratio": 1.6335472421383241, + "last_quartile_mean": 0.023617605213075876, + "mean": 0.06472041795495898, + "normalized": [ + 2.278996550367558, + 2.068514172728323, + 2.0761403775307596, + 1.9537250896945753, + 1.879231622178916, + 1.7055529409943957, + 1.5594087663016845, + 1.431318663899547, + 1.382107735687561, + 1.2644854472317082, + 1.1686221211602732, + 1.0808640120306183, + 1.0704040262086971, + 0.9820006724164564, + 0.9088834582355364, + 0.8405322479178425, + 0.8564063602524741, + 0.7807790810183837, + 0.713046234965646, + 0.6509099797949671, + 0.6919497851652032, + 0.6322822095960979, + 0.575354189755429, + 0.529145232117261, + 0.4371544964362311, + 0.41718141126177893, + 0.3982526899729607, + 0.38497247844984545, + 0.33031527661149607, + 0.3195220199067584, + 0.3168387288221828, + 0.3151019212888319 + ], + "peak_layer": 1, + "peak_normalized": 2.278996550367558, + "population_cv": 0.5877438959595169, + "rest_mean": 0.06983600149827975, + "spike_contrast": 0.5311911625115104, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.037096266821026805, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.14749760925769806, + 0.13387510180473328, + 0.1343686729669571, + 0.12644590437412262, + 0.121624656021595, + 0.11038409918546677, + 0.10092558711767197, + 0.09263554215431213, + 0.08945059031248093, + 0.08183802664279938, + 0.0756337121129036, + 0.06995397061109543, + 0.06927699595689774, + 0.06355549395084381, + 0.05882331728935242, + 0.05439959838986397, + 0.05542697757482529, + 0.050532348453998566, + 0.04614865034818649, + 0.04212716594338417, + 0.04478327929973602, + 0.040921568870544434, + 0.03723716363310814, + 0.034246500581502914, + 0.0282928217202425, + 0.027000155299901962, + 0.025775080546736717, + 0.024915579706430435, + 0.02137814275920391, + 0.020679598674178123, + 0.020505934953689575, + 0.020393528044223785 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.71177167280257e-06, + "first_to_last_ratio": 4.678295917137282, + "imbalance_abs_log_ratio": 1.542933923351862, + "last_quartile_mean": 1.6484146811990286e-06, + "mean": 4.007248069370917e-06, + "normalized": [ + 3.3590280906213086, + 2.3726088061078587, + 2.138143013037793, + 1.8393897753025719, + 1.595536022354422, + 1.4487646483149947, + 1.2955498112963995, + 1.3466260007113127, + 1.2739769202114613, + 1.0659628023122865, + 1.0478341793654378, + 1.0914579443825887, + 1.0411457104586188, + 0.9155146938805809, + 0.8066044523853629, + 0.7701707432135287, + 0.8215762550934921, + 0.7364785130598478, + 0.6730090976811546, + 0.6266208737012697, + 0.6693583504854638, + 0.6236588439557476, + 0.5897891291930596, + 0.5603290672032583, + 0.4588325813842428, + 0.4472228580494504, + 0.4388965999812878, + 0.43180771263272905, + 0.37877595830431615, + 0.3751711426274845, + 0.3796103005144005, + 0.3805491021762693 + ], + "peak_layer": 1, + "peak_normalized": 3.3590280906213086, + "population_cv": 0.6648044266247514, + "rest_mean": 4.318630839474348e-06, + "spike_contrast": 0.5385459413556807, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.325781110812386e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.3460458831104916e-05, + 9.507632057648152e-06, + 8.568069461034611e-06, + 7.370891125901835e-06, + 6.393708645191509e-06, + 5.805559339933097e-06, + 5.191589480091352e-06, + 5.396264441515086e-06, + 5.105141553940484e-06, + 4.271577381587122e-06, + 4.198931492283009e-06, + 4.373742740426678e-06, + 4.172129138169112e-06, + 3.6686944895336637e-06, + 3.232264134567231e-06, + 3.086265223828377e-06, + 3.292259862064384e-06, + 2.9512520995922387e-06, + 2.696914407351869e-06, + 2.5110252863669302e-06, + 2.682284957700176e-06, + 2.499155698387767e-06, + 2.3634313492948422e-06, + 2.2453775727626635e-06, + 1.8386559759164811e-06, + 1.7921329344972037e-06, + 1.7587675529284752e-06, + 1.730360622786975e-06, + 1.5178492276390898e-06, + 1.5034038369776681e-06, + 1.5211926438496448e-06, + 1.5249546549966908e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00011456928586994763, + "first_to_last_ratio": 4.836938296613537, + "imbalance_abs_log_ratio": 1.5762819372189676, + "last_quartile_mean": 2.3686323629590333e-05, + "mean": 6.211567796299278e-05, + "normalized": [ + 2.3488758065274955, + 2.0713117417995783, + 2.0294413849686572, + 1.9098281260734524, + 1.808891788235116, + 1.6555687759458058, + 1.5234656987314679, + 1.4082197237907075, + 1.3570554765296423, + 1.2477441770945124, + 1.1584743113969003, + 1.07706541625456, + 1.062967326057557, + 0.9794184805192363, + 0.9103158522005721, + 0.8467437815449563, + 0.8606016851807378, + 0.7903183885158978, + 0.726053695018778, + 0.6672539632450412, + 0.7127881461452974, + 0.6531207323660743, + 0.5952249360680494, + 0.548642506624817, + 0.4562346365956528, + 0.4371660140704956, + 0.4212530247856756, + 0.40928721260242446, + 0.3409968025795472, + 0.33116514257975266, + 0.32832896742236684, + 0.3261762785291743 + ], + "peak_layer": 1, + "peak_normalized": 2.3488758065274955, + "population_cv": 0.5738573169932581, + "rest_mean": 6.679503382690234e-05, + "spike_contrast": 0.5516451476522887, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6847156297881153e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014590201317332685, + 0.00012866093311458826, + 0.00012606012751348317, + 0.00011863026884384453, + 0.0001123605397879146, + 0.00010283677693223581, + 9.463110473006964e-05, + 8.747252286411822e-05, + 8.429442095803097e-05, + 7.750447548460215e-05, + 7.195941725512967e-05, + 6.690264854114503e-05, + 6.602693611057475e-05, + 6.08372429269366e-05, + 5.654488631989807e-05, + 5.259606405161321e-05, + 5.34568571310956e-05, + 4.9091162509284914e-05, + 4.509931750362739e-05, + 4.1446932300459594e-05, + 4.4275318941799924e-05, + 4.0569037082605064e-05, + 3.697280044434592e-05, + 3.4079301258316264e-05, + 2.833932376233861e-05, + 2.7154863346368074e-05, + 2.6166417228523642e-05, + 2.5423152692383155e-05, + 2.1181247575441375e-05, + 2.0570547349052504e-05, + 2.0394376406329684e-05, + 2.0260660676285625e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.518698971078265e-05, + "first_to_last_ratio": 4.0537764308209505, + "imbalance_abs_log_ratio": 1.399648898709643, + "last_quartile_mean": 2.3481065454689087e-05, + "mean": 5.345527347344614e-05, + "normalized": [ + 2.3490757134047047, + 2.03180856162592, + 1.9640798219543283, + 1.8355777168881, + 1.687288592851433, + 1.564711486655573, + 1.4531995506450504, + 1.3597364706007256, + 1.3148292531115187, + 1.211773745536977, + 1.127520479381137, + 1.0571877619151735, + 1.041978223119568, + 0.9643430432278, + 0.9029106186893467, + 0.848047824187126, + 0.863550005931762, + 0.7965670293846996, + 0.7398915500109849, + 0.6885201220036962, + 0.7527468103098911, + 0.696422774701012, + 0.6386525384719246, + 0.5954551044265526, + 0.5183504589787767, + 0.502904662044204, + 0.4927615213178117, + 0.48500410119101467, + 0.3866832502775355, + 0.3782793274311306, + 0.37617794688759315, + 0.3739639328369289 + ], + "peak_layer": 1, + "peak_normalized": 2.3490757134047047, + "population_cv": 0.5319468462674285, + "rest_mean": 5.701573583657456e-05, + "spike_contrast": 0.6003391206010799, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4228776712552644e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00012557048466987908, + 0.00010861088230740279, + 0.00010499042400624603, + 9.812130883801728e-05, + 9.019447315949947e-05, + 8.364208042621613e-05, + 7.768117939122021e-05, + 7.268508488778025e-05, + 7.028455729596317e-05, + 6.477569695562124e-05, + 6.027191557222977e-05, + 5.651226092595607e-05, + 5.5699230870231986e-05, + 5.154922109795734e-05, + 4.826533404411748e-05, + 4.5332628360483795e-05, + 4.6161301725078374e-05, + 4.2580708395689726e-05, + 3.955110514652915e-05, + 3.680503141367808e-05, + 4.0238286601379514e-05, + 3.722746987477876e-05, + 3.413934609852731e-05, + 3.1830215448280796e-05, + 2.7708565539796837e-05, + 2.6882906240643933e-05, + 2.6340701879234985e-05, + 2.5926026864908636e-05, + 2.0670258891186677e-05, + 2.022102489718236e-05, + 2.010869502555579e-05, + 1.9990344299003482e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002512551309337141, + "first_to_last_ratio": 5.815353323084221, + "imbalance_abs_log_ratio": 1.7605015447583472, + "last_quartile_mean": 4.320547986935708e-05, + "mean": 0.0001316397593882357, + "normalized": [ + 2.224752247026743, + 2.0950471769673564, + 2.148762457826491, + 2.0117752264198114, + 1.9594249362752976, + 1.7610216345575365, + 1.6099537081187654, + 1.4585175613957049, + 1.4124641586885858, + 1.2915649811078755, + 1.192264449118915, + 1.0889144888939652, + 1.0788791319896314, + 0.988941963176937, + 0.9104012729618977, + 0.8363983367426334, + 0.8532666616213362, + 0.773042726053889, + 0.7048431559693451, + 0.6396986756467042, + 0.668255590231208, + 0.6075232312555597, + 0.5571337058341255, + 0.5014728599784464, + 0.3945136003262428, + 0.3676589998419824, + 0.34703606615034643, + 0.33261343254293874, + 0.3038165004585062, + 0.2945782724854268, + 0.2930231193739135, + 0.29243967096188167 + ], + "peak_layer": 1, + "peak_normalized": 2.224752247026743, + "population_cv": 0.6145445602222892, + "rest_mean": 0.0001427126183073226, + "spike_contrast": 0.503433558134641, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.184632122516632e-05, + "top_five_layers": [ + 1, + 3, + 2, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00029286585049703717, + 0.0002757915062829852, + 0.00028286257293075323, + 0.0002648296067491174, + 0.00025793822715058923, + 0.00023182046425063163, + 0.0002119339187629521, + 0.00019199890084564686, + 0.00018593644199427217, + 0.00017002130334731191, + 0.00015694940520916134, + 0.00014334444131236523, + 0.00014202338934410363, + 0.00013018408208154142, + 0.00011984500451944768, + 0.0001101032758015208, + 0.00011232381802983582, + 0.00010176315845455974, + 9.278538345824927e-05, + 8.420977974310517e-05, + 8.796900510787964e-05, + 7.997421198524535e-05, + 7.334094698308036e-05, + 6.601376662729308e-05, + 5.193367542233318e-05, + 4.839854227611795e-05, + 4.5683744247071445e-05, + 4.3785152229247615e-05, + 3.999433101853356e-05, + 3.877821291098371e-05, + 3.857349292957224e-05, + 3.849668792099692e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.021530626341700554, + 0.03857670724391937, + 0.04954109713435173, + 0.057694368064403534, + 0.02140580676496029, + 0.025963041931390762, + 0.03339661285281181, + 0.040483180433511734, + 0.02151828072965145, + 0.035047177225351334, + 0.03725159168243408, + 0.043239567428827286, + 0.02124657854437828, + 0.03028978779911995, + 0.03979094699025154, + 0.05222286656498909, + 0.03916332125663757, + 0.04088718071579933, + 0.04756740853190422, + 0.05542457476258278, + 0.020541559904813766, + 0.03680417314171791, + 0.05126549303531647, + 0.06891486048698425, + 0.02905990369617939, + 0.05358703061938286, + 0.07880406081676483, + 0.10752817988395691, + 0.029148343950510025, + 0.054091889411211014, + 0.08628532290458679, + 0.11884573101997375 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.2954461630652077e-05, + "first_to_last_ratio": 5.105695837816317, + "imbalance_abs_log_ratio": 1.6303567474829335, + "last_quartile_mean": 6.454450613091467e-06, + "mean": 1.7350209816413553e-05, + "normalized": [ + 2.118074842246104, + 2.1574284903940977, + 2.139357285099935, + 2.0292905713578766, + 1.9547395369559644, + 1.7470193301554893, + 1.5858965602051887, + 1.4631504379843852, + 1.4022879982430452, + 1.2651865013786623, + 1.1596469721061315, + 1.075647057275218, + 1.0548378604659292, + 0.9617861976864589, + 0.8861851944905509, + 0.81666640942737, + 0.8226291628131417, + 0.7462778224571112, + 0.6809633158339518, + 0.623540506764385, + 0.6619488684789228, + 0.6046004431197245, + 0.5544183299835491, + 0.5123407388295048, + 0.4384090182726579, + 0.4193402639944847, + 0.4032847838635732, + 0.3940637985584808, + 0.33301498018274, + 0.33016832240151506, + 0.32877371956677426, + 0.3290246794070765 + ], + "peak_layer": 2, + "peak_normalized": 2.1574284903940977, + "population_cv": 0.5996208655691896, + "rest_mean": 1.8782105026157632e-05, + "spike_contrast": 0.5120818816847563, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.61797568379552e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.674904291983694e-05, + 3.7431836972245947e-05, + 3.711829776875675e-05, + 3.52086171915289e-05, + 3.391514110262506e-05, + 3.0311151931528002e-05, + 2.7515638066688552e-05, + 2.5385967092006467e-05, + 2.4329990992555395e-05, + 2.1951251255813986e-05, + 2.0120118279010057e-05, + 1.8662702132132836e-05, + 1.8301658201380633e-05, + 1.6687192328390665e-05, + 1.537549906061031e-05, + 1.4169333553581964e-05, + 1.4272788575908635e-05, + 1.2948076800967101e-05, + 1.1814856406999752e-05, + 1.0818558621394914e-05, + 1.148495175584685e-05, + 1.0489944543223828e-05, + 9.619274351280183e-06, + 8.889219316188246e-06, + 7.606488452438498e-06, + 7.27564156477456e-06, + 6.997075615799986e-06, + 6.837089586042566e-06, + 5.77787977817934e-06, + 5.728489668399561e-06, + 5.704293016606243e-06, + 5.708647222490981e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 0.00012894753399450565, + "first_to_last_ratio": 4.956313899688395, + "imbalance_abs_log_ratio": 1.600662299067145, + "last_quartile_mean": 2.601682149361295e-05, + "mean": 6.873563796716553e-05, + "normalized": [ + 2.0729820101052394, + 2.1343785185339303, + 2.1192055857825096, + 2.004033226215171, + 1.9290001532413559, + 1.7292283569198776, + 1.5695265126752398, + 1.449584785193964, + 1.392505223726082, + 1.2691464160417294, + 1.1657826244262586, + 1.0843620958713254, + 1.0620513069452608, + 0.9734787377496757, + 0.8984695878304712, + 0.827044661933948, + 0.8348303459591176, + 0.7567409969045322, + 0.6912396733798123, + 0.6326506836056272, + 0.674324856446311, + 0.615953502956551, + 0.5640541296457265, + 0.5213814868377082, + 0.4457146646251511, + 0.42575966145291916, + 0.4106923182898804, + 0.40125618166856064, + 0.3390529581992065, + 0.3361781427218294, + 0.33439592808042723, + 0.33499466603460004 + ], + "peak_layer": 2, + "peak_normalized": 2.1343785185339303, + "population_cv": 0.5868596491376813, + "rest_mean": 7.428176730981149e-05, + "spike_contrast": 0.5221542367874475, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.8786539516877386e-05, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00014248774095904082, + 0.00014670786913484335, + 0.00014566494792234153, + 0.00013774850231129676, + 0.00013259105617180467, + 0.00011885961430380121, + 0.00010788240615511313, + 9.963813499780372e-05, + 9.571473492542282e-05, + 8.723558858036995e-05, + 8.013081242097542e-05, + 7.453432044712827e-05, + 7.300077413674444e-05, + 6.691268208669499e-05, + 6.17568803136237e-05, + 5.684744246536866e-05, + 5.7382596423849463e-05, + 5.201507519814186e-05, + 4.7512799937976524e-05, + 4.3485648347996175e-05, + 4.6350149204954505e-05, + 4.233795698382892e-05, + 3.877062044921331e-05, + 3.5837489122059196e-05, + 3.0636481824330986e-05, + 2.926486195065081e-05, + 2.8229198505869135e-05, + 2.7580599635257386e-05, + 2.3305021386477165e-05, + 2.3107419110601768e-05, + 2.298491745023057e-05, + 2.3026072085485794e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.11435196176171303, + "first_to_last_ratio": 4.956313746493952, + "imbalance_abs_log_ratio": 1.6006622681581977, + "last_quartile_mean": 0.023071978008374572, + "mean": 0.06095545022981241, + "normalized": [ + 2.072982016679514, + 2.1343786009397814, + 2.119205465978219, + 2.004033333779568, + 1.9290000448154139, + 1.7292284201771326, + 1.5695265788612092, + 1.4495847902174488, + 1.392505122730498, + 1.2691463963100211, + 1.1657826534776023, + 1.0843621171351583, + 1.0620512430067517, + 0.97347872771027, + 0.8984695791578813, + 0.8270447011604714, + 0.8348303200871255, + 0.7567409772234059, + 0.691239667353491, + 0.6326506435390216, + 0.6743248169617974, + 0.6159534846579819, + 0.5640541661909846, + 0.5213814964456189, + 0.4457146897387812, + 0.4257596753388107, + 0.4106923303842507, + 0.4012562101149584, + 0.33905296941192775, + 0.3361781519455022, + 0.33439594785006815, + 0.3349946606193331 + ], + "peak_layer": 2, + "peak_normalized": 2.1343786009397814, + "population_cv": 0.586859647417989, + "rest_mean": 0.0658738130910529, + "spike_contrast": 0.5221542395241354, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03439629077911377, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.12635955214500427, + 0.1301020085811615, + 0.12917712330818176, + 0.12215675413608551, + 0.11758306622505188, + 0.10540589690208435, + 0.09567119926214218, + 0.08836009353399277, + 0.0848807767033577, + 0.07736138999462128, + 0.07106080651283264, + 0.06609778106212616, + 0.06473781168460846, + 0.05933883413672447, + 0.054766617715358734, + 0.05041288211941719, + 0.05088745802640915, + 0.04612748697400093, + 0.04213482514023781, + 0.038563504815101624, + 0.041103772819042206, + 0.03754572197794914, + 0.0343821756541729, + 0.031781043857336044, + 0.027168739587068558, + 0.025952372699975967, + 0.02503393590450287, + 0.0244587529450655, + 0.020667126402258873, + 0.020491890609264374, + 0.020383255556225777, + 0.02041975036263466 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 7.461771360794955e-06, + "first_to_last_ratio": 4.607249847489048, + "imbalance_abs_log_ratio": 1.5276311166276264, + "last_quartile_mean": 1.6195716767697377e-06, + "mean": 3.87299716564371e-06, + "normalized": [ + 3.0256593103213434, + 2.4199948075671043, + 2.2257260804728656, + 1.8933484975335824, + 1.6741432062616066, + 1.4907581305464577, + 1.3020525186737137, + 1.38123110006882, + 1.2950838303767327, + 1.0716197853648062, + 1.0585124142900662, + 1.1105323556195759, + 1.0395543790453834, + 0.9297870003858463, + 0.7930926998517134, + 0.7672014355923814, + 0.811551487187536, + 0.7239809147787968, + 0.6594474866461876, + 0.6140918356353618, + 0.649091321445497, + 0.6059854562252807, + 0.5735486080526876, + 0.5386444043869645, + 0.45665673160062437, + 0.44875544656608446, + 0.4444235140257583, + 0.44115113302107534, + 0.3820499025609541, + 0.38571480138297476, + 0.3910437033480733, + 0.3955657011641444 + ], + "peak_layer": 1, + "peak_normalized": 3.0256593103213434, + "population_cv": 0.6453809227293102, + "rest_mean": 4.185142588371322e-06, + "spike_contrast": 0.5226612562717605, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1874118829146026e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1718369933078066e-05, + 9.372633030579891e-06, + 8.620230801170692e-06, + 7.332933364523342e-06, + 6.483951892732875e-06, + 5.773702014266746e-06, + 5.042845714342548e-06, + 5.349504135665484e-06, + 5.015856004320085e-06, + 4.150380391365616e-06, + 4.099615580344107e-06, + 4.30108866567025e-06, + 4.026191163575277e-06, + 3.60106241714675e-06, + 3.0716457786184037e-06, + 2.9713689855270786e-06, + 3.1431366096512647e-06, + 2.8039760309184203e-06, + 2.5540382466715528e-06, + 2.3783759388606995e-06, + 2.5139288482023403e-06, + 2.3469799543818226e-06, + 2.2213521333469544e-06, + 2.086168251480558e-06, + 1.7686302271613386e-06, + 1.7380285726176226e-06, + 1.7212510101671796e-06, + 1.7085770878111362e-06, + 1.4796781897530309e-06, + 1.4938723325030878e-06, + 1.5145111547099077e-06, + 1.5320248394345981e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 0.00010837210174940992, + "first_to_last_ratio": 4.663743740878985, + "imbalance_abs_log_ratio": 1.5398185034733132, + "last_quartile_mean": 2.3237147615873255e-05, + "mean": 5.8729946545099665e-05, + "normalized": [ + 2.116910388942848, + 2.12576419551759, + 2.0713182282290723, + 1.958808009785277, + 1.85306829100999, + 1.6774795633398605, + 1.5342385858151337, + 1.4245037147377415, + 1.3659845454711594, + 1.2527089446494675, + 1.1583413114381589, + 1.0816008878212584, + 1.059050106445956, + 0.9749946937544519, + 0.9039297824077048, + 0.8370924379536968, + 0.8447837999101473, + 0.7717212504681601, + 0.708779947616627, + 0.6523240772837903, + 0.6962576204392495, + 0.6381480261388907, + 0.5854918741973255, + 0.5414119569273494, + 0.4665866617482466, + 0.4484909063213352, + 0.43561453714162146, + 0.42649489708721455, + 0.35024135478224894, + 0.347602098795276, + 0.34524713565577436, + 0.34501016816737656 + ], + "peak_layer": 2, + "peak_normalized": 2.12576419551759, + "population_cv": 0.5682337234037166, + "rest_mean": 6.323715206235647e-05, + "spike_contrast": 0.5438422767363214, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4391036751912904e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.0001243260339833796, + 0.00012484601757023484, + 0.00012164840882178396, + 0.00011504068970680237, + 0.0001088306016754359, + 9.851828508544713e-05, + 9.010575013235211e-05, + 8.366102701984346e-05, + 8.022419933695346e-05, + 7.357152935583144e-05, + 6.802932330174372e-05, + 6.352236232487485e-05, + 6.219795614015311e-05, + 5.726138624595478e-05, + 5.308774780132808e-05, + 4.916239413432777e-05, + 4.961410741088912e-05, + 4.5323147787712514e-05, + 4.1626608435763046e-05, + 3.8310958188958466e-05, + 4.089117283001542e-05, + 3.747839946299791e-05, + 3.4385906474199146e-05, + 3.1797095289221033e-05, + 2.740260970313102e-05, + 2.6339846954215318e-05, + 2.558361848059576e-05, + 2.5048022507689893e-05, + 2.0569656044244766e-05, + 2.041465268121101e-05, + 2.02763458219124e-05, + 2.026242873398587e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.029894954437623e-05, + "first_to_last_ratio": 3.896020389480804, + "imbalance_abs_log_ratio": 1.3599556192062257, + "last_quartile_mean": 2.3177227149062674e-05, + "mean": 5.08472848537167e-05, + "normalized": [ + 2.109310432443945, + 2.0823500317454187, + 1.994011612294058, + 1.875066462699875, + 1.727617067796283, + 1.5815194849270788, + 1.4631049290688447, + 1.374102981258909, + 1.3102874938711893, + 1.2100951410093623, + 1.1285247251248958, + 1.0633171446519065, + 1.0439918275480478, + 0.9649933174114446, + 0.9016610595399868, + 0.8411183538760414, + 0.8515897815206308, + 0.7853791814165375, + 0.7291252271433669, + 0.6771443554330082, + 0.737694287360078, + 0.6786209457138218, + 0.6314365240390994, + 0.5913748367488056, + 0.5313425485217884, + 0.5201071760800584, + 0.5115574693700321, + 0.5058856398361036, + 0.39753526518974286, + 0.39544594503370456, + 0.39314180453082337, + 0.39154694679511176 + ], + "peak_layer": 1, + "peak_normalized": 2.109310432443945, + "population_cv": 0.5213289902477497, + "rest_mean": 5.429271732282359e-05, + "spike_contrast": 0.5938540399226916, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.224194952053949e-05, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00010725270840339363, + 0.00010588184522930533, + 0.00010139007645193487, + 9.53420385485515e-05, + 8.78446371643804e-05, + 8.04159717517905e-05, + 7.439491309924051e-05, + 6.986940570641309e-05, + 6.662456144113094e-05, + 6.153005233500153e-05, + 5.7382418162887916e-05, + 5.406678974395618e-05, + 5.3084149840287864e-05, + 4.906729009235278e-05, + 4.584701673593372e-05, + 4.276858453522436e-05, + 4.3301028199493885e-05, + 3.993439895566553e-05, + 3.707403811858967e-05, + 3.4430951927788556e-05, + 3.750975156435743e-05, + 3.4506032534409314e-05, + 3.210683280485682e-05, + 3.006980477948673e-05, + 2.7017325919587165e-05, + 2.644603773660492e-05, + 2.601130836410448e-05, + 2.572291123215109e-05, + 2.0213588868500665e-05, + 2.0107352611375973e-05, + 1.9990193322882988e-05, + 1.9909099137294106e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0002369592275499599, + "first_to_last_ratio": 5.643806467094249, + "imbalance_abs_log_ratio": 1.7305587433768272, + "last_quartile_mean": 4.1985711050074315e-05, + "mean": 0.00012327438730608264, + "normalized": [ + 2.0536938202390447, + 2.174916037645403, + 2.1925488795748866, + 2.045836750341831, + 2.0263612272059848, + 1.7896425612539104, + 1.6137064052293353, + 1.480972211102483, + 1.4347200865083345, + 1.3015150111942828, + 1.1747028946305826, + 1.0857434163548976, + 1.0627794297334374, + 0.9758997849090596, + 0.8960919751718553, + 0.8230876844179131, + 0.8258333502423459, + 0.7456000475898096, + 0.6776422362024666, + 0.6227334054207839, + 0.6492856625079976, + 0.5954634942404196, + 0.5397936868798249, + 0.4867302228264425, + 0.3971938000026409, + 0.37230063475944564, + 0.35633216718416544, + 0.3460424570800059, + 0.31457528168990173, + 0.31217590063925915, + 0.31141034981968424, + 0.31466912740156466 + ], + "peak_layer": 3, + "peak_normalized": 2.1925488795748866, + "population_cv": 0.618597930641205, + "rest_mean": 0.00013391951021427909, + "spike_contrast": 0.49127064082412475, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.57907236018218e-05, + "top_five_layers": [ + 3, + 2, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.00025316784740425646, + 0.00026811144198291004, + 0.0002702851197682321, + 0.00025219927192665637, + 0.0002497984387446195, + 0.00022061709023546427, + 0.00019892866839654744, + 0.00018256594194099307, + 0.00017686423962004483, + 0.0001604434655746445, + 0.00014481077960226685, + 0.000133844354422763, + 0.00013101348304189742, + 0.00012030344805680215, + 0.00011046518920920789, + 0.00010146562999580055, + 0.00010180410026805475, + 9.191338904201984e-05, + 8.35359314805828e-05, + 7.67670790082775e-05, + 8.004029223229736e-05, + 7.34053974156268e-05, + 6.654273602180183e-05, + 6.0001370002282783e-05, + 4.896382233710028e-05, + 4.5895132643636316e-05, + 4.3926629587076604e-05, + 4.265817187842913e-05, + 3.877907511196099e-05, + 3.848329288302921e-05, + 3.838892007479444e-05, + 3.8790643884567544e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04393457621335983, + 0.024121688678860664, + 0.030756350606679916, + 0.03544071316719055, + 0.03840360417962074, + 0.026969486847519875, + 0.027978016063570976, + 0.029923241585493088, + 0.03152087330818176, + 0.024161184206604958, + 0.025710709393024445, + 0.02641460672020912, + 0.02742084674537182, + 0.021717891097068787, + 0.0224718376994133, + 0.023445574566721916, + 0.024849263951182365, + 0.01992032676935196, + 0.02058842033147812, + 0.020979218184947968, + 0.02214338816702366, + 0.01903437450528145, + 0.02008410543203354, + 0.021635910496115685, + 0.023218298330903053, + 0.022257626056671143, + 0.024358727037906647, + 0.025900650769472122, + 0.027197737246751785, + 0.02823900803923607, + 0.02913021855056286, + 0.029048049822449684 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.2108061784820165e-05, + "first_to_last_ratio": 4.5038225950610284, + "imbalance_abs_log_ratio": 1.5049265017537072, + "last_quartile_mean": 4.908732819330908e-06, + "mean": 1.424826272966584e-05, + "normalized": [ + 1.5393565072014856, + 1.6436990565699428, + 1.5746209905623882, + 1.4393049982912232, + 1.2296080351406573, + 1.5542168455156227, + 1.5210128581976592, + 1.9112374884411145, + 1.6656665742649008, + 1.4395591772020961, + 1.6324250537476421, + 1.3835491754103626, + 1.3477025444685555, + 1.5824225341301699, + 0.9384151980613074, + 0.8636925984699546, + 0.8034013225116797, + 0.8830691126060858, + 0.8605378947700126, + 0.6634146810472719, + 0.6587076476302224, + 0.7136661453539468, + 0.7077576665676224, + 0.6868400571413836, + 0.5793897205721035, + 0.6202776091342075, + 0.43119680892404477, + 0.3692582531909216, + 0.260043154103081, + 0.12977534522113043, + 0.15751431041740122, + 0.20866063513380237 + ], + "peak_layer": 8, + "peak_normalized": 1.9112374884411145, + "population_cv": 0.5186010202163185, + "rest_mean": 1.5120910120458575e-05, + "spike_contrast": 0.6306476755313104, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.535966819385067e-06, + "top_five_layers": [ + 8, + 9, + 2, + 11, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1933155949227512e-05, + 2.341985600651242e-05, + 2.243561357317958e-05, + 2.050759576377459e-05, + 1.751977833919227e-05, + 2.2144889953779057e-05, + 2.167179081880022e-05, + 2.723181387409568e-05, + 2.3732854970148765e-05, + 2.051121737167705e-05, + 2.3259221052285284e-05, + 1.9713172150659375e-05, + 1.9202419935027137e-05, + 2.2546772015630268e-05, + 1.3370786291488912e-05, + 1.2306119060667697e-05, + 1.1447073120507412e-05, + 1.2582200724864379e-05, + 1.2261170013516676e-05, + 9.452506674278993e-06, + 9.385439625475556e-06, + 1.0168502740270924e-05, + 1.0084317182190716e-05, + 9.786277587409131e-06, + 8.255296961579006e-06, + 8.837878340273164e-06, + 6.143805421743309e-06, + 5.26128860656172e-06, + 3.705163180711679e-06, + 1.8490732145437505e-06, + 2.244305278509273e-06, + 2.9730515507253585e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.226752566362848e-05, + "first_to_last_ratio": 3.2558095109526493, + "imbalance_abs_log_ratio": 1.1804409424216962, + "last_quartile_mean": 1.9125051835544582e-05, + "mean": 4.323053209986938e-05, + "normalized": [ + 1.0342606051842873, + 1.898717126163711, + 1.8089935381392723, + 1.359356421079101, + 1.274089744043777, + 1.682360782557305, + 1.2682690495969076, + 1.1968320759706532, + 1.2588398746694436, + 1.2880627600055157, + 1.2248689931301155, + 1.273330768170305, + 1.3149263319378108, + 1.3464942246682916, + 1.1005868831933079, + 1.0691381511055633, + 1.0591169598492518, + 1.0073228103092708, + 0.9327165506900205, + 0.7338249833201036, + 0.7729257473598179, + 0.8419776567654331, + 0.8893297062436448, + 0.8244833428272487, + 0.7679470877937514, + 0.8196268734117844, + 0.5402279437877997, + 0.47239671566976066, + 0.3242697700833044, + 0.15928933562535502, + 0.20350298326020644, + 0.25191420338787907 + ], + "peak_layer": 2, + "peak_normalized": 1.898717126163711, + "population_cv": 0.42885792050879057, + "rest_mean": 4.467689194387748e-05, + "spike_contrast": 0.7928078118486851, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.542018894222565e-05, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 4.471163629204966e-05, + 8.208255167119205e-05, + 7.82037532189861e-05, + 5.8765701396623626e-05, + 5.507957757799886e-05, + 7.272935181390494e-05, + 5.482794585986994e-05, + 5.173968747840263e-05, + 5.442031761049293e-05, + 5.568363849306479e-05, + 5.295173832564615e-05, + 5.50467666471377e-05, + 5.684496500180103e-05, + 5.820966180181131e-05, + 4.757895658258349e-05, + 4.621941116056405e-05, + 4.578618973027915e-05, + 4.354710108600557e-05, + 4.0321832784684375e-05, + 3.172364449710585e-05, + 3.341399133205414e-05, + 3.639914211817086e-05, + 3.8446196413133293e-05, + 3.564285361790098e-05, + 3.319876122986898e-05, + 3.543290586094372e-05, + 2.3354341465164907e-05, + 2.0421961380634457e-05, + 1.4018354704603553e-05, + 6.886162736918777e-06, + 8.797542250249535e-06, + 1.0890385055972729e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05521946307271719, + "first_to_last_ratio": 3.255809416547513, + "imbalance_abs_log_ratio": 1.180440913425793, + "last_quartile_mean": 0.016960287292022258, + "mean": 0.03833726760058198, + "normalized": [ + 1.0342606436254083, + 1.89871698336364, + 1.808993472457683, + 1.3593564915846248, + 1.2740897521241208, + 1.682360722725715, + 1.2682690813216775, + 1.1968320831111716, + 1.2588398492110833, + 1.2880628213558492, + 1.224868979767805, + 1.273330745435787, + 1.314926371992558, + 1.3464941899184253, + 1.1005869061077207, + 1.069138220456837, + 1.059116922317307, + 1.0073228539527392, + 0.932716509673011, + 0.7338250308305378, + 0.7729257302043212, + 0.841977653627328, + 0.8893297206164247, + 0.8244832831061564, + 0.7679470991481019, + 0.8196269423990945, + 0.5402279418758984, + 0.4723966869996653, + 0.32426976863400875, + 0.15928933802210024, + 0.20350299592243773, + 0.25191420811076204 + ], + "peak_layer": 2, + "peak_normalized": 1.89871698336364, + "population_cv": 0.42885790694027637, + "rest_mean": 0.039619914031829, + "spike_contrast": 0.7928077998002174, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.03141097687184811, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.039650727063417435, + 0.07279162108898163, + 0.06935186684131622, + 0.05211401358246803, + 0.04884511977434158, + 0.06449711322784424, + 0.048621971160173416, + 0.04588327184319496, + 0.04826048016548157, + 0.04938080906867981, + 0.04695812985301018, + 0.0488160215318203, + 0.0504106841981411, + 0.051620908081531525, + 0.042193494737148285, + 0.04098783805966377, + 0.040603648871183395, + 0.03861800581216812, + 0.03575780242681503, + 0.02813284657895565, + 0.029631860554218292, + 0.032279122620821, + 0.034094471484422684, + 0.03160843625664711, + 0.029440993443131447, + 0.03142225742340088, + 0.020710863173007965, + 0.018110398203134537, + 0.012431616894900799, + 0.006106717977672815, + 0.007801748812198639, + 0.009657702408730984 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2073289212821692e-05, + "first_to_last_ratio": 5.118173394564018, + "imbalance_abs_log_ratio": 1.6327976165236144, + "last_quartile_mean": 2.358905859978222e-06, + "mean": 9.608797252980139e-06, + "normalized": [ + 2.2116042312522914, + 1.2252124207577697, + 0.9166421564982928, + 1.186820314517719, + 0.9292558997947784, + 1.1346790160633606, + 1.0583515009014062, + 1.3892974727904446, + 1.5862793081829674, + 1.2655275776533215, + 1.4193695477429429, + 2.042377666614073, + 1.9979815760663124, + 2.116845169548275, + 1.4502152158936208, + 1.0590931016242418, + 1.4220084536973343, + 1.379051361973986, + 1.173597294405293, + 0.9241153337121745, + 0.6121286422299292, + 0.5710108338819657, + 0.461893587104583, + 0.5026871640197357, + 0.30513715456990576, + 0.2951723976971137, + 0.2866930192727529, + 0.2870900856457325, + 0.21497083513437823, + 0.17484004907583897, + 0.18455905309901366, + 0.21549255857844585 + ], + "peak_layer": 1, + "peak_normalized": 2.2116042312522914, + "population_cv": 0.6038429581123452, + "rest_mean": 1.0515277882337546e-05, + "spike_contrast": 0.44828124441370076, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.713801854450139e-06, + "top_five_layers": [ + 1, + 14, + 12, + 13, + 9 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1250856661936268e-05, + 1.1772817742894404e-05, + 8.807828635326587e-06, + 1.1403915777918883e-05, + 8.929031537263654e-06, + 1.0902900612563826e-05, + 1.0169484994548839e-05, + 1.3349477740121074e-05, + 1.5242236258927733e-05, + 1.2160197911725845e-05, + 1.3638434211316053e-05, + 1.9624792912509292e-05, + 1.919819987961091e-05, + 2.034033605013974e-05, + 1.3934823982708622e-05, + 1.017661088553723e-05, + 1.3663790923601482e-05, + 1.3251024938654155e-05, + 1.1276858458586503e-05, + 8.879636880010366e-06, + 5.8818200159294065e-06, + 5.4867273320269305e-06, + 4.43824183093966e-06, + 4.830219040741213e-06, + 2.9320010526134865e-06, + 2.836251724147587e-06, + 2.75477509603661e-06, + 2.7585904263105476e-06, + 2.06561117011006e-06, + 1.6800025832708343e-06, + 1.773390522430418e-06, + 2.0706243049062323e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.736539585792343e-05, + "first_to_last_ratio": 5.693189725390274, + "imbalance_abs_log_ratio": 1.7392706754966003, + "last_quartile_mean": 8.319658775235439e-06, + "mean": 3.0428698622131378e-05, + "normalized": [ + 0.9332234373969145, + 2.093244562439342, + 2.000219271237334, + 1.4660610383042352, + 1.4267640458493276, + 1.8877254776078727, + 1.397617962458032, + 1.2479661794130616, + 1.4262249611304942, + 1.4477791439571595, + 1.24181841376056, + 1.2462360629867504, + 1.2738256274260629, + 1.4269714780931013, + 1.1746071387256543, + 1.0297217541028905, + 1.0028420024769986, + 1.0837416535661664, + 0.9989109522318798, + 0.7645609375506638, + 0.8126373951104926, + 0.8905222845518224, + 0.8066853450678224, + 0.7327738890015353, + 0.50846646429871, + 0.4349609059082798, + 0.29076653368375877, + 0.2846228030965179, + 0.21016872042077955, + 0.130190170117833, + 0.14875731368946996, + 0.17938607433847711 + ], + "peak_layer": 2, + "peak_normalized": 2.093244562439342, + "population_cv": 0.5294278422744164, + "rest_mean": 3.183621146052718e-05, + "spike_contrast": 0.7170491791430023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.2828129294794054e-05, + "top_five_layers": [ + 2, + 3, + 6, + 4, + 10 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.83967747236602e-05, + 6.369470793288201e-05, + 6.086406938266009e-05, + 4.461032949620858e-05, + 4.3414573156042024e-05, + 5.744102963944897e-05, + 4.2527695768512785e-05, + 3.797398676397279e-05, + 4.339816950960085e-05, + 4.405403524287976e-05, + 3.7786918255733326e-05, + 3.7921341572655365e-05, + 3.876085611409508e-05, + 4.342088504927233e-05, + 3.5741766623687e-05, + 3.133309292024933e-05, + 3.051517705898732e-05, + 3.297684816061519e-05, + 3.0395560315810144e-05, + 2.3264594346983358e-05, + 2.4727498384891078e-05, + 2.7097434212919325e-05, + 2.4546385247958824e-05, + 2.2297355826594867e-05, + 1.547197280160617e-05, + 1.323529431829229e-05, + 8.847647222864907e-06, + 8.660701496410184e-06, + 6.39516065348289e-06, + 3.961517450079555e-06, + 4.52649146609474e-06, + 5.458484793052776e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.615552941482747e-05, + "first_to_last_ratio": 8.764492263828256, + "imbalance_abs_log_ratio": 2.1707085890088735, + "last_quartile_mean": 4.1252280595927004e-06, + "mean": 2.208462114339227e-05, + "normalized": [ + 0.7494933219758998, + 2.2634292925463795, + 2.1410570182439597, + 1.5739498957546751, + 1.5487374547204664, + 2.0376524711677733, + 1.534141636657325, + 1.2486273054515569, + 1.5321016320199576, + 1.6393386005987565, + 1.2261935965091952, + 1.1547399796224558, + 1.219246066181087, + 1.4546728804264264, + 1.2279261343623535, + 1.0312465593605784, + 0.9063241145323925, + 1.1172138768757267, + 1.0304062764426833, + 0.7847973944776799, + 0.8360866099062609, + 0.911401228786939, + 0.6978180640030948, + 0.6390634924568735, + 0.31508360869438684, + 0.26042370730894115, + 0.18982145136915554, + 0.23533143612154772, + 0.14063662904795424, + 0.10306654364310344, + 0.11411045319594963, + 0.13586126753846497 + ], + "peak_layer": 2, + "peak_normalized": 2.2634292925463795, + "population_cv": 0.6059056054769965, + "rest_mean": 2.3393786848289352e-05, + "spike_contrast": 0.6418424872519509, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.5015126336948015e-05, + "top_five_layers": [ + 2, + 3, + 6, + 10, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.6552276065340266e-05, + 4.998697841074318e-05, + 4.7284433094318956e-05, + 3.476008714642376e-05, + 3.420327993808314e-05, + 4.500078284763731e-05, + 3.388093682588078e-05, + 2.757546099019237e-05, + 3.383588409633376e-05, + 3.620417191996239e-05, + 2.708002102735918e-05, + 2.5501994969090447e-05, + 2.6926587452180684e-05, + 3.212589945178479e-05, + 2.7118283469462767e-05, + 2.277468956890516e-05, + 2.0015824702568352e-05, + 2.467324520694092e-05, + 2.2756132239010185e-05, + 1.7331953131360933e-05, + 1.8464656022842973e-05, + 2.0127950847381726e-05, + 1.5411047570523806e-05, + 1.4113475117483176e-05, + 6.958502126508392e-06, + 5.7513589126756415e-06, + 4.19213483837666e-06, + 5.1972056098748e-06, + 3.1059066714078654e-06, + 2.276185568916844e-06, + 2.520086127333343e-06, + 3.0004446216480574e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011569616435735952, + "first_to_last_ratio": 4.483785350026755, + "imbalance_abs_log_ratio": 1.500467633864892, + "last_quartile_mean": 2.5803234393606544e-05, + "mean": 7.895687571135568e-05, + "normalized": [ + 1.113643291734891, + 1.9315033327137208, + 1.8689084538217502, + 1.3013119102609747, + 1.3204401608174607, + 1.7172337521094045, + 1.2105960380108525, + 1.2588293628008012, + 1.3316624978487503, + 1.2436188212370092, + 1.3620564885728002, + 1.3794695319626817, + 1.4052872102652663, + 1.3953280795865155, + 1.1319795963186805, + 0.9796315470123751, + 1.1579414909870167, + 1.0808582581724016, + 0.9982944348550936, + 0.7435043404975695, + 0.8011357344053238, + 0.8526183967497576, + 0.9599446768973036, + 0.8397896829211625, + 0.7011434784757186, + 0.5094988193534132, + 0.32142473628464113, + 0.2742399520171878, + 0.26163364470536055, + 0.152278590465612, + 0.18046309892286552, + 0.21373058921563795 + ], + "peak_layer": 2, + "peak_normalized": 1.9315033327137208, + "population_cv": 0.47743629272692334, + "rest_mean": 8.142900973056115e-05, + "spike_contrast": 0.8056999860950416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.560735200764612e-05, + "top_five_layers": [ + 2, + 3, + 6, + 13, + 14 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.79297949722968e-05, + 0.00015250546857714653, + 0.00014756317250430584, + 0.00010274752276018262, + 0.00010425782966194674, + 0.00013558741193264723, + 9.558488090988249e-05, + 9.939323354046792e-05, + 0.00010514391033211723, + 9.819225670071319e-05, + 0.00010754372488008812, + 0.00010891860438277945, + 0.00011095708759967238, + 0.00011017074575647712, + 8.937757229432464e-05, + 7.734864630037919e-05, + 9.142744238488376e-05, + 8.53411911521107e-05, + 7.882220961619169e-05, + 5.870477980352007e-05, + 6.32551746093668e-05, + 6.732008478138596e-05, + 7.579423254355788e-05, + 6.630716961808503e-05, + 5.5360098485834897e-05, + 4.022843495476991e-05, + 2.5378692953381687e-05, + 2.165312980650924e-05, + 2.0657775166910142e-05, + 1.2023441740893759e-05, + 1.4248802472138777e-05, + 1.6875499568413943e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.018067941069602966, + 0.027584951370954514, + 0.03231721371412277, + 0.036101218312978745, + 0.025345133617520332, + 0.027166159823536873, + 0.028526438400149345, + 0.02979987859725952, + 0.02248864620923996, + 0.023894935846328735, + 0.02451697364449501, + 0.025183402001857758, + 0.020499059930443764, + 0.02105838619172573, + 0.0216353852301836, + 0.022715488448739052, + 0.01881815865635872, + 0.0193756353110075, + 0.02001863345503807, + 0.021088071167469025, + 0.018417155370116234, + 0.019219009205698967, + 0.020342160016298294, + 0.02150510996580124, + 0.02099662832915783, + 0.02211105264723301, + 0.02328667603433132, + 0.024160878732800484, + 0.02260657213628292, + 0.024989914149045944, + 0.02536177635192871, + 0.027097374200820923 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2429669027369528e-05, + "first_to_last_ratio": 1.5167074865147177, + "imbalance_abs_log_ratio": 0.4165418581180973, + "last_quartile_mean": 8.195165605684451e-06, + "mean": 1.1438181132916725e-05, + "normalized": [ + 2.3239055235296235, + 1.1190114244144547, + 0.8331917749372405, + 0.7813452753113945, + 1.003803456769089, + 0.9365675136194498, + 0.8842494964053754, + 0.8113839672107822, + 1.0847409217961617, + 1.0682117256400467, + 0.9883998597662562, + 0.9038042770049443, + 1.1938503302540149, + 1.1817190494107024, + 1.0722151723600664, + 0.9843285081524176, + 1.2863323428556752, + 1.156982982287669, + 1.0862279117485159, + 1.0234351273362494, + 1.2841852282798012, + 1.1754321217296801, + 1.070502203826642, + 1.0143774573484126, + 0.8953924191337335, + 0.8270637955966683, + 0.8133202903656793, + 0.7381861493144678, + 0.6730532700250402, + 0.5804717059865397, + 0.6082410306470007, + 0.5960676869362054 + ], + "peak_layer": 1, + "peak_normalized": 2.3239055235296235, + "population_cv": 0.3045451785807339, + "rest_mean": 1.1251827985419098e-05, + "spike_contrast": 1.1059970118215763, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2444488129403908e-05, + "top_five_layers": [ + 1, + 17, + 21, + 13, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.6581252313917503e-05, + 1.2799455362255685e-05, + 9.530198440188542e-06, + 8.937168786360417e-06, + 1.1481685760372784e-05, + 1.0712628863984719e-05, + 1.011420590657508e-05, + 9.280756785301492e-06, + 1.2407463145791553e-05, + 1.22183992061764e-05, + 1.1305496627755929e-05, + 1.0337877029087394e-05, + 1.3655476323037874e-05, + 1.3516716535377782e-05, + 1.2264191354915965e-05, + 1.125892777054105e-05, + 1.4713302334712353e-05, + 1.323378091910854e-05, + 1.2424471606209408e-05, + 1.1706236364261713e-05, + 1.4688743249280378e-05, + 1.3444805517792702e-05, + 1.2244598110555671e-05, + 1.1602633094298653e-05, + 1.0241660675092135e-05, + 9.460105502512306e-06, + 9.302904800279066e-06, + 8.443506885669194e-06, + 7.698505214648321e-06, + 6.639540515607223e-06, + 6.957171081012348e-06, + 6.817930170655018e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.8672892262402456e-05, + "first_to_last_ratio": 1.4790455122350374, + "imbalance_abs_log_ratio": 0.39139695555746173, + "last_quartile_mean": 3.290831273261574e-05, + "mean": 4.5425740495375067e-05, + "normalized": [ + 2.292867781479936, + 1.098844858197334, + 0.8288263395703189, + 0.7584336983627002, + 0.9778795698933019, + 0.9351432056276012, + 0.8751120479081964, + 0.8047536035443427, + 1.088349470440326, + 1.0783459726467945, + 0.9909402014503932, + 0.9019584923345375, + 1.2027204540239667, + 1.175772139842897, + 1.0782465054806212, + 0.9953149147774686, + 1.2799515830354413, + 1.1648392407100392, + 1.0907985090406367, + 1.0337452797487645, + 1.2844092657539465, + 1.1758215530808431, + 1.0768851187191208, + 1.0145044696518641, + 0.9072210422189393, + 0.8359231056438284, + 0.8227296910259355, + 0.7472276249111484, + 0.6761454407882689, + 0.5893866866897062, + 0.6130538257244701, + 0.6038483076763113 + ], + "peak_layer": 1, + "peak_normalized": 2.292867781479936, + "population_cv": 0.29963351102833546, + "rest_mean": 4.465376965796437e-05, + "spike_contrast": 1.1106426937137044, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.9594383017392826e-05, + "top_five_layers": [ + 1, + 21, + 17, + 13, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.00010415521683171391, + 4.9915841373149306e-05, + 3.765005021705292e-05, + 3.445241236477159e-05, + 4.442090357770212e-05, + 4.2479572584852576e-05, + 3.975261279265396e-05, + 3.655652835732326e-05, + 4.943908061250113e-05, + 4.898466431768611e-05, + 4.501419243752025e-05, + 4.097213241038844e-05, + 5.4634467232972383e-05, + 5.3410320106195286e-05, + 4.89801459480077e-05, + 4.5212917029857635e-05, + 5.814274845761247e-05, + 5.291368506732397e-05, + 4.9550330004421994e-05, + 4.695864481618628e-05, + 5.8345241995994e-05, + 5.341256473911926e-05, + 4.891830394626595e-05, + 4.608461676980369e-05, + 4.1211187635781243e-05, + 3.797242607106455e-05, + 3.737310544238426e-05, + 3.394336818018928e-05, + 3.071440733037889e-05, + 2.6773326680995524e-05, + 2.784842399705667e-05, + 2.7430256523075514e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.0431636068969965, + "first_to_last_ratio": 1.4790455355871048, + "imbalance_abs_log_ratio": 0.39139697134606793, + "last_quartile_mean": 0.02918341988697648, + "mean": 0.04028400045353919, + "normalized": [ + 2.2928676956325664, + 1.0988448461233453, + 0.8288263129280055, + 0.7584337335279033, + 0.9778795372444193, + 0.9351432892874852, + 0.8751120548755073, + 0.8047535990009899, + 1.0883495038762179, + 1.0783460397661777, + 0.9909401548295442, + 0.9019584843211922, + 1.2027204645548106, + 1.1757721269107668, + 1.0782465359354751, + 0.9953149017803241, + 1.2799516204237265, + 1.1648392822258575, + 1.090798537285597, + 1.0337452971674015, + 1.2844092255829724, + 1.1758215089234016, + 1.0768851089878908, + 1.014504529948594, + 0.9072209977912502, + 0.8359230814928427, + 0.8227296688325424, + 0.7472276232112787, + 0.676145406434352, + 0.5893866663000006, + 0.6130538743742001, + 0.6038482904233612 + ], + "peak_layer": 1, + "peak_normalized": 2.2928676956325664, + "population_cv": 0.29963350187956755, + "rest_mean": 0.0395994091199504, + "spike_contrast": 1.110642674533265, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.04398079365491867, + "top_five_layers": [ + 1, + 21, + 17, + 13, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 0.09236588329076767, + 0.04426586627960205, + 0.03338843956589699, + 0.03055274486541748, + 0.039392899721860886, + 0.03767131268978119, + 0.03525301441550255, + 0.03241869434714317, + 0.04384307190775871, + 0.043440092355012894, + 0.03991903364658356, + 0.03633449599146843, + 0.04845039173960686, + 0.04736480489373207, + 0.04343608394265175, + 0.040095265954732895, + 0.05156157165765762, + 0.04692438617348671, + 0.04394172877073288, + 0.04164339601993561, + 0.05174114182591438, + 0.04736679419875145, + 0.04338124021887779, + 0.04086830094456673, + 0.036546491086483, + 0.03367432579398155, + 0.03314284235239029, + 0.030101317912340164, + 0.027237841859459877, + 0.023742852732539177, + 0.024696262553334236, + 0.024325424805283546 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.470277960104795e-06, + "first_to_last_ratio": 1.5769587704873496, + "imbalance_abs_log_ratio": 0.4554981633698566, + "last_quartile_mean": 2.2006142614827695e-06, + "mean": 2.887214563429552e-06, + "normalized": [ + 2.7519535335351715, + 1.313168413337379, + 0.9399430071407856, + 0.8753790372729023, + 1.0813834937551676, + 0.9567319703631147, + 0.8229162441441861, + 0.8740977436183472, + 1.0566599601043785, + 0.9454608388064315, + 0.9224389765316489, + 0.8092920053718847, + 1.0583487949182648, + 1.0534209725986503, + 0.9874597862575576, + 0.8539122103086818, + 1.169801497382018, + 1.081662118029895, + 1.0127159978603695, + 0.961060648114576, + 1.1999729309980693, + 1.1228483457198037, + 1.0226342502719021, + 1.0291945210343731, + 0.8967756962809945, + 0.8510820239929227, + 0.8261498765197046, + 0.7721004685625728, + 0.7451436093586472, + 0.6285081646269207, + 0.6879375096468019, + 0.6898453535358771 + ], + "peak_layer": 1, + "peak_normalized": 2.7519535335351715, + "population_cv": 0.35016697743385605, + "rest_mean": 2.8581899574352e-06, + "spike_contrast": 1.0649912990844543, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.043947435799055e-06, + "top_five_layers": [ + 1, + 2, + 21, + 17, + 22 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.945480319904163e-06, + 3.7913989672233583e-06, + 2.7138171390106436e-06, + 2.5274071049352642e-06, + 3.12218617182225e-06, + 2.7622904781310353e-06, + 2.3759357645758428e-06, + 2.523707735235803e-06, + 3.0508040254062507e-06, + 2.729748302954249e-06, + 2.6632792469172273e-06, + 2.3365996639768127e-06, + 3.055680053876131e-06, + 3.041452373508946e-06, + 2.851008275683853e-06, + 2.4654277694935445e-06, + 3.3774679195630597e-06, + 3.122990619885968e-06, + 2.92392837764055e-06, + 2.774788299575448e-06, + 3.4645793220988708e-06, + 3.2419040962849976e-06, + 2.952564500446897e-06, + 2.9715054097323446e-06, + 2.589183850432164e-06, + 2.4572564143454656e-06, + 2.3852719550632173e-06, + 2.229219717264641e-06, + 2.1513894807867473e-06, + 1.814637926145224e-06, + 1.9862231965817045e-06, + 1.9917315512429923e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.1570810026314575e-05, + "first_to_last_ratio": 1.5044497617163743, + "imbalance_abs_log_ratio": 0.40842722451739255, + "last_quartile_mean": 2.763190309451602e-05, + "mean": 3.7813772848949156e-05, + "normalized": [ + 2.388009071075982, + 1.1470596649494917, + 0.8534302761319221, + 0.7776431816025992, + 0.9983567968228081, + 0.9421045940312636, + 0.8787540856715802, + 0.8094928604716568, + 1.0777400416324399, + 1.0654722038935143, + 0.9718539610468895, + 0.8865810688989143, + 1.168850821337844, + 1.1332939684204937, + 1.0417735360252487, + 0.9621498679953465, + 1.2282401671332226, + 1.126146404473492, + 1.059853477638917, + 1.0196054375892694, + 1.2877065753533457, + 1.1949291884271547, + 1.0946140188339069, + 1.0404469270679417, + 0.9334039480335107, + 0.8520038997158634, + 0.8392353967355263, + 0.7615998622978171, + 0.6682831346061853, + 0.5860512257947129, + 0.6091233405529298, + 0.5961909957382107 + ], + "peak_layer": 1, + "peak_normalized": 2.388009071075982, + "population_cv": 0.309055923286973, + "rest_mean": 3.704195080866123e-05, + "spike_contrast": 1.13335315635395, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.198161186650395e-05, + "top_five_layers": [ + 1, + 21, + 17, + 22, + 13 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 9.029963257489726e-05, + 4.337465361459181e-05, + 3.227141860406846e-05, + 2.9405622626654804e-05, + 3.775163713726215e-05, + 3.562452911864966e-05, + 3.322900738567114e-05, + 3.060997914872132e-05, + 4.0753417124506086e-05, + 4.0289523894898593e-05, + 3.674946492537856e-05, + 3.3524975151522085e-05, + 4.419865945237689e-05, + 4.2854120692936704e-05, + 3.9393387851305306e-05, + 3.638251655502245e-05, + 4.644439468393102e-05, + 4.258384433342144e-05, + 4.007705865660682e-05, + 3.855512841255404e-05, + 4.869304393650964e-05, + 4.518478090176359e-05, + 4.139148586546071e-05, + 3.934322376153432e-05, + 3.5295524867251515e-05, + 3.2217481930274516e-05, + 3.173465665895492e-05, + 2.879896419472061e-05, + 2.5270306650782004e-05, + 2.2160807930049486e-05, + 2.303325163666159e-05, + 2.2544230887433514e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.563517248039716e-05, + "first_to_last_ratio": 1.4352355648633195, + "imbalance_abs_log_ratio": 0.3613289924385252, + "last_quartile_mean": 2.4828796995279845e-05, + "mean": 3.227010705586508e-05, + "normalized": [ + 2.4014447290909864, + 1.1673853090110469, + 0.8593097269193264, + 0.7832036205871891, + 1.0067937101766118, + 0.9335226700834045, + 0.8775072266073977, + 0.8050578010660164, + 1.053006259394381, + 1.0264307263592312, + 0.9338631869266799, + 0.8595966381479759, + 1.1212921464302679, + 1.0758485643993823, + 0.9942738962286088, + 0.9189156755903263, + 1.1746749961886236, + 1.0800838025282244, + 1.019430881526982, + 0.9950392558872734, + 1.2898668670070823, + 1.2311059818213306, + 1.13592608464973, + 1.1011764563471198, + 0.9942343261534591, + 0.8955937948027416, + 0.9014365249588188, + 0.831025803802073, + 0.6870790019632969, + 0.6046383015495036, + 0.6312332814164576, + 0.610002752378451 + ], + "peak_layer": 1, + "peak_normalized": 2.4014447290909864, + "population_cv": 0.30444570497962015, + "rest_mean": 3.1370954645724315e-05, + "spike_contrast": 1.1834364140297289, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.7125530070625246e-05, + "top_five_layers": [ + 1, + 21, + 22, + 17, + 2 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 7.749487849650905e-05, + 3.7671648897230625e-05, + 2.7730016881832853e-05, + 2.527406468288973e-05, + 3.2489340810570866e-05, + 3.0124876502668485e-05, + 2.8317252144915983e-05, + 2.5979301426559687e-05, + 3.398062472115271e-05, + 3.312302942504175e-05, + 3.0135865017655306e-05, + 2.77392755378969e-05, + 3.618421760620549e-05, + 3.4717748349066824e-05, + 3.2085325074149296e-05, + 2.965350722661242e-05, + 3.790688788285479e-05, + 3.4854419936891645e-05, + 3.289714368293062e-05, + 3.211002331227064e-05, + 4.162414188613184e-05, + 3.972792183049023e-05, + 3.6656456359196454e-05, + 3.55350821337197e-05, + 3.2084048143588006e-05, + 2.8900907636852935e-05, + 2.9089453164488077e-05, + 2.6817291654879227e-05, + 2.2172112949192524e-05, + 1.9511742721078917e-05, + 2.03699655685341e-05, + 1.9684854123624973e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.83701450220542e-05, + "first_to_last_ratio": 1.5137756203780361, + "imbalance_abs_log_ratio": 0.41460694084716315, + "last_quartile_mean": 5.837730759594706e-05, + "mean": 8.360470872048609e-05, + "normalized": [ + 2.260874919572962, + 1.046876661671232, + 0.8108813008392315, + 0.7410270778432692, + 0.9650502462422242, + 0.9363082641888973, + 0.8839048255705971, + 0.8110736329048943, + 1.115418327104151, + 1.1154856868411749, + 1.0274711394986145, + 0.9306550068796721, + 1.2562444747790198, + 1.2247146350825069, + 1.1283567053258787, + 1.0398138975133757, + 1.305825331548614, + 1.176867029634062, + 1.1082958616806637, + 1.034638772497703, + 1.2882363465735436, + 1.1524683581694375, + 1.0651417678309785, + 0.9883391355795494, + 0.8670300800809992, + 0.8013878422949675, + 0.7767996236633684, + 0.7006926068874338, + 0.6674164616571702, + 0.5747719332171776, + 0.5954528519609661, + 0.6024791948656639 + ], + "peak_layer": 1, + "peak_normalized": 2.260874919572962, + "population_cv": 0.30484696934594563, + "rest_mean": 8.24862149278892e-05, + "spike_contrast": 1.0867825039478176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.964457520050928e-05, + "top_five_layers": [ + 1, + 17, + 21, + 13, + 14 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 0.0001890197891043499, + 8.752381836529821e-05, + 6.77934949635528e-05, + 6.195335299707949e-05, + 8.068274473771453e-05, + 7.82797797000967e-05, + 7.389860547846183e-05, + 6.780957482988015e-05, + 9.325422433903441e-05, + 9.325985593022779e-05, + 8.590142533648759e-05, + 7.780714076943696e-05, + 0.00010502795339561999, + 0.0001023919103317894, + 9.433593368157744e-05, + 8.693333802511916e-05, + 0.00010917314648395404, + 9.839162521529943e-05, + 9.265875269193202e-05, + 8.650067320559174e-05, + 0.00010770262451842427, + 9.635178139433265e-05, + 8.905086724553257e-05, + 8.262980554718524e-05, + 7.248779729707167e-05, + 6.69997971272096e-05, + 6.494410627055913e-05, + 5.858120130142197e-05, + 5.579915887210518e-05, + 4.8053640057332814e-05, + 4.978266224497929e-05, + 5.037009759689681e-05 + ] + } + } + } + } + } + }, + "step": 500 + }, + { + "modes": { + "learned": { + "bits_per_byte": 2.073383110809672, + "forward": { + "activation_sha256": { + "attention_branch_output": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", + "mlp_branch_output": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", + "post_attention_state": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", + "post_mlp_state": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", + "pre_attention_input": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", + "pre_mlp_input": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93" + }, + "logits_sha256": "9adeddfaa352b7946853a2fe6e55c7269e6a01f6272498cdfb911b26410503c6", + "loss_nats": 1.4371596574783325, + "mixer_summary_sha256": "739af9a4ced9bd8d3e1cd2555b418bfc5db56ce4da6de7d1c45506ae68b1e3e1" + }, + "loss_nats": 1.4371596574783325, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.4738388657569885, + "group": 1, + "latest_source_mass": 0.19323334097862244, + "layer": 1, + "max_source_mass": 0.8067666888237, + "mixer_index": 1, + "normalized_entropy": 0.6836049818080586, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8067666888237, + "median": 0.8265105485916138, + "p05": 0.6604017019271851, + "p95": 0.8949732184410095 + }, + { + "label": "current_group_partial", + "mean": 0.19323334097862244, + "median": 0.17348942160606384, + "p05": 0.10502681136131287, + "p95": 0.33959829807281494 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6344536542892456, + "group": 1, + "latest_source_mass": 0.5104025602340698, + "layer": 2, + "max_source_mass": 0.5104025602340698, + "mixer_index": 2, + "normalized_entropy": 0.9153231407169755, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4895974397659302, + "median": 0.45956116914749146, + "p05": 0.2515213191509247, + "p95": 0.7871550917625427 + }, + { + "label": "current_group_partial", + "mean": 0.5104025602340698, + "median": 0.5404388308525085, + "p05": 0.21284496784210205, + "p95": 0.7484787106513977 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.41773349046707153, + "group": 1, + "latest_source_mass": 0.15382957458496094, + "layer": 2, + "max_source_mass": 0.8461704254150391, + "mixer_index": 3, + "normalized_entropy": 0.6026620351100812, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8461704254150391, + "median": 0.8643104434013367, + "p05": 0.7285550832748413, + "p95": 0.9108630418777466 + }, + { + "label": "current_group_partial", + "mean": 0.15382957458496094, + "median": 0.13568954169750214, + "p05": 0.08913692831993103, + "p95": 0.2714449167251587 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6868985891342163, + "group": 1, + "latest_source_mass": 0.49918031692504883, + "layer": 3, + "max_source_mass": 0.500819742679596, + "mixer_index": 4, + "normalized_entropy": 0.9909851881375595, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.500819742679596, + "median": 0.50103759765625, + "p05": 0.41182032227516174, + "p95": 0.5924672484397888 + }, + { + "label": "current_group_partial", + "mean": 0.49918031692504883, + "median": 0.49896240234375, + "p05": 0.40753278136253357, + "p95": 0.5881797075271606 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5841774344444275, + "group": 1, + "latest_source_mass": 0.315071702003479, + "layer": 3, + "max_source_mass": 0.684928297996521, + "mixer_index": 5, + "normalized_entropy": 0.8427898876722131, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.684928297996521, + "median": 0.6976089477539062, + "p05": 0.4487886428833008, + "p95": 0.8860265612602234 + }, + { + "label": "current_group_partial", + "mean": 0.315071702003479, + "median": 0.3023911118507385, + "p05": 0.1139734610915184, + "p95": 0.5512113571166992 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6896515488624573, + "group": 1, + "latest_source_mass": 0.49074798822402954, + "layer": 4, + "max_source_mass": 0.5092520117759705, + "mixer_index": 6, + "normalized_entropy": 0.9949568694852597, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5092520117759705, + "median": 0.509276270866394, + "p05": 0.4409506916999817, + "p95": 0.5760948657989502 + }, + { + "label": "current_group_partial", + "mean": 0.49074798822402954, + "median": 0.49072372913360596, + "p05": 0.4239051342010498, + "p95": 0.5590493679046631 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5611414909362793, + "group": 1, + "latest_source_mass": 0.2719496786594391, + "layer": 4, + "max_source_mass": 0.7280503511428833, + "mixer_index": 7, + "normalized_entropy": 0.8095560462108093, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7280503511428833, + "median": 0.7264741063117981, + "p05": 0.5577248930931091, + "p95": 0.8873021602630615 + }, + { + "label": "current_group_partial", + "mean": 0.2719496786594391, + "median": 0.2735258936882019, + "p05": 0.11269781738519669, + "p95": 0.44227516651153564 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6817618608474731, + "group": 2, + "latest_source_mass": 0.4686949551105499, + "layer": 5, + "max_source_mass": 0.5313050746917725, + "mixer_index": 8, + "normalized_entropy": 0.9835744557118811, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5313050746917725, + "median": 0.532880961894989, + "p05": 0.4224134683609009, + "p95": 0.6363531351089478 + }, + { + "label": "completed_group_1", + "mean": 0.4686949551105499, + "median": 0.4671190679073334, + "p05": 0.363646924495697, + "p95": 0.5775865316390991 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0218309164047241, + "group": 2, + "latest_source_mass": 0.2602040767669678, + "layer": 5, + "max_source_mass": 0.4974328279495239, + "mixer_index": 9, + "normalized_entropy": 0.9301105830916285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4974328279495239, + "median": 0.4930803179740906, + "p05": 0.35206541419029236, + "p95": 0.6587448716163635 + }, + { + "label": "completed_group_1", + "mean": 0.2423631250858307, + "median": 0.23208677768707275, + "p05": 0.15309149026870728, + "p95": 0.3585616946220398 + }, + { + "label": "current_group_partial", + "mean": 0.2602040767669678, + "median": 0.261092871427536, + "p05": 0.1870650351047516, + "p95": 0.33141374588012695 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0932960510253906, + "group": 2, + "latest_source_mass": 0.34662574529647827, + "layer": 6, + "max_source_mass": 0.3533782958984375, + "mixer_index": 10, + "normalized_entropy": 0.9951609519595268, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3533782958984375, + "median": 0.3537653684616089, + "p05": 0.300750195980072, + "p95": 0.40760302543640137 + }, + { + "label": "completed_group_1", + "mean": 0.29999595880508423, + "median": 0.2998189330101013, + "p05": 0.277343213558197, + "p95": 0.3256012797355652 + }, + { + "label": "current_group_partial", + "mean": 0.34662574529647827, + "median": 0.34948140382766724, + "p05": 0.3000357747077942, + "p95": 0.3795154392719269 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0456881523132324, + "group": 2, + "latest_source_mass": 0.3350909352302551, + "layer": 6, + "max_source_mass": 0.4338536858558655, + "mixer_index": 11, + "normalized_entropy": 0.9518263750544431, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4338536858558655, + "median": 0.43320339918136597, + "p05": 0.2745198607444763, + "p95": 0.6123260259628296 + }, + { + "label": "completed_group_1", + "mean": 0.23105540871620178, + "median": 0.22906750440597534, + "p05": 0.15816110372543335, + "p95": 0.3054558038711548 + }, + { + "label": "current_group_partial", + "mean": 0.3350909352302551, + "median": 0.33941850066185, + "p05": 0.2199106216430664, + "p95": 0.4273326098918915 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0891339778900146, + "group": 2, + "latest_source_mass": 0.37785181403160095, + "layer": 7, + "max_source_mass": 0.37785181403160095, + "mixer_index": 12, + "normalized_entropy": 0.9913724697276178, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33998239040374756, + "median": 0.3426361680030823, + "p05": 0.2884981632232666, + "p95": 0.38276687264442444 + }, + { + "label": "completed_group_1", + "mean": 0.2821658253669739, + "median": 0.2775929868221283, + "p05": 0.2536381781101227, + "p95": 0.3205426335334778 + }, + { + "label": "current_group_partial", + "mean": 0.37785181403160095, + "median": 0.3805788457393646, + "p05": 0.34096643328666687, + "p95": 0.40283167362213135 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.038641333580017, + "group": 2, + "latest_source_mass": 0.4193001687526703, + "layer": 7, + "max_source_mass": 0.4193001687526703, + "mixer_index": 13, + "normalized_entropy": 0.9454120842205417, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34165239334106445, + "median": 0.308063805103302, + "p05": 0.2054935097694397, + "p95": 0.6325749754905701 + }, + { + "label": "completed_group_1", + "mean": 0.23904743790626526, + "median": 0.244267076253891, + "p05": 0.1361692249774933, + "p95": 0.30873122811317444 + }, + { + "label": "current_group_partial", + "mean": 0.4193001687526703, + "median": 0.44425007700920105, + "p05": 0.22311054170131683, + "p95": 0.5117462277412415 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0754427909851074, + "group": 2, + "latest_source_mass": 0.3816372752189636, + "layer": 8, + "max_source_mass": 0.3816372752189636, + "mixer_index": 14, + "normalized_entropy": 0.9789102143476917, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3781457245349884, + "median": 0.37959107756614685, + "p05": 0.3339657485485077, + "p95": 0.42053842544555664 + }, + { + "label": "completed_group_1", + "mean": 0.24021697044372559, + "median": 0.23580573499202728, + "p05": 0.21105526387691498, + "p95": 0.2833511531352997 + }, + { + "label": "current_group_partial", + "mean": 0.3816372752189636, + "median": 0.38216716051101685, + "p05": 0.3468884229660034, + "p95": 0.4153265357017517 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0433809757232666, + "group": 2, + "latest_source_mass": 0.4086577892303467, + "layer": 8, + "max_source_mass": 0.4086577892303467, + "mixer_index": 15, + "normalized_entropy": 0.9497262924195011, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.37802231311798096, + "median": 0.3628900945186615, + "p05": 0.27504366636276245, + "p95": 0.5507957339286804 + }, + { + "label": "completed_group_1", + "mean": 0.21331989765167236, + "median": 0.2002047300338745, + "p05": 0.13782089948654175, + "p95": 0.3198853135108948 + }, + { + "label": "current_group_partial", + "mean": 0.4086577892303467, + "median": 0.41623425483703613, + "p05": 0.29855865240097046, + "p95": 0.47824543714523315 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0922105312347412, + "group": 3, + "latest_source_mass": 0.3464643955230713, + "layer": 9, + "max_source_mass": 0.3464643955230713, + "mixer_index": 16, + "normalized_entropy": 0.994172869264798, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3460429310798645, + "median": 0.3446507453918457, + "p05": 0.28670766949653625, + "p95": 0.40976202487945557 + }, + { + "label": "completed_group_1", + "mean": 0.3074926435947418, + "median": 0.306111603975296, + "p05": 0.2859768867492676, + "p95": 0.3337686061859131 + }, + { + "label": "completed_group_2", + "mean": 0.3464643955230713, + "median": 0.3486866354942322, + "p05": 0.27475792169570923, + "p95": 0.40797942876815796 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3404325246810913, + "group": 3, + "latest_source_mass": 0.3108460605144501, + "layer": 9, + "max_source_mass": 0.3108460605144501, + "mixer_index": 17, + "normalized_entropy": 0.9669176780018418, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2880150079727173, + "median": 0.27969539165496826, + "p05": 0.18933802843093872, + "p95": 0.45879441499710083 + }, + { + "label": "completed_group_1", + "mean": 0.16260020434856415, + "median": 0.16053099930286407, + "p05": 0.11509333550930023, + "p95": 0.22219550609588623 + }, + { + "label": "completed_group_2", + "mean": 0.2385387420654297, + "median": 0.24041923880577087, + "p05": 0.1582493633031845, + "p95": 0.2989943325519562 + }, + { + "label": "current_group_partial", + "mean": 0.3108460605144501, + "median": 0.3127351701259613, + "p05": 0.2580314874649048, + "p95": 0.3545264005661011 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3765960931777954, + "group": 3, + "latest_source_mass": 0.24758204817771912, + "layer": 10, + "max_source_mass": 0.28703436255455017, + "mixer_index": 18, + "normalized_entropy": 0.9930041784673634, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28703436255455017, + "median": 0.2873149514198303, + "p05": 0.2421720325946808, + "p95": 0.32964712381362915 + }, + { + "label": "completed_group_1", + "mean": 0.2552259564399719, + "median": 0.2551601231098175, + "p05": 0.2324453592300415, + "p95": 0.27810582518577576 + }, + { + "label": "completed_group_2", + "mean": 0.2101576328277588, + "median": 0.20999819040298462, + "p05": 0.17491281032562256, + "p95": 0.24711720645427704 + }, + { + "label": "current_group_partial", + "mean": 0.24758204817771912, + "median": 0.2449823021888733, + "p05": 0.21694661676883698, + "p95": 0.2901700437068939 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3274900913238525, + "group": 3, + "latest_source_mass": 0.26038026809692383, + "layer": 10, + "max_source_mass": 0.33539247512817383, + "mixer_index": 19, + "normalized_entropy": 0.9575816857910796, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.33539247512817383, + "median": 0.3373180031776428, + "p05": 0.16021503508090973, + "p95": 0.5567492246627808 + }, + { + "label": "completed_group_1", + "mean": 0.1987970769405365, + "median": 0.19616791605949402, + "p05": 0.13216234743595123, + "p95": 0.2684130072593689 + }, + { + "label": "completed_group_2", + "mean": 0.20543015003204346, + "median": 0.20896214246749878, + "p05": 0.11321981251239777, + "p95": 0.2748231887817383 + }, + { + "label": "current_group_partial", + "mean": 0.26038026809692383, + "median": 0.2554967999458313, + "p05": 0.1821153312921524, + "p95": 0.3462901711463928 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3747133016586304, + "group": 3, + "latest_source_mass": 0.2833673655986786, + "layer": 11, + "max_source_mass": 0.2833673655986786, + "mixer_index": 20, + "normalized_entropy": 0.9916460314734998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27988842129707336, + "median": 0.28038889169692993, + "p05": 0.24265244603157043, + "p95": 0.31478387117385864 + }, + { + "label": "completed_group_1", + "mean": 0.2360810488462448, + "median": 0.2335505485534668, + "p05": 0.21358957886695862, + "p95": 0.2655458450317383 + }, + { + "label": "completed_group_2", + "mean": 0.20066317915916443, + "median": 0.2006566822528839, + "p05": 0.18401089310646057, + "p95": 0.21715889871120453 + }, + { + "label": "current_group_partial", + "mean": 0.2833673655986786, + "median": 0.28357356786727905, + "p05": 0.2585292458534241, + "p95": 0.30662211775779724 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3239198923110962, + "group": 3, + "latest_source_mass": 0.2723969519138336, + "layer": 11, + "max_source_mass": 0.3361120820045471, + "mixer_index": 21, + "normalized_entropy": 0.9550063315857346, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3361120820045471, + "median": 0.3591133654117584, + "p05": 0.14645838737487793, + "p95": 0.49824467301368713 + }, + { + "label": "completed_group_1", + "mean": 0.20296622812747955, + "median": 0.19982439279556274, + "p05": 0.13988786935806274, + "p95": 0.27185869216918945 + }, + { + "label": "completed_group_2", + "mean": 0.1885247528553009, + "median": 0.18602871894836426, + "p05": 0.12637801468372345, + "p95": 0.2539179027080536 + }, + { + "label": "current_group_partial", + "mean": 0.2723969519138336, + "median": 0.25681596994400024, + "p05": 0.19296783208847046, + "p95": 0.3959115445613861 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3692121505737305, + "group": 3, + "latest_source_mass": 0.2974509000778198, + "layer": 12, + "max_source_mass": 0.2974509000778198, + "mixer_index": 22, + "normalized_entropy": 0.9876777897788168, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28167375922203064, + "median": 0.27825769782066345, + "p05": 0.23325958847999573, + "p95": 0.3442809581756592 + }, + { + "label": "completed_group_1", + "mean": 0.21946553885936737, + "median": 0.21849599480628967, + "p05": 0.18336281180381775, + "p95": 0.2548777461051941 + }, + { + "label": "completed_group_2", + "mean": 0.20140981674194336, + "median": 0.2013496309518814, + "p05": 0.1822294294834137, + "p95": 0.2219773232936859 + }, + { + "label": "current_group_partial", + "mean": 0.2974509000778198, + "median": 0.297912061214447, + "p05": 0.26495662331581116, + "p95": 0.32783159613609314 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3342361450195312, + "group": 3, + "latest_source_mass": 0.23993611335754395, + "layer": 12, + "max_source_mass": 0.36468517780303955, + "mixer_index": 23, + "normalized_entropy": 0.9624479348972428, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.36468517780303955, + "median": 0.36720338463783264, + "p05": 0.24960748851299286, + "p95": 0.4882795810699463 + }, + { + "label": "completed_group_1", + "mean": 0.20593920350074768, + "median": 0.2046196460723877, + "p05": 0.16298335790634155, + "p95": 0.25410282611846924 + }, + { + "label": "completed_group_2", + "mean": 0.18943950533866882, + "median": 0.1881066858768463, + "p05": 0.13994796574115753, + "p95": 0.24192847311496735 + }, + { + "label": "current_group_partial", + "mean": 0.23993611335754395, + "median": 0.232496976852417, + "p05": 0.17571662366390228, + "p95": 0.320015549659729 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3645737171173096, + "group": 4, + "latest_source_mass": 0.30504196882247925, + "layer": 13, + "max_source_mass": 0.30504196882247925, + "mixer_index": 24, + "normalized_entropy": 0.9843318673062809, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2597482204437256, + "median": 0.2599886655807495, + "p05": 0.23272822797298431, + "p95": 0.2871353030204773 + }, + { + "label": "completed_group_1", + "mean": 0.22607669234275818, + "median": 0.21320533752441406, + "p05": 0.1705966293811798, + "p95": 0.32829806208610535 + }, + { + "label": "completed_group_2", + "mean": 0.209133118391037, + "median": 0.20912399888038635, + "p05": 0.19090500473976135, + "p95": 0.2265017032623291 + }, + { + "label": "completed_group_3", + "mean": 0.30504196882247925, + "median": 0.3161492347717285, + "p05": 0.20431193709373474, + "p95": 0.37756454944610596 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5910131931304932, + "group": 4, + "latest_source_mass": 0.19691801071166992, + "layer": 13, + "max_source_mass": 0.2617476284503937, + "mixer_index": 25, + "normalized_entropy": 0.988552078237214, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2617476284503937, + "median": 0.2677137851715088, + "p05": 0.20086795091629028, + "p95": 0.31516557931900024 + }, + { + "label": "completed_group_1", + "mean": 0.18299485743045807, + "median": 0.1818893998861313, + "p05": 0.1605435013771057, + "p95": 0.20783139765262604 + }, + { + "label": "completed_group_2", + "mean": 0.17935845255851746, + "median": 0.18049052357673645, + "p05": 0.14385172724723816, + "p95": 0.2110004425048828 + }, + { + "label": "completed_group_3", + "mean": 0.17898105084896088, + "median": 0.17628446221351624, + "p05": 0.14925207197666168, + "p95": 0.21800483763217926 + }, + { + "label": "current_group_partial", + "mean": 0.19691801071166992, + "median": 0.1964835375547409, + "p05": 0.16946497559547424, + "p95": 0.22626768052577972 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5884685516357422, + "group": 4, + "latest_source_mass": 0.26269519329071045, + "layer": 14, + "max_source_mass": 0.26269519329071045, + "mixer_index": 26, + "normalized_entropy": 0.9869710035805953, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2046908289194107, + "median": 0.2052469551563263, + "p05": 0.18031299114227295, + "p95": 0.2295369803905487 + }, + { + "label": "completed_group_1", + "mean": 0.17294223606586456, + "median": 0.16847725212574005, + "p05": 0.15012723207473755, + "p95": 0.21394622325897217 + }, + { + "label": "completed_group_2", + "mean": 0.15044090151786804, + "median": 0.1493840515613556, + "p05": 0.1370292603969574, + "p95": 0.1660556048154831 + }, + { + "label": "completed_group_3", + "mean": 0.20923087000846863, + "median": 0.20989257097244263, + "p05": 0.17921577394008636, + "p95": 0.23829039931297302 + }, + { + "label": "current_group_partial", + "mean": 0.26269519329071045, + "median": 0.2656410038471222, + "p05": 0.23202235996723175, + "p95": 0.28119558095932007 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.599259853363037, + "group": 4, + "latest_source_mass": 0.21076934039592743, + "layer": 14, + "max_source_mass": 0.21657392382621765, + "mixer_index": 27, + "normalized_entropy": 0.9936760163331371, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21657392382621765, + "median": 0.21635186672210693, + "p05": 0.15601962804794312, + "p95": 0.2927207350730896 + }, + { + "label": "completed_group_1", + "mean": 0.20580385625362396, + "median": 0.20571252703666687, + "p05": 0.18009206652641296, + "p95": 0.2311524748802185 + }, + { + "label": "completed_group_2", + "mean": 0.19366896152496338, + "median": 0.194598987698555, + "p05": 0.16221287846565247, + "p95": 0.22272710502147675 + }, + { + "label": "completed_group_3", + "mean": 0.1731838881969452, + "median": 0.17174121737480164, + "p05": 0.14892742037773132, + "p95": 0.19922053813934326 + }, + { + "label": "current_group_partial", + "mean": 0.21076934039592743, + "median": 0.20835596323013306, + "p05": 0.1771993488073349, + "p95": 0.2505151629447937 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5786192417144775, + "group": 4, + "latest_source_mass": 0.2778070569038391, + "layer": 15, + "max_source_mass": 0.2778070569038391, + "mixer_index": 28, + "normalized_entropy": 0.980851283245209, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20599736273288727, + "median": 0.20488622784614563, + "p05": 0.17115657031536102, + "p95": 0.25322195887565613 + }, + { + "label": "completed_group_1", + "mean": 0.20593789219856262, + "median": 0.20346379280090332, + "p05": 0.18622605502605438, + "p95": 0.2330799102783203 + }, + { + "label": "completed_group_2", + "mean": 0.13473063707351685, + "median": 0.13439436256885529, + "p05": 0.11930938065052032, + "p95": 0.15121281147003174 + }, + { + "label": "completed_group_3", + "mean": 0.17552703619003296, + "median": 0.1757315993309021, + "p05": 0.1576612889766693, + "p95": 0.1928577572107315 + }, + { + "label": "current_group_partial", + "mean": 0.2778070569038391, + "median": 0.2790508270263672, + "p05": 0.24338077008724213, + "p95": 0.3088063597679138 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.580681324005127, + "group": 4, + "latest_source_mass": 0.2450503408908844, + "layer": 15, + "max_source_mass": 0.2450503408908844, + "mixer_index": 29, + "normalized_entropy": 0.9821325270103262, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21277281641960144, + "median": 0.20640528202056885, + "p05": 0.12015408277511597, + "p95": 0.3659750819206238 + }, + { + "label": "completed_group_1", + "mean": 0.18522509932518005, + "median": 0.1863747239112854, + "p05": 0.1475280076265335, + "p95": 0.21663622558116913 + }, + { + "label": "completed_group_2", + "mean": 0.18489867448806763, + "median": 0.1888006329536438, + "p05": 0.12606219947338104, + "p95": 0.22459235787391663 + }, + { + "label": "completed_group_3", + "mean": 0.17205308377742767, + "median": 0.17103317379951477, + "p05": 0.13283535838127136, + "p95": 0.20961183309555054 + }, + { + "label": "current_group_partial", + "mean": 0.2450503408908844, + "median": 0.24511924386024475, + "p05": 0.19535338878631592, + "p95": 0.2939102053642273 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5705549716949463, + "group": 4, + "latest_source_mass": 0.28460633754730225, + "layer": 16, + "max_source_mass": 0.28460633754730225, + "mixer_index": 30, + "normalized_entropy": 0.9758406705603525, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2125052809715271, + "median": 0.21049857139587402, + "p05": 0.16735027730464935, + "p95": 0.2640833854675293 + }, + { + "label": "completed_group_1", + "mean": 0.20224158465862274, + "median": 0.20027190446853638, + "p05": 0.16782984137535095, + "p95": 0.24093782901763916 + }, + { + "label": "completed_group_2", + "mean": 0.12915468215942383, + "median": 0.12795840203762054, + "p05": 0.1122070699930191, + "p95": 0.14902588725090027 + }, + { + "label": "completed_group_3", + "mean": 0.1714920997619629, + "median": 0.1704360991716385, + "p05": 0.15070456266403198, + "p95": 0.19576270878314972 + }, + { + "label": "current_group_partial", + "mean": 0.28460633754730225, + "median": 0.28339242935180664, + "p05": 0.2418026626110077, + "p95": 0.33516958355903625 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5599372386932373, + "group": 4, + "latest_source_mass": 0.2475198209285736, + "layer": 16, + "max_source_mass": 0.2665032148361206, + "mixer_index": 31, + "normalized_entropy": 0.9692435021205642, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2665032148361206, + "median": 0.2523844242095947, + "p05": 0.16442818939685822, + "p95": 0.41760408878326416 + }, + { + "label": "completed_group_1", + "mean": 0.181569442152977, + "median": 0.18072238564491272, + "p05": 0.1432785987854004, + "p95": 0.2251572459936142 + }, + { + "label": "completed_group_2", + "mean": 0.15844660997390747, + "median": 0.15800738334655762, + "p05": 0.11516115069389343, + "p95": 0.20221972465515137 + }, + { + "label": "completed_group_3", + "mean": 0.14596089720726013, + "median": 0.14446985721588135, + "p05": 0.10661658644676208, + "p95": 0.18743650615215302 + }, + { + "label": "current_group_partial", + "mean": 0.2475198209285736, + "median": 0.24823713302612305, + "p05": 0.1857534945011139, + "p95": 0.30610787868499756 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.562828540802002, + "group": 5, + "latest_source_mass": 0.26428917050361633, + "layer": 17, + "max_source_mass": 0.26428917050361633, + "mixer_index": 32, + "normalized_entropy": 0.9710399691271056, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21806953847408295, + "median": 0.21765722334384918, + "p05": 0.1832873821258545, + "p95": 0.256405234336853 + }, + { + "label": "completed_group_1", + "mean": 0.22324788570404053, + "median": 0.2111721932888031, + "p05": 0.1587892323732376, + "p95": 0.33203551173210144 + }, + { + "label": "completed_group_2", + "mean": 0.11591021716594696, + "median": 0.11326268315315247, + "p05": 0.09832467138767242, + "p95": 0.13705819845199585 + }, + { + "label": "completed_group_3", + "mean": 0.17848318815231323, + "median": 0.1790149211883545, + "p05": 0.1343652456998825, + "p95": 0.21781212091445923 + }, + { + "label": "completed_group_4", + "mean": 0.26428917050361633, + "median": 0.27161818742752075, + "p05": 0.18240834772586823, + "p95": 0.3144339919090271 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.761409044265747, + "group": 5, + "latest_source_mass": 0.14106416702270508, + "layer": 17, + "max_source_mass": 0.20728391408920288, + "mixer_index": 33, + "normalized_entropy": 0.9830611053081897, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1612042486667633, + "median": 0.14052782952785492, + "p05": 0.09785228967666626, + "p95": 0.30037230253219604 + }, + { + "label": "completed_group_1", + "mean": 0.16519047319889069, + "median": 0.16112321615219116, + "p05": 0.13238948583602905, + "p95": 0.2119981199502945 + }, + { + "label": "completed_group_2", + "mean": 0.17529632151126862, + "median": 0.18006199598312378, + "p05": 0.12263078987598419, + "p95": 0.20985063910484314 + }, + { + "label": "completed_group_3", + "mean": 0.14996090531349182, + "median": 0.1468663513660431, + "p05": 0.11909312009811401, + "p95": 0.18974041938781738 + }, + { + "label": "completed_group_4", + "mean": 0.20728391408920288, + "median": 0.2042727917432785, + "p05": 0.1449805498123169, + "p95": 0.27828457951545715 + }, + { + "label": "current_group_partial", + "mean": 0.14106416702270508, + "median": 0.1405012309551239, + "p05": 0.10205969214439392, + "p95": 0.1835198849439621 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7366840839385986, + "group": 5, + "latest_source_mass": 0.2530929446220398, + "layer": 18, + "max_source_mass": 0.2530929446220398, + "mixer_index": 34, + "normalized_entropy": 0.9692618422085502, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18514017760753632, + "median": 0.18704235553741455, + "p05": 0.13617590069770813, + "p95": 0.22916007041931152 + }, + { + "label": "completed_group_1", + "mean": 0.1663871705532074, + "median": 0.16571956872940063, + "p05": 0.1462152749300003, + "p95": 0.18810859322547913 + }, + { + "label": "completed_group_2", + "mean": 0.09063515067100525, + "median": 0.08811916410923004, + "p05": 0.07649004459381104, + "p95": 0.11148498952388763 + }, + { + "label": "completed_group_3", + "mean": 0.11971023678779602, + "median": 0.11747527122497559, + "p05": 0.10390099883079529, + "p95": 0.1423199474811554 + }, + { + "label": "completed_group_4", + "mean": 0.18503433465957642, + "median": 0.18477493524551392, + "p05": 0.15245835483074188, + "p95": 0.22021599113941193 + }, + { + "label": "current_group_partial", + "mean": 0.2530929446220398, + "median": 0.25188589096069336, + "p05": 0.22554033994674683, + "p95": 0.2833186984062195 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.777672290802002, + "group": 5, + "latest_source_mass": 0.19777557253837585, + "layer": 18, + "max_source_mass": 0.19777557253837585, + "mixer_index": 35, + "normalized_entropy": 0.9921377960222963, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15483248233795166, + "median": 0.14712893962860107, + "p05": 0.12276758253574371, + "p95": 0.22418904304504395 + }, + { + "label": "completed_group_1", + "mean": 0.16576308012008667, + "median": 0.16587528586387634, + "p05": 0.13462601602077484, + "p95": 0.19553470611572266 + }, + { + "label": "completed_group_2", + "mean": 0.16957050561904907, + "median": 0.17099428176879883, + "p05": 0.14377880096435547, + "p95": 0.19032810628414154 + }, + { + "label": "completed_group_3", + "mean": 0.14225168526172638, + "median": 0.14081379771232605, + "p05": 0.12333962321281433, + "p95": 0.1642245650291443 + }, + { + "label": "completed_group_4", + "mean": 0.16980668902397156, + "median": 0.16821962594985962, + "p05": 0.13770604133605957, + "p95": 0.20901428163051605 + }, + { + "label": "current_group_partial", + "mean": 0.19777557253837585, + "median": 0.19705063104629517, + "p05": 0.1461065709590912, + "p95": 0.2508929371833801 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7414222955703735, + "group": 5, + "latest_source_mass": 0.25494134426116943, + "layer": 19, + "max_source_mass": 0.25494134426116943, + "mixer_index": 36, + "normalized_entropy": 0.9719062884710925, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18020270764827728, + "median": 0.180809885263443, + "p05": 0.1366642266511917, + "p95": 0.22788876295089722 + }, + { + "label": "completed_group_1", + "mean": 0.1900298148393631, + "median": 0.1896573156118393, + "p05": 0.17211011052131653, + "p95": 0.20767614245414734 + }, + { + "label": "completed_group_2", + "mean": 0.09944289177656174, + "median": 0.09800659120082855, + "p05": 0.08568179607391357, + "p95": 0.11607980728149414 + }, + { + "label": "completed_group_3", + "mean": 0.11960913985967636, + "median": 0.11915834993124008, + "p05": 0.10471149533987045, + "p95": 0.1360406130552292 + }, + { + "label": "completed_group_4", + "mean": 0.1557741016149521, + "median": 0.15596863627433777, + "p05": 0.13432671129703522, + "p95": 0.1770717203617096 + }, + { + "label": "current_group_partial", + "mean": 0.25494134426116943, + "median": 0.25551509857177734, + "p05": 0.22669723629951477, + "p95": 0.2819991707801819 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.733978509902954, + "group": 5, + "latest_source_mass": 0.2618475556373596, + "layer": 19, + "max_source_mass": 0.2618475556373596, + "mixer_index": 37, + "normalized_entropy": 0.9677518325883359, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15204599499702454, + "median": 0.1308840811252594, + "p05": 0.09369519352912903, + "p95": 0.32571133971214294 + }, + { + "label": "completed_group_1", + "mean": 0.14166373014450073, + "median": 0.14088967442512512, + "p05": 0.11298352479934692, + "p95": 0.17192427814006805 + }, + { + "label": "completed_group_2", + "mean": 0.1616208553314209, + "median": 0.1641538441181183, + "p05": 0.11296901851892471, + "p95": 0.1948811560869217 + }, + { + "label": "completed_group_3", + "mean": 0.1256796270608902, + "median": 0.122653529047966, + "p05": 0.09500046819448471, + "p95": 0.16056732833385468 + }, + { + "label": "completed_group_4", + "mean": 0.1571422517299652, + "median": 0.1528569459915161, + "p05": 0.11623523384332657, + "p95": 0.21299627423286438 + }, + { + "label": "current_group_partial", + "mean": 0.2618475556373596, + "median": 0.26793089509010315, + "p05": 0.15509119629859924, + "p95": 0.339982271194458 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7344707250595093, + "group": 5, + "latest_source_mass": 0.2615981698036194, + "layer": 20, + "max_source_mass": 0.2615981698036194, + "mixer_index": 38, + "normalized_entropy": 0.9680265430977588, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1829337775707245, + "median": 0.183739572763443, + "p05": 0.13839715719223022, + "p95": 0.2284226417541504 + }, + { + "label": "completed_group_1", + "mean": 0.18416839838027954, + "median": 0.18364182114601135, + "p05": 0.16665373742580414, + "p95": 0.20437687635421753 + }, + { + "label": "completed_group_2", + "mean": 0.09014187008142471, + "median": 0.08851997554302216, + "p05": 0.07715612649917603, + "p95": 0.10642829537391663 + }, + { + "label": "completed_group_3", + "mean": 0.12335794419050217, + "median": 0.12144016474485397, + "p05": 0.10556001961231232, + "p95": 0.14631223678588867 + }, + { + "label": "completed_group_4", + "mean": 0.15779981017112732, + "median": 0.15796968340873718, + "p05": 0.1340107023715973, + "p95": 0.18188069760799408 + }, + { + "label": "current_group_partial", + "mean": 0.2615981698036194, + "median": 0.262384295463562, + "p05": 0.230574831366539, + "p95": 0.28982529044151306 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.735896348953247, + "group": 5, + "latest_source_mass": 0.27010154724121094, + "layer": 20, + "max_source_mass": 0.27010154724121094, + "mixer_index": 39, + "normalized_entropy": 0.9688221989423192, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17276549339294434, + "median": 0.16886113584041595, + "p05": 0.10281073302030563, + "p95": 0.2923668622970581 + }, + { + "label": "completed_group_1", + "mean": 0.15101772546768188, + "median": 0.14904022216796875, + "p05": 0.13034172356128693, + "p95": 0.17705516517162323 + }, + { + "label": "completed_group_2", + "mean": 0.13220664858818054, + "median": 0.1315174400806427, + "p05": 0.10168404132127762, + "p95": 0.16267849504947662 + }, + { + "label": "completed_group_3", + "mean": 0.11932596564292908, + "median": 0.11731794476509094, + "p05": 0.09613680094480515, + "p95": 0.1465223878622055 + }, + { + "label": "completed_group_4", + "mean": 0.15458261966705322, + "median": 0.15350666642189026, + "p05": 0.12296099960803986, + "p95": 0.19313591718673706 + }, + { + "label": "current_group_partial", + "mean": 0.27010154724121094, + "median": 0.27393096685409546, + "p05": 0.19484001398086548, + "p95": 0.3264923095703125 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7644665241241455, + "group": 6, + "latest_source_mass": 0.2094498574733734, + "layer": 21, + "max_source_mass": 0.2094498574733734, + "mixer_index": 40, + "normalized_entropy": 0.9847675173076283, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19160009920597076, + "median": 0.196849524974823, + "p05": 0.12982682883739471, + "p95": 0.24405300617218018 + }, + { + "label": "completed_group_1", + "mean": 0.1915181279182434, + "median": 0.1907939314842224, + "p05": 0.17317205667495728, + "p95": 0.2111247181892395 + }, + { + "label": "completed_group_2", + "mean": 0.11520625650882721, + "median": 0.11408726871013641, + "p05": 0.10055835545063019, + "p95": 0.13332504034042358 + }, + { + "label": "completed_group_3", + "mean": 0.1341419219970703, + "median": 0.13221079111099243, + "p05": 0.11641807854175568, + "p95": 0.15773889422416687 + }, + { + "label": "completed_group_4", + "mean": 0.1580837219953537, + "median": 0.15754032135009766, + "p05": 0.13662610948085785, + "p95": 0.18392926454544067 + }, + { + "label": "completed_group_5", + "mean": 0.2094498574733734, + "median": 0.20958366990089417, + "p05": 0.1805649697780609, + "p95": 0.23726987838745117 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6835086345672607, + "group": 6, + "latest_source_mass": 0.43279585242271423, + "layer": 21, + "max_source_mass": 0.43279585242271423, + "mixer_index": 41, + "normalized_entropy": 0.8651522966692777, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10065847635269165, + "median": 0.09005473554134369, + "p05": 0.05753578990697861, + "p95": 0.19214311242103577 + }, + { + "label": "completed_group_1", + "mean": 0.09339895844459534, + "median": 0.09201203286647797, + "p05": 0.07428346574306488, + "p95": 0.11678388714790344 + }, + { + "label": "completed_group_2", + "mean": 0.09682720899581909, + "median": 0.09625889360904694, + "p05": 0.07657647132873535, + "p95": 0.11835116147994995 + }, + { + "label": "completed_group_3", + "mean": 0.08155890554189682, + "median": 0.08032877743244171, + "p05": 0.06701995432376862, + "p95": 0.09893998503684998 + }, + { + "label": "completed_group_4", + "mean": 0.09126688539981842, + "median": 0.09069330245256424, + "p05": 0.07231517881155014, + "p95": 0.1128317192196846 + }, + { + "label": "completed_group_5", + "mean": 0.10349369794130325, + "median": 0.10402766615152359, + "p05": 0.07336798310279846, + "p95": 0.1318768411874771 + }, + { + "label": "current_group_partial", + "mean": 0.43279585242271423, + "median": 0.4367910623550415, + "p05": 0.3485929071903229, + "p95": 0.5040479302406311 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7937740087509155, + "group": 6, + "latest_source_mass": 0.3452628254890442, + "layer": 22, + "max_source_mass": 0.3452628254890442, + "mixer_index": 42, + "normalized_entropy": 0.9218174896830382, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13723835349082947, + "median": 0.1412133425474167, + "p05": 0.08910952508449554, + "p95": 0.17562061548233032 + }, + { + "label": "completed_group_1", + "mean": 0.12937891483306885, + "median": 0.13051339983940125, + "p05": 0.10541269928216934, + "p95": 0.1487061083316803 + }, + { + "label": "completed_group_2", + "mean": 0.07450413703918457, + "median": 0.07415075600147247, + "p05": 0.06578464806079865, + "p95": 0.08426731824874878 + }, + { + "label": "completed_group_3", + "mean": 0.08537043631076813, + "median": 0.08443015813827515, + "p05": 0.07344960421323776, + "p95": 0.09956531971693039 + }, + { + "label": "completed_group_4", + "mean": 0.09880410879850388, + "median": 0.09825065732002258, + "p05": 0.08438277244567871, + "p95": 0.11530908197164536 + }, + { + "label": "completed_group_5", + "mean": 0.12944120168685913, + "median": 0.12968069314956665, + "p05": 0.10858328640460968, + "p95": 0.15059794485569 + }, + { + "label": "current_group_partial", + "mean": 0.3452628254890442, + "median": 0.33866745233535767, + "p05": 0.288330078125, + "p95": 0.42612025141716003 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5790722370147705, + "group": 6, + "latest_source_mass": 0.49195876717567444, + "layer": 22, + "max_source_mass": 0.49195876717567444, + "mixer_index": 43, + "normalized_entropy": 0.8114826050839847, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09500400722026825, + "median": 0.07923799753189087, + "p05": 0.045484717935323715, + "p95": 0.2259966880083084 + }, + { + "label": "completed_group_1", + "mean": 0.08892714232206345, + "median": 0.0876917839050293, + "p05": 0.06900187581777573, + "p95": 0.11303731799125671 + }, + { + "label": "completed_group_2", + "mean": 0.09081116318702698, + "median": 0.0898551493883133, + "p05": 0.06699725985527039, + "p95": 0.11749734729528427 + }, + { + "label": "completed_group_3", + "mean": 0.07538282871246338, + "median": 0.07381957024335861, + "p05": 0.05869410187005997, + "p95": 0.09544046223163605 + }, + { + "label": "completed_group_4", + "mean": 0.07685283571481705, + "median": 0.07580418884754181, + "p05": 0.056799523532390594, + "p95": 0.10076010227203369 + }, + { + "label": "completed_group_5", + "mean": 0.08106326311826706, + "median": 0.08067160844802856, + "p05": 0.05696551501750946, + "p95": 0.10726240277290344 + }, + { + "label": "current_group_partial", + "mean": 0.49195876717567444, + "median": 0.49396610260009766, + "p05": 0.3897285759449005, + "p95": 0.583945631980896 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8034920692443848, + "group": 6, + "latest_source_mass": 0.33595213294029236, + "layer": 23, + "max_source_mass": 0.33595213294029236, + "mixer_index": 44, + "normalized_entropy": 0.926811584861681, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.142653226852417, + "median": 0.14694267511367798, + "p05": 0.08785572648048401, + "p95": 0.18721188604831696 + }, + { + "label": "completed_group_1", + "mean": 0.13419847190380096, + "median": 0.13438312709331512, + "p05": 0.11451410502195358, + "p95": 0.15392597019672394 + }, + { + "label": "completed_group_2", + "mean": 0.07437049597501755, + "median": 0.07391499727964401, + "p05": 0.06574448198080063, + "p95": 0.08430380374193192 + }, + { + "label": "completed_group_3", + "mean": 0.09070361405611038, + "median": 0.08887749910354614, + "p05": 0.07508289068937302, + "p95": 0.11117327958345413 + }, + { + "label": "completed_group_4", + "mean": 0.1013825386762619, + "median": 0.10140319168567657, + "p05": 0.08447366952896118, + "p95": 0.11889617145061493 + }, + { + "label": "completed_group_5", + "mean": 0.12073953449726105, + "median": 0.12033741921186447, + "p05": 0.10022275149822235, + "p95": 0.1415012627840042 + }, + { + "label": "current_group_partial", + "mean": 0.33595213294029236, + "median": 0.330989271402359, + "p05": 0.27680477499961853, + "p95": 0.41842275857925415 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5278494358062744, + "group": 6, + "latest_source_mass": 0.51820969581604, + "layer": 23, + "max_source_mass": 0.51820969581604, + "mixer_index": 45, + "normalized_entropy": 0.7851592924514033, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10045768320560455, + "median": 0.08486579358577728, + "p05": 0.04896760731935501, + "p95": 0.2206544280052185 + }, + { + "label": "completed_group_1", + "mean": 0.08844950795173645, + "median": 0.08638796210289001, + "p05": 0.06730383634567261, + "p95": 0.11565113067626953 + }, + { + "label": "completed_group_2", + "mean": 0.08561251312494278, + "median": 0.08505219221115112, + "p05": 0.0625130906701088, + "p95": 0.11068116128444672 + }, + { + "label": "completed_group_3", + "mean": 0.06598107516765594, + "median": 0.06433478742837906, + "p05": 0.04859655350446701, + "p95": 0.08642670512199402 + }, + { + "label": "completed_group_4", + "mean": 0.06767018139362335, + "median": 0.06678327918052673, + "p05": 0.047009289264678955, + "p95": 0.09129223972558975 + }, + { + "label": "completed_group_5", + "mean": 0.07361933588981628, + "median": 0.07290077209472656, + "p05": 0.049975596368312836, + "p95": 0.09868258237838745 + }, + { + "label": "current_group_partial", + "mean": 0.51820969581604, + "median": 0.5205609798431396, + "p05": 0.40315571427345276, + "p95": 0.6213710308074951 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8435695171356201, + "group": 6, + "latest_source_mass": 0.29524505138397217, + "layer": 24, + "max_source_mass": 0.29524505138397217, + "mixer_index": 46, + "normalized_entropy": 0.9474073188993969, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14839638769626617, + "median": 0.15232303738594055, + "p05": 0.08978340029716492, + "p95": 0.19894656538963318 + }, + { + "label": "completed_group_1", + "mean": 0.1482572853565216, + "median": 0.14857065677642822, + "p05": 0.13154153525829315, + "p95": 0.16430144011974335 + }, + { + "label": "completed_group_2", + "mean": 0.08113027364015579, + "median": 0.08067925274372101, + "p05": 0.07102686911821365, + "p95": 0.09267604351043701 + }, + { + "label": "completed_group_3", + "mean": 0.09534022957086563, + "median": 0.09320184588432312, + "p05": 0.0778813511133194, + "p95": 0.11805401742458344 + }, + { + "label": "completed_group_4", + "mean": 0.10612697899341583, + "median": 0.10644470155239105, + "p05": 0.08815888315439224, + "p95": 0.12415381520986557 + }, + { + "label": "completed_group_5", + "mean": 0.12550383806228638, + "median": 0.1254296898841858, + "p05": 0.10509618371725082, + "p95": 0.14616522192955017 + }, + { + "label": "current_group_partial", + "mean": 0.29524505138397217, + "median": 0.29100558161735535, + "p05": 0.24107839167118073, + "p95": 0.37045177817344666 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.6421003341674805, + "group": 6, + "latest_source_mass": 0.4530346691608429, + "layer": 24, + "max_source_mass": 0.4530346691608429, + "mixer_index": 47, + "normalized_entropy": 0.8438726397334819, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11422763019800186, + "median": 0.09945468604564667, + "p05": 0.06323201954364777, + "p95": 0.23352834582328796 + }, + { + "label": "completed_group_1", + "mean": 0.10543527454137802, + "median": 0.1044958084821701, + "p05": 0.08478721231222153, + "p95": 0.1284603774547577 + }, + { + "label": "completed_group_2", + "mean": 0.09285669028759003, + "median": 0.09177995473146439, + "p05": 0.06902404874563217, + "p95": 0.12035554647445679 + }, + { + "label": "completed_group_3", + "mean": 0.07746599614620209, + "median": 0.07653152942657471, + "p05": 0.058040909469127655, + "p95": 0.09812526404857635 + }, + { + "label": "completed_group_4", + "mean": 0.07571054995059967, + "median": 0.07457360625267029, + "p05": 0.05648855119943619, + "p95": 0.0989091694355011 + }, + { + "label": "completed_group_5", + "mean": 0.08126918971538544, + "median": 0.08049014210700989, + "p05": 0.05931335315108299, + "p95": 0.10596252232789993 + }, + { + "label": "current_group_partial", + "mean": 0.4530346691608429, + "median": 0.455755352973938, + "p05": 0.3413957357406616, + "p95": 0.5519019365310669 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8733552694320679, + "group": 7, + "latest_source_mass": 0.26451510190963745, + "layer": 25, + "max_source_mass": 0.26451510190963745, + "mixer_index": 48, + "normalized_entropy": 0.9627141676307774, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14798463881015778, + "median": 0.14657661318778992, + "p05": 0.09472498297691345, + "p95": 0.2258487045764923 + }, + { + "label": "completed_group_1", + "mean": 0.160526841878891, + "median": 0.16084568202495575, + "p05": 0.1358838975429535, + "p95": 0.18423213064670563 + }, + { + "label": "completed_group_2", + "mean": 0.09668850898742676, + "median": 0.09659601747989655, + "p05": 0.08555105328559875, + "p95": 0.1084243580698967 + }, + { + "label": "completed_group_3", + "mean": 0.10477185994386673, + "median": 0.10316876322031021, + "p05": 0.08876212686300278, + "p95": 0.12562818825244904 + }, + { + "label": "completed_group_4", + "mean": 0.10857057571411133, + "median": 0.1074000895023346, + "p05": 0.09068692475557327, + "p95": 0.12911471724510193 + }, + { + "label": "completed_group_5", + "mean": 0.11694245785474777, + "median": 0.11679476499557495, + "p05": 0.0976894274353981, + "p95": 0.13528725504875183 + }, + { + "label": "completed_group_6", + "mean": 0.26451510190963745, + "median": 0.26442092657089233, + "p05": 0.21035388112068176, + "p95": 0.31906524300575256 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.784506916999817, + "group": 7, + "latest_source_mass": 0.27427971363067627, + "layer": 25, + "max_source_mass": 0.29875272512435913, + "mixer_index": 49, + "normalized_entropy": 0.8581664265292297, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08589775860309601, + "median": 0.07514797151088715, + "p05": 0.03144541755318642, + "p95": 0.20497030019760132 + }, + { + "label": "completed_group_1", + "mean": 0.08786046504974365, + "median": 0.08646859228610992, + "p05": 0.06123892217874527, + "p95": 0.11937689781188965 + }, + { + "label": "completed_group_2", + "mean": 0.07714234292507172, + "median": 0.07644426077604294, + "p05": 0.05296650156378746, + "p95": 0.10448302328586578 + }, + { + "label": "completed_group_3", + "mean": 0.06314892321825027, + "median": 0.06132982298731804, + "p05": 0.044174306094646454, + "p95": 0.08466912060976028 + }, + { + "label": "completed_group_4", + "mean": 0.05684571713209152, + "median": 0.055630408227443695, + "p05": 0.036537639796733856, + "p95": 0.08176163583993912 + }, + { + "label": "completed_group_5", + "mean": 0.056072384119033813, + "median": 0.0549161471426487, + "p05": 0.03825453296303749, + "p95": 0.07769857347011566 + }, + { + "label": "completed_group_6", + "mean": 0.29875272512435913, + "median": 0.29761478304862976, + "p05": 0.17369259893894196, + "p95": 0.4214131534099579 + }, + { + "label": "current_group_partial", + "mean": 0.27427971363067627, + "median": 0.26517564058303833, + "p05": 0.11691330373287201, + "p95": 0.45670589804649353 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9675533771514893, + "group": 7, + "latest_source_mass": 0.24859297275543213, + "layer": 26, + "max_source_mass": 0.24859297275543213, + "mixer_index": 50, + "normalized_entropy": 0.9461931666335953, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11549945175647736, + "median": 0.1186310350894928, + "p05": 0.0548042394220829, + "p95": 0.17459861934185028 + }, + { + "label": "completed_group_1", + "mean": 0.12592050433158875, + "median": 0.12912830710411072, + "p05": 0.08620737493038177, + "p95": 0.15710318088531494 + }, + { + "label": "completed_group_2", + "mean": 0.07530814409255981, + "median": 0.07495854794979095, + "p05": 0.06174474209547043, + "p95": 0.0894969180226326 + }, + { + "label": "completed_group_3", + "mean": 0.08277833461761475, + "median": 0.08158879727125168, + "p05": 0.06734651327133179, + "p95": 0.10064717382192612 + }, + { + "label": "completed_group_4", + "mean": 0.08296641707420349, + "median": 0.08213280141353607, + "p05": 0.06705781072378159, + "p95": 0.10095006227493286 + }, + { + "label": "completed_group_5", + "mean": 0.09016789495944977, + "median": 0.08980530500411987, + "p05": 0.07541801035404205, + "p95": 0.10646115988492966 + }, + { + "label": "completed_group_6", + "mean": 0.17876628041267395, + "median": 0.17772917449474335, + "p05": 0.13528534770011902, + "p95": 0.2264900654554367 + }, + { + "label": "current_group_partial", + "mean": 0.24859297275543213, + "median": 0.245039701461792, + "p05": 0.14620795845985413, + "p95": 0.3620130121707916 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7119876146316528, + "group": 7, + "latest_source_mass": 0.3658455014228821, + "layer": 26, + "max_source_mass": 0.3658455014228821, + "mixer_index": 51, + "normalized_entropy": 0.8232920138974705, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08804735541343689, + "median": 0.08122549951076508, + "p05": 0.054039862006902695, + "p95": 0.1443495899438858 + }, + { + "label": "completed_group_1", + "mean": 0.07527871429920197, + "median": 0.0738755539059639, + "p05": 0.05371527001261711, + "p95": 0.10149700939655304 + }, + { + "label": "completed_group_2", + "mean": 0.05666808784008026, + "median": 0.05508599430322647, + "p05": 0.03937675803899765, + "p95": 0.07947445660829544 + }, + { + "label": "completed_group_3", + "mean": 0.050419021397829056, + "median": 0.04877372086048126, + "p05": 0.03257015347480774, + "p95": 0.0718536451458931 + }, + { + "label": "completed_group_4", + "mean": 0.05142981931567192, + "median": 0.0498797781765461, + "p05": 0.034862153232097626, + "p95": 0.07353642582893372 + }, + { + "label": "completed_group_5", + "mean": 0.05313318967819214, + "median": 0.051505643874406815, + "p05": 0.037266045808792114, + "p95": 0.07490143924951553 + }, + { + "label": "completed_group_6", + "mean": 0.2591783106327057, + "median": 0.2531449794769287, + "p05": 0.15121260285377502, + "p95": 0.386227548122406 + }, + { + "label": "current_group_partial", + "mean": 0.3658455014228821, + "median": 0.3656759262084961, + "p05": 0.19218485057353973, + "p95": 0.5332037210464478 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9094492197036743, + "group": 7, + "latest_source_mass": 0.2766643464565277, + "layer": 27, + "max_source_mass": 0.2766643464565277, + "mixer_index": 52, + "normalized_entropy": 0.918250973365264, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10337945818901062, + "median": 0.10516610741615295, + "p05": 0.05207033082842827, + "p95": 0.1573200225830078 + }, + { + "label": "completed_group_1", + "mean": 0.10583682358264923, + "median": 0.10595200955867767, + "p05": 0.0643514022231102, + "p95": 0.14548130333423615 + }, + { + "label": "completed_group_2", + "mean": 0.08642244338989258, + "median": 0.08549235016107559, + "p05": 0.06607823818922043, + "p95": 0.11075147986412048 + }, + { + "label": "completed_group_3", + "mean": 0.08043928444385529, + "median": 0.07886679470539093, + "p05": 0.05936145409941673, + "p95": 0.10334464907646179 + }, + { + "label": "completed_group_4", + "mean": 0.0717030018568039, + "median": 0.07051880657672882, + "p05": 0.05472247302532196, + "p95": 0.09299077093601227 + }, + { + "label": "completed_group_5", + "mean": 0.0645938515663147, + "median": 0.06373226642608643, + "p05": 0.04421529173851013, + "p95": 0.08712958544492722 + }, + { + "label": "completed_group_6", + "mean": 0.21096080541610718, + "median": 0.20807480812072754, + "p05": 0.14586198329925537, + "p95": 0.2868158519268036 + }, + { + "label": "current_group_partial", + "mean": 0.2766643464565277, + "median": 0.27722808718681335, + "p05": 0.16094130277633667, + "p95": 0.3963583707809448 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7860023975372314, + "group": 7, + "latest_source_mass": 0.3244447112083435, + "layer": 27, + "max_source_mass": 0.3244447112083435, + "mixer_index": 53, + "normalized_entropy": 0.8588856006475877, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09247160702943802, + "median": 0.08733023703098297, + "p05": 0.04943867400288582, + "p95": 0.16303153336048126 + }, + { + "label": "completed_group_1", + "mean": 0.0784597396850586, + "median": 0.07711629569530487, + "p05": 0.05584436655044556, + "p95": 0.10506026446819305 + }, + { + "label": "completed_group_2", + "mean": 0.07236532866954803, + "median": 0.07097338140010834, + "p05": 0.050456635653972626, + "p95": 0.09917968511581421 + }, + { + "label": "completed_group_3", + "mean": 0.05986309051513672, + "median": 0.057845089584589005, + "p05": 0.04075033217668533, + "p95": 0.08416998386383057 + }, + { + "label": "completed_group_4", + "mean": 0.05389959365129471, + "median": 0.05301224812865257, + "p05": 0.03848717734217644, + "p95": 0.07268592715263367 + }, + { + "label": "completed_group_5", + "mean": 0.06059400737285614, + "median": 0.05963316559791565, + "p05": 0.04356392100453377, + "p95": 0.08111730217933655 + }, + { + "label": "completed_group_6", + "mean": 0.2579019069671631, + "median": 0.2530352473258972, + "p05": 0.16421708464622498, + "p95": 0.3691956102848053 + }, + { + "label": "current_group_partial", + "mean": 0.3244447112083435, + "median": 0.3252047896385193, + "p05": 0.17263495922088623, + "p95": 0.47815465927124023 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9232951402664185, + "group": 7, + "latest_source_mass": 0.2509572505950928, + "layer": 28, + "max_source_mass": 0.2509572505950928, + "mixer_index": 54, + "normalized_entropy": 0.9249094536760685, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1160542219877243, + "median": 0.1222282201051712, + "p05": 0.061863381415605545, + "p95": 0.16598135232925415 + }, + { + "label": "completed_group_1", + "mean": 0.11346358060836792, + "median": 0.11177559196949005, + "p05": 0.05926967412233353, + "p95": 0.16970883309841156 + }, + { + "label": "completed_group_2", + "mean": 0.08816123753786087, + "median": 0.08707751333713531, + "p05": 0.06823235750198364, + "p95": 0.1114775538444519 + }, + { + "label": "completed_group_3", + "mean": 0.08138670027256012, + "median": 0.0797908753156662, + "p05": 0.06149819865822792, + "p95": 0.10523255914449692 + }, + { + "label": "completed_group_4", + "mean": 0.07161372900009155, + "median": 0.07095701992511749, + "p05": 0.055086471140384674, + "p95": 0.09069444239139557 + }, + { + "label": "completed_group_5", + "mean": 0.06013647839426994, + "median": 0.05960702896118164, + "p05": 0.04596705362200737, + "p95": 0.076435886323452 + }, + { + "label": "completed_group_6", + "mean": 0.21822679042816162, + "median": 0.21632584929466248, + "p05": 0.14715570211410522, + "p95": 0.2940847873687744 + }, + { + "label": "current_group_partial", + "mean": 0.2509572505950928, + "median": 0.24700899422168732, + "p05": 0.14031530916690826, + "p95": 0.37019768357276917 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8294003009796143, + "group": 7, + "latest_source_mass": 0.20826560258865356, + "layer": 28, + "max_source_mass": 0.31431835889816284, + "mixer_index": 55, + "normalized_entropy": 0.879755580674689, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08831655979156494, + "median": 0.06431648135185242, + "p05": 0.03252303972840309, + "p95": 0.2706584930419922 + }, + { + "label": "completed_group_1", + "mean": 0.10072161257266998, + "median": 0.1009141132235527, + "p05": 0.06054181233048439, + "p95": 0.14067479968070984 + }, + { + "label": "completed_group_2", + "mean": 0.0818098932504654, + "median": 0.08032949268817902, + "p05": 0.054586175829172134, + "p95": 0.1145491972565651 + }, + { + "label": "completed_group_3", + "mean": 0.07514279335737228, + "median": 0.07337591052055359, + "p05": 0.04998338222503662, + "p95": 0.10431315004825592 + }, + { + "label": "completed_group_4", + "mean": 0.06651485711336136, + "median": 0.06524930149316788, + "p05": 0.04414192587137222, + "p95": 0.09315875917673111 + }, + { + "label": "completed_group_5", + "mean": 0.06491032987833023, + "median": 0.06368150562047958, + "p05": 0.04332352057099342, + "p95": 0.09065432846546173 + }, + { + "label": "completed_group_6", + "mean": 0.31431835889816284, + "median": 0.3147735297679901, + "p05": 0.18699732422828674, + "p95": 0.44058555364608765 + }, + { + "label": "current_group_partial", + "mean": 0.20826560258865356, + "median": 0.1914007067680359, + "p05": 0.08225662261247635, + "p95": 0.38793477416038513 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9393219947814941, + "group": 8, + "latest_source_mass": 0.15344023704528809, + "layer": 29, + "max_source_mass": 0.2604207992553711, + "mixer_index": 56, + "normalized_entropy": 0.9326167415193847, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11041727662086487, + "median": 0.10045937448740005, + "p05": 0.024797730147838593, + "p95": 0.279465913772583 + }, + { + "label": "completed_group_1", + "mean": 0.12080886960029602, + "median": 0.12561748921871185, + "p05": 0.0680447518825531, + "p95": 0.16192926466464996 + }, + { + "label": "completed_group_2", + "mean": 0.09995828568935394, + "median": 0.09907965362071991, + "p05": 0.07490158826112747, + "p95": 0.1276247799396515 + }, + { + "label": "completed_group_3", + "mean": 0.09934850037097931, + "median": 0.09762611985206604, + "p05": 0.06941570341587067, + "p95": 0.134530171751976 + }, + { + "label": "completed_group_4", + "mean": 0.08031868189573288, + "median": 0.07758662104606628, + "p05": 0.05694210156798363, + "p95": 0.11248748749494553 + }, + { + "label": "completed_group_5", + "mean": 0.075287364423275, + "median": 0.07397153228521347, + "p05": 0.05265223979949951, + "p95": 0.10342685133218765 + }, + { + "label": "completed_group_6", + "mean": 0.2604207992553711, + "median": 0.2616186738014221, + "p05": 0.16666486859321594, + "p95": 0.34887272119522095 + }, + { + "label": "completed_group_7", + "mean": 0.15344023704528809, + "median": 0.13995957374572754, + "p05": 0.05727577582001686, + "p95": 0.29449376463890076 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.9925593137741089, + "group": 8, + "latest_source_mass": 0.18462148308753967, + "layer": 29, + "max_source_mass": 0.2503892779350281, + "mixer_index": 57, + "normalized_entropy": 0.9068528243889232, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08698172122240067, + "median": 0.07842238247394562, + "p05": 0.02795225754380226, + "p95": 0.1876923143863678 + }, + { + "label": "completed_group_1", + "mean": 0.08857265114784241, + "median": 0.08714614808559418, + "p05": 0.06365615874528885, + "p95": 0.11900493502616882 + }, + { + "label": "completed_group_2", + "mean": 0.07299254834651947, + "median": 0.07189398258924484, + "p05": 0.05586059391498566, + "p95": 0.0933830514550209 + }, + { + "label": "completed_group_3", + "mean": 0.060478877276182175, + "median": 0.05862775444984436, + "p05": 0.04474102333188057, + "p95": 0.07903359085321426 + }, + { + "label": "completed_group_4", + "mean": 0.054115522652864456, + "median": 0.052696481347084045, + "p05": 0.03536852449178696, + "p95": 0.07748772948980331 + }, + { + "label": "completed_group_5", + "mean": 0.056821923702955246, + "median": 0.05573669448494911, + "p05": 0.0398341529071331, + "p95": 0.07687154412269592 + }, + { + "label": "completed_group_6", + "mean": 0.2503892779350281, + "median": 0.2475910186767578, + "p05": 0.15396791696548462, + "p95": 0.357724130153656 + }, + { + "label": "completed_group_7", + "mean": 0.14502598345279694, + "median": 0.13307344913482666, + "p05": 0.06183449178934097, + "p95": 0.2673647403717041 + }, + { + "label": "current_group_partial", + "mean": 0.18462148308753967, + "median": 0.17473986744880676, + "p05": 0.08267536759376526, + "p95": 0.3240666389465332 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.068819999694824, + "group": 8, + "latest_source_mass": 0.15494346618652344, + "layer": 30, + "max_source_mass": 0.21415996551513672, + "mixer_index": 58, + "normalized_entropy": 0.9415605582761754, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09429103136062622, + "median": 0.0943363606929779, + "p05": 0.03544093295931816, + "p95": 0.16596338152885437 + }, + { + "label": "completed_group_1", + "mean": 0.09028280526399612, + "median": 0.090491384267807, + "p05": 0.05409309267997742, + "p95": 0.12526515126228333 + }, + { + "label": "completed_group_2", + "mean": 0.09188266098499298, + "median": 0.09010649472475052, + "p05": 0.06878700107336044, + "p95": 0.12181076407432556 + }, + { + "label": "completed_group_3", + "mean": 0.0735517293214798, + "median": 0.07178769260644913, + "p05": 0.0548262856900692, + "p95": 0.09550250321626663 + }, + { + "label": "completed_group_4", + "mean": 0.06746294349431992, + "median": 0.06608930230140686, + "p05": 0.050633110105991364, + "p95": 0.08874846994876862 + }, + { + "label": "completed_group_5", + "mean": 0.060166649520397186, + "median": 0.060052454471588135, + "p05": 0.039401471614837646, + "p95": 0.08220525085926056 + }, + { + "label": "completed_group_6", + "mean": 0.21415996551513672, + "median": 0.2118481993675232, + "p05": 0.15205052495002747, + "p95": 0.2836942672729492 + }, + { + "label": "completed_group_7", + "mean": 0.15325874090194702, + "median": 0.14970678091049194, + "p05": 0.08526452630758286, + "p95": 0.23864519596099854 + }, + { + "label": "current_group_partial", + "mean": 0.15494346618652344, + "median": 0.1473558098077774, + "p05": 0.0697651207447052, + "p95": 0.2689872086048126 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0139355659484863, + "group": 8, + "latest_source_mass": 0.1372738927602768, + "layer": 30, + "max_source_mass": 0.24272148311138153, + "mixer_index": 59, + "normalized_entropy": 0.916581576012616, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0956040769815445, + "median": 0.09956102818250656, + "p05": 0.03933291882276535, + "p95": 0.16697832942008972 + }, + { + "label": "completed_group_1", + "mean": 0.09831400960683823, + "median": 0.09554082155227661, + "p05": 0.05786377936601639, + "p95": 0.14979025721549988 + }, + { + "label": "completed_group_2", + "mean": 0.0714857280254364, + "median": 0.06985784322023392, + "p05": 0.04893886297941208, + "p95": 0.10043824464082718 + }, + { + "label": "completed_group_3", + "mean": 0.0677608773112297, + "median": 0.06590595096349716, + "p05": 0.044468365609645844, + "p95": 0.09555256366729736 + }, + { + "label": "completed_group_4", + "mean": 0.05654233694076538, + "median": 0.054802894592285156, + "p05": 0.037319183349609375, + "p95": 0.08175043016672134 + }, + { + "label": "completed_group_5", + "mean": 0.05388312786817551, + "median": 0.05288306996226311, + "p05": 0.036485109478235245, + "p95": 0.07530424743890762 + }, + { + "label": "completed_group_6", + "mean": 0.24272148311138153, + "median": 0.23883119225502014, + "p05": 0.1493627429008484, + "p95": 0.35197100043296814 + }, + { + "label": "completed_group_7", + "mean": 0.17641445994377136, + "median": 0.17445214092731476, + "p05": 0.0891575887799263, + "p95": 0.276222825050354 + }, + { + "label": "current_group_partial", + "mean": 0.1372738927602768, + "median": 0.12184794992208481, + "p05": 0.05821039155125618, + "p95": 0.2647005021572113 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0349321365356445, + "group": 8, + "latest_source_mass": 0.13534162938594818, + "layer": 31, + "max_source_mass": 0.2261163890361786, + "mixer_index": 60, + "normalized_entropy": 0.9261375270991514, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10256819427013397, + "median": 0.10616284608840942, + "p05": 0.019877789542078972, + "p95": 0.19627508521080017 + }, + { + "label": "completed_group_1", + "mean": 0.0938093364238739, + "median": 0.09388002753257751, + "p05": 0.034650709480047226, + "p95": 0.15219232439994812 + }, + { + "label": "completed_group_2", + "mean": 0.0912778377532959, + "median": 0.08955617249011993, + "p05": 0.06106609106063843, + "p95": 0.1285511702299118 + }, + { + "label": "completed_group_3", + "mean": 0.08394120633602142, + "median": 0.0820196121931076, + "p05": 0.05809478089213371, + "p95": 0.11390723288059235 + }, + { + "label": "completed_group_4", + "mean": 0.06161237880587578, + "median": 0.059880051761865616, + "p05": 0.044582173228263855, + "p95": 0.08485961705446243 + }, + { + "label": "completed_group_5", + "mean": 0.053358763456344604, + "median": 0.052702393382787704, + "p05": 0.03554790839552879, + "p95": 0.07320070266723633 + }, + { + "label": "completed_group_6", + "mean": 0.2261163890361786, + "median": 0.22380158305168152, + "p05": 0.1451866328716278, + "p95": 0.3134670853614807 + }, + { + "label": "completed_group_7", + "mean": 0.15197426080703735, + "median": 0.1527099758386612, + "p05": 0.0639471560716629, + "p95": 0.2459363490343094 + }, + { + "label": "current_group_partial", + "mean": 0.13534162938594818, + "median": 0.11519233882427216, + "p05": 0.04325530678033829, + "p95": 0.2904844284057617 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.046657085418701, + "group": 8, + "latest_source_mass": 0.09365127235651016, + "layer": 31, + "max_source_mass": 0.2572976052761078, + "mixer_index": 61, + "normalized_entropy": 0.9314737813009277, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08380690217018127, + "median": 0.06641843914985657, + "p05": 0.039923232048749924, + "p95": 0.23694062232971191 + }, + { + "label": "completed_group_1", + "mean": 0.1217331737279892, + "median": 0.12164830416440964, + "p05": 0.08828366547822952, + "p95": 0.1561325639486313 + }, + { + "label": "completed_group_2", + "mean": 0.10242430865764618, + "median": 0.10224691033363342, + "p05": 0.06860878318548203, + "p95": 0.13651791214942932 + }, + { + "label": "completed_group_3", + "mean": 0.09225838631391525, + "median": 0.09045033901929855, + "p05": 0.0655013769865036, + "p95": 0.1235189214348793 + }, + { + "label": "completed_group_4", + "mean": 0.06788390874862671, + "median": 0.0664045438170433, + "p05": 0.047665324062108994, + "p95": 0.09357043355703354 + }, + { + "label": "completed_group_5", + "mean": 0.06336672604084015, + "median": 0.06275692582130432, + "p05": 0.044327184557914734, + "p95": 0.08375845849514008 + }, + { + "label": "completed_group_6", + "mean": 0.2572976052761078, + "median": 0.25880685448646545, + "p05": 0.14942821860313416, + "p95": 0.36036571860313416 + }, + { + "label": "completed_group_7", + "mean": 0.11757771670818329, + "median": 0.10582176595926285, + "p05": 0.05813705176115036, + "p95": 0.21660257875919342 + }, + { + "label": "current_group_partial", + "mean": 0.09365127235651016, + "median": 0.08298523724079132, + "p05": 0.04494771361351013, + "p95": 0.18065699934959412 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.094015121459961, + "group": 8, + "latest_source_mass": 0.09723767638206482, + "layer": 32, + "max_source_mass": 0.2018478661775589, + "mixer_index": 62, + "normalized_entropy": 0.9530273523513089, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10284869372844696, + "median": 0.09173205494880676, + "p05": 0.03822793811559677, + "p95": 0.22231025993824005 + }, + { + "label": "completed_group_1", + "mean": 0.13057854771614075, + "median": 0.13079363107681274, + "p05": 0.0711527168750763, + "p95": 0.19180890917778015 + }, + { + "label": "completed_group_2", + "mean": 0.10152992606163025, + "median": 0.09965524077415466, + "p05": 0.07341106235980988, + "p95": 0.13697651028633118 + }, + { + "label": "completed_group_3", + "mean": 0.10079469531774521, + "median": 0.10007210820913315, + "p05": 0.07479676604270935, + "p95": 0.12990058958530426 + }, + { + "label": "completed_group_4", + "mean": 0.0796234980225563, + "median": 0.07812212407588959, + "p05": 0.05766180902719498, + "p95": 0.10730832070112228 + }, + { + "label": "completed_group_5", + "mean": 0.07438632845878601, + "median": 0.07310014963150024, + "p05": 0.05309801548719406, + "p95": 0.10082833468914032 + }, + { + "label": "completed_group_6", + "mean": 0.2018478661775589, + "median": 0.20017066597938538, + "p05": 0.13963642716407776, + "p95": 0.2733708918094635 + }, + { + "label": "completed_group_7", + "mean": 0.1111527606844902, + "median": 0.10317055881023407, + "p05": 0.05416924133896828, + "p95": 0.1888933926820755 + }, + { + "label": "current_group_partial", + "mean": 0.09723767638206482, + "median": 0.08349506556987762, + "p05": 0.03931424394249916, + "p95": 0.1855989694595337 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.057201385498047, + "group": 8, + "latest_source_mass": 0.09480181336402893, + "layer": 32, + "max_source_mass": 0.24551761150360107, + "mixer_index": 63, + "normalized_entropy": 0.9362726990757002, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09265108406543732, + "median": 0.07600092887878418, + "p05": 0.04942645877599716, + "p95": 0.20456968247890472 + }, + { + "label": "completed_group_1", + "mean": 0.11481083929538727, + "median": 0.1135297492146492, + "p05": 0.08239293098449707, + "p95": 0.15063267946243286 + }, + { + "label": "completed_group_2", + "mean": 0.09007447212934494, + "median": 0.0894046500325203, + "p05": 0.06493785232305527, + "p95": 0.11702875047922134 + }, + { + "label": "completed_group_3", + "mean": 0.08496076613664627, + "median": 0.08339293301105499, + "p05": 0.06288711726665497, + "p95": 0.11049411445856094 + }, + { + "label": "completed_group_4", + "mean": 0.06770835816860199, + "median": 0.06692098081111908, + "p05": 0.0488700196146965, + "p95": 0.09013402462005615 + }, + { + "label": "completed_group_5", + "mean": 0.05925413966178894, + "median": 0.058933183550834656, + "p05": 0.04426975175738335, + "p95": 0.07536415755748749 + }, + { + "label": "completed_group_6", + "mean": 0.24551761150360107, + "median": 0.24382448196411133, + "p05": 0.15310630202293396, + "p95": 0.3403708338737488 + }, + { + "label": "completed_group_7", + "mean": 0.15022093057632446, + "median": 0.14221730828285217, + "p05": 0.0825236365199089, + "p95": 0.24877692759037018 + }, + { + "label": "current_group_partial", + "mean": 0.09480181336402893, + "median": 0.08117073774337769, + "p05": 0.04724843427538872, + "p95": 0.17635159194469452 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.8435449600219727, + "group": null, + "latest_source_mass": 0.20717504620552063, + "layer": null, + "max_source_mass": 0.31774085760116577, + "mixer_index": null, + "normalized_entropy": 0.839033469331102, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1112452894449234, + "median": 0.11452806740999222, + "p05": 0.04704529047012329, + "p95": 0.1625424325466156 + }, + { + "label": "completed_group_1", + "mean": 0.06898629665374756, + "median": 0.06820575892925262, + "p05": 0.04699244350194931, + "p95": 0.09449291229248047 + }, + { + "label": "completed_group_2", + "mean": 0.05835210159420967, + "median": 0.05662037432193756, + "p05": 0.039408810436725616, + "p95": 0.081703320145607 + }, + { + "label": "completed_group_3", + "mean": 0.042265649884939194, + "median": 0.040285609662532806, + "p05": 0.026811392977833748, + "p95": 0.06341734528541565 + }, + { + "label": "completed_group_4", + "mean": 0.041034016758203506, + "median": 0.039777740836143494, + "p05": 0.027301941066980362, + "p95": 0.05862141400575638 + }, + { + "label": "completed_group_5", + "mean": 0.04826357960700989, + "median": 0.047516219317913055, + "p05": 0.03253086656332016, + "p95": 0.06693053245544434 + }, + { + "label": "completed_group_6", + "mean": 0.10493715852499008, + "median": 0.09417732059955597, + "p05": 0.04745320975780487, + "p95": 0.19990575313568115 + }, + { + "label": "completed_group_7", + "mean": 0.31774085760116577, + "median": 0.3171391487121582, + "p05": 0.10361962765455246, + "p95": 0.5363480448722839 + }, + { + "label": "completed_group_8", + "mean": 0.20717504620552063, + "median": 0.19020375609397888, + "p05": 0.03896218538284302, + "p95": 0.4246461093425751 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", + "optimizer_state_before": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.04804428666830063, + 0.12356939166784286, + 0.05923242121934891, + 0.04508936405181885, + 0.0799286738038063, + 0.03290615975856781, + 0.03170505166053772, + 0.03961387276649475, + 0.04508088901638985, + 0.0312613770365715, + 0.029994890093803406, + 0.03058313950896263, + 0.0497574545443058, + 0.03759258985519409, + 0.030690157786011696, + 0.03000541403889656, + 0.079553984105587, + 0.0414140485227108, + 0.04122982919216156, + 0.05830265209078789, + 0.03440643846988678, + 0.033897366374731064, + 0.02957344427704811, + 0.03406442329287529, + 0.025225287303328514, + 0.01915411651134491, + 0.014526215381920338, + 0.011566666886210442, + 0.0242468249052763, + 0.013597902841866016, + 0.020146220922470093, + 0.02710210345685482 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.406048522767378e-05, + "first_to_last_ratio": 1.2121174427344374, + "imbalance_abs_log_ratio": 0.19236878289853168, + "last_quartile_mean": 1.1599936385664478e-05, + "mean": 1.385052358671146e-05, + "normalized": [ + 1.1035462446531923, + 1.2453829183001823, + 1.1319013131596403, + 1.0360400584890102, + 0.9887153549881315, + 0.9324565992202007, + 0.881809246461942, + 0.8014211622598989, + 0.7197327796271318, + 0.6642367228932815, + 0.63144486698815, + 0.6054650987106983, + 0.6612979512556375, + 0.6318736594713652, + 0.6080689788224007, + 0.5826508433334544, + 0.71610531341614, + 0.685873407734308, + 0.733241450138544, + 0.7670951091759252, + 2.8592552563843903, + 2.5014923465112644, + 2.101621489466362, + 1.7092008525685705, + 1.57037557605485, + 1.4158002854110978, + 1.11083466622777, + 0.8417375710978994, + 0.5124454971351892, + 0.45441040448215947, + 0.4041964735899746, + 0.3902705019712373 + ], + "peak_layer": 21, + "peak_normalized": 2.8592552563843903, + "population_cv": 0.5792971940134687, + "rest_mean": 1.0905006850710683e-05, + "spike_contrast": 2.7286836559095273, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.9756313961115665e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.5284693290595897e-05, + 1.724920548440423e-05, + 1.5677425835747272e-05, + 1.4349697266879957e-05, + 1.369422534480691e-05, + 1.2915012121084146e-05, + 1.2213519767101388e-05, + 1.1100102710770443e-05, + 9.96867584035499e-06, + 9.20002639759332e-06, + 8.745842023927253e-06, + 8.38600863062311e-06, + 9.159322871710174e-06, + 8.75178102432983e-06, + 8.422073733527213e-06, + 8.070019248407334e-06, + 9.91843353403965e-06, + 9.499705811322201e-06, + 1.0155777999898419e-05, + 1.0624668902892154e-05, + 3.960218236898072e-05, + 3.464697874733247e-05, + 2.910855801019352e-05, + 2.3673326722928323e-05, + 2.1750523956143297e-05, + 1.9609575247159228e-05, + 1.538564174552448e-05, + 1.165850608231267e-05, + 7.0976384449750185e-06, + 6.293822025327245e-06, + 5.598332791123539e-06, + 5.405450792750344e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.784132736152969e-05, + "first_to_last_ratio": 1.247704596204738, + "imbalance_abs_log_ratio": 0.22130554016992907, + "last_quartile_mean": 4.635819050236023e-05, + "mean": 5.578696777774894e-05, + "normalized": [ + 1.1172279996665548, + 1.2820128122064964, + 1.1617993468115064, + 1.0568066843619655, + 1.0159130322231686, + 0.9528514141416237, + 0.8952059773271221, + 0.8127833779089165, + 0.726905325923399, + 0.6687158255328219, + 0.6353938715547037, + 0.6079598423303039, + 0.663249561462659, + 0.632156813919224, + 0.6072625305936415, + 0.5816757612181556, + 0.7067342161190907, + 0.6745011066249388, + 0.7095591340205663, + 0.7592836625042008, + 2.8322187742699385, + 2.4767130392204146, + 2.0818487421172924, + 1.6933329620054323, + 1.5551938962366243, + 1.4028337924372432, + 1.095017319874792, + 0.8328031307001514, + 0.5108569494896386, + 0.4540995368567512, + 0.4053097243347225, + 0.3917738360059398 + ], + "peak_layer": 21, + "peak_normalized": 2.8322187742699385, + "population_cv": 0.5731363623317645, + "rest_mean": 4.413512107459138e-05, + "spike_contrast": 2.6896253388356377, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011870693997479975, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.2326762417797e-05, + 7.151960744522512e-05, + 6.481326272478327e-05, + 5.8956040447810665e-05, + 5.667470759362914e-05, + 5.315669113770127e-05, + 4.994082701159641e-05, + 4.534272011369467e-05, + 4.055184399476275e-05, + 3.730562821147032e-05, + 3.5446697438601404e-05, + 3.391623613424599e-05, + 3.700068191392347e-05, + 3.526611180859618e-05, + 3.387733522686176e-05, + 3.2449926948174834e-05, + 3.942655894206837e-05, + 3.762837150134146e-05, + 3.9584152546012774e-05, + 4.235813321429305e-05, + 0.00015800089749973267, + 0.00013816831051371992, + 0.00011614002869464457, + 9.446591138839722e-05, + 8.675955177750438e-05, + 7.825984357623383e-05, + 6.108769593993202e-05, + 4.645956141757779e-05, + 2.8499160180217586e-05, + 2.5332836230518296e-05, + 2.2611000531469472e-05, + 2.1855874365428463e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.051294267643243074, + "first_to_last_ratio": 1.247704590273494, + "imbalance_abs_log_ratio": 0.22130553541620454, + "last_quartile_mean": 0.04111090721562505, + "mean": 0.04947244143113494, + "normalized": [ + 1.1172280253192406, + 1.2820128195205216, + 1.1617993344751307, + 1.056806678091042, + 1.0159130361962296, + 0.9528513588115517, + 0.895205957602592, + 0.8127833650766507, + 0.726905316511733, + 0.6687157952448587, + 0.6353938256699311, + 0.6079598134280393, + 0.6632495949619723, + 0.6321568158374328, + 0.6072625701847152, + 0.5816757499080466, + 0.7067342453085952, + 0.6745011182519967, + 0.7095591365309404, + 0.759283697871088, + 2.8322188463523386, + 2.476713119523555, + 2.0818486723693663, + 1.6933329451602226, + 1.555193919529888, + 1.4028338274869276, + 1.095017294552204, + 0.832803082651722, + 0.5108569355338682, + 0.4540995509098777, + 0.40530972999858744, + 0.391773821129134 + ], + "peak_layer": 21, + "peak_normalized": 2.8322188463523386, + "population_cv": 0.5731363736273818, + "rest_mean": 0.039139466981093086, + "spike_contrast": 2.6896253725737616, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10527050346136094, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.055271998047828674, + 0.06342430412769318, + 0.05747704952955246, + 0.05228280648589134, + 0.05025969818234444, + 0.047139883041381836, + 0.0442880243062973, + 0.04021037742495537, + 0.035961780697107315, + 0.033083003014326096, + 0.03143448382616043, + 0.0300772562623024, + 0.03281257674098015, + 0.03127434104681015, + 0.03004276193678379, + 0.028776919469237328, + 0.03496386855840683, + 0.033369217067956924, + 0.03510362282395363, + 0.037563618272542953, + 0.14011678099632263, + 0.1225290447473526, + 0.10299413651227951, + 0.08377331495285034, + 0.07693924009799957, + 0.06940161436796188, + 0.05417317897081375, + 0.041200801730155945, + 0.025273339822888374, + 0.022465413436293602, + 0.020051661878824234, + 0.01938200742006302 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.2204174854323355e-06, + "first_to_last_ratio": 0.735976396978067, + "imbalance_abs_log_ratio": 0.306557230090771, + "last_quartile_mean": 3.0169683356007226e-06, + "mean": 2.8222587751258743e-06, + "normalized": [ + 1.1437692142315412, + 0.8934875077918236, + 0.843634317775851, + 0.8332025934148901, + 0.6691897239669076, + 0.649206320818976, + 0.6345003713598615, + 0.6270252408101402, + 0.548073507234572, + 0.5298155928110359, + 0.5235851424549667, + 0.5213985029212593, + 0.5319259781097652, + 0.5365763585413705, + 0.538492744687615, + 0.5418387466813859, + 0.5975925562684307, + 0.6133961155504078, + 0.6328406634942718, + 0.6938494895757952, + 2.792643580637443, + 2.792207888188673, + 2.5604162828657895, + 2.199406095874773, + 1.9619910931916569, + 1.8148930082375383, + 1.4357221480925244, + 1.0849940773120685, + 0.6411781557251384, + 0.5785168725769138, + 0.5245362458580646, + 0.5100938629385489 + ], + "peak_layer": 21, + "peak_normalized": 2.792643580637443, + "population_cv": 0.7010168394314351, + "rest_mean": 2.0585069512014378e-06, + "spike_contrast": 3.3745422235584535, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.946518624317832e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.2280127015837934e-06, + 2.5216529593308223e-06, + 2.3809543563402258e-06, + 2.3515133307228098e-06, + 1.8886265706896666e-06, + 1.8322282357985387e-06, + 1.7907242408909951e-06, + 1.7696274881018326e-06, + 1.5468052652067854e-06, + 1.495276706009463e-06, + 1.4776927628190606e-06, + 1.4715215002070181e-06, + 1.5012327594376984e-06, + 1.5143573364184704e-06, + 1.5197658740362385e-06, + 1.5292091575247468e-06, + 1.6865608358784812e-06, + 1.7311625697402633e-06, + 1.7860401158031891e-06, + 1.958222810571897e-06, + 7.881562851252966e-06, + 7.880333214416169e-06, + 7.2261573222931474e-06, + 6.207293154147919e-06, + 5.5372465794789605e-06, + 5.122097718412988e-06, + 4.051979431096697e-06, + 3.062134055653587e-06, + 1.809570676414296e-06, + 1.6327243201885722e-06, + 1.4803770227445057e-06, + 1.4396168808161747e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.569237398754922e-05, + "first_to_last_ratio": 1.4673112290466532, + "imbalance_abs_log_ratio": 0.38343163006552816, + "last_quartile_mean": 3.114020603334211e-05, + "mean": 4.183441066629712e-05, + "normalized": [ + 1.1974296567066158, + 1.344289107904746, + 1.230408462214659, + 1.1234591616583915, + 1.0623659972849115, + 0.9983070144562474, + 0.9335099902486312, + 0.8479894563327729, + 0.7756304429380606, + 0.7156796904410139, + 0.6795815361308393, + 0.6491943107046658, + 0.7109629912124332, + 0.6750948063174392, + 0.6474411689349395, + 0.6200717213988419, + 0.7662657439847727, + 0.7307653188389935, + 0.7545343036423542, + 0.7927280102241547, + 2.8287575533506253, + 2.4215604626110183, + 1.9807235727824308, + 1.5583037155212895, + 1.4117479335225582, + 1.2389201568217543, + 0.9856291732269274, + 0.776844076198889, + 0.43766749474230543, + 0.39060885635754505, + 0.36143095960303334, + 0.352097153686139 + ], + "peak_layer": 21, + "peak_normalized": 2.8287575533506253, + "population_cv": 0.551495636352156, + "rest_mean": 3.3775719169137085e-05, + "spike_contrast": 2.5270030320761303, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.535134475096129e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.009376400266774e-05, + 5.623754259431735e-05, + 5.1473412895575166e-05, + 4.699925193563104e-05, + 4.444345540832728e-05, + 4.176358561380766e-05, + 3.905284029315226e-05, + 3.547513915691525e-05, + 3.244804247515276e-05, + 2.994003807543777e-05, + 2.842989306373056e-05, + 2.7158661396242678e-05, + 2.9742717742919922e-05, + 2.8242193366168067e-05, + 2.708531974349171e-05, + 2.5940335035556927e-05, + 3.205627581337467e-05, + 3.0571136448998004e-05, + 3.156549792038277e-05, + 3.316330912639387e-05, + 0.00011833940516225994, + 0.00010130455484613776, + 8.286240336019546e-05, + 6.519071757793427e-05, + 5.905964280827902e-05, + 5.1829494623234496e-05, + 4.1233215597458184e-05, + 3.249881410738453e-05, + 1.830956171033904e-05, + 1.63408913067542e-05, + 1.512025119154714e-05, + 1.4729776921740267e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.5050525639235275e-05, + "first_to_last_ratio": 1.5890428234254472, + "imbalance_abs_log_ratio": 0.46313183711305195, + "last_quartile_mean": 2.2057634396333015e-05, + "mean": 3.131621542706853e-05, + "normalized": [ + 1.247642034455602, + 1.3675088708957686, + 1.2617968995203173, + 1.1676693179472686, + 1.0797799047255963, + 1.0154085918080462, + 0.9450439951530737, + 0.8691124556208352, + 0.8151155724977966, + 0.7531778294737046, + 0.7147915273060139, + 0.6827252927557514, + 0.7417922051240886, + 0.7100839458764344, + 0.679513505581043, + 0.6487030016923234, + 0.8192115232383504, + 0.7896113859496233, + 0.7975027577835305, + 0.8129328118460493, + 2.8268246118214386, + 2.3622917424367986, + 1.850934565013104, + 1.4060108194738914, + 1.2902617154939733, + 1.123282471468924, + 0.9231284485726192, + 0.7885511099194735, + 0.4159101173561302, + 0.376982553975174, + 0.35968054987980413, + 0.3570178653374513 + ], + "peak_layer": 21, + "peak_normalized": 2.8268246118214386, + "population_cv": 0.5277183722759345, + "rest_mean": 2.5822744107615478e-05, + "spike_contrast": 2.3615213122965844, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.0980960552114996e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 2 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.907142672687769e-05, + 4.282520239939913e-05, + 3.951470353058539e-05, + 3.656698390841484e-05, + 3.3814620110206306e-05, + 3.179875420755707e-05, + 2.959520134027116e-05, + 2.721731289057061e-05, + 2.5526334866299294e-05, + 2.3586679162690416e-05, + 2.238456545455847e-05, + 2.138037234544754e-05, + 2.3230124497786164e-05, + 2.223714182036929e-05, + 2.1279791326378472e-05, + 2.03149229491828e-05, + 2.5654604542069137e-05, + 2.4727640266064554e-05, + 2.4974768166430295e-05, + 2.5457979063503444e-05, + 8.852544851833954e-05, + 7.397803710773587e-05, + 5.796426557935774e-05, + 4.403093771543354e-05, + 4.040611383970827e-05, + 3.517695586197078e-05, + 2.8908889362355694e-05, + 2.469443643349223e-05, + 1.3024730833421927e-05, + 1.1805666872533038e-05, + 1.1263833584962413e-05, + 1.1180448382219765e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011513851586641977, + "first_to_last_ratio": 1.1690722131114868, + "imbalance_abs_log_ratio": 0.15621045398792427, + "last_quartile_mean": 9.848708623394486e-05, + "mean": 0.00011408905686494109, + "normalized": [ + 1.0878105091943224, + 1.2333698943486997, + 1.1325669787296346, + 1.0350609428916697, + 0.9847532971789725, + 0.9327565235282874, + 0.875161061795294, + 0.7921095504711437, + 0.7054552974920509, + 0.6480774339061697, + 0.6173745394266138, + 0.5959457136906282, + 0.6486626275163402, + 0.6167276122272274, + 0.5953045899588231, + 0.5685580660104519, + 0.6801746911900812, + 0.6508152675683995, + 0.6710893308044551, + 0.7221277670118063, + 2.8258795357154587, + 2.493070695082837, + 2.1580463328702972, + 1.8231222239658644, + 1.644278740520026, + 1.5034127662796515, + 1.171932855948766, + 0.8973146194608761, + 0.48937000596181734, + 0.4306716621780954, + 0.38968515069330856, + 0.3793137163819305 + ], + "peak_layer": 21, + "peak_normalized": 2.8258795357154587, + "population_cv": 0.5949686549165712, + "rest_mean": 8.89708825087399e-05, + "spike_contrast": 2.806841871708938, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002497271983884275, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012410727504175156, + 0.00014071400801185519, + 0.0001292134984396398, + 0.00011808912677224725, + 0.00011234957491979003, + 0.00010641731205396354, + 9.98463001451455e-05, + 9.037103154696524e-05, + 8.048472955124453e-05, + 7.393854320980608e-05, + 7.043567893560976e-05, + 6.799088441766798e-05, + 7.400530739687383e-05, + 7.036187162157148e-05, + 6.79177392157726e-05, + 6.486625352408737e-05, + 7.760048902127892e-05, + 7.425090007018298e-05, + 7.656394882360473e-05, + 8.238687587436289e-05, + 0.0003224019310437143, + 0.00028443208429962397, + 0.0002462094707880169, + 0.00020799829508177936, + 0.00018759421072900295, + 0.00017152294458355755, + 0.00013370471424423158, + 0.00010237377864541486, + 5.583176243817434e-05, + 4.913492375635542e-05, + 4.4458811316872016e-05, + 4.327554415795021e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.07808538526296616, + 0.03881331905722618, + 0.01480295043438673, + 0.02500341460108757, + 0.015180142596364021, + 0.013721407391130924, + 0.016902483999729156, + 0.03325798362493515, + 0.026172807440161705, + 0.027887089177966118, + 0.03240044787526131, + 0.03682815283536911, + 0.029301028698682785, + 0.0338764488697052, + 0.0393521711230278, + 0.046694882214069366, + 0.029503758996725082, + 0.03615153953433037, + 0.030687781050801277, + 0.05279085040092468, + 0.024021649733185768, + 0.03336174041032791, + 0.04255227372050285, + 0.0520954355597496, + 0.03984410688281059, + 0.04249191656708717, + 0.05309459939599037, + 0.06231515854597092, + 0.06388300657272339, + 0.0749494656920433, + 0.10156872868537903, + 0.11044161766767502 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.387307077038713e-05, + "first_to_last_ratio": 1.3885610977109948, + "imbalance_abs_log_ratio": 0.32826802946929257, + "last_quartile_mean": 9.990968919737497e-06, + "mean": 1.2854331032485788e-05, + "normalized": [ + 1.1519258940103256, + 1.3312041641813053, + 1.2167309231197612, + 1.1117514471382886, + 1.0587713135165344, + 0.9928224052891267, + 0.9274023828418168, + 0.8434126323121538, + 0.7336839389102785, + 0.6915061646833663, + 0.6596107083150127, + 0.6376340360074726, + 0.6952642607649582, + 0.6629197998266377, + 0.6345516401532968, + 0.6122226131721931, + 0.7648990252904823, + 0.729764382313612, + 0.7866676809835089, + 0.8716429632898921, + 2.8889885153646824, + 2.3385372141657035, + 1.899664892928511, + 1.5404582652892223, + 1.5315814746576397, + 1.1978299957950485, + 0.9082536044284508, + 0.816087598131252, + 0.48962660632970234, + 0.43639989519040423, + 0.42103372910248354, + 0.4171498324968764 + ], + "peak_layer": 21, + "peak_normalized": 2.8889885153646824, + "population_cv": 0.5419279223583527, + "rest_mean": 1.0379048506837097e-05, + "spike_contrast": 2.5263256698064365, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6220856670988724e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.4807236766500864e-05, + 1.7111738998210058e-05, + 1.5640262063243426e-05, + 1.4290821127360687e-05, + 1.3609796951641329e-05, + 1.2762067854055203e-05, + 1.1921137229364831e-05, + 1.0841505172720645e-05, + 9.4310162239708e-06, + 8.888849151844624e-06, + 8.478854397253599e-06, + 8.196358976420015e-06, + 8.937156962929294e-06, + 8.521390554960817e-06, + 8.156736839737277e-06, + 7.869712135288864e-06, + 9.832265277509578e-06, + 9.380632945976686e-06, + 1.0112086783919949e-05, + 1.120438719226513e-05, + 3.7136014725547284e-05, + 3.0060331482673064e-05, + 2.441892138449475e-05, + 1.9801560483756475e-05, + 1.9687455278472044e-05, + 1.5397303286590613e-05, + 1.1674992492771707e-05, + 1.0490260137885343e-05, + 6.293822480074596e-06, + 5.609628715319559e-06, + 5.412106929725269e-06, + 5.3621820370608475e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.7066908539127326e-05, + "first_to_last_ratio": 1.4278402189707298, + "imbalance_abs_log_ratio": 0.3561629661799943, + "last_quartile_mean": 3.99672931052919e-05, + "mean": 5.182235634038079e-05, + "normalized": [ + 1.1683197832075227, + 1.369358043457492, + 1.2477885257362629, + 1.1320828936276526, + 1.0861971638477104, + 1.0136921452638399, + 0.9398491667147963, + 0.8523323061186183, + 0.7376381293423032, + 0.6943242114792568, + 0.6618878610984286, + 0.637951161315391, + 0.6961169332186252, + 0.6619150990682322, + 0.6320240246440271, + 0.6099217459665006, + 0.753292310075495, + 0.715605419809301, + 0.7584607148457314, + 0.8619138236046655, + 2.8636215539276253, + 2.323250279031787, + 1.8821006100447468, + 1.5304640359562964, + 1.5177704400182797, + 1.1794085853576253, + 0.8974069340864187, + 0.8090474514501561, + 0.4887596394765995, + 0.4369840848318672, + 0.4222231761264232, + 0.4182917472503225 + ], + "peak_layer": 21, + "peak_normalized": 2.8636215539276253, + "population_cv": 0.5384897216371255, + "rest_mean": 4.2000662991389014e-05, + "spike_contrast": 2.4966153616773794, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010485950042493641, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.0545084124896675e-05, + 7.09633604856208e-05, + 6.466334161814302e-05, + 5.866720312042162e-05, + 5.6289296480827034e-05, + 5.2531915571307763e-05, + 4.870519842370413e-05, + 4.416986848809756e-05, + 3.822614598902874e-05, + 3.598151670303196e-05, + 3.430058859521523e-05, + 3.306013240944594e-05, + 3.607441976782866e-05, + 3.430200013099238e-05, + 3.275297422078438e-05, + 3.1607582059223205e-05, + 3.9037382521200925e-05, + 3.708435906446539e-05, + 3.930522143491544e-05, + 4.466640530154109e-05, + 0.00014839961659163237, + 0.00012039630382787436, + 9.753488848218694e-05, + 7.931225263746455e-05, + 7.865444058552384e-05, + 6.111973198130727e-05, + 4.650574192055501e-05, + 4.192674532532692e-05, + 2.5328676201752387e-05, + 2.264554495923221e-05, + 2.188059988839086e-05, + 2.1676863980246708e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05060750572010875, + "first_to_last_ratio": 1.4278402181986947, + "imbalance_abs_log_ratio": 0.356162965639293, + "last_quartile_mean": 0.03544339560903609, + "mean": 0.045956584566738456, + "normalized": [ + 1.1683197838664416, + 1.3693581481609842, + 1.2477884983122534, + 1.1320828879264566, + 1.086197129718159, + 1.0136921293230559, + 0.9398491347700728, + 0.852332271168493, + 0.7376380598562373, + 0.6943242075392044, + 0.6618878602537814, + 0.6379511317995364, + 0.6961168732850856, + 0.66191505624554, + 0.6320240673482492, + 0.6099217105646539, + 0.7532923321079824, + 0.7156054155999519, + 0.7584607053972269, + 0.8619137716807654, + 2.8636216906024767, + 2.3232503851911614, + 1.8821005956556751, + 1.530464123018414, + 1.5177704445282252, + 1.1794086151290946, + 0.8974069322276481, + 0.8090474387174796, + 0.48875963581745097, + 0.43698406072332996, + 0.42222316261516873, + 0.41829174084974413 + ], + "peak_layer": 21, + "peak_normalized": 2.8636216906024767, + "population_cv": 0.5384897545714166, + "rest_mean": 0.03724660800287017, + "spike_contrast": 2.496615477158658, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0929904580116272, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.053691986948251724, + 0.06293102353811264, + 0.05734409764409065, + 0.0520266629755497, + 0.04991791024804115, + 0.04658582806587219, + 0.04319225624203682, + 0.0391702800989151, + 0.033899325877428055, + 0.03190876916050911, + 0.03041810542345047, + 0.029318055137991905, + 0.031991153955459595, + 0.030419355258345604, + 0.029045667499303818, + 0.028029918670654297, + 0.034618742763996124, + 0.03288678079843521, + 0.03485626354813576, + 0.0396106131374836, + 0.13160227239131927, + 0.1067686527967453, + 0.08649491518735886, + 0.07033490389585495, + 0.0697515457868576, + 0.05420159175992012, + 0.041241757571697235, + 0.037181057035923004, + 0.022461723536252975, + 0.020082294940948486, + 0.019403934478759766, + 0.019223259761929512 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.187219166671639e-06, + "first_to_last_ratio": 0.8437183001333121, + "imbalance_abs_log_ratio": 0.16993660771354646, + "last_quartile_mean": 2.5923571485009234e-06, + "mean": 2.5683367930184886e-06, + "normalized": [ + 1.1919543817128102, + 0.9792836739870964, + 0.924042037649071, + 0.9164810837361373, + 0.7326981846944463, + 0.7055637920842305, + 0.6863211882787902, + 0.6765290243634249, + 0.5799994679651287, + 0.5635357621627565, + 0.5597470962238182, + 0.5639314892075872, + 0.5739189935534712, + 0.5756380159008986, + 0.5748207554502599, + 0.5781987328520684, + 0.6522355446992087, + 0.6638052489880939, + 0.6844497180630856, + 0.7655738065181559, + 2.843883820310657, + 2.5944552048335017, + 2.2989551083840256, + 2.0391579131550635, + 1.9644428765002167, + 1.5269231604355886, + 1.179167971687766, + 1.0934476508277742, + 0.6272800411331483, + 0.5710254497958045, + 0.5563528736513774, + 0.556179931194537 + ], + "peak_layer": 21, + "peak_normalized": 2.843883820310657, + "population_cv": 0.6360421157910927, + "rest_mean": 1.9271187023116874e-06, + "spike_contrast": 3.129498185867219, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.030914482835214e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0613402941526147e-06, + 2.5151302907033823e-06, + 2.3732511635898845e-06, + 2.35383208746498e-06, + 1.8818157059286023e-06, + 1.8121254470315762e-06, + 1.7627039596845862e-06, + 1.7375543848174857e-06, + 1.4896339735059883e-06, + 1.4473496321443236e-06, + 1.4376190620168927e-06, + 1.4483659924735548e-06, + 1.4740172673555207e-06, + 1.4784322956984397e-06, + 1.4763332956135855e-06, + 1.4850090792606352e-06, + 1.675160547165433e-06, + 1.7048754443749203e-06, + 1.757897393872554e-06, + 1.9662513750517974e-06, + 7.304051450773841e-06, + 6.663434760412201e-06, + 5.9044909903605e-06, + 5.2372442951309495e-06, + 5.045350917498581e-06, + 3.921652933058795e-06, + 3.028500486834673e-06, + 2.808341832860606e-06, + 1.6110664091684157e-06, + 1.4665856724604964e-06, + 1.4289015553003992e-06, + 1.4284573808254208e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.514474267125479e-05, + "first_to_last_ratio": 1.655519660200222, + "imbalance_abs_log_ratio": 0.5041149536218862, + "last_quartile_mean": 2.726922775764251e-05, + "mean": 3.928292341015549e-05, + "normalized": [ + 1.2428531532199, + 1.4224719349524337, + 1.3075943550391356, + 1.1908940314839356, + 1.1240062299860272, + 1.0512639226040563, + 0.9724687394121546, + 0.8822119975062471, + 0.7835780621436017, + 0.7378068411250925, + 0.7017901972736155, + 0.675554292822624, + 0.7400896696370771, + 0.7012250467261758, + 0.6687293301443596, + 0.6449026664130145, + 0.8086596332888067, + 0.7678950765711053, + 0.7972794784799546, + 0.884192918474659, + 2.849023696568842, + 2.282552164711599, + 1.7952274549858118, + 1.4143282104887471, + 1.3505434814026465, + 1.05691913246549, + 0.8328089954359738, + 0.7560703953177289, + 0.41728160395883906, + 0.38644818475535425, + 0.376537560284359, + 0.3767915423206315 + ], + "peak_layer": 21, + "peak_normalized": 2.849023696568842, + "population_cv": 0.5250932744673414, + "rest_mean": 3.245689711345929e-05, + "spike_contrast": 2.34598720716096, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.614346541231498e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.8822905228007585e-05, + 5.587885607383214e-05, + 5.136612890055403e-05, + 4.6781799028394744e-05, + 4.415425064507872e-05, + 4.129672015551478e-05, + 3.820141500909813e-05, + 3.4655866329558194e-05, + 3.0781237001065165e-05, + 2.898320963140577e-05, + 2.7568370569497347e-05, + 2.6537747544352897e-05, + 2.907288580900058e-05, + 2.754616980382707e-05, + 2.6269643058185466e-05, + 2.5333662051707506e-05, + 3.176651443936862e-05, + 3.016516347997822e-05, + 3.131946868961677e-05, + 3.4733682696241885e-05, + 0.0001119179796660319, + 8.966532186605036e-05, + 7.0521782618016e-05, + 5.555894676945172e-05, + 5.305329614202492e-05, + 4.151887333136983e-05, + 3.271517198299989e-05, + 2.9700655431952327e-05, + 1.639204128878191e-05, + 1.51808144437382e-05, + 1.479149614169728e-05, + 1.4801473298575729e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.469924399723823e-05, + "first_to_last_ratio": 1.7510149859622042, + "imbalance_abs_log_ratio": 0.5601956117834839, + "last_quartile_mean": 1.981664593131427e-05, + "mean": 2.9772959720730796e-05, + "normalized": [ + 1.2826990789388135, + 1.4313521308000512, + 1.325603166494873, + 1.2222410753351367, + 1.1310087328328617, + 1.056511628684211, + 0.9758416336275024, + 0.898436085327916, + 0.8214143991142518, + 0.7703245121438669, + 0.7298179894195511, + 0.7040392340287683, + 0.7681958890441348, + 0.7313866125384519, + 0.6969281018261846, + 0.6697043803055042, + 0.8568340618774071, + 0.8208415080145076, + 0.8328878615188123, + 0.8883662275011015, + 2.8312616617599446, + 2.239848895619399, + 1.6996416088520014, + 1.290076953440798, + 1.215112714243769, + 0.9719493099553681, + 0.8309033623504332, + 0.7782938507320865, + 0.3989932047584423, + 0.3786265198784562, + 0.3759890641537673, + 0.37486854488162713 + ], + "peak_layer": 21, + "peak_normalized": 2.8312616617599446, + "population_cv": 0.5072717351311633, + "rest_mean": 2.505786920260612e-05, + "spike_contrast": 2.2042755539987984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.523444851860404e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 3 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.8189748011063784e-05, + 4.261558933649212e-05, + 3.9467129681725055e-05, + 3.638973430497572e-05, + 3.367347744642757e-05, + 3.14554781652987e-05, + 2.905369365180377e-05, + 2.67491013801191e-05, + 2.445593781885691e-05, + 2.293484067195095e-05, + 2.172884160245303e-05, + 2.096133175655268e-05, + 2.287146526214201e-05, + 2.177554415538907e-05, + 2.0749612303916365e-05, + 1.9939081539632753e-05, + 2.55104860116262e-05, + 2.443888115521986e-05, + 2.4797536752885208e-05, + 2.6449291908647865e-05, + 8.429503941442817e-05, + 6.668693094979972e-05, + 5.0603361160028726e-05, + 3.840940917143598e-05, + 3.61775018973276e-05, + 2.8937807655893266e-05, + 2.473845233907923e-05, + 2.3172111468738876e-05, + 1.1879208614118397e-05, + 1.1272832125541754e-05, + 1.1194307262485381e-05, + 1.1160946087329648e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011354441994626541, + "first_to_last_ratio": 1.3378984746048481, + "imbalance_abs_log_ratio": 0.2911000803554069, + "last_quartile_mean": 8.486774004268227e-05, + "mean": 0.00010590371198304638, + "normalized": [ + 1.1416862116188895, + 1.318287506150763, + 1.2188213004978723, + 1.1129317281051134, + 1.0503469451456144, + 0.9936779473327889, + 0.9152461482494265, + 0.8261837163725879, + 0.7163781808438098, + 0.6746444347634984, + 0.649936107552919, + 0.6258996320477875, + 0.6816391399615429, + 0.6448073899118816, + 0.6202888218188445, + 0.5944257945762098, + 0.72764205396545, + 0.6903920368963828, + 0.7174952313478369, + 0.8067894001918988, + 2.8841338501609886, + 2.3452358192551364, + 1.972412467214386, + 1.6597621298267584, + 1.6194268438614279, + 1.2623302767875173, + 0.9611591980513982, + 0.8707097795471348, + 0.4633879176416832, + 0.4198531088291576, + 0.4079853661334264, + 0.40608351533986714 + ], + "peak_layer": 21, + "peak_normalized": 2.8841338501609886, + "population_cv": 0.5599416174982532, + "rest_mean": 8.440537176620974e-05, + "spike_contrast": 2.630102142892712, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022199474915396422, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012090880773030221, + 0.00013961154036223888, + 0.0001290776999667287, + 0.00011786360119003803, + 0.00011123564036097378, + 0.0001052341831382364, + 9.692796447779983e-05, + 8.749592234380543e-05, + 7.586710853502154e-05, + 7.144734991015866e-05, + 6.883064634166658e-05, + 6.62850943626836e-05, + 7.218811515485868e-05, + 6.828749610576779e-05, + 6.569088873220608e-05, + 6.295189814409241e-05, + 7.70599945099093e-05, + 7.311507943086326e-05, + 7.598540832987055e-05, + 8.544199226889759e-05, + 0.000305440480588004, + 0.00024836917873471975, + 0.00020888580183964223, + 0.00017577497055754066, + 0.00017150331404991448, + 0.00013368546206038445, + 0.0001017903268802911, + 9.221139771398157e-05, + 4.9074500566348433e-05, + 4.4464002712629735e-05, + 4.320716470829211e-05, + 4.3005751649616286e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.04804428666830063, + 0.18500657379627228, + 0.22299833595752716, + 0.2331935465335846, + 0.0799286738038063, + 0.09402555972337723, + 0.10843002796173096, + 0.12457633018493652, + 0.04508088901638985, + 0.0760042667388916, + 0.10498681664466858, + 0.13159531354904175, + 0.0497574545443058, + 0.07735268771648407, + 0.10224150121212006, + 0.13362281024456024, + 0.079553984105587, + 0.10536550730466843, + 0.14260481297969818, + 0.19080305099487305, + 0.03440643846988678, + 0.060867179185152054, + 0.08364810794591904, + 0.11119203269481659, + 0.025225287303328514, + 0.05530283972620964, + 0.08037404716014862, + 0.11502843350172043, + 0.0242468249052763, + 0.07164724916219711, + 0.1371813863515854, + 0.23144805431365967 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.406055810093676e-05, + "first_to_last_ratio": 1.2121225310488972, + "imbalance_abs_log_ratio": 0.19237298076220427, + "last_quartile_mean": 1.1599947811191669e-05, + "mean": 1.3850583115981863e-05, + "normalized": [ + 1.1035219335667126, + 1.245398841062786, + 1.1319237648162956, + 1.0360361966114253, + 0.9887036854141024, + 0.9324573194178568, + 0.8818082800629979, + 0.8014300627506455, + 0.7197404552580381, + 0.6642341963531577, + 0.63144215306254, + 0.605460329509273, + 0.6612975386173722, + 0.6318610283296698, + 0.6080617688348535, + 0.5826465005104303, + 0.7161183891450303, + 0.6858808349031477, + 0.733242501237421, + 0.767094241825568, + 2.859288144745241, + 2.5015026079052487, + 2.101601950422847, + 1.7091984970083665, + 1.5703665940407991, + 1.4157861892600545, + 1.1108502479674358, + 0.8417170775086074, + 0.5124528160490821, + 0.4544107825414821, + 0.40419536018403524, + 0.39026971107747666 + ], + "peak_layer": 21, + "peak_normalized": 2.859288144745241, + "population_cv": 0.5793000123245652, + "rest_mean": 1.0905047424279877e-05, + "spike_contrast": 2.7286883489310063, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.975647585117258e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.5284422261174768e-05, + 1.7249500160687603e-05, + 1.567780418554321e-05, + 1.4349705452332273e-05, + 1.369412257190561e-05, + 1.2915077604702674e-05, + 1.2213558875373565e-05, + 1.1100273695774376e-05, + 9.968824997486081e-06, + 9.200030945066828e-06, + 8.745842023927253e-06, + 8.385978617297951e-06, + 9.15935652301414e-06, + 8.751643690629862e-06, + 8.42201006889809e-06, + 8.069993782555684e-06, + 9.918657269736286e-06, + 9.499849511485081e-06, + 1.0155836207559332e-05, + 1.062470255419612e-05, + 3.960280810133554e-05, + 3.4647269785637036e-05, + 2.9108412491041236e-05, + 2.3673395844525658e-05, + 2.1750493033323437e-05, + 1.960946428880561e-05, + 1.538592368888203e-05, + 1.1658272342174314e-05, + 7.0977703217067756e-06, + 6.293854312389158e-06, + 5.598341431323206e-06, + 5.405463070928818e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.784146014775615e-05, + "first_to_last_ratio": 1.2477003862842722, + "imbalance_abs_log_ratio": 0.22130216603186706, + "last_quartile_mean": 4.635845334632904e-05, + "mean": 5.5787147289265704e-05, + "normalized": [ + 1.1172174922096436, + 1.2820167732139756, + 1.1618030425233523, + 1.0568058922472814, + 1.015898285952758, + 0.952837653331755, + 0.8952147696466347, + 0.8127990870482149, + 0.7269196158944218, + 0.6687128912021425, + 0.6353920226237901, + 0.6079547558759868, + 0.6632448840034066, + 0.632152432145331, + 0.6072547727004206, + 0.5816761067060003, + 0.7067464190080565, + 0.6745056530323595, + 0.7095637632538165, + 0.7592902185109378, + 2.8322130517871535, + 2.4767089823758397, + 2.0818425648522476, + 1.6933243830446072, + 1.5552023255692202, + 1.402829278407864, + 1.095030881819868, + 0.8327931471930783, + 0.5108623485290222, + 0.45409986898200716, + 0.4053123002322658, + 0.3917743360765406 + ], + "peak_layer": 21, + "peak_normalized": 2.8322130517871535, + "population_cv": 0.5731345570971216, + "rest_mean": 4.41352963712739e-05, + "spike_contrast": 2.6896192391648626, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011870714224642143, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.232637679204345e-05, + 7.15200585545972e-05, + 6.481367745436728e-05, + 5.895618596696295e-05, + 5.667406730935909e-05, + 5.315609450917691e-05, + 4.994147820980288e-05, + 4.534374238573946e-05, + 4.055277167935856e-05, + 3.7305584555724636e-05, + 3.5446708352537826e-05, + 3.391606151126325e-05, + 3.7000540032749996e-05, + 3.5265980841359124e-05, + 3.387701144674793e-05, + 3.2450050639454275e-05, + 3.942736657336354e-05, + 3.762874621315859e-05, + 3.9584538171766326e-05, + 4.2358635255368426e-05, + 0.00015800108667463064, + 0.00013816852879244834, + 0.00011614005779847503, + 9.446573676541448e-05, + 8.676030120113865e-05, + 7.825984357623383e-05, + 6.108864909037948e-05, + 4.6459153963951394e-05, + 2.8499553081928752e-05, + 2.533293627493549e-05, + 2.2611216991208494e-05, + 2.1855972590856254e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.051294384989887476, + "first_to_last_ratio": 1.2477003995429623, + "imbalance_abs_log_ratio": 0.2213021766583686, + "last_quartile_mean": 0.04111113934777677, + "mean": 0.04947259998880327, + "normalized": [ + 1.117217517042371, + 1.282016692529182, + 1.1618030656626175, + 1.0568059265711638, + 1.0158982593254349, + 0.9528376876517187, + 0.8952147600149789, + 0.8127990580563501, + 0.7269196281255159, + 0.6687128990372552, + 0.6353920151588079, + 0.6079547776386781, + 0.6632449090716109, + 0.6321524554983756, + 0.6072547505273581, + 0.5816761070130537, + 0.706746437867777, + 0.674505658209711, + 0.7095637900293472, + 0.7592902251062212, + 2.8322130824094, + 2.4767089467624213, + 2.081842602529718, + 1.693324355494102, + 1.5552023386093694, + 1.4028291808614948, + 1.0950308028824434, + 0.8327931847449266, + 0.5108623388148443, + 0.45409990274101625, + 0.40531230895107145, + 0.3917743350616629 + ], + "peak_layer": 21, + "peak_normalized": 2.8322130824094, + "population_cv": 0.5731345524386317, + "rest_mean": 0.03913962206354848, + "spike_contrast": 2.689619246048363, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10527068078517914, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.055271655321121216, + 0.06342469900846481, + 0.05747741833329201, + 0.05228293687105179, + 0.05025912821292877, + 0.04713935777544975, + 0.044288601726293564, + 0.040211282670497894, + 0.035962603986263275, + 0.03308296576142311, + 0.031434495002031326, + 0.030077103525400162, + 0.03281245008111, + 0.0312742255628109, + 0.03004247136414051, + 0.028777029365301132, + 0.034964583814144135, + 0.03336954861879349, + 0.03510396555066109, + 0.03756406158208847, + 0.14011694490909576, + 0.12252923101186752, + 0.1029941663146019, + 0.0837731584906578, + 0.0769399031996727, + 0.06940160691738129, + 0.054174020886421204, + 0.04120044410228729, + 0.02527368813753128, + 0.022465502843260765, + 0.020051853731274605, + 0.019382094964385033 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.2204297493999547e-06, + "first_to_last_ratio": 0.7359606295430359, + "imbalance_abs_log_ratio": 0.3065786541527352, + "last_quartile_mean": 3.0170496359005483e-06, + "mean": 2.8223206918198684e-06, + "normalized": [ + 1.1437255119946472, + 0.8934281083094181, + 0.843667370045445, + 0.833234424409991, + 0.6691350034907739, + 0.6491575975670105, + 0.6344872974587108, + 0.6270766601877794, + 0.5480904457549984, + 0.5298089242035497, + 0.5236022556745512, + 0.5213819889179987, + 0.5319070176791533, + 0.5365159271613865, + 0.5384760973551012, + 0.5417839198031603, + 0.5976312076631317, + 0.6133914803428778, + 0.632878138794897, + 0.6938486884695898, + 2.7927450515890637, + 2.792273921126163, + 2.5604223062656137, + 2.1993623563278324, + 1.9620758229843447, + 1.8148751057591102, + 1.435693067783232, + 1.0849680187180202, + 0.6411889832806804, + 0.5785351975720235, + 0.5245373868045311, + 0.5100947165052133 + ], + "peak_layer": 21, + "peak_normalized": 2.7927450515890637, + "population_cv": 0.7010322041721825, + "rest_mean": 2.058529688568982e-06, + "spike_contrast": 3.374637804812921, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.946792109374656e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.227960178264766e-06, + 2.5215406367351534e-06, + 2.3810998754925095e-06, + 2.3516547571489355e-06, + 1.8885135659729713e-06, + 1.832130919865449e-06, + 1.7907266283145873e-06, + 1.7698114334052661e-06, + 1.5468870060431072e-06, + 1.4952906894905027e-06, + 1.477773480473843e-06, + 1.4715071756654652e-06, + 1.5012121821200708e-06, + 1.5142200027185027e-06, + 1.519752231615712e-06, + 1.5290879673557356e-06, + 1.6867069234649534e-06, + 1.7311874671577243e-06, + 1.7861850665212842e-06, + 1.958263510459801e-06, + 7.882022146077361e-06, + 7.880692464823369e-06, + 7.22633285477059e-06, + 6.207305887073744e-06, + 5.5376071941282135e-06, + 5.1221595640527084e-06, + 4.0519862523069605e-06, + 3.0621276891906746e-06, + 1.809640934880008e-06, + 1.6328118590536178e-06, + 1.4804127204115503e-06, + 1.4396508731806534e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.56924158243055e-05, + "first_to_last_ratio": 1.4673115279536288, + "imbalance_abs_log_ratio": 0.38343183377619244, + "last_quartile_mean": 3.1140228202275466e-05, + "mean": 4.183447202876778e-05, + "normalized": [ + 1.1974206825409346, + 1.3442935712461421, + 1.2304058748098903, + 1.1234622096860003, + 1.0623591343774659, + 0.9983032021931795, + 0.9335117515893174, + 0.8479976043257199, + 0.7756355664609215, + 0.7156793363778972, + 0.6795817133031917, + 0.6491928801846335, + 0.7109607744004338, + 0.6750928595197824, + 0.6474397844670364, + 0.6200735511625727, + 0.7662731422367131, + 0.7307687689442663, + 0.7545377188834692, + 0.7927345870100259, + 2.8287570565248865, + 2.421556562840323, + 1.9807276243744367, + 1.558292733686825, + 1.4117490803489046, + 1.238912861023533, + 0.9856350322605589, + 0.7768378929765343, + 0.43766980945827866, + 0.39060663115145766, + 0.36143264697135674, + 0.3520973546633113 + ], + "peak_layer": 21, + "peak_normalized": 2.8287570565248865, + "population_cv": 0.551494708887264, + "rest_mean": 3.377578448433067e-05, + "spike_contrast": 2.5269993301954115, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.535138476872817e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.009346205042675e-05, + 5.623781180474907e-05, + 5.14733801537659e-05, + 4.699944838648662e-05, + 4.444323349162005e-05, + 4.176348738837987e-05, + 3.905297126038931e-05, + 3.547553205862641e-05, + 3.244830440962687e-05, + 2.9940067179268226e-05, + 2.8429942176444456e-05, + 2.715864138735924e-05, + 2.9742668630206026e-05, + 2.824215334840119e-05, + 2.7085301553597674e-05, + 2.594044963188935e-05, + 3.205663233529776e-05, + 3.057132562389597e-05, + 3.156568709528074e-05, + 3.31636329065077e-05, + 0.00011833955795736983, + 0.00010130454029422253, + 8.286269439850003e-05, + 6.519035378005356e-05, + 5.9059777413494885e-05, + 5.182926543056965e-05, + 4.123352118767798e-05, + 3.249860310461372e-05, + 1.830968540161848e-05, + 1.6340822185156867e-05, + 1.5120343960006721e-05, + 1.4729806935065426e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.505123777358676e-05, + "first_to_last_ratio": 1.5889493494930538, + "imbalance_abs_log_ratio": 0.4630730113341737, + "last_quartile_mean": 2.2059380171413068e-05, + "mean": 3.131689976498819e-05, + "normalized": [ + 1.2479593211749789, + 1.367487352044708, + 1.261680923885286, + 1.1674855830446973, + 1.0794756507539984, + 1.0156514953231304, + 0.945160188314607, + 0.8690478102601445, + 0.8149681766862981, + 0.7530914967772463, + 0.7148635553828698, + 0.6827279730652124, + 0.7417389963838543, + 0.7102014979770842, + 0.6794125192711595, + 0.6486601330546842, + 0.8192809210261347, + 0.7895373258521271, + 0.7975483511256357, + 0.812938977938045, + 2.826829287271584, + 2.3623544295272914, + 1.8510256189591003, + 1.4057347513501408, + 1.2902085448265628, + 1.123507683724877, + 0.9230685473509727, + 0.7887132397564032, + 0.4159196155359273, + 0.3769758263062851, + 0.35976736593467973, + 0.35697684011427405 + ], + "peak_layer": 21, + "peak_normalized": 2.826829287271584, + "population_cv": 0.5277213122295441, + "rest_mean": 2.5823506533655477e-05, + "spike_contrast": 2.3614617610009216, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.0981223214184865e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 2 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.908221697201952e-05, + 4.282546433387324e-05, + 3.9511935028713197e-05, + 3.656202898127958e-05, + 3.380583075340837e-05, + 3.180705607519485e-05, + 2.9599486879305914e-05, + 2.7215883164899424e-05, + 2.5522276700939983e-05, + 2.358449091843795e-05, + 2.238731030956842e-05, + 2.1380923499236815e-05, + 2.3228965801536106e-05, + 2.2241309125092812e-05, + 2.1277093765093014e-05, + 2.031402436841745e-05, + 2.5657338483142667e-05, + 2.4725861294427887e-05, + 2.4976741769933142e-05, + 2.5458728487137705e-05, + 8.85275294422172e-05, + 7.398161687888205e-05, + 5.796838377136737e-05, + 4.402325430419296e-05, + 4.0405331674264744e-05, + 3.518477751640603e-05, + 2.890764517360367e-05, + 2.4700053472770378e-05, + 1.3025312910031062e-05, + 1.180571416625753e-05, + 1.1266798537690192e-05, + 1.1179407920280937e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011513973367982544, + "first_to_last_ratio": 1.1689761361783517, + "imbalance_abs_log_ratio": 0.15612826840504843, + "last_quartile_mean": 9.849622256297152e-05, + "mean": 0.00011409382295823889, + "normalized": [ + 1.087599643787199, + 1.2332404432021893, + 1.1325955558216252, + 1.0348100003758545, + 0.9846631201892967, + 0.9329597002283465, + 0.8753802278299959, + 0.7920881953113688, + 0.705656362899583, + 0.6479209049109275, + 0.6172280934926675, + 0.5959629720755263, + 0.6485619380890737, + 0.616624813163851, + 0.5952030046676504, + 0.5686106501058988, + 0.6801552059812119, + 0.6508138445342289, + 0.6710903132095474, + 0.7219939084470286, + 2.8256801162653837, + 2.4931864357424143, + 2.15835309890828, + 1.8232898013061003, + 1.6443057108717645, + 1.5033517491402877, + 1.1720956223235564, + 0.8974197290603315, + 0.48941489739710603, + 0.4307568860388074, + 0.3896112863482277, + 0.3793757682746689 + ], + "peak_layer": 21, + "peak_normalized": 2.8256801162653837, + "population_cv": 0.5949861940504896, + "rest_mean": 8.897283448647984e-05, + "spike_contrast": 2.80700465650208, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00024974716070573776, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012408840120770037, + 0.00014070511679165065, + 0.00012922215682920069, + 0.00011806542897829786, + 0.00011234397970838472, + 0.0001064449388650246, + 9.987547673517838e-05, + 9.037237032316625e-05, + 8.05110321380198e-05, + 7.392377301584929e-05, + 7.042191282380372e-05, + 6.799569382565096e-05, + 7.399691094178706e-05, + 7.035308226477355e-05, + 6.790898623876274e-05, + 6.487496284535155e-05, + 7.76015076553449e-05, + 7.425383955705911e-05, + 7.656725938431919e-05, + 8.237504516728222e-05, + 0.0003223926469217986, + 0.00028445717180147767, + 0.0002462547563482076, + 0.00020802610379178077, + 0.00018760512466542423, + 0.00017152314831037074, + 0.00013372887042351067, + 0.00010239004768664017, + 5.583921665675007e-05, + 4.9146699893753976e-05, + 4.445224112714641e-05, + 4.328443174017593e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.09451264888048172, + 0.2031717449426651, + 0.22856928408145905, + 0.2430499792098999, + 0.08243431895971298, + 0.09709372371435165, + 0.11282631754875183, + 0.13645486533641815, + 0.05442482605576515, + 0.0885990634560585, + 0.12215005606412888, + 0.15440259873867035, + 0.060873307287693024, + 0.09471380710601807, + 0.1260664463043213, + 0.163865864276886, + 0.08853588253259659, + 0.12131853401660919, + 0.15660560131072998, + 0.2097783088684082, + 0.04455159604549408, + 0.07411240041255951, + 0.10008715838193893, + 0.13061410188674927, + 0.04960336163640022, + 0.07874798029661179, + 0.11358420550823212, + 0.1580832600593567, + 0.06955241411924362, + 0.13302887976169586, + 0.22457914054393768, + 0.32859006524086 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.3873141369913355e-05, + "first_to_last_ratio": 1.3885721694739201, + "imbalance_abs_log_ratio": 0.32827600298864523, + "last_quartile_mean": 9.990940100124135e-06, + "mean": 1.2854364840109156e-05, + "normalized": [ + 1.1519312840854947, + 1.3311758992301272, + 1.2167315437597699, + 1.1117686172453958, + 1.058772632613286, + 0.9928324590403058, + 0.927417632168206, + 0.8434123244463263, + 0.7336779055664342, + 0.6915156665898262, + 0.6596055773251193, + 0.6376390806056413, + 0.6952626444464219, + 0.6629130327970861, + 0.6345504665270513, + 0.6122283613875565, + 0.7648834288449639, + 0.7297709534484661, + 0.7866551404509706, + 0.8716621799696268, + 2.8889843133608784, + 2.338532478770988, + 1.899676736110628, + 1.5404711947102903, + 1.5315734843108408, + 1.1978159493624418, + 0.9082426544704836, + 0.8160818433380076, + 0.4896280426071342, + 0.4364018959793389, + 0.4210420673902388, + 0.4171425090406535 + ], + "peak_layer": 21, + "peak_normalized": 2.8889843133608784, + "population_cv": 0.5419273520370343, + "rest_mean": 1.037907206948687e-05, + "spike_contrast": 2.5263285220415503, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6220945801469498e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.4807344996370375e-05, + 1.7111420675064437e-05, + 1.5640311175957322e-05, + 1.429107942385599e-05, + 1.3609849702334031e-05, + 1.2762230653606821e-05, + 1.1921364603040274e-05, + 1.0841529729077592e-05, + 9.430963473278098e-06, + 8.888994670996908e-06, + 8.478810741507914e-06, + 8.196445378416684e-06, + 8.937159691413399e-06, + 8.521325980836991e-06, + 8.15674320620019e-06, + 7.869806722737849e-06, + 9.832090654526837e-06, + 9.380742085340898e-06, + 1.0111952178704087e-05, + 1.120466367865447e-05, + 3.713605838129297e-05, + 3.00603496725671e-05, + 2.4419137844233774e-05, + 1.98017787624849e-05, + 1.9687404346768744e-05, + 1.539716322440654e-05, + 1.1674882443912793e-05, + 1.0490213753655553e-05, + 6.293857495620614e-06, + 5.609669187833788e-06, + 5.4122283472679555e-06, + 5.3621020015270915e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.706737192667788e-05, + "first_to_last_ratio": 1.4278557447018398, + "imbalance_abs_log_ratio": 0.3561738396982594, + "last_quartile_mean": 3.996718305643299e-05, + "mean": 5.182250379220932e-05, + "normalized": [ + 1.1683241108409874, + 1.3693593420455452, + 1.2478029467661282, + 1.1320859203517522, + 1.0862154844943164, + 1.0136862423485675, + 0.9398593392770029, + 0.8523331101914637, + 0.7376332926932397, + 0.6943335382230071, + 0.6618887156422048, + 0.6379567874173152, + 0.6961101086882286, + 0.6619092142621851, + 0.6320190672968202, + 0.6099299790483219, + 0.7532799174046017, + 0.7156061917099562, + 0.758449009476584, + 0.8619189528616946, + 2.863607509132394, + 2.3232710469251585, + 1.8820910428129736, + 1.530471615417879, + 1.517753064129064, + 1.1793980690808934, + 0.8973983434078874, + 0.8090471852696426, + 0.4887601793176453, + 0.43698385938136236, + 0.4222309955613731, + 0.41828581852380436 + ], + "peak_layer": 21, + "peak_normalized": 2.863607509132394, + "population_cv": 0.5384894543542103, + "rest_mean": 4.200080675892187e-05, + "spike_contrast": 2.4966108002124767, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010485966777196154, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 6.054548066458665e-05, + 7.096362969605252e-05, + 6.466427294071764e-05, + 5.866752690053545e-05, + 5.6290406064363196e-05, + 5.253175913821906e-05, + 4.870586417382583e-05, + 4.417003583512269e-05, + 3.822600410785526e-05, + 3.59821024176199e-05, + 3.430073047638871e-05, + 3.306051803519949e-05, + 3.607416874729097e-05, + 3.430179276620038e-05, + 3.275281051173806e-05, + 3.160809865221381e-05, + 3.903685137629509e-05, + 3.708450458361767e-05, + 3.9304726669797674e-05, + 4.4666798203252256e-05, + 0.00014839931100141257, + 0.00012039772263960913, + 9.753467020345852e-05, + 7.931287109386176e-05, + 7.865376392146572e-05, + 6.111936090746894e-05, + 4.65054290543776e-05, + 4.1926850826712325e-05, + 2.5328776246169582e-05, + 2.2645597709924914e-05, + 2.1881067368667573e-05, + 2.167661841667723e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05060791550204158, + "first_to_last_ratio": 1.4278557661609772, + "imbalance_abs_log_ratio": 0.3561738547271846, + "last_quartile_mean": 0.035443296656012535, + "mean": 0.045956714369822294, + "normalized": [ + 1.1683241037091443, + 1.3693593062308924, + 1.2478029694871473, + 1.1320859320845278, + 1.0862154618577222, + 1.013686266934737, + 0.9398592878142475, + 0.8523331062206766, + 0.7376333014160883, + 0.6943335139035525, + 0.6618887468442457, + 0.6379567875276879, + 0.6961101245379498, + 0.6619092147086973, + 0.632019080315496, + 0.6099299583472245, + 0.75327990973307, + 0.7156062315279661, + 0.7584489979720741, + 0.861918956956209, + 2.863607441781645, + 2.3232710596995285, + 1.8820912266878924, + 1.5304716351616434, + 1.5177530257827039, + 1.179397988455519, + 0.8973983179704046, + 0.8090471801133733, + 0.48876020079385396, + 0.43698383973821214, + 0.4222310083457578, + 0.41828581734010906 + ], + "peak_layer": 21, + "peak_normalized": 2.863607441781645, + "population_cv": 0.538489454771283, + "rest_mean": 0.037246735076661465, + "spike_contrast": 2.496610840158122, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09299060255289078, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.05369233712553978, + 0.06293125450611115, + 0.05734492465853691, + 0.05202694982290268, + 0.04991889372467995, + 0.04658569023013115, + 0.043192844837903976, + 0.03917042911052704, + 0.033899202942848206, + 0.031909286975860596, + 0.030418232083320618, + 0.029318397864699364, + 0.031990934163331985, + 0.03041917271912098, + 0.02904552035033703, + 0.028030376881361008, + 0.03461826965212822, + 0.03288691118359566, + 0.03485582396388054, + 0.03961096331477165, + 0.1316019892692566, + 0.10676990449428558, + 0.08649472892284393, + 0.07033544778823853, + 0.06975094228982925, + 0.05420125648379326, + 0.04124147817492485, + 0.037181150168180466, + 0.02246181294322014, + 0.020082341507077217, + 0.019404349848628044, + 0.019223041832447052 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.187229483752162e-06, + "first_to_last_ratio": 0.843721752674449, + "imbalance_abs_log_ratio": 0.16993251566757275, + "last_quartile_mean": 2.592358768538361e-06, + "mean": 2.5683487088201673e-06, + "normalized": [ + 1.191975321874382, + 0.9792375219295126, + 0.9240659028379828, + 0.9165026822465744, + 0.7327411746199551, + 0.7055071798348216, + 0.68634443005073, + 0.6764996809984228, + 0.5799709708281261, + 0.5635565193402196, + 0.5596940376847175, + 0.5639570693866405, + 0.5739412518179093, + 0.5756289711426894, + 0.5747974170292999, + 0.5782423953100819, + 0.652186394989175, + 0.6637611802901628, + 0.6844201166273622, + 0.7655503355998897, + 2.843845838013164, + 2.5945154076809365, + 2.2989812705463573, + 2.039289390897498, + 1.9644018920207635, + 1.5269105875005642, + 1.1791879088220734, + 1.0934639133198125, + 0.6272666844428548, + 0.5710249694984988, + 0.5563916798315591, + 0.5561399029872623 + ], + "peak_layer": 21, + "peak_normalized": 2.843845838013164, + "population_cv": 0.6360528130844023, + "rest_mean": 1.9271144327393374e-06, + "spike_contrast": 3.129556655898081, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.031013799656648e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.061408278881572e-06, + 2.515023425075924e-06, + 2.3733234684186755e-06, + 2.3538984805782093e-06, + 1.8819348497345345e-06, + 1.8119884543921216e-06, + 1.762771830726706e-06, + 1.7374870822095545e-06, + 1.4895676940795965e-06, + 1.4474096587946406e-06, + 1.43748945902189e-06, + 1.4484384109891835e-06, + 1.474081273045158e-06, + 1.4784159247938078e-06, + 1.4762802038603695e-06, + 1.4851281093797297e-06, + 1.675042085480527e-06, + 1.7047701703631901e-06, + 1.7578295228304341e-06, + 1.9662002159748226e-06, + 7.303987786144717e-06, + 6.663620297331363e-06, + 5.904585577809485e-06, + 5.237606274022255e-06, + 5.0452690629754215e-06, + 3.921638835890917e-06, + 3.0285657430795254e-06, + 2.8083966299163876e-06, + 1.6110395790747134e-06, + 1.4665912431155448e-06, + 1.429007852493669e-06, + 1.4283612017607084e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.514497459240374e-05, + "first_to_last_ratio": 1.6555269986325893, + "imbalance_abs_log_ratio": 0.5041193863183254, + "last_quartile_mean": 2.7269246970718086e-05, + "mean": 3.9283020413449776e-05, + "normalized": [ + 1.2428521215956514, + 1.4224732380710257, + 1.3076049249463846, + 1.1908965547141857, + 1.1240104927458778, + 1.0512648458306981, + 0.9724745802930264, + 0.8822121342574466, + 0.7835766828764287, + 0.7378119649346744, + 0.7017902701963611, + 0.6755581812134592, + 0.7400830264097682, + 0.701223315160251, + 0.6687269842506862, + 0.6449060748365787, + 0.8086534690047941, + 0.7678965143140377, + 0.7972719531557511, + 0.8841907350981015, + 2.849012586533336, + 2.2825637536620915, + 1.7952254297971346, + 1.4143290706697607, + 1.3505415355877917, + 1.056910410344958, + 0.832807309381792, + 0.7560735292299187, + 0.4172799715855694, + 0.3864473693959841, + 0.37654214074567177, + 0.3767888291608037 + ], + "peak_layer": 21, + "peak_normalized": 2.849012586533336, + "population_cv": 0.5250931691969987, + "rest_mean": 3.245698109013675e-05, + "spike_contrast": 2.345986293299445, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.61436327593401e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.882298526354134e-05, + 5.587904524873011e-05, + 5.136667095939629e-05, + 4.678201366914436e-05, + 4.415452713146806e-05, + 4.1296858398709446e-05, + 3.820173878921196e-05, + 3.465595727902837e-05, + 3.078125882893801e-05, + 2.8983482479816303e-05, + 2.7568441510084085e-05, + 2.6537965823081322e-05, + 2.9072696634102613e-05, + 2.754616980382707e-05, + 2.6269615773344412e-05, + 2.533385850256309e-05, + 3.17663507303223e-05, + 3.0165294447215274e-05, + 3.1319250410888344e-05, + 3.4733682696241885e-05, + 0.00011191781959496439, + 8.966599853010848e-05, + 7.052187720546499e-05, + 5.5559117754455656e-05, + 5.3053350711707026e-05, + 4.1518633224768564e-05, + 3.271518653491512e-05, + 2.970085188280791e-05, + 1.6392017641919665e-05, + 1.518081990070641e-05, + 1.4791712601436302e-05, + 1.4801403267483693e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.469784724074998e-05, + "first_to_last_ratio": 1.7508907324210148, + "imbalance_abs_log_ratio": 0.5601246483988392, + "last_quartile_mean": 1.9817254496956593e-05, + "mean": 2.9772389552817913e-05, + "normalized": [ + 1.2825657703659485, + 1.4312331552120288, + 1.325820640442136, + 1.2221823686225868, + 1.130955243942597, + 1.0564145565086833, + 0.9757410003641811, + 0.8985840377367545, + 0.8214467481888739, + 0.7702253806339946, + 0.7297568784639149, + 0.7040532057825352, + 0.7682184821342395, + 0.7314383768963997, + 0.6969099228187515, + 0.6697241707472397, + 0.8568233441617242, + 0.8208320560978474, + 0.8328912261837936, + 0.8882556709751984, + 2.8316971252422305, + 2.2395765326860197, + 1.6996117178665944, + 1.2900403186629539, + 1.2152436368088633, + 0.9720044593739222, + 0.8310137912106149, + 0.7782207156702814, + 0.3990111711474371, + 0.3786791961781316, + 0.37596186733999515, + 0.3748672315335271 + ], + "peak_layer": 21, + "peak_normalized": 2.8316971252422305, + "population_cv": 0.507296052444767, + "rest_mean": 2.505713847414073e-05, + "spike_contrast": 2.204351683440534, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5234745377674696e-05, + "top_five_layers": [ + 21, + 22, + 23, + 2, + 3 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.818504774244502e-05, + 4.2611231037881225e-05, + 3.94728485844098e-05, + 3.638728958321735e-05, + 3.367124008946121e-05, + 3.145198570564389e-05, + 2.9050141165498644e-05, + 2.675299401744269e-05, + 2.4456432583974674e-05, + 2.2931450075702742e-05, + 2.1726606064476073e-05, + 2.096134630846791e-05, + 2.2871699911775067e-05, + 2.1776668290840462e-05, + 2.0748673705384135e-05, + 1.9939288904424757e-05, + 2.5509678380331025e-05, + 2.44381317315856e-05, + 2.4797162041068077e-05, + 2.644549385877326e-05, + 8.43063899083063e-05, + 6.667754496447742e-05, + 5.06015021528583e-05, + 3.840758290607482e-05, + 3.618070695665665e-05, + 2.8938895411556587e-05, + 2.4741266315686516e-05, + 2.3169490305008367e-05, + 1.1879516023327596e-05, + 1.127418454416329e-05, + 1.1193283171451185e-05, + 1.1160693247802556e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011355107744748238, + "first_to_last_ratio": 1.3380154347456796, + "imbalance_abs_log_ratio": 0.2911874973238657, + "last_quartile_mean": 8.486529713991331e-05, + "mean": 0.00010590487272565952, + "normalized": [ + 1.14177249304275, + 1.318239805267508, + 1.2190343862407116, + 1.1130801571737208, + 1.0501716457006294, + 0.9937227743286174, + 0.9154089043884999, + 0.8261602336216923, + 0.7161833889440711, + 0.6746260480574641, + 0.6498212581625004, + 0.6259761084931398, + 0.6817894792861978, + 0.6448396206511546, + 0.6202148320012568, + 0.5943358056681826, + 0.7276123000587434, + 0.6905927768196939, + 0.7174064355718048, + 0.8068892453627287, + 2.8834514867746446, + 2.3459559521323445, + 1.9722659474769593, + 1.659767709595541, + 1.6192051847503666, + 1.2622818151535096, + 0.9611298389601171, + 0.8706981752777411, + 0.4632426508072287, + 0.41992689699434765, + 0.4080288833982593, + 0.4061677598378732 + ], + "peak_layer": 21, + "peak_normalized": 2.8834514867746446, + "population_cv": 0.5599097168924846, + "rest_mean": 8.44075709917686e-05, + "spike_contrast": 2.6299809303873802, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022199030208867044, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.00012091927055735141, + 0.00013960801879875362, + 0.00012910168152302504, + 0.00011788061237893999, + 0.00011121829447802156, + 0.0001052400839398615, + 9.69462635111995e-05, + 8.749439439270645e-05, + 7.584731065435335e-05, + 7.14461857569404e-05, + 6.881923764012754e-05, + 6.62939200992696e-05, + 7.220482802949846e-05, + 6.82916579535231e-05, + 6.56837728456594e-05, + 6.294305785559118e-05, + 7.70576880313456e-05, + 7.313714013434947e-05, + 7.597683725180104e-05, + 8.545350283384323e-05, + 0.00030537156271748245, + 0.00024844816653057933, + 0.00020887257414869964, + 0.00017577748803887516, + 0.00017148171900771558, + 0.0001336817949777469, + 0.00010178833326790482, + 9.221117943525314e-05, + 4.905965397483669e-05, + 4.447230458026752e-05, + 4.321224696468562e-05, + 4.30151449108962e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.05595625191926956, + 0.0632273480296135, + 0.11160287261009216, + 0.12218338996171951, + 0.12365243583917618, + 0.0926642045378685, + 0.093480683863163, + 0.08749273419380188, + 0.09927791357040405, + 0.07820390164852142, + 0.07669360190629959, + 0.07840896397829056, + 0.08920575678348541, + 0.06850729882717133, + 0.07440610229969025, + 0.07867594808340073, + 0.09058429300785065, + 0.07234266400337219, + 0.07771008461713791, + 0.08416424691677094, + 0.08947562426328659, + 0.056279782205820084, + 0.057985786348581314, + 0.06238175556063652, + 0.06515856087207794, + 0.05041429027915001, + 0.05083387345075607, + 0.05600840970873833, + 0.06128782033920288, + 0.056301049888134, + 0.06424970924854279, + 0.06844019144773483 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 9.178191703540506e-06, + "first_to_last_ratio": 2.0927096232778837, + "imbalance_abs_log_ratio": 0.7384596967524485, + "last_quartile_mean": 4.385793232586366e-06, + "mean": 9.837752486419049e-06, + "normalized": [ + 1.3396520059806467, + 1.2909822759863991, + 0.8248929730174986, + 0.8734689592802515, + 0.4594300774035242, + 0.8015983028871521, + 0.902358022333612, + 0.9712665986901348, + 0.6517279620702138, + 0.5032471363266136, + 0.3886238838550298, + 0.3309759496712648, + 0.3194854781922982, + 0.5459377912807375, + 0.43221827830345944, + 0.4611789434475917, + 1.0400100603479894, + 1.0312967006898324, + 1.4985429159426815, + 1.6347346226131838, + 2.6918571666331577, + 4.297443405515621, + 2.6798531616309957, + 2.4627171634441143, + 1.010352002858167, + 0.8524077751550129, + 0.3921416774715023, + 0.37035603642720544, + 0.29698661283582345, + 0.19544726726737693, + 0.19990393116510585, + 0.24890486127580272 + ], + "peak_layer": 22, + "peak_normalized": 4.297443405515621, + "population_cv": 0.8934604862364398, + "rest_mean": 6.871042353816813e-06, + "spike_contrast": 3.763328163463758, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.5857987202471124e-05, + "top_five_layers": [ + 22, + 21, + 23, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.3179164852772374e-05, + 1.2700364095508121e-05, + 8.115092896332499e-06, + 8.592971425969154e-06, + 4.5197593863122165e-06, + 7.88592569733737e-06, + 8.877174877852667e-06, + 9.555080396239646e-06, + 6.411538379325066e-06, + 4.950820766680408e-06, + 3.823185579676647e-06, + 3.2560594718233915e-06, + 3.1430190574610606e-06, + 5.370800863602199e-06, + 4.252056442055618e-06, + 4.536964297585655e-06, + 1.0231361557089258e-05, + 1.014564168144716e-05, + 1.4742294297320768e-05, + 1.6082114598248154e-05, + 2.6481824534130283e-05, + 4.227718454785645e-05, + 2.636373210407328e-05, + 2.4227601898019202e-05, + 9.939592928276397e-06, + 8.385776709474158e-06, + 3.8577927625738084e-06, + 3.6434710182220442e-06, + 2.9216807888587937e-06, + 1.922761839523446e-06, + 1.9666053958644625e-06, + 2.4486644178978167e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.4749664109767764e-05, + "first_to_last_ratio": 2.002663558914822, + "imbalance_abs_log_ratio": 0.6944780739856734, + "last_quartile_mean": 1.735172338612756e-05, + "mean": 3.858046457594355e-05, + "normalized": [ + 0.8375866313273315, + 1.2899140602905792, + 0.8694344042805302, + 0.9158393863926719, + 0.4754536346534661, + 0.8638099380048725, + 0.9821647500595908, + 0.9714468918970042, + 0.6838803886511606, + 0.5360010655021746, + 0.3851151654093992, + 0.3481231986646324, + 0.3212694297422333, + 0.5688836414280135, + 0.4682021401391152, + 0.46212538368136635, + 1.0036599670842112, + 1.0844213697059686, + 1.4666068093127032, + 1.6452318694807484, + 2.683098703810988, + 4.385873690297744, + 2.767792621757238, + 2.3860317964922024, + 1.0350167413971205, + 0.871092691603006, + 0.39147360723073643, + 0.341414306210179, + 0.30371057059916196, + 0.20069739154416044, + 0.20302718339269, + 0.25160056995699964 + ], + "peak_layer": 22, + "peak_normalized": 4.385873690297744, + "population_cv": 0.8995046127848129, + "rest_mean": 2.6780824454261972e-05, + "spike_contrast": 3.8198421190406635, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010229852123302407, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.2314481359208e-05, + 4.9765483709052205e-05, + 3.354318323545158e-05, + 3.533350900397636e-05, + 1.8343222109251656e-05, + 3.332618871354498e-05, + 3.7892372347414494e-05, + 3.7478872400242835e-05, + 2.6384423108538613e-05, + 2.0679170120274648e-05, + 1.4857921996735968e-05, + 1.3430754734145012e-05, + 1.239472385350382e-05, + 2.194779517594725e-05, + 1.8063456082018092e-05, + 1.7829011994763277e-05, + 3.8721667806385085e-05, + 4.183748023933731e-05, + 5.658237205352634e-05, + 6.34738098597154e-05, + 0.00010351519449613988, + 0.00016920904454309493, + 0.00010678272519726306, + 9.205421520164236e-05, + 3.9931426726980135e-05, + 3.3607160730753094e-05, + 1.5103233636182267e-05, + 1.3171922546462156e-05, + 1.171729491034057e-05, + 7.742998604953755e-06, + 7.832883056835271e-06, + 9.706866876513232e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.030816350365057588, + "first_to_last_ratio": 2.002663649499559, + "imbalance_abs_log_ratio": 0.6944781192178017, + "last_quartile_mean": 0.015387681487482041, + "mean": 0.03421354205056559, + "normalized": [ + 0.8375866167785538, + 1.2899141113192616, + 0.8694344424609419, + 0.915839393595925, + 0.475453642772802, + 0.8638099562451347, + 0.9821647993180688, + 0.9714468517291538, + 0.6838804035566444, + 0.5360010365403223, + 0.3851151782472681, + 0.3481232059212625, + 0.321269430974011, + 0.5688836365906816, + 0.46820216481780275, + 0.4621253765317527, + 1.0036599271213709, + 1.0844213253993487, + 1.4666067752392928, + 1.645231903565359, + 2.6830987807805085, + 4.385873655623207, + 2.767792506758732, + 2.3860319203463485, + 1.0350167426483041, + 0.8710926293236593, + 0.3914735937130686, + 0.34141432168112545, + 0.3037105506770517, + 0.20069737952502648, + 0.20302717344287544, + 0.25160056675513426 + ], + "peak_layer": 22, + "peak_normalized": 4.385873655623207, + "population_cv": 0.8995046133337352, + "rest_mean": 0.023749503048343792, + "spike_contrast": 3.8198421448186797, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09071935266256333, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.02865680493414402, + 0.0441325306892395, + 0.02974643185734749, + 0.03133410960435867, + 0.016266953200101852, + 0.02955399826169014, + 0.03360333666205406, + 0.03323663771152496, + 0.02339797094464302, + 0.01833849400281906, + 0.013176154345273972, + 0.01191052794456482, + 0.010991765186190605, + 0.019463524222373962, + 0.016018854454159737, + 0.015810946002602577, + 0.03433876112103462, + 0.03710189461708069, + 0.050177812576293945, + 0.05628921091556549, + 0.0917983129620552, + 0.15005627274513245, + 0.09469598531723022, + 0.08163460344076157, + 0.035411588847637177, + 0.029803164303302765, + 0.013393698260188103, + 0.011680993251502514, + 0.010391013696789742, + 0.006866568233817816, + 0.006946278735995293, + 0.008608146570622921 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.8313342426145027e-06, + "first_to_last_ratio": 0.9841100436889928, + "imbalance_abs_log_ratio": 0.01601755516546022, + "last_quartile_mean": 2.87705044854647e-06, + "mean": 2.8823285838086576e-06, + "normalized": [ + 3.860850182514526, + 1.3779145113400497, + 0.37073650468044217, + 0.4054583128312923, + 0.3411145989186331, + 0.46814184382330143, + 0.46288850930954656, + 0.5713590261909561, + 0.3765635727895922, + 0.3810005221832118, + 0.35155981621084625, + 0.393783350864445, + 0.33878649322165827, + 0.47478509983905887, + 0.4584607500663275, + 0.5226177027680562, + 0.4037194811557118, + 0.6360221076907124, + 0.6562906885789336, + 0.9724019253458765, + 1.6772106521914107, + 3.0942458588650434, + 2.9339099985078207, + 2.4848281321882983, + 1.7824259625811416, + 1.6267190981510171, + 1.2975773719544916, + 1.1633314414023308, + 0.5790326834473365, + 0.5326514143129327, + 0.49977516979896786, + 0.503837216276031 + ], + "peak_layer": 1, + "peak_normalized": 3.860850182514526, + "population_cv": 0.9094699254837627, + "rest_mean": 2.1379810404040554e-06, + "spike_contrast": 3.2281882709722924, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.90180531819351e-06, + "top_five_layers": [ + 1, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.112823883886449e-05, + 3.971602382080164e-06, + 1.0685844245017506e-06, + 1.1686640846164664e-06, + 9.83204358817602e-07, + 1.3493386177287903e-06, + 1.334196781499486e-06, + 1.6468444528072723e-06, + 1.0853799494725536e-06, + 1.0981686955346959e-06, + 1.0133109071830404e-06, + 1.1350130080245435e-06, + 9.764939932210837e-07, + 1.3684866644325666e-06, + 1.3214345244705328e-06, + 1.5063559430927853e-06, + 1.163652200375509e-06, + 1.8332247009311686e-06, + 1.8916454109785263e-06, + 2.8027818643749924e-06, + 4.834272203879664e-06, + 8.918633284338284e-06, + 8.456492651021108e-06, + 7.16209115125821e-06, + 5.137537300470285e-06, + 4.688738954428118e-06, + 3.74004434888775e-06, + 3.3531034659972647e-06, + 1.6689624544596882e-06, + 1.535276396680274e-06, + 1.4405162573893904e-06, + 1.4522244100589887e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.3673139480706595e-05, + "first_to_last_ratio": 2.257300425332895, + "imbalance_abs_log_ratio": 0.8141695972328834, + "last_quartile_mean": 1.0487367660516611e-05, + "mean": 2.6041475749138954e-05, + "normalized": [ + 0.6771788519354571, + 1.2979325532283608, + 0.9303271057055151, + 0.9639489120456851, + 0.5271438854731791, + 0.9092278846604233, + 1.028243959121235, + 0.9384386122687313, + 0.6892881365249813, + 0.5698927083153338, + 0.44101071224628413, + 0.3889922048793135, + 0.37294517808508393, + 0.6424365166247796, + 0.5244231346818359, + 0.4778644993700143, + 1.0150499086102556, + 1.1538261154181413, + 1.4466435534537925, + 1.6137729430394836, + 2.6468294071482537, + 4.220741079232897, + 2.9670008977788065, + 2.335098275492907, + 0.997666272782542, + 0.8091786594449493, + 0.3691954363618722, + 0.3097208222153953, + 0.23815117281925108, + 0.1573714206968098, + 0.14298175278225358, + 0.19747742755617637 + ], + "peak_layer": 22, + "peak_normalized": 4.220741079232897, + "population_cv": 0.8898631795325547, + "rest_mean": 1.816408757804311e-05, + "spike_contrast": 3.7755473033479423, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.857937187305651e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.7634736650506966e-05, + 3.380007910891436e-05, + 2.4227090761996806e-05, + 2.5102652216446586e-05, + 1.3727604709856678e-05, + 2.3677635908825323e-05, + 2.6776990125654265e-05, + 2.443832636345178e-05, + 1.7950080291484483e-05, + 1.4840847143204883e-05, + 1.1484569768072106e-05, + 1.0129931069968734e-05, + 9.712042810861021e-06, + 1.6729994968045503e-05, + 1.3656752344104461e-05, + 1.2444296771718655e-05, + 2.643339757923968e-05, + 3.004733480338473e-05, + 3.767273301491514e-05, + 4.202502896077931e-05, + 6.892734381835908e-05, + 0.00010991432645823807, + 7.726508192718029e-05, + 6.080940511310473e-05, + 2.5980702048400417e-05, + 2.1072206436656415e-05, + 9.614394002710469e-06, + 8.065587280725595e-06, + 6.201807991601527e-06, + 4.098184035683516e-06, + 3.723455847648438e-06, + 5.142603640706511e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.5893040995251795e-05, + "first_to_last_ratio": 2.4728739688838175, + "imbalance_abs_log_ratio": 0.9053810243954281, + "last_quartile_mean": 6.426951472349174e-06, + "mean": 1.7339248245207273e-05, + "normalized": [ + 0.5512149265120582, + 1.2985625489006958, + 0.9783452688543264, + 0.9971647574760376, + 0.5962146785243507, + 0.9290000522923118, + 1.0861697535899135, + 0.8960754411880637, + 0.726825196328683, + 0.6059411861934751, + 0.5099587939140718, + 0.44057324634523176, + 0.4391978251197769, + 0.7191858964012756, + 0.5811495621714656, + 0.4881322826391351, + 1.0011626163335512, + 1.1990369599246455, + 1.4118886136322046, + 1.554688514867236, + 2.6196553178384634, + 3.9391535171303675, + 3.1321858798359785, + 2.3332437535315793, + 0.9448004540825558, + 0.7676606562766751, + 0.3665181536820326, + 0.2919617681864151, + 0.18930565608900826, + 0.13559488549785165, + 0.11350226730002422, + 0.15592956934053803 + ], + "peak_layer": 22, + "peak_normalized": 3.9391535171303675, + "population_cv": 0.8670378924383861, + "rest_mean": 1.2221576239594637e-05, + "spike_contrast": 3.679940802546367, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.49746770755155e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.55765244725626e-06, + 2.2516098397318274e-05, + 1.6963771486189216e-05, + 1.7290087271248922e-05, + 1.0337914318370167e-05, + 1.610816252650693e-05, + 1.8833366993931122e-05, + 1.5537274521193467e-05, + 1.2602602510014549e-05, + 1.0506564649404027e-05, + 8.842302122502588e-06, + 7.639208888576832e-06, + 7.615360118506942e-06, + 1.2470142792153638e-05, + 1.0076696526084561e-05, + 8.463846825179644e-06, + 1.735940713842865e-05, + 2.0790399503312074e-05, + 2.4481087166350335e-05, + 2.6957130103255622e-05, + 4.542285387287848e-05, + 6.830196070950478e-05, + 5.430974852060899e-05, + 4.0456692659063265e-05, + 1.638212961552199e-05, + 1.3310658687260002e-05, + 6.355149253067793e-06, + 5.062397576693911e-06, + 3.282417765149148e-06, + 2.351113380427705e-06, + 1.968043989108992e-06, + 2.7037015115638496e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.000305004112306e-05, + "first_to_last_ratio": 2.193525307703265, + "imbalance_abs_log_ratio": 0.7855099791473917, + "last_quartile_mean": 3.191349094322504e-05, + "mean": 7.690838336316119e-05, + "normalized": [ + 0.8923720562068232, + 1.3111973164248405, + 0.8460717362092315, + 0.9445716926262052, + 0.4741848629196519, + 0.8656844065975011, + 0.9924506003335006, + 0.9551754577499273, + 0.6234358560310297, + 0.5152936103919111, + 0.3846693562402596, + 0.34526537659024253, + 0.30979982474526946, + 0.5791064055325631, + 0.47493901082375733, + 0.4350256040989574, + 1.011874248892453, + 1.1362510856108166, + 1.416617477431854, + 1.6892273518644711, + 2.692715672842754, + 4.577499824936365, + 2.8907687343273323, + 2.31616526773872, + 1.0250323630610032, + 0.8340050859269844, + 0.36589411026523266, + 0.29901912364743294, + 0.2532838824218693, + 0.16506308776319403, + 0.15276161475417316, + 0.22457789499367237 + ], + "peak_layer": 22, + "peak_normalized": 4.577499824936365, + "population_cv": 0.9353823752601559, + "rest_mean": 5.269026995072771e-05, + "spike_contrast": 3.9416422801499356, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000207686195790302, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 6.863089220132679e-05, + 0.00010084206587634981, + 6.507000944111496e-05, + 7.264548185048625e-05, + 3.646879122243263e-05, + 6.657838821411133e-05, + 7.632777123944834e-05, + 7.346100028371438e-05, + 4.7947443817975e-05, + 3.9630398532608524e-05, + 2.9584298317786306e-05, + 2.6553801944828592e-05, + 2.3826203687349334e-05, + 4.453813744476065e-05, + 3.652679151855409e-05, + 3.34571159328334e-05, + 7.782161264913157e-05, + 8.738723408896476e-05, + 0.00010894976003328338, + 0.00012991574476473033, + 0.00020709240925498307, + 0.0003520481113810092, + 0.00022232435003388673, + 0.00017813252634368837, + 7.883358193794265e-05, + 6.414198287529871e-05, + 2.8140324502601288e-05, + 2.2997077394393273e-05, + 1.9479653929010965e-05, + 1.2694735232798848e-05, + 1.174864883068949e-05, + 1.7271922843065113e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.047033343464136124, + 0.060984931886196136, + 0.09079429507255554, + 0.08531255275011063, + 0.0786692351102829, + 0.08076046407222748, + 0.08904207497835159, + 0.08403915911912918, + 0.06012389436364174, + 0.06841155141592026, + 0.07131828367710114, + 0.07358616590499878, + 0.06597626209259033, + 0.07235056161880493, + 0.07261744141578674, + 0.07352335751056671, + 0.07520031183958054, + 0.07378771901130676, + 0.07715436071157455, + 0.08649503439664841, + 0.04556623473763466, + 0.04546353965997696, + 0.05167657509446144, + 0.058827146887779236, + 0.04798740893602371, + 0.046526361256837845, + 0.051699329167604446, + 0.059578344225883484, + 0.05090610682964325, + 0.05821637064218521, + 0.06188955530524254, + 0.06796324998140335 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.364372159670893e-06, + "first_to_last_ratio": 0.2681115118992595, + "imbalance_abs_log_ratio": 1.3163522958360352, + "last_quartile_mean": 1.627819756322424e-05, + "mean": 1.0259845446114468e-05, + "normalized": [ + 1.0134993414800566, + 0.522056581911947, + 0.2644061262673726, + 0.3833634649429024, + 0.2440106441209935, + 0.2461172286895863, + 0.27872130983193855, + 0.4508957935090263, + 0.4833194134691065, + 0.39025737906993885, + 0.3708598519154414, + 0.36981229946243677, + 0.42398498232424564, + 0.41928891386718925, + 0.4397458055599365, + 0.45615006609958, + 0.4114730365687318, + 0.5108641768312265, + 0.48188059954330154, + 0.7614115358485232, + 2.335493650100753, + 2.8449380275824225, + 2.7201337953566025, + 2.484573069357449, + 2.4148997615580035, + 2.4501695400243877, + 2.1889991494613, + 1.6914475251239836, + 1.1580207584918483, + 1.0537166329047938, + 0.8749014303613178, + 0.8605881083636573 + ], + "peak_layer": 22, + "peak_normalized": 2.8449380275824225, + "population_cv": 0.8504187150143712, + "rest_mean": 7.295875539731006e-06, + "spike_contrast": 3.6000179550148346, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6265282940585166e-05, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.0398346603324171e-05, + 5.356219844543375e-06, + 2.7127659905090695e-06, + 3.933249900001101e-06, + 2.503511495888233e-06, + 2.525124727981165e-06, + 2.8596375614142744e-06, + 4.626121153705753e-06, + 4.9587824832997285e-06, + 4.00398039346328e-06, + 3.804964762821328e-06, + 3.7942170365568018e-06, + 4.350020390120335e-06, + 4.301839453546563e-06, + 4.511724000622053e-06, + 4.680029178416589e-06, + 4.221649760438595e-06, + 5.241387498244876e-06, + 4.944020474795252e-06, + 7.811964678694494e-06, + 2.3961803890415467e-05, + 2.9188624466769397e-05, + 2.7908152333111502e-05, + 2.5491335691185668e-05, + 2.4776498321443796e-05, + 2.5138360797427595e-05, + 2.2458792955148965e-05, + 1.735399018798489e-05, + 1.1881114005518612e-05, + 1.081096979760332e-05, + 8.976353456091601e-06, + 8.829500984575134e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.76496694166417e-05, + "first_to_last_ratio": 0.27059749062388033, + "imbalance_abs_log_ratio": 1.3071228366665688, + "last_quartile_mean": 6.522480816784082e-05, + "mean": 4.117927500146834e-05, + "normalized": [ + 1.007463472142432, + 0.5209651751221482, + 0.2643970395144652, + 0.3829104157055737, + 0.2536844256534051, + 0.25437837481906395, + 0.28207653195017285, + 0.46296966564632774, + 0.49565891190391403, + 0.39385639261196637, + 0.3771016062127897, + 0.37530908825337306, + 0.4278590182802805, + 0.42183720924091433, + 0.44371630887119135, + 0.4598267075919064, + 0.41216967133849985, + 0.5079852336529339, + 0.47800548121281095, + 0.7481511452590028, + 2.3310574965471607, + 2.8352891763903005, + 2.718777452615949, + 2.4731685677354274, + 2.4074108105775847, + 2.4550622811024825, + 2.180382977806297, + 1.6846495469817935, + 1.161664542480338, + 1.0519342613419045, + 0.8725636415138176, + 0.8577173699237725 + ], + "peak_layer": 22, + "peak_normalized": 2.8352891763903005, + "population_cv": 0.8467657487851442, + "rest_mean": 2.9335347587928278e-05, + "spike_contrast": 3.5839521832647088, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010513648303458467, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.148661537328735e-05, + 2.1452968212543055e-05, + 1.0887678399740253e-05, + 1.576797330926638e-05, + 1.0446540727571119e-05, + 1.0475117051100824e-05, + 1.1615707080636639e-05, + 1.906475517898798e-05, + 2.0410874640219845e-05, + 1.6218720702454448e-05, + 1.552877074573189e-05, + 1.5454956155736e-05, + 1.761892417562194e-05, + 1.7370950445183553e-05, + 1.8271915905643255e-05, + 1.8935330444946885e-05, + 1.697284824331291e-05, + 2.0918463633279316e-05, + 1.968391916307155e-05, + 3.080832175328396e-05, + 9.599125769454986e-05, + 0.00011675515270326287, + 0.00011195728438906372, + 0.00010184328857576475, + 9.913543181028217e-05, + 0.0001010976848192513, + 8.978659025160596e-05, + 6.937264697626233e-05, + 4.783650365425274e-05, + 4.3317890231264755e-05, + 3.593153815018013e-05, + 3.5320179449627176e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.015651903115212917, + "first_to_last_ratio": 0.2705974872956941, + "imbalance_abs_log_ratio": 1.307122848965967, + "last_quartile_mean": 0.05784201202914119, + "mean": 0.03651819290826097, + "normalized": [ + 1.0074634186476772, + 0.5209651919181703, + 0.2643970221469926, + 0.382910419856966, + 0.2536844208302265, + 0.25437835679028203, + 0.2820765262777231, + 0.4629696917045884, + 0.4956588954855505, + 0.393856398901423, + 0.3771015816519749, + 0.37530907952450476, + 0.42785903141736564, + 0.4218372178649866, + 0.4437163224543097, + 0.4598267050189124, + 0.41216967747329053, + 0.5079852485753572, + 0.4780054808986325, + 0.7481511542287224, + 2.331057435590359, + 2.835289207255424, + 2.7187775380829278, + 2.4731685834004287, + 2.4074106915648685, + 2.455062286976735, + 2.180383000719517, + 1.6846496365458605, + 1.1616645069657927, + 1.0519342829703067, + 0.8725635941676015, + 0.857717394092523 + ], + "peak_layer": 22, + "peak_normalized": 2.835289207255424, + "population_cv": 0.8467657515631118, + "rest_mean": 0.026014879690828146, + "spike_contrast": 3.5839521608577614, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09323608428239823, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.036790743470191956, + 0.01902470737695694, + 0.009655301459133625, + 0.013983196578919888, + 0.00926409661769867, + 0.009289437904953957, + 0.010300925001502037, + 0.016906816512346268, + 0.018100567162036896, + 0.01438292395323515, + 0.013771068304777145, + 0.013705609366297722, + 0.01562463864684105, + 0.015404732897877693, + 0.01620371825993061, + 0.016792040318250656, + 0.01505169179290533, + 0.018550703302025795, + 0.017455896362662315, + 0.02732112817466259, + 0.08512600511312485, + 0.10353963822126389, + 0.09928484261035919, + 0.09031564742326736, + 0.08791428804397583, + 0.08965443819761276, + 0.07962364703416824, + 0.06152036041021347, + 0.042421888560056686, + 0.038414739072322845, + 0.03186444565653801, + 0.03132228925824165 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.036145782506992e-06, + "first_to_last_ratio": 0.23813875303874632, + "imbalance_abs_log_ratio": 1.434901779225667, + "last_quartile_mean": 4.35101708262664e-06, + "mean": 2.601542721691885e-06, + "normalized": [ + 1.0445105925849996, + 0.4868956651441851, + 0.2653781288407282, + 0.38704215458190727, + 0.2311435484604591, + 0.22762453648926426, + 0.22559249684218238, + 0.3180631703688383, + 0.41371995096278646, + 0.35456417415204133, + 0.35816018513093856, + 0.3384166475711158, + 0.38409460459920464, + 0.37901778296480176, + 0.4237575276229331, + 0.4302249635723777, + 0.38747010650156044, + 0.47861136725136255, + 0.44057460034262785, + 0.49825323727916515, + 2.2524328318137035, + 2.8621193651579846, + 2.8149870576895797, + 2.617539402453915, + 2.4626516595822796, + 2.6269385253668815, + 2.362864616381361, + 1.8136241394740558, + 1.1783341687752071, + 1.0893413201795874, + 0.9454878584900481, + 0.9005636133719171 + ], + "peak_layer": 22, + "peak_normalized": 2.8621193651579846, + "population_cv": 0.9010662730896349, + "rest_mean": 1.8297776991689548e-06, + "spike_contrast": 3.6993968427913813, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.76907384331571e-06, + "top_five_layers": [ + 22, + 23, + 26, + 24, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.717338929869584e-06, + 1.266679873879184e-06, + 6.903925395818078e-07, + 1.0069067002405063e-06, + 6.013298161633429e-07, + 5.921749561821343e-07, + 5.868885182280792e-07, + 8.274549259112973e-07, + 1.0763101272459608e-06, + 9.224138466379372e-07, + 9.317690228272113e-07, + 8.80405366388004e-07, + 9.992385230361833e-07, + 9.860309546638746e-07, + 1.1024233117495896e-06, + 1.1192486226718756e-06, + 1.0080200354423141e-06, + 1.245127918991784e-06, + 1.1461736448836746e-06, + 1.296227083003032e-06, + 5.859800239704782e-06, + 7.445925803040154e-06, + 7.323309091589181e-06, + 6.809640581195708e-06, + 6.406693501048721e-06, + 6.8340928010002244e-06, + 6.147093245090218e-06, + 4.718220679933438e-06, + 3.0654866804979974e-06, + 2.8339679829514353e-06, + 2.4597270567028318e-06, + 2.342854713788256e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.2745702917982271e-05, + "first_to_last_ratio": 0.3040242464136832, + "imbalance_abs_log_ratio": 1.1906478228166633, + "last_quartile_mean": 4.1923310618585674e-05, + "mean": 2.8222844818515114e-05, + "normalized": [ + 1.092738547299169, + 0.5627549631144219, + 0.24809914749976855, + 0.3821239330736069, + 0.2717732575074452, + 0.2708922129031054, + 0.29822547774823177, + 0.4862679168736732, + 0.544040724965591, + 0.43215299071277247, + 0.4170763968939518, + 0.42015115877685144, + 0.47871065526968887, + 0.46826286484802104, + 0.4900402959326792, + 0.5070735564042542, + 0.4633460623422733, + 0.5693312811282049, + 0.5194930950029935, + 0.7472397737749802, + 2.399371006579266, + 2.8986292719666995, + 2.739777767666698, + 2.408916709158605, + 2.333408286273547, + 2.300446003604439, + 2.062640270367703, + 1.6162771645238014, + 1.0435838635901262, + 0.9412033869198291, + 0.8018738201820883, + 0.7840781370955142 + ], + "peak_layer": 22, + "peak_normalized": 2.8986292719666995, + "population_cv": 0.8223755795225939, + "rest_mean": 2.0090376640129965e-05, + "spike_contrast": 3.5906829560228823, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.213817298179492e-05, + "top_five_layers": [ + 22, + 23, + 24, + 21, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0840190447634086e-05, + 1.5882545994827524e-05, + 7.0020637394918595e-06, + 1.0784624464577064e-05, + 7.670214472454973e-06, + 7.6453488873085e-06, + 8.416771379415877e-06, + 1.3723863958148286e-05, + 1.5354376955656335e-05, + 1.219658679474378e-05, + 1.1771082427003421e-05, + 1.1857860954478383e-05, + 1.3510576536646113e-05, + 1.3215710168879014e-05, + 1.3830331226927228e-05, + 1.4311058293969836e-05, + 1.3076944014756009e-05, + 1.606814839760773e-05, + 1.4661573004559614e-05, + 2.1089232177473605e-05, + 6.771707558073103e-05, + 8.18075641291216e-05, + 7.732432277407497e-05, + 6.79864824633114e-05, + 6.58554199617356e-05, + 6.492513057310134e-05, + 5.821357626700774e-05, + 4.561593959806487e-05, + 2.9452905437210575e-05, + 2.6563437131699175e-05, + 2.263116039102897e-05, + 2.2128915588837117e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.982105043742195e-06, + "first_to_last_ratio": 0.3255748540372994, + "imbalance_abs_log_ratio": 1.122162877451286, + "last_quartile_mean": 2.758844834716001e-05, + "mean": 1.9313689406885715e-05, + "normalized": [ + 1.153292392223287, + 0.5797609618245384, + 0.23076786066825913, + 0.37581241874572735, + 0.2819306093436648, + 0.2851688939012761, + 0.30549556090944335, + 0.5082847079850745, + 0.583859969769007, + 0.45907810601554777, + 0.45116352890905354, + 0.46235698273638653, + 0.522619204122844, + 0.5090842134923546, + 0.5336639455934807, + 0.5440001607440246, + 0.5095787126873097, + 0.6267236930541729, + 0.5514908746704607, + 0.7356132069066491, + 2.430614280460028, + 2.906404914690865, + 2.7136484598097654, + 2.3120655741144565, + 2.2482332201724016, + 2.1318721582988647, + 1.9232163840966459, + 1.5686764250532175, + 1.0367548344938609, + 0.9157520446280638, + 0.8238220024989565, + 0.7791936973803123 + ], + "peak_layer": 22, + "peak_normalized": 2.906404914690865, + "population_cv": 0.7900897792080465, + "rest_mean": 1.3869398959219257e-05, + "spike_contrast": 3.5122544219484153, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.8712857824284585e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2274331058724783e-05, + 1.119732314691646e-05, + 4.456978786038235e-06, + 7.258324330905452e-06, + 5.4451202231575735e-06, + 5.5076634453143924e-06, + 5.900246378587326e-06, + 9.816852980293334e-06, + 1.1276490113232285e-05, + 8.866491953085642e-06, + 8.713632269063964e-06, + 8.92981915967539e-06, + 1.0093704986502416e-05, + 9.832294381340034e-06, + 1.0307019692845643e-05, + 1.0506650141905993e-05, + 9.841844985203352e-06, + 1.2104346751584671e-05, + 1.0651323464117013e-05, + 1.4207405001798179e-05, + 4.694412928074598e-05, + 5.613340181298554e-05, + 5.2410563512239605e-05, + 4.4654516386799514e-05, + 4.3421678128652275e-05, + 4.117431672057137e-05, + 3.714440390467644e-05, + 3.0296929253381677e-05, + 2.0023560864501633e-05, + 1.768655056366697e-05, + 1.5911042282823473e-05, + 1.5049105059006251e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.531509560161794e-05, + "first_to_last_ratio": 0.2551422661109217, + "imbalance_abs_log_ratio": 1.36593398307824, + "last_quartile_mean": 0.00013841334930475568, + "mean": 8.573219730578785e-05, + "normalized": [ + 0.9822788357613834, + 0.5147377658277125, + 0.24077984042300446, + 0.3633612864660074, + 0.2505573845679934, + 0.23871348492801034, + 0.2720502738875204, + 0.4329080853308244, + 0.4899097166580031, + 0.3804980486342757, + 0.36593190899850525, + 0.3643305688887499, + 0.412819132135824, + 0.40296081445560644, + 0.4305724210670739, + 0.4502160271371867, + 0.3962072388182212, + 0.4765863055500514, + 0.454434200643566, + 0.7197132038245179, + 2.2718289409811727, + 2.8618271883484865, + 2.7794936015049534, + 2.53140346971279, + 2.4848609397336863, + 2.575834812444881, + 2.3585524326140215, + 1.8181964500308416, + 1.0732112926751614, + 0.9812549829334379, + 0.8189418899610229, + 0.8050274550555072 + ], + "peak_layer": 22, + "peak_normalized": 2.8618271883484865, + "population_cv": 0.8817284503784036, + "rest_mean": 6.055419369119961e-05, + "spike_contrast": 3.6610745401896616, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022169341682456433, + "top_five_layers": [ + 22, + 23, + 26, + 24, + 25 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.42129229567945e-05, + 4.4129599700681865e-05, + 2.064258478640113e-05, + 3.115176150458865e-05, + 2.1480835130205378e-05, + 2.0465431589400396e-05, + 2.3323467758018523e-05, + 3.7114161386853084e-05, + 4.200103649054654e-05, + 3.262093377998099e-05, + 3.137214662274346e-05, + 3.123486021650024e-05, + 3.539189128787257e-05, + 3.454671605140902e-05, + 3.691391975735314e-05, + 3.859800926875323e-05, + 3.3967717172345147e-05, + 4.08587911806535e-05, + 3.895964255207218e-05, + 6.170259439386427e-05, + 0.00019476888701319695, + 0.00024535073316656053, + 0.00023829209385439754, + 0.0002170227817259729, + 0.0002130325883626938, + 0.00022083197836764157, + 0.0002022038825089112, + 0.00015587797679472715, + 9.200876229442656e-05, + 8.412514580413699e-05, + 7.020968769211322e-05, + 6.9016772613395e-05 + ] + } + } + } + } + } + }, + "step": 2000 + }, + { + "modes": { + "learned": { + "bits_per_byte": 1.9433676664104056, + "forward": { + "activation_sha256": { + "attention_branch_output": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", + "mlp_branch_output": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", + "post_attention_state": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", + "post_mlp_state": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", + "pre_attention_input": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", + "pre_mlp_input": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c" + }, + "logits_sha256": "000b42d2e473ee11592a1ced2fe607cfcfbbe21b9cb8637453b9d2bc3414fcfe", + "loss_nats": 1.347039818763733, + "mixer_summary_sha256": "7935548f1014624ed4c9fa535e6d7a55432ce9be32776c3985f675572d6d61e8" + }, + "loss_nats": 1.347039818763733, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.24809812009334564, + "group": 1, + "latest_source_mass": 0.07119978964328766, + "layer": 1, + "max_source_mass": 0.9288002252578735, + "mixer_index": 1, + "normalized_entropy": 0.35792992751254427, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9288002252578735, + "median": 0.9386287331581116, + "p05": 0.858244001865387, + "p95": 0.9685006141662598 + }, + { + "label": "current_group_partial", + "mean": 0.07119978964328766, + "median": 0.06137128174304962, + "p05": 0.031499382108449936, + "p95": 0.14175599813461304 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6067080497741699, + "group": 1, + "latest_source_mass": 0.43957531452178955, + "layer": 2, + "max_source_mass": 0.5604246854782104, + "mixer_index": 2, + "normalized_entropy": 0.8752946946766094, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5604246854782104, + "median": 0.5413482189178467, + "p05": 0.26798251271247864, + "p95": 0.8787311315536499 + }, + { + "label": "current_group_partial", + "mean": 0.43957531452178955, + "median": 0.4586517810821533, + "p05": 0.1212688684463501, + "p95": 0.7320175170898438 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3847596049308777, + "group": 1, + "latest_source_mass": 0.13518528640270233, + "layer": 2, + "max_source_mass": 0.8648147583007812, + "mixer_index": 3, + "normalized_entropy": 0.555090773968174, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8648147583007812, + "median": 0.8810019493103027, + "p05": 0.7563575506210327, + "p95": 0.9309777617454529 + }, + { + "label": "current_group_partial", + "mean": 0.13518528640270233, + "median": 0.11899800598621368, + "p05": 0.06902224570512772, + "p95": 0.24364244937896729 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6843481659889221, + "group": 1, + "latest_source_mass": 0.5056107044219971, + "layer": 3, + "max_source_mass": 0.5056107044219971, + "mixer_index": 4, + "normalized_entropy": 0.9873057053136751, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.49438929557800293, + "median": 0.4973145127296448, + "p05": 0.3897787928581238, + "p95": 0.6022077798843384 + }, + { + "label": "current_group_partial", + "mean": 0.5056107044219971, + "median": 0.502685546875, + "p05": 0.3977922201156616, + "p95": 0.6102211475372314 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5728601813316345, + "group": 1, + "latest_source_mass": 0.30453476309776306, + "layer": 3, + "max_source_mass": 0.6954652070999146, + "mixer_index": 5, + "normalized_entropy": 0.8264625427299015, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6954652070999146, + "median": 0.7084143161773682, + "p05": 0.453019917011261, + "p95": 0.9048184752464294 + }, + { + "label": "current_group_partial", + "mean": 0.30453476309776306, + "median": 0.29158568382263184, + "p05": 0.09518152475357056, + "p95": 0.5469801425933838 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.686518132686615, + "group": 1, + "latest_source_mass": 0.4736926555633545, + "layer": 4, + "max_source_mass": 0.5263072848320007, + "mixer_index": 6, + "normalized_entropy": 0.9904363055073309, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5263072848320007, + "median": 0.5291417241096497, + "p05": 0.4370424747467041, + "p95": 0.6044135093688965 + }, + { + "label": "current_group_partial", + "mean": 0.4736926555633545, + "median": 0.47085827589035034, + "p05": 0.3955864906311035, + "p95": 0.5629575252532959 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5044870376586914, + "group": 1, + "latest_source_mass": 0.2221563160419464, + "layer": 4, + "max_source_mass": 0.777843713760376, + "mixer_index": 7, + "normalized_entropy": 0.7278209474229579, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.777843713760376, + "median": 0.7819972634315491, + "p05": 0.6222297549247742, + "p95": 0.9289263486862183 + }, + { + "label": "current_group_partial", + "mean": 0.2221563160419464, + "median": 0.21800270676612854, + "p05": 0.07107366621494293, + "p95": 0.3777702748775482 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6724032163619995, + "group": 2, + "latest_source_mass": 0.4305936098098755, + "layer": 5, + "max_source_mass": 0.5694063901901245, + "mixer_index": 8, + "normalized_entropy": 0.9700727857232454, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5694063901901245, + "median": 0.5728721022605896, + "p05": 0.45058131217956543, + "p95": 0.6837627291679382 + }, + { + "label": "completed_group_1", + "mean": 0.4305936098098755, + "median": 0.4271278977394104, + "p05": 0.31623730063438416, + "p95": 0.5494186878204346 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9718273282051086, + "group": 2, + "latest_source_mass": 0.23954525589942932, + "layer": 5, + "max_source_mass": 0.5544795989990234, + "mixer_index": 9, + "normalized_entropy": 0.8845953556402437, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5544795989990234, + "median": 0.5475102663040161, + "p05": 0.40941980481147766, + "p95": 0.7249865531921387 + }, + { + "label": "completed_group_1", + "mean": 0.20597517490386963, + "median": 0.20141024887561798, + "p05": 0.11906205117702484, + "p95": 0.3046763241291046 + }, + { + "label": "current_group_partial", + "mean": 0.23954525589942932, + "median": 0.2397485226392746, + "p05": 0.14606745541095734, + "p95": 0.33606359362602234 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0856361389160156, + "group": 2, + "latest_source_mass": 0.36206769943237305, + "layer": 6, + "max_source_mass": 0.3703838884830475, + "mixer_index": 10, + "normalized_entropy": 0.9881885994850598, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3703838884830475, + "median": 0.3719092607498169, + "p05": 0.31337594985961914, + "p95": 0.42367395758628845 + }, + { + "label": "completed_group_1", + "mean": 0.26754841208457947, + "median": 0.2691181004047394, + "p05": 0.24584907293319702, + "p95": 0.28693482279777527 + }, + { + "label": "current_group_partial", + "mean": 0.36206769943237305, + "median": 0.3628925681114197, + "p05": 0.31465041637420654, + "p95": 0.4040696620941162 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.022249460220337, + "group": 2, + "latest_source_mass": 0.32249319553375244, + "layer": 6, + "max_source_mass": 0.46441033482551575, + "mixer_index": 11, + "normalized_entropy": 0.9304915580906614, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46441033482551575, + "median": 0.47390004992485046, + "p05": 0.25647592544555664, + "p95": 0.6490365266799927 + }, + { + "label": "completed_group_1", + "mean": 0.21309645473957062, + "median": 0.2148953229188919, + "p05": 0.13959459960460663, + "p95": 0.27812671661376953 + }, + { + "label": "current_group_partial", + "mean": 0.32249319553375244, + "median": 0.31052517890930176, + "p05": 0.20389017462730408, + "p95": 0.47258833050727844 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0742855072021484, + "group": 2, + "latest_source_mass": 0.36498188972473145, + "layer": 7, + "max_source_mass": 0.38770636916160583, + "mixer_index": 12, + "normalized_entropy": 0.9778568092521033, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38770636916160583, + "median": 0.4003474712371826, + "p05": 0.2879825532436371, + "p95": 0.45421719551086426 + }, + { + "label": "completed_group_1", + "mean": 0.24731171131134033, + "median": 0.24118143320083618, + "p05": 0.21154659986495972, + "p95": 0.2994430363178253 + }, + { + "label": "current_group_partial", + "mean": 0.36498188972473145, + "median": 0.35881954431533813, + "p05": 0.31933724880218506, + "p95": 0.42228373885154724 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0207322835922241, + "group": 2, + "latest_source_mass": 0.4325985312461853, + "layer": 7, + "max_source_mass": 0.4325985312461853, + "mixer_index": 13, + "normalized_entropy": 0.9291105644100317, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3404029607772827, + "median": 0.2972515821456909, + "p05": 0.1890696883201599, + "p95": 0.675995409488678 + }, + { + "label": "completed_group_1", + "mean": 0.22699852287769318, + "median": 0.2352103292942047, + "p05": 0.11332616209983826, + "p95": 0.2928347587585449 + }, + { + "label": "current_group_partial", + "mean": 0.4325985312461853, + "median": 0.4621005654335022, + "p05": 0.2054932415485382, + "p95": 0.5461148023605347 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0628657341003418, + "group": 2, + "latest_source_mass": 0.361979603767395, + "layer": 8, + "max_source_mass": 0.41766518354415894, + "mixer_index": 14, + "normalized_entropy": 0.9674620838156608, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.41766518354415894, + "median": 0.4145643711090088, + "p05": 0.36852169036865234, + "p95": 0.47784724831581116 + }, + { + "label": "completed_group_1", + "mean": 0.22035519778728485, + "median": 0.2179521918296814, + "p05": 0.191657155752182, + "p95": 0.25349321961402893 + }, + { + "label": "current_group_partial", + "mean": 0.361979603767395, + "median": 0.363964319229126, + "p05": 0.3155593276023865, + "p95": 0.39730241894721985 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.029329538345337, + "group": 2, + "latest_source_mass": 0.3858758807182312, + "layer": 8, + "max_source_mass": 0.42153143882751465, + "mixer_index": 15, + "normalized_entropy": 0.936936122927619, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42153143882751465, + "median": 0.41038113832473755, + "p05": 0.29627615213394165, + "p95": 0.6068807244300842 + }, + { + "label": "completed_group_1", + "mean": 0.19259268045425415, + "median": 0.18828678131103516, + "p05": 0.12331291288137436, + "p95": 0.2707396447658539 + }, + { + "label": "current_group_partial", + "mean": 0.3858758807182312, + "median": 0.39191579818725586, + "p05": 0.2700805068016052, + "p95": 0.4676054120063782 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0849454402923584, + "group": 3, + "latest_source_mass": 0.3364688754081726, + "layer": 9, + "max_source_mass": 0.37970075011253357, + "mixer_index": 16, + "normalized_entropy": 0.9875598985040298, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.37970075011253357, + "median": 0.382883220911026, + "p05": 0.29113224148750305, + "p95": 0.46186232566833496 + }, + { + "label": "completed_group_1", + "mean": 0.2838304042816162, + "median": 0.282575786113739, + "p05": 0.25098085403442383, + "p95": 0.31989675760269165 + }, + { + "label": "completed_group_2", + "mean": 0.3364688754081726, + "median": 0.33596307039260864, + "p05": 0.28113189339637756, + "p95": 0.3942520022392273 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.3144235610961914, + "group": 3, + "latest_source_mass": 0.30927807092666626, + "layer": 9, + "max_source_mass": 0.3205832242965698, + "mixer_index": 17, + "normalized_entropy": 0.9481561766105434, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3205832242965698, + "median": 0.3228584825992584, + "p05": 0.20318874716758728, + "p95": 0.49334216117858887 + }, + { + "label": "completed_group_1", + "mean": 0.13048580288887024, + "median": 0.1293262243270874, + "p05": 0.08803658187389374, + "p95": 0.17583128809928894 + }, + { + "label": "completed_group_2", + "mean": 0.2396528720855713, + "median": 0.23850613832473755, + "p05": 0.15605276823043823, + "p95": 0.3097888231277466 + }, + { + "label": "current_group_partial", + "mean": 0.30927807092666626, + "median": 0.3110916018486023, + "p05": 0.24646314978599548, + "p95": 0.36450129747390747 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3647149801254272, + "group": 3, + "latest_source_mass": 0.24869562685489655, + "layer": 10, + "max_source_mass": 0.3188267946243286, + "mixer_index": 18, + "normalized_entropy": 0.9844337670269171, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3188267946243286, + "median": 0.32193225622177124, + "p05": 0.2530626356601715, + "p95": 0.38645845651626587 + }, + { + "label": "completed_group_1", + "mean": 0.2379944920539856, + "median": 0.23598769307136536, + "p05": 0.21630848944187164, + "p95": 0.2653774321079254 + }, + { + "label": "completed_group_2", + "mean": 0.19448311626911163, + "median": 0.19229988753795624, + "p05": 0.16299879550933838, + "p95": 0.23252975940704346 + }, + { + "label": "current_group_partial", + "mean": 0.24869562685489655, + "median": 0.24343957006931305, + "p05": 0.21172824501991272, + "p95": 0.29676955938339233 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2778496742248535, + "group": 3, + "latest_source_mass": 0.2590479850769043, + "layer": 10, + "max_source_mass": 0.37896889448165894, + "mixer_index": 19, + "normalized_entropy": 0.9217736940028869, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.37896889448165894, + "median": 0.3914150297641754, + "p05": 0.14146459102630615, + "p95": 0.6288160681724548 + }, + { + "label": "completed_group_1", + "mean": 0.16353653371334076, + "median": 0.162285715341568, + "p05": 0.08837783336639404, + "p95": 0.23340767621994019 + }, + { + "label": "completed_group_2", + "mean": 0.1984466314315796, + "median": 0.19906753301620483, + "p05": 0.09471487253904343, + "p95": 0.29035234451293945 + }, + { + "label": "current_group_partial", + "mean": 0.2590479850769043, + "median": 0.2469818890094757, + "p05": 0.1562758982181549, + "p95": 0.3916419744491577 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3567583560943604, + "group": 3, + "latest_source_mass": 0.3088001608848572, + "layer": 11, + "max_source_mass": 0.3088001608848572, + "mixer_index": 20, + "normalized_entropy": 0.978694276010998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29770222306251526, + "median": 0.29406338930130005, + "p05": 0.24821004271507263, + "p95": 0.3566911518573761 + }, + { + "label": "completed_group_1", + "mean": 0.21492168307304382, + "median": 0.21014970541000366, + "p05": 0.18107125163078308, + "p95": 0.26316314935684204 + }, + { + "label": "completed_group_2", + "mean": 0.17857596278190613, + "median": 0.17864474654197693, + "p05": 0.15527817606925964, + "p95": 0.2019544541835785 + }, + { + "label": "current_group_partial", + "mean": 0.3088001608848572, + "median": 0.309349000453949, + "p05": 0.2659364938735962, + "p95": 0.3488244414329529 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2773678302764893, + "group": 3, + "latest_source_mass": 0.24117937684059143, + "layer": 11, + "max_source_mass": 0.3856483995914459, + "mixer_index": 21, + "normalized_entropy": 0.9214261170654932, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3856483995914459, + "median": 0.4236947298049927, + "p05": 0.12469026446342468, + "p95": 0.588103175163269 + }, + { + "label": "completed_group_1", + "mean": 0.18032220005989075, + "median": 0.17527258396148682, + "p05": 0.10318915545940399, + "p95": 0.2606950104236603 + }, + { + "label": "completed_group_2", + "mean": 0.1928499937057495, + "median": 0.18568618595600128, + "p05": 0.12013933062553406, + "p95": 0.28692901134490967 + }, + { + "label": "current_group_partial", + "mean": 0.24117937684059143, + "median": 0.2192210555076599, + "p05": 0.14662782847881317, + "p95": 0.4063720405101776 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3655540943145752, + "group": 3, + "latest_source_mass": 0.29187142848968506, + "layer": 12, + "max_source_mass": 0.29819542169570923, + "mixer_index": 22, + "normalized_entropy": 0.9850390599666288, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29819542169570923, + "median": 0.29867103695869446, + "p05": 0.2453947365283966, + "p95": 0.34859126806259155 + }, + { + "label": "completed_group_1", + "mean": 0.20063453912734985, + "median": 0.19907723367214203, + "p05": 0.16811034083366394, + "p95": 0.23844538629055023 + }, + { + "label": "completed_group_2", + "mean": 0.20929867029190063, + "median": 0.21011404693126678, + "p05": 0.18459835648536682, + "p95": 0.23381994664669037 + }, + { + "label": "current_group_partial", + "mean": 0.29187142848968506, + "median": 0.29417213797569275, + "p05": 0.24766536056995392, + "p95": 0.32690057158470154 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2754871845245361, + "group": 3, + "latest_source_mass": 0.22957558929920197, + "layer": 12, + "max_source_mass": 0.42651012539863586, + "mixer_index": 23, + "normalized_entropy": 0.9200695179154873, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42651012539863586, + "median": 0.44201791286468506, + "p05": 0.24115881323814392, + "p95": 0.5910159945487976 + }, + { + "label": "completed_group_1", + "mean": 0.1713070571422577, + "median": 0.170497864484787, + "p05": 0.12993231415748596, + "p95": 0.21631143987178802 + }, + { + "label": "completed_group_2", + "mean": 0.1726072132587433, + "median": 0.16530022025108337, + "p05": 0.11561842262744904, + "p95": 0.2528795301914215 + }, + { + "label": "current_group_partial", + "mean": 0.22957558929920197, + "median": 0.2182570993900299, + "p05": 0.14037305116653442, + "p95": 0.3425685167312622 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3596348762512207, + "group": 4, + "latest_source_mass": 0.3104133605957031, + "layer": 13, + "max_source_mass": 0.3104133605957031, + "mixer_index": 24, + "normalized_entropy": 0.9807692466936578, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2750997543334961, + "median": 0.2747533321380615, + "p05": 0.2329672873020172, + "p95": 0.31678152084350586 + }, + { + "label": "completed_group_1", + "mean": 0.22746169567108154, + "median": 0.215274840593338, + "p05": 0.18571145832538605, + "p95": 0.3119947016239166 + }, + { + "label": "completed_group_2", + "mean": 0.18702521920204163, + "median": 0.1861300766468048, + "p05": 0.16768434643745422, + "p95": 0.20858913660049438 + }, + { + "label": "completed_group_3", + "mean": 0.3104133605957031, + "median": 0.32007908821105957, + "p05": 0.2161477506160736, + "p95": 0.3817093074321747 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5752925872802734, + "group": 4, + "latest_source_mass": 0.20874352753162384, + "layer": 13, + "max_source_mass": 0.28211504220962524, + "mixer_index": 25, + "normalized_entropy": 0.9787843166300303, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28211504220962524, + "median": 0.2998196482658386, + "p05": 0.19197332859039307, + "p95": 0.3606031835079193 + }, + { + "label": "completed_group_1", + "mean": 0.1675817370414734, + "median": 0.16741523146629333, + "p05": 0.14403808116912842, + "p95": 0.19205224514007568 + }, + { + "label": "completed_group_2", + "mean": 0.1759152114391327, + "median": 0.17603915929794312, + "p05": 0.138168603181839, + "p95": 0.21206450462341309 + }, + { + "label": "completed_group_3", + "mean": 0.16564446687698364, + "median": 0.16148409247398376, + "p05": 0.12781348824501038, + "p95": 0.21234022080898285 + }, + { + "label": "current_group_partial", + "mean": 0.20874352753162384, + "median": 0.2069983333349228, + "p05": 0.17109251022338867, + "p95": 0.24858909845352173 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5704567432403564, + "group": 4, + "latest_source_mass": 0.29884350299835205, + "layer": 14, + "max_source_mass": 0.29884350299835205, + "mixer_index": 26, + "normalized_entropy": 0.975779637789948, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21000900864601135, + "median": 0.21329987049102783, + "p05": 0.17762595415115356, + "p95": 0.2425374686717987 + }, + { + "label": "completed_group_1", + "mean": 0.18079037964344025, + "median": 0.17888139188289642, + "p05": 0.15963314473628998, + "p95": 0.20616433024406433 + }, + { + "label": "completed_group_2", + "mean": 0.13544103503227234, + "median": 0.1345825046300888, + "p05": 0.12005693465471268, + "p95": 0.1533820480108261 + }, + { + "label": "completed_group_3", + "mean": 0.1749160885810852, + "median": 0.17272356152534485, + "p05": 0.15442612767219543, + "p95": 0.20184364914894104 + }, + { + "label": "current_group_partial", + "mean": 0.29884350299835205, + "median": 0.3006042242050171, + "p05": 0.2674207091331482, + "p95": 0.3227018713951111 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.579707384109497, + "group": 4, + "latest_source_mass": 0.24509160220623016, + "layer": 14, + "max_source_mass": 0.24509160220623016, + "mixer_index": 27, + "normalized_entropy": 0.98152738412901, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21335941553115845, + "median": 0.21055345237255096, + "p05": 0.12470410019159317, + "p95": 0.33470356464385986 + }, + { + "label": "completed_group_1", + "mean": 0.18788450956344604, + "median": 0.18773096799850464, + "p05": 0.14719606935977936, + "p95": 0.22808396816253662 + }, + { + "label": "completed_group_2", + "mean": 0.20250459015369415, + "median": 0.20350772142410278, + "p05": 0.15922458469867706, + "p95": 0.24294495582580566 + }, + { + "label": "completed_group_3", + "mean": 0.1511598825454712, + "median": 0.14913730323314667, + "p05": 0.12029191851615906, + "p95": 0.1872422844171524 + }, + { + "label": "current_group_partial", + "mean": 0.24509160220623016, + "median": 0.23680704832077026, + "p05": 0.18374761939048767, + "p95": 0.3342677652835846 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5663042068481445, + "group": 4, + "latest_source_mass": 0.29828590154647827, + "layer": 15, + "max_source_mass": 0.29828590154647827, + "mixer_index": 28, + "normalized_entropy": 0.9731995218624366, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21676908433437347, + "median": 0.2142370641231537, + "p05": 0.1767536848783493, + "p95": 0.2722740173339844 + }, + { + "label": "completed_group_1", + "mean": 0.19467836618423462, + "median": 0.19520726799964905, + "p05": 0.16570945084095, + "p95": 0.2210082858800888 + }, + { + "label": "completed_group_2", + "mean": 0.13713446259498596, + "median": 0.13748224079608917, + "p05": 0.12025479972362518, + "p95": 0.15284715592861176 + }, + { + "label": "completed_group_3", + "mean": 0.15313217043876648, + "median": 0.153159961104393, + "p05": 0.13474825024604797, + "p95": 0.1712232530117035 + }, + { + "label": "current_group_partial", + "mean": 0.29828590154647827, + "median": 0.29950791597366333, + "p05": 0.2589420676231384, + "p95": 0.33298033475875854 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5384478569030762, + "group": 4, + "latest_source_mass": 0.29070425033569336, + "layer": 15, + "max_source_mass": 0.29070425033569336, + "mixer_index": 29, + "normalized_entropy": 0.9558913984922479, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21904908120632172, + "median": 0.20143723487854004, + "p05": 0.09117002040147781, + "p95": 0.4469224512577057 + }, + { + "label": "completed_group_1", + "mean": 0.15864285826683044, + "median": 0.15922614932060242, + "p05": 0.10943489521741867, + "p95": 0.2047940045595169 + }, + { + "label": "completed_group_2", + "mean": 0.18178996443748474, + "median": 0.18768036365509033, + "p05": 0.10846641659736633, + "p95": 0.23086166381835938 + }, + { + "label": "completed_group_3", + "mean": 0.14981383085250854, + "median": 0.14754682779312134, + "p05": 0.10115746408700943, + "p95": 0.19903366267681122 + }, + { + "label": "current_group_partial", + "mean": 0.29070425033569336, + "median": 0.2898399233818054, + "p05": 0.20691633224487305, + "p95": 0.37558960914611816 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.552668571472168, + "group": 4, + "latest_source_mass": 0.30334532260894775, + "layer": 16, + "max_source_mass": 0.30334532260894775, + "mixer_index": 30, + "normalized_entropy": 0.9647272252484255, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22710487246513367, + "median": 0.2245989590883255, + "p05": 0.16175971925258636, + "p95": 0.2988150715827942 + }, + { + "label": "completed_group_1", + "mean": 0.1922147125005722, + "median": 0.1916719675064087, + "p05": 0.15179888904094696, + "p95": 0.23553937673568726 + }, + { + "label": "completed_group_2", + "mean": 0.13047492504119873, + "median": 0.12986470758914948, + "p05": 0.10791919380426407, + "p95": 0.15449906885623932 + }, + { + "label": "completed_group_3", + "mean": 0.14686015248298645, + "median": 0.14501844346523285, + "p05": 0.12235601246356964, + "p95": 0.17693395912647247 + }, + { + "label": "current_group_partial", + "mean": 0.30334532260894775, + "median": 0.29679566621780396, + "p05": 0.24902349710464478, + "p95": 0.3774981200695038 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5210195779800415, + "group": 4, + "latest_source_mass": 0.282525897026062, + "layer": 16, + "max_source_mass": 0.282525897026062, + "mixer_index": 31, + "normalized_entropy": 0.9450625999481175, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2812899947166443, + "median": 0.2745441794395447, + "p05": 0.15230709314346313, + "p95": 0.4724813997745514 + }, + { + "label": "completed_group_1", + "mean": 0.14511607587337494, + "median": 0.14084157347679138, + "p05": 0.10224629938602448, + "p95": 0.2002197802066803 + }, + { + "label": "completed_group_2", + "mean": 0.15909329056739807, + "median": 0.15865719318389893, + "p05": 0.10898765176534653, + "p95": 0.21094731986522675 + }, + { + "label": "completed_group_3", + "mean": 0.1319747418165207, + "median": 0.1301102191209793, + "p05": 0.08913854509592056, + "p95": 0.17862913012504578 + }, + { + "label": "current_group_partial", + "mean": 0.282525897026062, + "median": 0.28236913681030273, + "p05": 0.19806379079818726, + "p95": 0.36864838004112244 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5376527309417725, + "group": 5, + "latest_source_mass": 0.287123441696167, + "layer": 17, + "max_source_mass": 0.287123441696167, + "mixer_index": 32, + "normalized_entropy": 0.9553973589551147, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.23905543982982635, + "median": 0.23701313138008118, + "p05": 0.18620719015598297, + "p95": 0.2925828695297241 + }, + { + "label": "completed_group_1", + "mean": 0.21673071384429932, + "median": 0.2044925093650818, + "p05": 0.15778198838233948, + "p95": 0.321443647146225 + }, + { + "label": "completed_group_2", + "mean": 0.0950920432806015, + "median": 0.09378060698509216, + "p05": 0.07889896631240845, + "p95": 0.11442621052265167 + }, + { + "label": "completed_group_3", + "mean": 0.16199833154678345, + "median": 0.16117054224014282, + "p05": 0.12811943888664246, + "p95": 0.19840528070926666 + }, + { + "label": "completed_group_4", + "mean": 0.287123441696167, + "median": 0.29111331701278687, + "p05": 0.2107674777507782, + "p95": 0.34516024589538574 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7209875583648682, + "group": 5, + "latest_source_mass": 0.1365262269973755, + "layer": 17, + "max_source_mass": 0.25328749418258667, + "mixer_index": 33, + "normalized_entropy": 0.9605014444859178, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1703755259513855, + "median": 0.1390259861946106, + "p05": 0.08010496944189072, + "p95": 0.3672885000705719 + }, + { + "label": "completed_group_1", + "mean": 0.14151468873023987, + "median": 0.1361694037914276, + "p05": 0.10555511713027954, + "p95": 0.19257718324661255 + }, + { + "label": "completed_group_2", + "mean": 0.17087554931640625, + "median": 0.17344608902931213, + "p05": 0.10519347339868546, + "p95": 0.22031483054161072 + }, + { + "label": "completed_group_3", + "mean": 0.12742051482200623, + "median": 0.12381769716739655, + "p05": 0.0888485461473465, + "p95": 0.17263755202293396 + }, + { + "label": "completed_group_4", + "mean": 0.25328749418258667, + "median": 0.25078505277633667, + "p05": 0.163196861743927, + "p95": 0.35207411646842957 + }, + { + "label": "current_group_partial", + "mean": 0.1365262269973755, + "median": 0.13269345462322235, + "p05": 0.0852486714720726, + "p95": 0.20064347982406616 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7200093269348145, + "group": 5, + "latest_source_mass": 0.27521812915802, + "layer": 18, + "max_source_mass": 0.27521812915802, + "mixer_index": 34, + "normalized_entropy": 0.9599554831295783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18992580473423004, + "median": 0.19383516907691956, + "p05": 0.1371862292289734, + "p95": 0.23474527895450592 + }, + { + "label": "completed_group_1", + "mean": 0.16036251187324524, + "median": 0.1590055525302887, + "p05": 0.14324960112571716, + "p95": 0.1809857338666916 + }, + { + "label": "completed_group_2", + "mean": 0.08439569175243378, + "median": 0.08315694332122803, + "p05": 0.07186377048492432, + "p95": 0.10051116347312927 + }, + { + "label": "completed_group_3", + "mean": 0.11250905692577362, + "median": 0.11015664041042328, + "p05": 0.09692119807004929, + "p95": 0.13486211001873016 + }, + { + "label": "completed_group_4", + "mean": 0.1775888353586197, + "median": 0.1758042573928833, + "p05": 0.1488552987575531, + "p95": 0.21211811900138855 + }, + { + "label": "current_group_partial", + "mean": 0.27521812915802, + "median": 0.2752543091773987, + "p05": 0.25022760033607483, + "p95": 0.3000151515007019 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.753357172012329, + "group": 5, + "latest_source_mass": 0.24494445323944092, + "layer": 18, + "max_source_mass": 0.24494445323944092, + "mixer_index": 35, + "normalized_entropy": 0.978567269839924, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15539543330669403, + "median": 0.14567650854587555, + "p05": 0.1150466799736023, + "p95": 0.23550888895988464 + }, + { + "label": "completed_group_1", + "mean": 0.1388990879058838, + "median": 0.1368958055973053, + "p05": 0.11385980248451233, + "p95": 0.17140795290470123 + }, + { + "label": "completed_group_2", + "mean": 0.15745997428894043, + "median": 0.15778976678848267, + "p05": 0.13043276965618134, + "p95": 0.1834889054298401 + }, + { + "label": "completed_group_3", + "mean": 0.12427005916833878, + "median": 0.12222100794315338, + "p05": 0.10104347765445709, + "p95": 0.1512138545513153 + }, + { + "label": "completed_group_4", + "mean": 0.17903099954128265, + "median": 0.1781059056520462, + "p05": 0.14208883047103882, + "p95": 0.21939483284950256 + }, + { + "label": "current_group_partial", + "mean": 0.24494445323944092, + "median": 0.24745827913284302, + "p05": 0.16898170113563538, + "p95": 0.3117744028568268 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7199265956878662, + "group": 5, + "latest_source_mass": 0.2730581760406494, + "layer": 19, + "max_source_mass": 0.2730581760406494, + "mixer_index": 36, + "normalized_entropy": 0.9599093099415087, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19207912683486938, + "median": 0.19417056441307068, + "p05": 0.14205873012542725, + "p95": 0.2466108500957489 + }, + { + "label": "completed_group_1", + "mean": 0.17904852330684662, + "median": 0.17943383753299713, + "p05": 0.15607136487960815, + "p95": 0.2006712406873703 + }, + { + "label": "completed_group_2", + "mean": 0.08410340547561646, + "median": 0.08364531397819519, + "p05": 0.07357755303382874, + "p95": 0.09571512043476105 + }, + { + "label": "completed_group_3", + "mean": 0.1121990978717804, + "median": 0.11130240559577942, + "p05": 0.0962604433298111, + "p95": 0.13100717961788177 + }, + { + "label": "completed_group_4", + "mean": 0.15951167047023773, + "median": 0.1591540426015854, + "p05": 0.13084900379180908, + "p95": 0.1905163675546646 + }, + { + "label": "current_group_partial", + "mean": 0.2730581760406494, + "median": 0.27290621399879456, + "p05": 0.241956889629364, + "p95": 0.3052297830581665 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6756346225738525, + "group": 5, + "latest_source_mass": 0.3124367296695709, + "layer": 19, + "max_source_mass": 0.3124367296695709, + "mixer_index": 37, + "normalized_entropy": 0.9351894890756556, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15863081812858582, + "median": 0.1342058777809143, + "p05": 0.07548131048679352, + "p95": 0.3739146590232849 + }, + { + "label": "completed_group_1", + "mean": 0.11242728680372238, + "median": 0.11213450133800507, + "p05": 0.08389876782894135, + "p95": 0.14075449109077454 + }, + { + "label": "completed_group_2", + "mean": 0.15970337390899658, + "median": 0.16262288391590118, + "p05": 0.10122501850128174, + "p95": 0.2050495743751526 + }, + { + "label": "completed_group_3", + "mean": 0.09774985909461975, + "median": 0.09510388225317001, + "p05": 0.06857988238334656, + "p95": 0.12727102637290955 + }, + { + "label": "completed_group_4", + "mean": 0.15905189514160156, + "median": 0.1553158164024353, + "p05": 0.11221202462911606, + "p95": 0.2187761515378952 + }, + { + "label": "current_group_partial", + "mean": 0.3124367296695709, + "median": 0.3196638822555542, + "p05": 0.18551334738731384, + "p95": 0.4139381945133209 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7271867990493774, + "group": 5, + "latest_source_mass": 0.2614278793334961, + "layer": 20, + "max_source_mass": 0.2614278793334961, + "mixer_index": 38, + "normalized_entropy": 0.9639613065884912, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19560003280639648, + "median": 0.1968384087085724, + "p05": 0.1501731276512146, + "p95": 0.2440420687198639 + }, + { + "label": "completed_group_1", + "mean": 0.18333327770233154, + "median": 0.18228237330913544, + "p05": 0.1655699908733368, + "p95": 0.2043052464723587 + }, + { + "label": "completed_group_2", + "mean": 0.08475088328123093, + "median": 0.08410285413265228, + "p05": 0.07304102182388306, + "p95": 0.09892263263463974 + }, + { + "label": "completed_group_3", + "mean": 0.11598999798297882, + "median": 0.1141754686832428, + "p05": 0.10139824450016022, + "p95": 0.1362377256155014 + }, + { + "label": "completed_group_4", + "mean": 0.15889792144298553, + "median": 0.15817126631736755, + "p05": 0.13111311197280884, + "p95": 0.18958830833435059 + }, + { + "label": "current_group_partial", + "mean": 0.2614278793334961, + "median": 0.2622074484825134, + "p05": 0.2309473156929016, + "p95": 0.2893397808074951 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6916961669921875, + "group": 5, + "latest_source_mass": 0.3091394901275635, + "layer": 20, + "max_source_mass": 0.3091394901275635, + "mixer_index": 39, + "normalized_entropy": 0.9441536076943532, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19065655767917633, + "median": 0.1910099983215332, + "p05": 0.09996911883354187, + "p95": 0.313973069190979 + }, + { + "label": "completed_group_1", + "mean": 0.12983909249305725, + "median": 0.12771058082580566, + "p05": 0.10737630724906921, + "p95": 0.15882958471775055 + }, + { + "label": "completed_group_2", + "mean": 0.11857748031616211, + "median": 0.11902420967817307, + "p05": 0.08811743557453156, + "p95": 0.14735448360443115 + }, + { + "label": "completed_group_3", + "mean": 0.09883742034435272, + "median": 0.0951075479388237, + "p05": 0.07493072748184204, + "p95": 0.1301351934671402 + }, + { + "label": "completed_group_4", + "mean": 0.1529499888420105, + "median": 0.15091747045516968, + "p05": 0.11475001275539398, + "p95": 0.19942505657672882 + }, + { + "label": "current_group_partial", + "mean": 0.3091394901275635, + "median": 0.3146688640117645, + "p05": 0.2252732366323471, + "p95": 0.3756124973297119 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7504222393035889, + "group": 6, + "latest_source_mass": 0.2152354121208191, + "layer": 21, + "max_source_mass": 0.2152354121208191, + "mixer_index": 40, + "normalized_entropy": 0.9769292527069633, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20809361338615417, + "median": 0.21471115946769714, + "p05": 0.13097207248210907, + "p95": 0.2735797166824341 + }, + { + "label": "completed_group_1", + "mean": 0.19103102385997772, + "median": 0.19057458639144897, + "p05": 0.16917173564434052, + "p95": 0.2132101207971573 + }, + { + "label": "completed_group_2", + "mean": 0.10743936896324158, + "median": 0.10638970136642456, + "p05": 0.09155704826116562, + "p95": 0.12698283791542053 + }, + { + "label": "completed_group_3", + "mean": 0.1194620281457901, + "median": 0.11578401178121567, + "p05": 0.10080886632204056, + "p95": 0.14768129587173462 + }, + { + "label": "completed_group_4", + "mean": 0.15873853862285614, + "median": 0.15696901082992554, + "p05": 0.13252565264701843, + "p95": 0.19312383234500885 + }, + { + "label": "completed_group_5", + "mean": 0.2152354121208191, + "median": 0.21616290509700775, + "p05": 0.18420100212097168, + "p95": 0.24399688839912415 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5379457473754883, + "group": 6, + "latest_source_mass": 0.515034556388855, + "layer": 21, + "max_source_mass": 0.515034556388855, + "mixer_index": 41, + "normalized_entropy": 0.7903477702308708, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09571269154548645, + "median": 0.08428268879652023, + "p05": 0.04900683835148811, + "p95": 0.1901935338973999 + }, + { + "label": "completed_group_1", + "mean": 0.07078731805086136, + "median": 0.06967149674892426, + "p05": 0.05498572438955307, + "p95": 0.08918046206235886 + }, + { + "label": "completed_group_2", + "mean": 0.0824701264500618, + "median": 0.08152244985103607, + "p05": 0.06332080066204071, + "p95": 0.10414593666791916 + }, + { + "label": "completed_group_3", + "mean": 0.060012347996234894, + "median": 0.058421529829502106, + "p05": 0.04666804522275925, + "p95": 0.07636430114507675 + }, + { + "label": "completed_group_4", + "mean": 0.08145605027675629, + "median": 0.0806138813495636, + "p05": 0.061518602073192596, + "p95": 0.10454171150922775 + }, + { + "label": "completed_group_5", + "mean": 0.09452696144580841, + "median": 0.09397821128368378, + "p05": 0.06939592957496643, + "p95": 0.12165899574756622 + }, + { + "label": "current_group_partial", + "mean": 0.515034556388855, + "median": 0.5195302963256836, + "p05": 0.41491252183914185, + "p95": 0.5988897681236267 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7319592237472534, + "group": 6, + "latest_source_mass": 0.3903183937072754, + "layer": 22, + "max_source_mass": 0.3903183937072754, + "mixer_index": 42, + "normalized_entropy": 0.8900509741357138, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13678354024887085, + "median": 0.14219121634960175, + "p05": 0.0800691545009613, + "p95": 0.18056681752204895 + }, + { + "label": "completed_group_1", + "mean": 0.11740624904632568, + "median": 0.11779546737670898, + "p05": 0.09323577582836151, + "p95": 0.13856223225593567 + }, + { + "label": "completed_group_2", + "mean": 0.06668868660926819, + "median": 0.06673324108123779, + "p05": 0.05682401731610298, + "p95": 0.07633707672357559 + }, + { + "label": "completed_group_3", + "mean": 0.07360264658927917, + "median": 0.07287117838859558, + "p05": 0.06238633394241333, + "p95": 0.0858106017112732 + }, + { + "label": "completed_group_4", + "mean": 0.09287553280591965, + "median": 0.0919380784034729, + "p05": 0.07644987106323242, + "p95": 0.11259186267852783 + }, + { + "label": "completed_group_5", + "mean": 0.12232495099306107, + "median": 0.12153243273496628, + "p05": 0.10421743988990784, + "p95": 0.14276890456676483 + }, + { + "label": "current_group_partial", + "mean": 0.3903183937072754, + "median": 0.382642537355423, + "p05": 0.32760486006736755, + "p95": 0.4824257791042328 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4663677215576172, + "group": 6, + "latest_source_mass": 0.5484944581985474, + "layer": 22, + "max_source_mass": 0.5484944581985474, + "mixer_index": 43, + "normalized_entropy": 0.7535639414129677, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09485072642564774, + "median": 0.07708277553319931, + "p05": 0.041287682950496674, + "p95": 0.2252470850944519 + }, + { + "label": "completed_group_1", + "mean": 0.07167200744152069, + "median": 0.07091978192329407, + "p05": 0.05350632965564728, + "p95": 0.09384822845458984 + }, + { + "label": "completed_group_2", + "mean": 0.0841417908668518, + "median": 0.08349095284938812, + "p05": 0.060824211686849594, + "p95": 0.11007672548294067 + }, + { + "label": "completed_group_3", + "mean": 0.05784624069929123, + "median": 0.056044802069664, + "p05": 0.04343996196985245, + "p95": 0.07539206743240356 + }, + { + "label": "completed_group_4", + "mean": 0.07052354514598846, + "median": 0.06955709308385849, + "p05": 0.048022691160440445, + "p95": 0.09658979624509811 + }, + { + "label": "completed_group_5", + "mean": 0.0724712610244751, + "median": 0.07230593264102936, + "p05": 0.05036160722374916, + "p95": 0.09567554295063019 + }, + { + "label": "current_group_partial", + "mean": 0.5484944581985474, + "median": 0.5530485510826111, + "p05": 0.4294937252998352, + "p95": 0.6537213325500488 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.749481201171875, + "group": 6, + "latest_source_mass": 0.37420469522476196, + "layer": 23, + "max_source_mass": 0.37420469522476196, + "mixer_index": 44, + "normalized_entropy": 0.8990554892892669, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14530612528324127, + "median": 0.1497679054737091, + "p05": 0.08296400308609009, + "p95": 0.20158618688583374 + }, + { + "label": "completed_group_1", + "mean": 0.12230369448661804, + "median": 0.12149038910865784, + "p05": 0.10498286038637161, + "p95": 0.14227813482284546 + }, + { + "label": "completed_group_2", + "mean": 0.0660901814699173, + "median": 0.06578399986028671, + "p05": 0.05611872673034668, + "p95": 0.07736622542142868 + }, + { + "label": "completed_group_3", + "mean": 0.07630561292171478, + "median": 0.07523077726364136, + "p05": 0.06339914351701736, + "p95": 0.09089048206806183 + }, + { + "label": "completed_group_4", + "mean": 0.09453228116035461, + "median": 0.09349468350410461, + "p05": 0.07686570286750793, + "p95": 0.11586424708366394 + }, + { + "label": "completed_group_5", + "mean": 0.12125740945339203, + "median": 0.12067118287086487, + "p05": 0.10250546038150787, + "p95": 0.14243635535240173 + }, + { + "label": "current_group_partial", + "mean": 0.37420469522476196, + "median": 0.36897724866867065, + "p05": 0.30640295147895813, + "p95": 0.4604528248310089 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.472255825996399, + "group": 6, + "latest_source_mass": 0.5438836812973022, + "layer": 23, + "max_source_mass": 0.5438836812973022, + "mixer_index": 45, + "normalized_entropy": 0.7565898285237576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10688216984272003, + "median": 0.0909440815448761, + "p05": 0.04993019625544548, + "p95": 0.22096660733222961 + }, + { + "label": "completed_group_1", + "mean": 0.07465942203998566, + "median": 0.07390505075454712, + "p05": 0.05582176893949509, + "p95": 0.09690531343221664 + }, + { + "label": "completed_group_2", + "mean": 0.08327433466911316, + "median": 0.08253603428602219, + "p05": 0.05906296521425247, + "p95": 0.10942579060792923 + }, + { + "label": "completed_group_3", + "mean": 0.054589346051216125, + "median": 0.052618082612752914, + "p05": 0.039012178778648376, + "p95": 0.07247936725616455 + }, + { + "label": "completed_group_4", + "mean": 0.0637175589799881, + "median": 0.062460802495479584, + "p05": 0.04167243465781212, + "p95": 0.09087979793548584 + }, + { + "label": "completed_group_5", + "mean": 0.07299347221851349, + "median": 0.07232271879911423, + "p05": 0.05079510435461998, + "p95": 0.09688537567853928 + }, + { + "label": "current_group_partial", + "mean": 0.5438836812973022, + "median": 0.5471436977386475, + "p05": 0.42356419563293457, + "p95": 0.6585544347763062 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7924723625183105, + "group": 6, + "latest_source_mass": 0.3345995247364044, + "layer": 24, + "max_source_mass": 0.3345995247364044, + "mixer_index": 46, + "normalized_entropy": 0.9211485758417507, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15522946417331696, + "median": 0.16076582670211792, + "p05": 0.0863914042711258, + "p95": 0.2195497751235962 + }, + { + "label": "completed_group_1", + "mean": 0.13724806904792786, + "median": 0.13687172532081604, + "p05": 0.1200050488114357, + "p95": 0.1560649424791336 + }, + { + "label": "completed_group_2", + "mean": 0.06889298558235168, + "median": 0.06847138702869415, + "p05": 0.058477431535720825, + "p95": 0.08076588809490204 + }, + { + "label": "completed_group_3", + "mean": 0.08198929578065872, + "median": 0.08025878667831421, + "p05": 0.06727518886327744, + "p95": 0.10007832199335098 + }, + { + "label": "completed_group_4", + "mean": 0.09816981852054596, + "median": 0.09709671884775162, + "p05": 0.07776231318712234, + "p95": 0.12178758531808853 + }, + { + "label": "completed_group_5", + "mean": 0.123870849609375, + "median": 0.12361469119787216, + "p05": 0.1038728728890419, + "p95": 0.14525073766708374 + }, + { + "label": "current_group_partial", + "mean": 0.3345995247364044, + "median": 0.3317973017692566, + "p05": 0.265953928232193, + "p95": 0.412749707698822 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5755163431167603, + "group": 6, + "latest_source_mass": 0.48653730750083923, + "layer": 24, + "max_source_mass": 0.48653730750083923, + "mixer_index": 47, + "normalized_entropy": 0.8096552371041545, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12274101376533508, + "median": 0.10600537061691284, + "p05": 0.06405961513519287, + "p95": 0.25983333587646484 + }, + { + "label": "completed_group_1", + "mean": 0.08900541067123413, + "median": 0.08780360221862793, + "p05": 0.06876012682914734, + "p95": 0.11320354044437408 + }, + { + "label": "completed_group_2", + "mean": 0.08551642298698425, + "median": 0.08451470732688904, + "p05": 0.06253526359796524, + "p95": 0.1114322617650032 + }, + { + "label": "completed_group_3", + "mean": 0.06285787373781204, + "median": 0.061114855110645294, + "p05": 0.045210886746644974, + "p95": 0.08213447034358978 + }, + { + "label": "completed_group_4", + "mean": 0.07429315149784088, + "median": 0.07252945005893707, + "p05": 0.049904488027095795, + "p95": 0.10469712316989899 + }, + { + "label": "completed_group_5", + "mean": 0.07904879748821259, + "median": 0.07821422815322876, + "p05": 0.05642634630203247, + "p95": 0.10449491441249847 + }, + { + "label": "current_group_partial", + "mean": 0.48653730750083923, + "median": 0.49373340606689453, + "p05": 0.3549031913280487, + "p95": 0.5985680222511292 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.8390922546386719, + "group": 7, + "latest_source_mass": 0.29349833726882935, + "layer": 25, + "max_source_mass": 0.29349833726882935, + "mixer_index": 48, + "normalized_entropy": 0.945106461123861, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15884169936180115, + "median": 0.15805873274803162, + "p05": 0.09690988063812256, + "p95": 0.2504432201385498 + }, + { + "label": "completed_group_1", + "mean": 0.14726480841636658, + "median": 0.14679506421089172, + "p05": 0.12764519453048706, + "p95": 0.16903650760650635 + }, + { + "label": "completed_group_2", + "mean": 0.08053630590438843, + "median": 0.08051620423793793, + "p05": 0.06807594001293182, + "p95": 0.09329371899366379 + }, + { + "label": "completed_group_3", + "mean": 0.0909707322716713, + "median": 0.08902134001255035, + "p05": 0.07499626278877258, + "p95": 0.11060230433940887 + }, + { + "label": "completed_group_4", + "mean": 0.10738551616668701, + "median": 0.10607586801052094, + "p05": 0.08672098070383072, + "p95": 0.1325560212135315 + }, + { + "label": "completed_group_5", + "mean": 0.12150255590677261, + "median": 0.12205702066421509, + "p05": 0.0993877649307251, + "p95": 0.14230164885520935 + }, + { + "label": "completed_group_6", + "mean": 0.29349833726882935, + "median": 0.29394760727882385, + "p05": 0.22660650312900543, + "p95": 0.3548165559768677 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.6550344228744507, + "group": 7, + "latest_source_mass": 0.3589511513710022, + "layer": 25, + "max_source_mass": 0.3589511513710022, + "mixer_index": 49, + "normalized_entropy": 0.795903318127166, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0844201073050499, + "median": 0.0718768984079361, + "p05": 0.02839931659400463, + "p95": 0.21728233993053436 + }, + { + "label": "completed_group_1", + "mean": 0.06695197522640228, + "median": 0.06580687314271927, + "p05": 0.04618899151682854, + "p95": 0.09124172478914261 + }, + { + "label": "completed_group_2", + "mean": 0.061310961842536926, + "median": 0.06096746027469635, + "p05": 0.040395740419626236, + "p95": 0.08348730951547623 + }, + { + "label": "completed_group_3", + "mean": 0.0447731651365757, + "median": 0.042184896767139435, + "p05": 0.02882835641503334, + "p95": 0.06167382374405861 + }, + { + "label": "completed_group_4", + "mean": 0.04748315364122391, + "median": 0.04516277089715004, + "p05": 0.0278231929987669, + "p95": 0.0743657797574997 + }, + { + "label": "completed_group_5", + "mean": 0.045562684535980225, + "median": 0.044303566217422485, + "p05": 0.03103901632130146, + "p95": 0.06420586258172989 + }, + { + "label": "completed_group_6", + "mean": 0.2905467748641968, + "median": 0.2895723581314087, + "p05": 0.1524062305688858, + "p95": 0.43639296293258667 + }, + { + "label": "current_group_partial", + "mean": 0.3589511513710022, + "median": 0.3606260418891907, + "p05": 0.16963356733322144, + "p95": 0.546379804611206 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8679088354110718, + "group": 7, + "latest_source_mass": 0.32435470819473267, + "layer": 26, + "max_source_mass": 0.32435470819473267, + "mixer_index": 50, + "normalized_entropy": 0.8982742712267442, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11674155294895172, + "median": 0.12052157521247864, + "p05": 0.05360257253050804, + "p95": 0.17403334379196167 + }, + { + "label": "completed_group_1", + "mean": 0.1088850274682045, + "median": 0.1090676486492157, + "p05": 0.08014046400785446, + "p95": 0.13653376698493958 + }, + { + "label": "completed_group_2", + "mean": 0.05779466778039932, + "median": 0.05755443871021271, + "p05": 0.04364454001188278, + "p95": 0.07319164276123047 + }, + { + "label": "completed_group_3", + "mean": 0.06273135542869568, + "median": 0.06075543537735939, + "p05": 0.04851231724023819, + "p95": 0.07996035367250443 + }, + { + "label": "completed_group_4", + "mean": 0.07066236436367035, + "median": 0.06902407109737396, + "p05": 0.052817195653915405, + "p95": 0.09365298599004745 + }, + { + "label": "completed_group_5", + "mean": 0.08129343390464783, + "median": 0.08064926415681839, + "p05": 0.06491829454898834, + "p95": 0.09953147917985916 + }, + { + "label": "completed_group_6", + "mean": 0.17753681540489197, + "median": 0.17456364631652832, + "p05": 0.11964619904756546, + "p95": 0.24453631043434143 + }, + { + "label": "current_group_partial", + "mean": 0.32435470819473267, + "median": 0.32046452164649963, + "p05": 0.19449321925640106, + "p95": 0.4612511992454529 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5923268795013428, + "group": 7, + "latest_source_mass": 0.4404740333557129, + "layer": 26, + "max_source_mass": 0.4404740333557129, + "mixer_index": 51, + "normalized_entropy": 0.7657473641769285, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08656083047389984, + "median": 0.0795329213142395, + "p05": 0.050312869250774384, + "p95": 0.14976806938648224 + }, + { + "label": "completed_group_1", + "mean": 0.05770102143287659, + "median": 0.056408509612083435, + "p05": 0.04085284471511841, + "p95": 0.07934759557247162 + }, + { + "label": "completed_group_2", + "mean": 0.05105586349964142, + "median": 0.049396127462387085, + "p05": 0.034691184759140015, + "p95": 0.07344280183315277 + }, + { + "label": "completed_group_3", + "mean": 0.039375390857458115, + "median": 0.036708764731884, + "p05": 0.024915706366300583, + "p95": 0.05899723991751671 + }, + { + "label": "completed_group_4", + "mean": 0.04410157352685928, + "median": 0.04195208102464676, + "p05": 0.027429571375250816, + "p95": 0.06846393644809723 + }, + { + "label": "completed_group_5", + "mean": 0.04500962793827057, + "median": 0.043660376220941544, + "p05": 0.030392050743103027, + "p95": 0.06387433409690857 + }, + { + "label": "completed_group_6", + "mean": 0.2357216626405716, + "median": 0.2248685657978058, + "p05": 0.11950507014989853, + "p95": 0.3818710744380951 + }, + { + "label": "current_group_partial", + "mean": 0.4404740333557129, + "median": 0.4484519362449646, + "p05": 0.24098733067512512, + "p95": 0.6138508915901184 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8469831943511963, + "group": 7, + "latest_source_mass": 0.33607518672943115, + "layer": 27, + "max_source_mass": 0.33607518672943115, + "mixer_index": 52, + "normalized_entropy": 0.8882111650319092, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10824646055698395, + "median": 0.11040513217449188, + "p05": 0.05085912346839905, + "p95": 0.16382771730422974 + }, + { + "label": "completed_group_1", + "mean": 0.09116974472999573, + "median": 0.09135521948337555, + "p05": 0.056257300078868866, + "p95": 0.1266864538192749 + }, + { + "label": "completed_group_2", + "mean": 0.0701109915971756, + "median": 0.06960364431142807, + "p05": 0.05100060999393463, + "p95": 0.0908425822854042 + }, + { + "label": "completed_group_3", + "mean": 0.0629766583442688, + "median": 0.06064727157354355, + "p05": 0.04317091405391693, + "p95": 0.08672259747982025 + }, + { + "label": "completed_group_4", + "mean": 0.07049155235290527, + "median": 0.06874367594718933, + "p05": 0.050685517489910126, + "p95": 0.09697030484676361 + }, + { + "label": "completed_group_5", + "mean": 0.07297641038894653, + "median": 0.07156282663345337, + "p05": 0.051110126078128815, + "p95": 0.09974705427885056 + }, + { + "label": "completed_group_6", + "mean": 0.18795298039913177, + "median": 0.1818247139453888, + "p05": 0.1166272684931755, + "p95": 0.27661576867103577 + }, + { + "label": "current_group_partial", + "mean": 0.33607518672943115, + "median": 0.3355332612991333, + "p05": 0.20117856562137604, + "p95": 0.4717060327529907 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6590619087219238, + "group": 7, + "latest_source_mass": 0.4179539084434509, + "layer": 27, + "max_source_mass": 0.4179539084434509, + "mixer_index": 53, + "normalized_entropy": 0.7978401294136326, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09444728493690491, + "median": 0.09060665965080261, + "p05": 0.047914937138557434, + "p95": 0.15662360191345215 + }, + { + "label": "completed_group_1", + "mean": 0.0654030293226242, + "median": 0.06396090984344482, + "p05": 0.04467916116118431, + "p95": 0.09097769111394882 + }, + { + "label": "completed_group_2", + "mean": 0.060137998312711716, + "median": 0.05839194357395172, + "p05": 0.04035535454750061, + "p95": 0.08539871126413345 + }, + { + "label": "completed_group_3", + "mean": 0.0449567511677742, + "median": 0.04218417406082153, + "p05": 0.02788582444190979, + "p95": 0.06733109056949615 + }, + { + "label": "completed_group_4", + "mean": 0.044405773282051086, + "median": 0.04241851717233658, + "p05": 0.02826966717839241, + "p95": 0.06812064349651337 + }, + { + "label": "completed_group_5", + "mean": 0.05244143307209015, + "median": 0.05142951011657715, + "p05": 0.035461653023958206, + "p95": 0.07271309196949005 + }, + { + "label": "completed_group_6", + "mean": 0.2202538251876831, + "median": 0.21207934617996216, + "p05": 0.11892802268266678, + "p95": 0.34954455494880676 + }, + { + "label": "current_group_partial", + "mean": 0.4179539084434509, + "median": 0.4249612092971802, + "p05": 0.2265779674053192, + "p95": 0.586824893951416 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.884438157081604, + "group": 7, + "latest_source_mass": 0.2829248011112213, + "layer": 28, + "max_source_mass": 0.2829248011112213, + "mixer_index": 54, + "normalized_entropy": 0.9062231946945226, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11613816022872925, + "median": 0.12147872895002365, + "p05": 0.04899268597364426, + "p95": 0.1770799458026886 + }, + { + "label": "completed_group_1", + "mean": 0.09775452315807343, + "median": 0.09778773784637451, + "p05": 0.05219751223921776, + "p95": 0.14079101383686066 + }, + { + "label": "completed_group_2", + "mean": 0.08394932001829147, + "median": 0.08419600129127502, + "p05": 0.05889316648244858, + "p95": 0.10900868475437164 + }, + { + "label": "completed_group_3", + "mean": 0.07057419419288635, + "median": 0.06822949647903442, + "p05": 0.05265102535486221, + "p95": 0.0926743820309639 + }, + { + "label": "completed_group_4", + "mean": 0.06920696794986725, + "median": 0.06799854338169098, + "p05": 0.05105053633451462, + "p95": 0.09136901795864105 + }, + { + "label": "completed_group_5", + "mean": 0.06340071558952332, + "median": 0.0628691017627716, + "p05": 0.0437660738825798, + "p95": 0.0860220268368721 + }, + { + "label": "completed_group_6", + "mean": 0.21605131030082703, + "median": 0.21297892928123474, + "p05": 0.14316481351852417, + "p95": 0.2999357581138611 + }, + { + "label": "current_group_partial", + "mean": 0.2829248011112213, + "median": 0.2790382504463196, + "p05": 0.14648588001728058, + "p95": 0.4251174330711365 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7424441576004028, + "group": 7, + "latest_source_mass": 0.278835266828537, + "layer": 28, + "max_source_mass": 0.3030073344707489, + "mixer_index": 55, + "normalized_entropy": 0.8379385150653496, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08763164281845093, + "median": 0.05824008211493492, + "p05": 0.027796253561973572, + "p95": 0.2880856990814209 + }, + { + "label": "completed_group_1", + "mean": 0.07771545648574829, + "median": 0.07654327154159546, + "p05": 0.046035416424274445, + "p95": 0.11438068747520447 + }, + { + "label": "completed_group_2", + "mean": 0.07502467930316925, + "median": 0.07301308214664459, + "p05": 0.04458038881421089, + "p95": 0.11157117038965225 + }, + { + "label": "completed_group_3", + "mean": 0.06006643548607826, + "median": 0.05717916786670685, + "p05": 0.036717794835567474, + "p95": 0.08720126748085022 + }, + { + "label": "completed_group_4", + "mean": 0.06314835697412491, + "median": 0.061049528419971466, + "p05": 0.03865773230791092, + "p95": 0.09540070593357086 + }, + { + "label": "completed_group_5", + "mean": 0.054570816457271576, + "median": 0.05328378826379776, + "p05": 0.03375566378235817, + "p95": 0.07989827543497086 + }, + { + "label": "completed_group_6", + "mean": 0.3030073344707489, + "median": 0.30074357986450195, + "p05": 0.14776623249053955, + "p95": 0.469026654958725 + }, + { + "label": "current_group_partial", + "mean": 0.278835266828537, + "median": 0.26309943199157715, + "p05": 0.10582277178764343, + "p95": 0.5049530863761902 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8953055143356323, + "group": 8, + "latest_source_mass": 0.18233531713485718, + "layer": 29, + "max_source_mass": 0.2779633104801178, + "mixer_index": 56, + "normalized_entropy": 0.911449288833841, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1135263666510582, + "median": 0.0953834131360054, + "p05": 0.02703639306128025, + "p95": 0.31321749091148376 + }, + { + "label": "completed_group_1", + "mean": 0.10274903476238251, + "median": 0.10363410413265228, + "p05": 0.06506763398647308, + "p95": 0.13818706572055817 + }, + { + "label": "completed_group_2", + "mean": 0.0858360230922699, + "median": 0.08501487225294113, + "p05": 0.059830889105796814, + "p95": 0.11487221717834473 + }, + { + "label": "completed_group_3", + "mean": 0.08475606143474579, + "median": 0.08265408128499985, + "p05": 0.05679357051849365, + "p95": 0.11555421352386475 + }, + { + "label": "completed_group_4", + "mean": 0.08078759908676147, + "median": 0.07859720289707184, + "p05": 0.055059194564819336, + "p95": 0.11340422928333282 + }, + { + "label": "completed_group_5", + "mean": 0.07204629480838776, + "median": 0.07071015238761902, + "p05": 0.0498509518802166, + "p95": 0.09922727942466736 + }, + { + "label": "completed_group_6", + "mean": 0.2779633104801178, + "median": 0.2749147415161133, + "p05": 0.15705733001232147, + "p95": 0.39374127984046936 + }, + { + "label": "completed_group_7", + "mean": 0.18233531713485718, + "median": 0.16221198439598083, + "p05": 0.07201667875051498, + "p95": 0.35018864274024963 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.801485300064087, + "group": 8, + "latest_source_mass": 0.3215197026729584, + "layer": 29, + "max_source_mass": 0.3215197026729584, + "mixer_index": 57, + "normalized_entropy": 0.8198912931549752, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07824120670557022, + "median": 0.06902759522199631, + "p05": 0.017845120280981064, + "p95": 0.17217300832271576 + }, + { + "label": "completed_group_1", + "mean": 0.060231711715459824, + "median": 0.05764210224151611, + "p05": 0.03765189275145531, + "p95": 0.0922410786151886 + }, + { + "label": "completed_group_2", + "mean": 0.054100219160318375, + "median": 0.0524706169962883, + "p05": 0.034336626529693604, + "p95": 0.07949143648147583 + }, + { + "label": "completed_group_3", + "mean": 0.039871927350759506, + "median": 0.037151023745536804, + "p05": 0.0235070139169693, + "p95": 0.060298819094896317 + }, + { + "label": "completed_group_4", + "mean": 0.04207881912589073, + "median": 0.040273066610097885, + "p05": 0.021681729704141617, + "p95": 0.06873302906751633 + }, + { + "label": "completed_group_5", + "mean": 0.045095667243003845, + "median": 0.044285234063863754, + "p05": 0.025416042655706406, + "p95": 0.06805071234703064 + }, + { + "label": "completed_group_6", + "mean": 0.2047339677810669, + "median": 0.1937887817621231, + "p05": 0.08607597649097443, + "p95": 0.3561563789844513 + }, + { + "label": "completed_group_7", + "mean": 0.15412676334381104, + "median": 0.13271009922027588, + "p05": 0.05149175971746445, + "p95": 0.32324373722076416 + }, + { + "label": "current_group_partial", + "mean": 0.3215197026729584, + "median": 0.31218427419662476, + "p05": 0.08214674144983292, + "p95": 0.5716250538825989 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.008213520050049, + "group": 8, + "latest_source_mass": 0.1996053159236908, + "layer": 30, + "max_source_mass": 0.21496367454528809, + "mixer_index": 58, + "normalized_entropy": 0.9139773606959576, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09438058733940125, + "median": 0.09028880298137665, + "p05": 0.02690170146524906, + "p95": 0.18525993824005127 + }, + { + "label": "completed_group_1", + "mean": 0.07153058052062988, + "median": 0.070882648229599, + "p05": 0.041449472308158875, + "p95": 0.10379751026630402 + }, + { + "label": "completed_group_2", + "mean": 0.07423336803913116, + "median": 0.0730072408914566, + "p05": 0.048166655004024506, + "p95": 0.10590565949678421 + }, + { + "label": "completed_group_3", + "mean": 0.06403130292892456, + "median": 0.06157837435603142, + "p05": 0.04371092468500137, + "p95": 0.08951176702976227 + }, + { + "label": "completed_group_4", + "mean": 0.0645662397146225, + "median": 0.0623253732919693, + "p05": 0.043447867035865784, + "p95": 0.09214113652706146 + }, + { + "label": "completed_group_5", + "mean": 0.05577655881643295, + "median": 0.055099695920944214, + "p05": 0.03416105732321739, + "p95": 0.079859159886837 + }, + { + "label": "completed_group_6", + "mean": 0.21496367454528809, + "median": 0.2118406593799591, + "p05": 0.13403329253196716, + "p95": 0.31010040640830994 + }, + { + "label": "completed_group_7", + "mean": 0.1609123945236206, + "median": 0.15419161319732666, + "p05": 0.07973575592041016, + "p95": 0.27241089940071106 + }, + { + "label": "current_group_partial", + "mean": 0.1996053159236908, + "median": 0.19533705711364746, + "p05": 0.0843900740146637, + "p95": 0.33647382259368896 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.8966764211654663, + "group": 8, + "latest_source_mass": 0.21615305542945862, + "layer": 30, + "max_source_mass": 0.21615305542945862, + "mixer_index": 59, + "normalized_entropy": 0.8632146393815058, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09610478579998016, + "median": 0.0941460132598877, + "p05": 0.030834276229143143, + "p95": 0.17860566079616547 + }, + { + "label": "completed_group_1", + "mean": 0.07267358899116516, + "median": 0.06876465678215027, + "p05": 0.03731950744986534, + "p95": 0.11969097703695297 + }, + { + "label": "completed_group_2", + "mean": 0.060228411108255386, + "median": 0.05793646723031998, + "p05": 0.0349978432059288, + "p95": 0.09366682916879654 + }, + { + "label": "completed_group_3", + "mean": 0.046177469193935394, + "median": 0.04286886006593704, + "p05": 0.02664034441113472, + "p95": 0.07394801080226898 + }, + { + "label": "completed_group_4", + "mean": 0.04206448048353195, + "median": 0.03912021219730377, + "p05": 0.022682487964630127, + "p95": 0.07153408974409103 + }, + { + "label": "completed_group_5", + "mean": 0.04096413776278496, + "median": 0.039336949586868286, + "p05": 0.023794004693627357, + "p95": 0.06405855715274811 + }, + { + "label": "completed_group_6", + "mean": 0.21562576293945312, + "median": 0.20660808682441711, + "p05": 0.10648336261510849, + "p95": 0.3508927822113037 + }, + { + "label": "completed_group_7", + "mean": 0.21000832319259644, + "median": 0.204543337225914, + "p05": 0.08861437439918518, + "p95": 0.3495482802391052 + }, + { + "label": "current_group_partial", + "mean": 0.21615305542945862, + "median": 0.19288693368434906, + "p05": 0.07421350479125977, + "p95": 0.43364205956459045 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.981720209121704, + "group": 8, + "latest_source_mass": 0.17098912596702576, + "layer": 31, + "max_source_mass": 0.23084521293640137, + "mixer_index": 60, + "normalized_entropy": 0.9019197352708571, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10501759499311447, + "median": 0.10529834032058716, + "p05": 0.012508269399404526, + "p95": 0.22564347088336945 + }, + { + "label": "completed_group_1", + "mean": 0.06946458667516708, + "median": 0.06936883926391602, + "p05": 0.025395508855581284, + "p95": 0.11417273432016373 + }, + { + "label": "completed_group_2", + "mean": 0.07816962152719498, + "median": 0.07627303898334503, + "p05": 0.042986176908016205, + "p95": 0.11851722747087479 + }, + { + "label": "completed_group_3", + "mean": 0.07385894656181335, + "median": 0.07061562687158585, + "p05": 0.04739787429571152, + "p95": 0.1065390408039093 + }, + { + "label": "completed_group_4", + "mean": 0.06364607810974121, + "median": 0.06080539897084236, + "p05": 0.0418400801718235, + "p95": 0.09455618262290955 + }, + { + "label": "completed_group_5", + "mean": 0.05328303202986717, + "median": 0.05173730105161667, + "p05": 0.03466875106096268, + "p95": 0.07786844670772552 + }, + { + "label": "completed_group_6", + "mean": 0.23084521293640137, + "median": 0.2265024036169052, + "p05": 0.1294252872467041, + "p95": 0.3483029305934906 + }, + { + "label": "completed_group_7", + "mean": 0.1547257900238037, + "median": 0.15095551311969757, + "p05": 0.057022616267204285, + "p95": 0.2705056667327881 + }, + { + "label": "current_group_partial", + "mean": 0.17098912596702576, + "median": 0.14655888080596924, + "p05": 0.04728491231799126, + "p95": 0.3703039288520813 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0028607845306396, + "group": 8, + "latest_source_mass": 0.12469469010829926, + "layer": 31, + "max_source_mass": 0.2620389461517334, + "mixer_index": 61, + "normalized_entropy": 0.911541225776195, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0842779129743576, + "median": 0.06333573907613754, + "p05": 0.03144961595535278, + "p95": 0.24860140681266785 + }, + { + "label": "completed_group_1", + "mean": 0.10298711061477661, + "median": 0.10017631947994232, + "p05": 0.06670551002025604, + "p95": 0.14914412796497345 + }, + { + "label": "completed_group_2", + "mean": 0.09207960218191147, + "median": 0.09137493371963501, + "p05": 0.056820835918188095, + "p95": 0.13043460249900818 + }, + { + "label": "completed_group_3", + "mean": 0.07771934568881989, + "median": 0.0746275931596756, + "p05": 0.04656440392136574, + "p95": 0.11502407491207123 + }, + { + "label": "completed_group_4", + "mean": 0.06265503168106079, + "median": 0.0610337033867836, + "p05": 0.038839153945446014, + "p95": 0.09315080940723419 + }, + { + "label": "completed_group_5", + "mean": 0.055270493030548096, + "median": 0.0542943961918354, + "p05": 0.035113874822854996, + "p95": 0.07895103842020035 + }, + { + "label": "completed_group_6", + "mean": 0.2620389461517334, + "median": 0.25993645191192627, + "p05": 0.13434287905693054, + "p95": 0.3996090292930603 + }, + { + "label": "completed_group_7", + "mean": 0.1382768601179123, + "median": 0.12482025474309921, + "p05": 0.058253247290849686, + "p95": 0.2677440643310547 + }, + { + "label": "current_group_partial", + "mean": 0.12469469010829926, + "median": 0.11286871135234833, + "p05": 0.04615473002195358, + "p95": 0.24487172067165375 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0655057430267334, + "group": 8, + "latest_source_mass": 0.09924688190221786, + "layer": 32, + "max_source_mass": 0.21871674060821533, + "mixer_index": 62, + "normalized_entropy": 0.9400521750629723, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10472239553928375, + "median": 0.08473113924264908, + "p05": 0.030345402657985687, + "p95": 0.2588862180709839 + }, + { + "label": "completed_group_1", + "mean": 0.11792458593845367, + "median": 0.11655837297439575, + "p05": 0.07099610567092896, + "p95": 0.17131033539772034 + }, + { + "label": "completed_group_2", + "mean": 0.0928182601928711, + "median": 0.09008213877677917, + "p05": 0.06447342038154602, + "p95": 0.12974750995635986 + }, + { + "label": "completed_group_3", + "mean": 0.10183011740446091, + "median": 0.10004840791225433, + "p05": 0.0707390084862709, + "p95": 0.1374078094959259 + }, + { + "label": "completed_group_4", + "mean": 0.08287280797958374, + "median": 0.08090724050998688, + "p05": 0.05679214745759964, + "p95": 0.11534547060728073 + }, + { + "label": "completed_group_5", + "mean": 0.06570011377334595, + "median": 0.06424097716808319, + "p05": 0.04414675012230873, + "p95": 0.09294299781322479 + }, + { + "label": "completed_group_6", + "mean": 0.21871674060821533, + "median": 0.21457019448280334, + "p05": 0.13963721692562103, + "p95": 0.3112483620643616 + }, + { + "label": "completed_group_7", + "mean": 0.11616809666156769, + "median": 0.10701116174459457, + "p05": 0.05174187570810318, + "p95": 0.21264401078224182 + }, + { + "label": "current_group_partial", + "mean": 0.09924688190221786, + "median": 0.08725139498710632, + "p05": 0.032173603773117065, + "p95": 0.20341788232326508 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0258567333221436, + "group": 8, + "latest_source_mass": 0.12343063950538635, + "layer": 32, + "max_source_mass": 0.22868779301643372, + "mixer_index": 63, + "normalized_entropy": 0.9220071330979595, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09674200415611267, + "median": 0.07686861604452133, + "p05": 0.04638649895787239, + "p95": 0.2122267782688141 + }, + { + "label": "completed_group_1", + "mean": 0.09482249617576599, + "median": 0.09244489669799805, + "p05": 0.06637479364871979, + "p95": 0.13019198179244995 + }, + { + "label": "completed_group_2", + "mean": 0.08996410667896271, + "median": 0.08937586098909378, + "p05": 0.06193297728896141, + "p95": 0.12055040895938873 + }, + { + "label": "completed_group_3", + "mean": 0.06836152076721191, + "median": 0.06606560945510864, + "p05": 0.04674140736460686, + "p95": 0.09358662366867065 + }, + { + "label": "completed_group_4", + "mean": 0.06293784081935883, + "median": 0.061281464993953705, + "p05": 0.04231695458292961, + "p95": 0.08961650729179382 + }, + { + "label": "completed_group_5", + "mean": 0.05016915872693062, + "median": 0.049648597836494446, + "p05": 0.03570414334535599, + "p95": 0.06624776124954224 + }, + { + "label": "completed_group_6", + "mean": 0.22868779301643372, + "median": 0.2242993861436844, + "p05": 0.1197199895977974, + "p95": 0.3530009686946869 + }, + { + "label": "completed_group_7", + "mean": 0.1848844289779663, + "median": 0.17580115795135498, + "p05": 0.10029035806655884, + "p95": 0.3003215491771698 + }, + { + "label": "current_group_partial", + "mean": 0.12343063950538635, + "median": 0.10480014234781265, + "p05": 0.05131315439939499, + "p95": 0.2450288087129593 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.7394249439239502, + "group": null, + "latest_source_mass": 0.2556654214859009, + "layer": null, + "max_source_mass": 0.3314756751060486, + "mixer_index": null, + "normalized_entropy": 0.7916464078663832, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10602210462093353, + "median": 0.10817556083202362, + "p05": 0.03750653192400932, + "p95": 0.16905617713928223 + }, + { + "label": "completed_group_1", + "mean": 0.05902107432484627, + "median": 0.05735805630683899, + "p05": 0.039041999727487564, + "p95": 0.08384283632040024 + }, + { + "label": "completed_group_2", + "mean": 0.049884356558322906, + "median": 0.04808305203914642, + "p05": 0.03180687129497528, + "p95": 0.07381068915128708 + }, + { + "label": "completed_group_3", + "mean": 0.034865088760852814, + "median": 0.03287554159760475, + "p05": 0.02153801918029785, + "p95": 0.05384448915719986 + }, + { + "label": "completed_group_4", + "mean": 0.03282830864191055, + "median": 0.0315137580037117, + "p05": 0.020427687093615532, + "p95": 0.050021201372146606 + }, + { + "label": "completed_group_5", + "mean": 0.043302543461322784, + "median": 0.041625380516052246, + "p05": 0.027595922350883484, + "p95": 0.06415434926748276 + }, + { + "label": "completed_group_6", + "mean": 0.08693542331457138, + "median": 0.07748770713806152, + "p05": 0.03693632781505585, + "p95": 0.17101936042308807 + }, + { + "label": "completed_group_7", + "mean": 0.3314756751060486, + "median": 0.32931262254714966, + "p05": 0.0907490998506546, + "p95": 0.5930418968200684 + }, + { + "label": "completed_group_8", + "mean": 0.2556654214859009, + "median": 0.23854607343673706, + "p05": 0.04562881588935852, + "p95": 0.5189820528030396 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", + "optimizer_state_before": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.08528144657611847, + 0.2063225358724594, + 0.07479097694158554, + 0.053203217685222626, + 0.1070452407002449, + 0.03943018242716789, + 0.041509829461574554, + 0.06257687509059906, + 0.061002109199762344, + 0.04113767296075821, + 0.040399398654699326, + 0.03907898813486099, + 0.06714949756860733, + 0.04437125846743584, + 0.0412614606320858, + 0.046277787536382675, + 0.10073095560073853, + 0.05776162073016167, + 0.05185067281126976, + 0.07827401161193848, + 0.04170985147356987, + 0.041845910251140594, + 0.038240060210227966, + 0.046402644366025925, + 0.030182205140590668, + 0.025695007294416428, + 0.024306239560246468, + 0.017157377675175667, + 0.030125634744763374, + 0.01829664409160614, + 0.024464696645736694, + 0.032671861350536346 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1464898420854297e-05, + "first_to_last_ratio": 0.8455323660117368, + "imbalance_abs_log_ratio": 0.16778883105478307, + "last_quartile_mean": 1.355938445612992e-05, + "mean": 1.3058880313110421e-05, + "normalized": [ + 0.8914129879074366, + 1.077571175585955, + 0.9537440677326601, + 0.8574453930361242, + 0.9048950001587052, + 0.8425968627438281, + 0.7839582579962869, + 0.7118870894358409, + 0.5788084225246481, + 0.530693141917732, + 0.5023627734883869, + 0.4844869540829288, + 0.6659476097626051, + 0.6212422516778212, + 0.5836177358524666, + 0.5530522631862488, + 0.6705030645582619, + 0.6390296210597277, + 0.6539820627182751, + 0.682372291616315, + 3.0705747439001825, + 2.5957154007666214, + 2.1136981524200404, + 1.7237888584415353, + 1.9773452676094638, + 1.7509357725203292, + 1.3561771885891878, + 0.9354730786944212, + 0.8232148160475435, + 0.5856284416831322, + 0.45203530207589593, + 0.4258039502093924 + ], + "peak_layer": 21, + "peak_normalized": 3.0705747439001825, + "population_cv": 0.6442856671110848, + "rest_mean": 9.924206164281781e-06, + "spike_contrast": 3.0215132797934157, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.998612071678508e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.1640855518635362e-05, + 1.4071873010834679e-05, + 1.2454829629859887e-05, + 1.119727676268667e-05, + 1.1816915503004566e-05, + 1.100337158277398e-05, + 1.0237617061648052e-05, + 9.29644829739118e-06, + 7.558589913969627e-06, + 6.930258223292185e-06, + 6.560295332747046e-06, + 6.326857146632392e-06, + 8.696530130691826e-06, + 8.112728210107889e-06, + 7.6213941611058544e-06, + 7.222243311844068e-06, + 8.756019269640092e-06, + 8.345011337951291e-06, + 8.540273483959027e-06, + 8.91101808520034e-06, + 4.009826807305217e-05, + 3.389713674550876e-05, + 2.7602531190495938e-05, + 2.2510752387461253e-05, + 2.5821915187407285e-05, + 2.2865260689286515e-05, + 1.7710155589156784e-05, + 1.2216230970807374e-05, + 1.0750263754744083e-05, + 7.64765172789339e-06, + 5.90307490710984e-06, + 5.560522822634084e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.568297117657494e-05, + "first_to_last_ratio": 0.8467277881503474, + "imbalance_abs_log_ratio": 0.16637601951453293, + "last_quartile_mean": 5.3952370308252284e-05, + "mean": 5.2044935273443116e-05, + "normalized": [ + 0.8954256553649687, + 1.0662758237422765, + 0.9521993752570108, + 0.8636540973713456, + 0.9057610370429903, + 0.8375327549345261, + 0.7860647798171227, + 0.7151678277570597, + 0.5850341337918961, + 0.5346127123576437, + 0.503507273341017, + 0.4847402912324663, + 0.6665718638058081, + 0.6225508325294531, + 0.5847778078263528, + 0.5518762696320233, + 0.6703631394314256, + 0.6385628522390465, + 0.6481962908175356, + 0.6867108224315941, + 3.0618207183884687, + 2.6065455407648237, + 2.120445260989786, + 1.7184046595544948, + 1.970749079693583, + 1.7371379145623163, + 1.3426067482179225, + 0.9360274925849964, + 0.8279776750708923, + 0.5898304420688857, + 0.45715187233946364, + 0.4317169550408049 + ], + "peak_layer": 21, + "peak_normalized": 3.0618207183884687, + "population_cv": 0.6425897668413451, + "rest_mean": 3.955807295386438e-05, + "spike_contrast": 3.0202176920627024, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011947399179916829, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.6602370275650173e-05, + 5.549425623030402e-05, + 4.95571548526641e-05, + 4.494882159633562e-05, + 4.714027454610914e-05, + 4.35893380199559e-05, + 4.091069058631547e-05, + 3.7220863305265084e-05, + 3.044806362595409e-05, + 2.7823884011013433e-05, + 2.620500345074106e-05, + 2.5228277081623673e-05, + 3.4691689506871626e-05, + 3.240061778342351e-05, + 3.0434723157668486e-05, + 2.872236473194789e-05, + 3.4889006201410666e-05, + 3.32339623128064e-05, + 3.373533400008455e-05, + 3.5739820305025205e-05, + 0.00015935226110741496, + 0.00013565749395638704, + 0.00011035843635909259, + 8.943425928009674e-05, + 0.00010256750829285011, + 9.04092303244397e-05, + 6.987588130868971e-05, + 4.8715490265749395e-05, + 4.30920445069205e-05, + 3.0697687179781497e-05, + 2.3792439606040716e-05, + 2.246868098154664e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.040512115228921175, + "first_to_last_ratio": 0.8467277882597376, + "imbalance_abs_log_ratio": 0.16637601938534127, + "last_quartile_mean": 0.047845500987023115, + "mean": 0.046153969364240766, + "normalized": [ + 0.8954256214236297, + 1.0662758259809275, + 0.9521993286286553, + 0.8636540898502394, + 0.9057610206102777, + 0.8375327282419698, + 0.7860647826245731, + 0.7151678297957529, + 0.5850341696121162, + 0.5346127266472548, + 0.5035072907230911, + 0.484740303083307, + 0.666571864231768, + 0.6225508316512409, + 0.5847778206533832, + 0.5518762470155281, + 0.6703631006740365, + 0.6385628347715187, + 0.6481963011119384, + 0.6867108353729822, + 3.0618206630891383, + 2.6065457195235173, + 2.120445297956706, + 1.7184047348201537, + 1.9707489675025585, + 1.7371379031231275, + 1.3426067663321783, + 0.9360274706240039, + 0.8279776759485932, + 0.5898304215013819, + 0.45715187640138183, + 0.4317169504730694 + ], + "peak_layer": 21, + "peak_normalized": 3.0618206630891383, + "population_cv": 0.6425897745941355, + "rest_mean": 0.03508049470406992, + "spike_contrast": 3.0202177427352903, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10595073252916336, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.041327446699142456, + 0.04921286180615425, + 0.04394777864217758, + 0.03986106440424919, + 0.041804466396570206, + 0.03865545988082886, + 0.036280009895563126, + 0.03300783410668373, + 0.027001649141311646, + 0.02467449940741062, + 0.023238860070705414, + 0.022372689098119736, + 0.03076493740081787, + 0.02873319201171398, + 0.02698981761932373, + 0.02547127939760685, + 0.030939918011426926, + 0.02947220951318741, + 0.029916832223534584, + 0.031694430857896805, + 0.14131517708301544, + 0.12030243128538132, + 0.0978669673204422, + 0.07931119948625565, + 0.0909578874707222, + 0.08017580956220627, + 0.061966631561517715, + 0.04320138320326805, + 0.03821445629000664, + 0.027223015204072, + 0.021099373698234558, + 0.019925450906157494 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8178302667593016e-06, + "first_to_last_ratio": 0.5015852279301855, + "imbalance_abs_log_ratio": 0.6899817399970322, + "last_quartile_mean": 3.6241702616734983e-06, + "mean": 2.789410469716813e-06, + "normalized": [ + 0.8037687132877044, + 0.7183737285046393, + 0.691319672118338, + 0.6807254875963729, + 0.6052756387601809, + 0.5818941580160492, + 0.565889251881305, + 0.5662715078678539, + 0.4369805018739323, + 0.41687613565271625, + 0.40783237228271835, + 0.41545507589336067, + 0.5056083509177715, + 0.5034210266522742, + 0.5059866535158252, + 0.503281516870639, + 0.5409761795262856, + 0.5584199152903219, + 0.570327765976957, + 0.6037643031707576, + 2.871505340690978, + 2.8594035701851728, + 2.496246607238993, + 2.1963141894218743, + 2.393616152828075, + 2.14842944375899, + 1.7201180299928378, + 1.2992827591669334, + 1.010592310992817, + 0.7253855724579202, + 0.5639386418465228, + 0.532719425762883 + ], + "peak_layer": 21, + "peak_normalized": 2.871505340690978, + "population_cv": 0.7722487160005334, + "rest_mean": 1.9818156126213445e-06, + "spike_contrast": 3.6080161305089784, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.150422698032344e-06, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.242040864075534e-06, + 2.0038391994603444e-06, + 1.9283743313280866e-06, + 1.8988228021044051e-06, + 1.6883622038221802e-06, + 1.6231416566370171e-06, + 1.5784974038979271e-06, + 1.5795636727489182e-06, + 1.2189179869892541e-06, + 1.162838657364773e-06, + 1.1376118891348597e-06, + 1.1588747383939335e-06, + 1.4103492276262841e-06, + 1.4042478824194404e-06, + 1.4114044688540162e-06, + 1.4038587323739193e-06, + 1.5090046190380235e-06, + 1.5576623582091997e-06, + 1.5908782415863243e-06, + 1.6841464685057872e-06, + 8.009807061171159e-06, + 7.976050255820155e-06, + 6.96305642122752e-06, + 6.126421794760972e-06, + 6.676777957181912e-06, + 5.9928515838691965e-06, + 4.798115242010681e-06, + 3.6242329315427924e-06, + 2.818956772898673e-06, + 2.0233981103956467e-06, + 1.5730563518445706e-06, + 1.4859731436445145e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.489204118523048e-05, + "first_to_last_ratio": 1.0102007473277157, + "imbalance_abs_log_ratio": 0.01014907083348142, + "last_quartile_mean": 3.45397103274081e-05, + "mean": 3.719139070312849e-05, + "normalized": [ + 0.978746778311648, + 1.126838378619792, + 1.0180419362177582, + 0.9373857091569082, + 0.9573174487244173, + 0.8927610480088312, + 0.8376961987226147, + 0.7566142687378048, + 0.6421710022561828, + 0.5898929811647327, + 0.5545314669590201, + 0.532480114188549, + 0.7191190555706168, + 0.6685869838104541, + 0.6276517153974418, + 0.5907065315366481, + 0.7404063048506205, + 0.7009976783342637, + 0.7076013069062036, + 0.7352296418703005, + 3.079471174742761, + 2.569668642671924, + 2.031249327096867, + 1.5752201563285502, + 1.8426406293315767, + 1.5558607485184148, + 1.2001391008391664, + 0.8699295985429139, + 0.6815919538112388, + 0.4881760018644189, + 0.4044368033178356, + 0.3868393135895246 + ], + "peak_layer": 21, + "peak_normalized": 3.079471174742761, + "population_cv": 0.6121845820849332, + "rest_mean": 2.8791301971338084e-05, + "spike_contrast": 2.867250322231956, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.255186985479668e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.640095383161679e-05, + 4.1908686398528516e-05, + 3.786239540204406e-05, + 3.486267814878374e-05, + 3.560396726243198e-05, + 3.320302494103089e-05, + 3.115508661721833e-05, + 2.8139536880189553e-05, + 2.3883232643129304e-05, + 2.1938940335530788e-05, + 2.0623796444851905e-05, + 1.98036759684328e-05, + 2.674503775779158e-05, + 2.4865679733920842e-05, + 2.3343240172835067e-05, + 2.196919740526937e-05, + 2.7536740162759088e-05, + 2.6071078536915593e-05, + 2.631667666719295e-05, + 2.7344212867319584e-05, + 0.00011452981561888009, + 9.55695504671894e-05, + 7.554498733952641e-05, + 5.858462827745825e-05, + 6.853036757092923e-05, + 5.786462497781031e-05, + 4.463484219741076e-05, + 3.235389158362523e-05, + 2.5349352654302493e-05, + 1.8155944417230785e-05, + 1.5041567166917957e-05, + 1.4387092051038053e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6185193746641744e-05, + "first_to_last_ratio": 1.1813045118653667, + "imbalance_abs_log_ratio": 0.16661934637193965, + "last_quartile_mean": 2.2166336862028402e-05, + "mean": 2.6416844121968097e-05, + "normalized": [ + 1.0582483794330777, + 1.1839113566742743, + 1.0890809034246702, + 1.013584100454415, + 1.0023428903115454, + 0.935888205711016, + 0.8664359614724408, + 0.7803558787188484, + 0.6948576391403528, + 0.6454578452882075, + 0.6088646607200737, + 0.5814486923097771, + 0.764934624577467, + 0.7118430529936276, + 0.6645388585723574, + 0.629085742341511, + 0.8117172258565646, + 0.7682825984070001, + 0.7655811246450389, + 0.7857710139637165, + 3.1099202343035985, + 2.532060860240954, + 1.9118190653037337, + 1.3711803024378004, + 1.6503311681253898, + 1.3613514907810353, + 1.0865408308937656, + 0.8606461743085547, + 0.5744935662233659, + 0.4301536296331042, + 0.37929000098103194, + 0.3699819217516835 + ], + "peak_layer": 21, + "peak_normalized": 3.1099202343035985, + "population_cv": 0.583027433434022, + "rest_mean": 2.096195011189492e-05, + "spike_contrast": 2.6654615376008266, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5873271776363254e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.795558248180896e-05, + 3.127520176349208e-05, + 2.8770080461981706e-05, + 2.6775693186209537e-05, + 2.6478735890123062e-05, + 2.4723212845856324e-05, + 2.2888503735885024e-05, + 2.061453960777726e-05, + 1.835594594012946e-05, + 1.7050959286279976e-05, + 1.608428283361718e-05, + 1.5360039469669573e-05, + 2.020715874095913e-05, + 1.8804646970238537e-05, + 1.755501943989657e-05, + 1.661845999478828e-05, + 2.1443007426569238e-05, + 2.0295601643738337e-05, + 2.0224237232469022e-05, + 2.0757590391440317e-05, + 8.215427806135267e-05, + 6.688905705232173e-05, + 5.050422623753548e-05, + 3.6222256312612444e-05, + 4.3596541217993945e-05, + 3.59626101271715e-05, + 2.8702979761874303e-05, + 2.2735555830877274e-05, + 1.5176306987996213e-05, + 1.136330138251651e-05, + 1.0019644832937047e-05, + 9.77375475486042e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.068558847502572e-05, + "first_to_last_ratio": 0.7914793598068353, + "imbalance_abs_log_ratio": 0.2338514773139948, + "last_quartile_mean": 0.00011457732580311131, + "mean": 0.00010754580421235005, + "normalized": [ + 0.8538738720411836, + 1.0095472825344882, + 0.8946236658075398, + 0.8160443171140711, + 0.8628188117624934, + 0.8243645351821149, + 0.7801447421092808, + 0.7044034320375209, + 0.5665887799168225, + 0.525608047514618, + 0.4958157795364288, + 0.4792984745903937, + 0.6722545548598279, + 0.6294814179611099, + 0.5936561952518512, + 0.5594680423070397, + 0.6646720414600984, + 0.6282112723775689, + 0.6412680496471301, + 0.6738175767739937, + 3.007412745002923, + 2.616093975094252, + 2.1689004360837822, + 1.8085787931443134, + 2.061415837562002, + 1.8302883103495597, + 1.4233838379602504, + 0.9902275928882596, + 0.8088632655175125, + 0.5559016015107706, + 0.43871981288186035, + 0.4142529012189394 + ], + "peak_layer": 21, + "peak_normalized": 3.007412745002923, + "population_cv": 0.6573728454605334, + "rest_mean": 8.100827242877265e-05, + "spike_contrast": 3.096578513808316, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000250848475843668, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.183055226458237e-05, + 0.00010857257439056411, + 9.621302160667256e-05, + 8.776214235695079e-05, + 9.279254300054163e-05, + 8.865694690030068e-05, + 8.390129369217902e-05, + 7.575563358841464e-05, + 6.093424599384889e-05, + 5.6526940170442685e-05, + 5.3322906751418486e-05, + 5.1546539907576516e-05, + 7.229815673781559e-05, + 6.769808533135802e-05, + 6.384523294400424e-05, + 6.016844054101966e-05, + 7.148268923629075e-05, + 6.756148650310934e-05, + 6.896568811498582e-05, + 7.246625318657607e-05, + 0.00032343462225981057, + 0.00028134993044659495, + 0.00023325614165514708, + 0.00019450506079010665, + 0.00022169662406668067, + 0.00019683982827700675, + 0.00015307895955629647, + 0.00010649482283042744, + 8.698985038790852e-05, + 5.978488479740918e-05, + 4.7182475100271404e-05, + 4.455116140889004e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13266965746879578, + 0.045500293374061584, + 0.018729008734226227, + 0.03815881907939911, + 0.02209169790148735, + 0.01640710048377514, + 0.02340921014547348, + 0.05481670796871185, + 0.037687551230192184, + 0.04149388521909714, + 0.048693906515836716, + 0.06190986558794975, + 0.04065176844596863, + 0.049332331866025925, + 0.0589165985584259, + 0.08173932880163193, + 0.043543290346860886, + 0.05228494107723236, + 0.04403071105480194, + 0.0687810555100441, + 0.030229199677705765, + 0.04699210822582245, + 0.06082639843225479, + 0.07667754590511322, + 0.050203632563352585, + 0.05539015680551529, + 0.06946423649787903, + 0.07630576938390732, + 0.07239200174808502, + 0.09628339111804962, + 0.12468792498111725, + 0.15056027472019196 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1373777851986233e-05, + "first_to_last_ratio": 1.0318498984683846, + "imbalance_abs_log_ratio": 0.03135320926034069, + "last_quartile_mean": 1.1022705791674525e-05, + "mean": 1.1832125267119409e-05, + "normalized": [ + 0.9800012201604696, + 1.1859002007221142, + 1.0527348906308314, + 0.9435939146843411, + 0.9942658078795703, + 0.9213667974504651, + 0.8458121244297144, + 0.7664247312681599, + 0.6009152433386855, + 0.5652338271700392, + 0.5379286363125223, + 0.5220099956402732, + 0.7166619388130793, + 0.6623381156132455, + 0.6162294438595727, + 0.5904010132472348, + 0.7324356475968063, + 0.6882143187892091, + 0.7092129547489621, + 0.8089361635707597, + 3.115397457088153, + 2.4321274256844663, + 1.9642577727784198, + 1.5948693957730091, + 1.9522870367789025, + 1.5037975945274742, + 1.0368577963312928, + 0.8807302172788337, + 0.6474760441659657, + 0.5016437023911682, + 0.4711656138967313, + 0.45877295737952767 + ], + "peak_layer": 21, + "peak_normalized": 3.115397457088153, + "population_cv": 0.603271891680543, + "rest_mean": 9.176935405034951e-06, + "spike_contrast": 2.851730928390011, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.617015052237548e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1595497198868543e-05, + 1.4031719729246106e-05, + 1.2456091099011246e-05, + 1.1164721399836708e-05, + 1.1764277587644756e-05, + 1.0901727364398539e-05, + 1.000775500870077e-05, + 9.068433428183198e-06, + 7.110104434104869e-06, + 6.687917448289227e-06, + 6.364839009620482e-06, + 6.176487659104168e-06, + 8.479633834213018e-06, + 7.836867553123739e-06, + 7.291303973033791e-06, + 6.985698746575508e-06, + 8.66627033246914e-06, + 8.143038030539174e-06, + 8.391496521653607e-06, + 9.571434020472225e-06, + 3.686177296913229e-05, + 2.8777236366295256e-05, + 2.3241344024427235e-05, + 1.8870694475481287e-05, + 2.309970477654133e-05, + 1.7793121514841914e-05, + 1.226823133038124e-05, + 1.0420910257380456e-05, + 7.661017662030645e-06, + 5.9355111261538696e-06, + 5.574890565185342e-06, + 5.428259100881405e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.52927956757776e-05, + "first_to_last_ratio": 1.0314200550157464, + "imbalance_abs_log_ratio": 0.03093654690539193, + "last_quartile_mean": 4.3913045374210924e-05, + "mean": 4.717359485084671e-05, + "normalized": [ + 0.9832135136979314, + 1.173164387917132, + 1.0505394497504255, + 0.9501840008496486, + 0.9944561477628384, + 0.9147691984159712, + 0.8464742625690872, + 0.7682410828301822, + 0.6067449473404415, + 0.5681699639617173, + 0.5386389915171271, + 0.5211401199814687, + 0.7167243803511351, + 0.6632392162921593, + 0.6172595012574109, + 0.588538933455825, + 0.7318717018485573, + 0.6893632656633977, + 0.7026754642712614, + 0.8079854981931587, + 3.1150142181403937, + 2.4538347722826446, + 1.966137677039432, + 1.5845641436792013, + 1.9375330190421414, + 1.4892879811658426, + 1.035008152926599, + 0.8840423036226741, + 0.6517018335432763, + 0.5065002027138268, + 0.477654085237329, + 0.4653275826797635 + ], + "peak_layer": 21, + "peak_normalized": 3.1150142181403937, + "population_cv": 0.6027267551758898, + "rest_mean": 3.659083860704278e-05, + "spike_contrast": 2.8509999371074524, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010432047856738791, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.638171594706364e-05, + 5.534238152904436e-05, + 4.9557722377358004e-05, + 4.482359508983791e-05, + 4.691207141149789e-05, + 4.315295154810883e-05, + 3.993123391410336e-05, + 3.6240693589206785e-05, + 2.862234032363631e-05, + 2.6802619686350226e-05, + 2.5409537556697614e-05, + 2.458405288052745e-05, + 3.381046553840861e-05, + 3.1287378078559414e-05, + 2.9118349630152807e-05, + 2.7763497200794518e-05, + 3.4525019145803526e-05, + 3.251974339946173e-05, + 3.3147727663163096e-05, + 3.8115580537123606e-05, + 0.00014694641868118197, + 0.00011575620737858117, + 9.274978219764307e-05, + 7.47495869291015e-05, + 9.140039765043184e-05, + 7.025506783975288e-05, + 4.8825055273482576e-05, + 4.1703453462105244e-05, + 3.074311825912446e-05, + 2.3893435354693793e-05, + 2.253266029583756e-05, + 2.1951174858259037e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.0401661042124033, + "first_to_last_ratio": 1.0314200458773342, + "imbalance_abs_log_ratio": 0.030936538045362262, + "last_quartile_mean": 0.03894252819009125, + "mean": 0.041834015981294215, + "normalized": [ + 0.9832134785727225, + 1.1731643857154097, + 1.0505394772834944, + 0.9501839314909589, + 0.9944561321819699, + 0.9147691986147233, + 0.8464742543512297, + 0.7682411398349651, + 0.6067449159473433, + 0.5681700011232179, + 0.5386389788463947, + 0.5211401213699254, + 0.7167244214988735, + 0.663239179094363, + 0.6172595023900571, + 0.5885389178924195, + 0.7318717092957638, + 0.6893632192579286, + 0.7026754233788964, + 0.8079854502108175, + 3.1150143563768617, + 2.4538347987960454, + 1.9661377249833973, + 1.5845640989337733, + 1.937532949114683, + 1.4892880148885772, + 1.0350081198083678, + 0.8840423776654689, + 0.6517018613150984, + 0.5065001920223166, + 0.4776540925208594, + 0.46532757522307666 + ], + "peak_layer": 21, + "peak_normalized": 3.1150143563768617, + "population_cv": 0.6027267719407451, + "rest_mean": 0.03244912169045872, + "spike_contrast": 2.850999975725323, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09251244515180587, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04113176837563515, + 0.04907817766070366, + 0.04394828528165817, + 0.03975000977516174, + 0.04160209372639656, + 0.03826846927404404, + 0.03541141748428345, + 0.03213861212134361, + 0.025382576510310173, + 0.02376883290708065, + 0.02253343164920807, + 0.021801384165883064, + 0.029983460903167725, + 0.027745958417654037, + 0.025822443887591362, + 0.024620946496725082, + 0.030617132782936096, + 0.028838831931352615, + 0.029395734891295433, + 0.03380127623677254, + 0.13031356036663055, + 0.10265376418828964, + 0.08225143700838089, + 0.06628867983818054, + 0.08105478435754776, + 0.06230289861559868, + 0.043298546224832535, + 0.03698304295539856, + 0.027263306081295013, + 0.02118893712759018, + 0.019982188940048218, + 0.019466521218419075 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.809132328389751e-06, + "first_to_last_ratio": 0.5997744339081645, + "imbalance_abs_log_ratio": 0.5112016379368517, + "last_quartile_mean": 3.016354526153009e-06, + "mean": 2.487152045205221e-06, + "normalized": [ + 0.890408531440174, + 0.809673961899388, + 0.7756270411999202, + 0.7664197016433177, + 0.6818557608603003, + 0.6497146124011205, + 0.6267527377763467, + 0.6186766662305586, + 0.4670749596929491, + 0.4522141597899267, + 0.449235216497964, + 0.45731293359098185, + 0.5550856819872789, + 0.5467983404610478, + 0.5410652082029204, + 0.536255547908205, + 0.6011915384635866, + 0.6092812314836655, + 0.6241321581033094, + 0.6880744209704973, + 2.915841858020288, + 2.6264204515100484, + 2.3560883070579246, + 2.052603139729255, + 2.421562802944245, + 1.9231629319277221, + 1.4565088666484294, + 1.2845058483948424, + 0.8068415180205192, + 0.6295503966151642, + 0.5969673730151487, + 0.5830960955129549 + ], + "peak_layer": 21, + "peak_normalized": 2.915841858020288, + "population_cv": 0.7106082711854734, + "rest_mean": 1.8080198363653228e-06, + "spike_contrast": 3.4039814437617255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.154465972940671e-06, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.214581400039606e-06, + 2.013782250287477e-06, + 1.9291023818368558e-06, + 1.906202328427753e-06, + 1.695878950158658e-06, + 1.6159390270331642e-06, + 1.5588293535984121e-06, + 1.5387429357360816e-06, + 1.1616864412644645e-06, + 1.1247253723922768e-06, + 1.1173162874911213e-06, + 1.1374067980796099e-06, + 1.3805824892187957e-06, + 1.3599706107925158e-06, + 1.3457114391712821e-06, + 1.3337490827325382e-06, + 1.4952547644497827e-06, + 1.5153750609897543e-06, + 1.5523115735049942e-06, + 1.7113457033701707e-06, + 7.252142040670151e-06, + 6.532306997542037e-06, + 5.859949851583224e-06, + 5.105136096972274e-06, + 6.022794877935667e-06, + 4.783198619406903e-06, + 3.6225590065441793e-06, + 3.1947613479132997e-06, + 2.0067375317012193e-06, + 1.5657875565011636e-06, + 1.484748622715415e-06, + 1.4502486465062248e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.464051883383945e-05, + "first_to_last_ratio": 1.2085017018903634, + "imbalance_abs_log_ratio": 0.18938132942715577, + "last_quartile_mean": 2.866402155632386e-05, + "mean": 3.421727353725146e-05, + "normalized": [ + 1.059950361009556, + 1.2224231923999602, + 1.1066505484293667, + 1.0166204096806029, + 1.0363170309532028, + 0.9614969179125695, + 0.8913049122896795, + 0.8041913955656627, + 0.6595104635459774, + 0.6198402043713972, + 0.5858567866753434, + 0.5655998592669403, + 0.7633893605247403, + 0.7040501232645258, + 0.6548491311439005, + 0.6226536428847976, + 0.7974262571555777, + 0.7481683838617732, + 0.7573132817097481, + 0.8440618552058439, + 3.1217920974447857, + 2.4268766121530727, + 1.8727793487823996, + 1.45522837174708, + 1.7605498171411489, + 1.332061600542984, + 0.9562588529424351, + 0.830045802534728, + 0.533682078716892, + 0.4444300159412478, + 0.423319578555294, + 0.4213017056467665 + ], + "peak_layer": 21, + "peak_normalized": 3.1217920974447857, + "population_cv": 0.5796313993689626, + "rest_mean": 2.7073180481912225e-05, + "spike_contrast": 2.6888372455803045, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.279537603608333e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.626861143857241e-05, + 4.182798875262961e-05, + 3.7866564525756985e-05, + 3.478597864159383e-05, + 3.5459943319438025e-05, + 3.28998030454386e-05, + 3.0498023988911882e-05, + 2.751723695837427e-05, + 2.2566649931832217e-05, + 2.1209241822361946e-05, + 2.0046421923325397e-05, + 1.9353285097167827e-05, + 2.6121102564502507e-05, + 2.4090675651677884e-05, + 2.2407151845982298e-05, + 2.1305510017555207e-05, + 2.7285752366879024e-05, + 2.5600282242521644e-05, + 2.5913195713656023e-05, + 2.8881495381938294e-05, + 0.00010681921412469819, + 8.30411008791998e-05, + 6.408140325220302e-05, + 4.979394725523889e-05, + 6.0241214669076726e-05, + 4.557951615424827e-05, + 3.2720570743549615e-05, + 2.84019042737782e-05, + 1.826114566938486e-05, + 1.5207183423626702e-05, + 1.4484841813100502e-05, + 1.4415795703826007e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6040052716780338e-05, + "first_to_last_ratio": 1.3601380691640932, + "imbalance_abs_log_ratio": 0.3075862160391937, + "last_quartile_mean": 1.9145153942190518e-05, + "mean": 2.476174321941471e-05, + "normalized": [ + 1.1242415044070515, + 1.2621466755459967, + 1.1626111778912407, + 1.0807522798319469, + 1.0668131611583063, + 0.9898612219145286, + 0.9055963728518246, + 0.8209725998576762, + 0.7059309419194623, + 0.6698532100359812, + 0.6332217391849956, + 0.605864404334209, + 0.7980357739095889, + 0.7372559068278083, + 0.6856394720285257, + 0.6565553942525388, + 0.8578727127097449, + 0.8064680621295278, + 0.8069038983319957, + 0.8750777012450611, + 3.1375211320361442, + 2.3884634889966896, + 1.7586693778594196, + 1.27827401077098, + 1.559779974845964, + 1.1976668654320728, + 0.9315461470291763, + 0.8389979859505794, + 0.4605019751312927, + 0.40913221789275306, + 0.3951692615572719, + 0.3926033521296457 + ], + "peak_layer": 21, + "peak_normalized": 3.1375211320361442, + "population_cv": 0.5588199044745877, + "rest_mean": 2.0063699526805e-05, + "spike_contrast": 2.4986009729926506, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.0131179159507154e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.78381794487359e-05, + 3.1252951885107905e-05, + 2.8788279450964183e-05, + 2.67613104369957e-05, + 2.6416153559694067e-05, + 2.451068939990364e-05, + 2.2424144844990224e-05, + 2.0328712707851082e-05, + 1.7480080714449286e-05, + 1.6586733181611635e-05, + 1.5679674106650054e-05, + 1.5002258805907331e-05, + 1.9760756913456134e-05, + 1.8255741451866925e-05, + 1.697762854746543e-05, + 1.6257456081802957e-05, + 2.124242382706143e-05, + 1.9969555069110356e-05, + 1.9980347133241594e-05, + 2.1668449335265905e-05, + 7.769049261696637e-05, + 5.914251960348338e-05, + 4.3547719542402774e-05, + 3.165229281876236e-05, + 3.8622871215920895e-05, + 2.96563193842303e-05, + 2.3066706489771605e-05, + 2.0775052689714357e-05, + 1.140283166023437e-05, + 1.013082692224998e-05, + 9.785079782886896e-06, + 9.721543392515741e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.001170565170469e-05, + "first_to_last_ratio": 0.9626745767731016, + "imbalance_abs_log_ratio": 0.038039850801722765, + "last_quartile_mean": 9.350169602839742e-05, + "mean": 9.76294164729552e-05, + "normalized": [ + 0.9359776868957951, + 1.1142596124196091, + 0.9861551159867753, + 0.8961521822246734, + 0.949684039356357, + 0.8995149572091795, + 0.846411402631371, + 0.747630614092772, + 0.5888085074137973, + 0.5541267695979373, + 0.5294131458395803, + 0.5184084456985816, + 0.7260183643779575, + 0.6675529417749002, + 0.6222711070136198, + 0.5862023232639773, + 0.7253041044725564, + 0.6816202250286443, + 0.6942528027900181, + 0.79595263681847, + 3.0863426882796725, + 2.474889704855176, + 2.017495070776267, + 1.693791348848989, + 2.023091547543398, + 1.5725916380980733, + 1.090726894893622, + 0.9642815025941395, + 0.6146236296489728, + 0.48585096781736253, + 0.45630461264981964, + 0.45429340908793464 + ], + "peak_layer": 21, + "peak_normalized": 3.0863426882796725, + "population_cv": 0.6173495462344238, + "rest_mean": 7.486509181304697e-05, + "spike_contrast": 2.9460562232025813, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022055676963645966, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.137895540334284e-05, + 0.00010878451575990766, + 9.627774852560833e-05, + 8.749081462156028e-05, + 9.271709859604016e-05, + 8.781912038102746e-05, + 8.263465133495629e-05, + 7.299074059119448e-05, + 5.7485030993120745e-05, + 5.409907316789031e-05, + 5.168629650142975e-05, + 5.06119140482042e-05, + 7.088074926286936e-05, + 6.517280417028815e-05, + 6.075196506571956e-05, + 5.723059075535275e-05, + 7.081101648509502e-05, + 6.654618482571095e-05, + 6.777949602110311e-05, + 7.770839147269726e-05, + 0.0003013178356923163, + 0.00024162203771993518, + 0.0001969668664969504, + 0.0001653638610150665, + 0.00019751324725802988, + 0.00015353120397776365, + 0.00010648703027982265, + 9.414224041393027e-05, + 6.0005346313118935e-05, + 4.743334648082964e-05, + 4.4548753066919744e-05, + 4.435240043676458e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.08528144657611847, + 0.3133305609226227, + 0.3510810136795044, + 0.36267346143722534, + 0.1070452407002449, + 0.1260577291250229, + 0.14502130448818207, + 0.17329032719135284, + 0.061002109199762344, + 0.0987050011754036, + 0.13416865468025208, + 0.17569874227046967, + 0.06714949756860733, + 0.09735818952322006, + 0.13312570750713348, + 0.17941679060459137, + 0.10073095560073853, + 0.13780468702316284, + 0.17965665459632874, + 0.24453195929527283, + 0.04170985147356987, + 0.07352267950773239, + 0.10462787002325058, + 0.14256280660629272, + 0.030182205140590668, + 0.06915032118558884, + 0.10093247890472412, + 0.1392749696969986, + 0.030125634744763374, + 0.08267941325902939, + 0.15948866307735443, + 0.2670789062976837 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1464836006780388e-05, + "first_to_last_ratio": 0.8455234102400129, + "imbalance_abs_log_ratio": 0.16779942298421527, + "last_quartile_mean": 1.355945425984828e-05, + "mean": 1.3058895916628899e-05, + "normalized": [ + 0.8914048885899613, + 1.0775847922014457, + 0.9537205719054163, + 0.8574486865386339, + 0.9048781790315702, + 0.8425943237574262, + 0.7839480584125434, + 0.7118847066289794, + 0.5788136508075115, + 0.5307043127450849, + 0.5023599445771596, + 0.4844905191035793, + 0.6659490427095206, + 0.6212508418929359, + 0.5836098650152179, + 0.5530587758649329, + 0.6705037956060123, + 0.6390296236121042, + 0.6539787044155942, + 0.6823628402237571, + 3.070552131396758, + 2.5957343072679153, + 2.1136920052807127, + 1.723798777803649, + 1.977372713282607, + 1.75094231645673, + 1.356176403898088, + 0.9354566389044705, + 0.8232143895880623, + 0.5856320599681389, + 0.4520406818345609, + 0.4258114506789209 + ], + "peak_layer": 21, + "peak_normalized": 3.070552131396758, + "population_cv": 0.6442860381062553, + "rest_mean": 9.924204715827256e-06, + "spike_contrast": 3.0215245714485643, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.998622840095777e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.1640763659670483e-05, + 1.4072067642700858e-05, + 1.2454537682060618e-05, + 1.119733315135818e-05, + 1.1816709957201965e-05, + 1.100335157389054e-05, + 1.0237496098852716e-05, + 9.296428288507741e-06, + 7.558667221019277e-06, + 6.9304123826441355e-06, + 6.560266228916589e-06, + 6.3269112615671474e-06, + 8.696559234522283e-06, + 8.112850082397927e-06, + 7.621300483151572e-06, + 7.22233698979835e-06, + 8.75603927852353e-06, + 8.345021342393011e-06, + 8.540239832655061e-06, + 8.91090530785732e-06, + 4.0098020690493286e-05, + 3.389742414583452e-05, + 2.7602483896771446e-05, + 2.2510908820549957e-05, + 2.5822304451139644e-05, + 2.2865373466629535e-05, + 1.7710166503093205e-05, + 1.2216030881972983e-05, + 1.0750271030701697e-05, + 7.6477081165649e-06, + 5.9031522141594905e-06, + 5.560627414524788e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.568271015159553e-05, + "first_to_last_ratio": 0.8467191676221869, + "imbalance_abs_log_ratio": 0.16638620055756023, + "last_quartile_mean": 5.3952611324348254e-05, + "mean": 5.204496915212076e-05, + "normalized": [ + 0.8954358371924881, + 1.066255627359817, + 0.9521818394578381, + 0.863653814778354, + 0.9057514302498563, + 0.8375363338827018, + 0.7860525947143857, + 0.7151691796362102, + 0.5850370033455559, + 0.5346182010583443, + 0.5035057922216662, + 0.4847392766839134, + 0.6665845013282112, + 0.6225484700608501, + 0.5847766932080282, + 0.5518779375078734, + 0.6703635418666527, + 0.6385663510046552, + 0.6481965678809122, + 0.6867100259137117, + 3.061813692446919, + 2.6065600609918396, + 2.1204217920704185, + 1.7184136066569389, + 1.970772681476859, + 1.7371423758267366, + 1.3426072722615594, + 0.9360215708260244, + 0.8279779050062409, + 0.5898318056357381, + 0.4571555590952484, + 0.4317206583534514 + ], + "peak_layer": 21, + "peak_normalized": 3.061813692446919, + "population_cv": 0.6425899958232327, + "rest_mean": 3.9558066755826415e-05, + "spike_contrast": 3.0202244925053914, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011947424209211022, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.6602930524386466e-05, + 5.549324123421684e-05, + 4.955627446179278e-05, + 4.494883614825085e-05, + 4.7139805246843025e-05, + 4.358955266070552e-05, + 4.0910083043854684e-05, + 3.722095789271407e-05, + 3.044823279196862e-05, + 2.7824187782243825e-05, + 2.6204943424090743e-05, + 2.5228240701835603e-05, + 3.469236980890855e-05, + 3.2400515920016915e-05, + 3.0434684958891012e-05, + 2.8722470233333297e-05, + 3.488904985715635e-05, + 3.3234166039619595e-05, + 3.373537037987262e-05, + 3.573980211513117e-05, + 0.00015935199917294085, + 0.0001356583379674703, + 0.00011035728675778955, + 8.943478314904496e-05, + 0.00010256880341330543, + 9.040952136274427e-05, + 6.987595406826586e-05, + 4.8715213779360056e-05, + 4.309208452468738e-05, + 3.0697778129251674e-05, + 2.379264697083272e-05, + 2.2468888346338645e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.04051188472658396, + "first_to_last_ratio": 0.846719171620572, + "imbalance_abs_log_ratio": 0.16638619583535103, + "last_quartile_mean": 0.047845715656876564, + "mean": 0.04615399963222444, + "normalized": [ + 0.8954358499233408, + 1.0662556745533454, + 0.9521818348722626, + 0.8636537656053828, + 0.9057513866007281, + 0.8375363761309916, + 0.7860525635386049, + 0.7151692172158937, + 0.5850370145184794, + 0.53461818747956, + 0.5035057498056084, + 0.48473925875942064, + 0.666584502816651, + 0.6225484458781422, + 0.5847766703673021, + 0.551877902951673, + 0.670363548905047, + 0.6385663306455772, + 0.6481965620941408, + 0.6867099814534051, + 3.0618138122732117, + 2.606560153011009, + 2.1204217916267223, + 1.7184136164632142, + 1.9707726965272079, + 1.7371424139057434, + 1.3426073387034982, + 0.9360215296259088, + 0.8279778593866045, + 0.5898318104039583, + 0.45715553160267397, + 0.43172062235469105 + ], + "peak_layer": 21, + "peak_normalized": 3.0618138122732117, + "population_cv": 0.6425900249685824, + "rest_mean": 0.03508048918512133, + "spike_contrast": 3.020224589442675, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10595095604658127, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.04132794588804245, + 0.04921196401119232, + 0.0439470000565052, + 0.03986107558012009, + 0.04180404916405678, + 0.03865565359592438, + 0.03627946972846985, + 0.033007919788360596, + 0.027001798152923584, + 0.02467476762831211, + 0.023238804191350937, + 0.02237265557050705, + 0.030765540897846222, + 0.02873310074210167, + 0.026989782229065895, + 0.02547137252986431, + 0.03093995898962021, + 0.029472390189766884, + 0.02991686388850212, + 0.03169441223144531, + 0.14131495356559753, + 0.12030317634344101, + 0.09786594659090042, + 0.07931166142225266, + 0.09095904231071472, + 0.08017607033252716, + 0.06196669861674309, + 0.04320113733410835, + 0.038214489817619324, + 0.027223097160458565, + 0.021099556237459183, + 0.019925633445382118 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8178741925112263e-06, + "first_to_last_ratio": 0.5015983374707057, + "imbalance_abs_log_ratio": 0.68995560412126, + "last_quartile_mean": 3.6241631136135766e-06, + "mean": 2.789416313930815e-06, + "normalized": [ + 0.8037911571270303, + 0.7184217017994549, + 0.6913125177981712, + 0.6806867284307945, + 0.605325927592527, + 0.5818942838314957, + 0.5658827679202331, + 0.5663181288215361, + 0.4369798716346615, + 0.41689441779300684, + 0.4078213694495809, + 0.41540374891403675, + 0.5056054983122913, + 0.5034202979673874, + 0.5059732849415142, + 0.5032640783142611, + 0.5409407291356678, + 0.5584123873096231, + 0.5703010167375573, + 0.6038059140356934, + 2.8715612743645713, + 2.85957495161325, + 2.4960596032794165, + 2.196308283631167, + 2.3935907596217825, + 2.1483375605717328, + 1.7201321959397329, + 1.2992909597305788, + 1.010652795641223, + 0.7253891064801847, + 0.563940476297644, + 0.5327062049621921 + ], + "peak_layer": 21, + "peak_normalized": 2.8715612743645713, + "population_cv": 0.7722499515786573, + "rest_mean": 1.981818833748413e-06, + "spike_contrast": 3.608020362482595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.150442706915783e-06, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.2421081666834652e-06, + 2.0039772152813384e-06, + 1.9283584151708055e-06, + 1.8987186649610521e-06, + 1.688506017671898e-06, + 1.623145408302662e-06, + 1.5784826246090233e-06, + 1.5796970274095656e-06, + 1.2189187827971182e-06, + 1.1628920901785023e-06, + 1.137583581112267e-06, + 1.1587339940888342e-06, + 1.4103442254054244e-06, + 1.4042487919141422e-06, + 1.4113701354290242e-06, + 1.4038130302651552e-06, + 1.508908894720662e-06, + 1.5576446230625152e-06, + 1.5908069599390728e-06, + 1.6842660670590703e-06, + 8.009979865164496e-06, + 7.97654502093792e-06, + 6.962549377931282e-06, + 6.126418156782165e-06, + 6.676721113763051e-06, + 5.992607839289121e-06, + 4.798164809471928e-06, + 3.6242633996153018e-06, + 2.8191313958814135e-06, + 2.023412207563524e-06, + 1.573064764670562e-06, + 1.4859393786537112e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.4892004123321385e-05, + "first_to_last_ratio": 1.010196136459353, + "imbalance_abs_log_ratio": 0.010144506514065088, + "last_quartile_mean": 3.453983129020344e-05, + "mean": 3.719143947478187e-05, + "normalized": [ + 0.9787509726025058, + 1.1268290755091765, + 1.0180314063323372, + 0.9373900555049763, + 0.9573129653471193, + 0.8927633987058169, + 0.8376905027658148, + 0.756615575252793, + 0.6421721653954596, + 0.5898966583006985, + 0.5545320603024807, + 0.5324801984524263, + 0.7191238348729887, + 0.6685886503056807, + 0.6276517726739922, + 0.5907083001617862, + 0.7404052849968692, + 0.7009964656156153, + 0.7076026287863885, + 0.7352288733491559, + 3.079466745153095, + 2.56966898996169, + 2.031234533994799, + 1.57522845930543, + 1.8426476034454873, + 1.5558698594237106, + 1.200138700827154, + 0.8699239581339129, + 0.6815952172432139, + 0.48817942111819174, + 0.40443641967892363, + 0.3868392464803102 + ], + "peak_layer": 21, + "peak_normalized": 3.079466745153095, + "population_cv": 0.6121837817284164, + "rest_mean": 2.879134468390408e-05, + "spike_contrast": 2.8672488989954323, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.255195134552196e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.6401157558429986e-05, + 4.190839536022395e-05, + 3.786205343203619e-05, + 3.486288551357575e-05, + 3.5603847209131345e-05, + 3.3203155908267945e-05, + 3.11549156322144e-05, + 2.813962237269152e-05, + 2.388330722169485e-05, + 2.193910586356651e-05, + 2.06238455575658e-05, + 1.9803705072263256e-05, + 2.6745250579551794e-05, + 2.4865774321369827e-05, + 2.334327291464433e-05, + 2.1969291992718354e-05, + 2.7536738343769684e-05, + 2.607106762297917e-05, + 2.6316760340705514e-05, + 2.7344220143277198e-05, + 0.00011452980106696486, + 9.556968871038407e-05, + 7.554453623015434e-05, + 5.85850139032118e-05, + 6.853071681689471e-05, + 5.786503970739432e-05, + 4.463488585315645e-05, + 3.23537242366001e-05, + 2.5349507268401794e-05, + 1.815609539335128e-05, + 1.5041572623886168e-05, + 1.4387108421942685e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6184012767771492e-05, + "first_to_last_ratio": 1.1812258678199312, + "imbalance_abs_log_ratio": 0.1665527702583449, + "last_quartile_mean": 2.2166812868817942e-05, + "mean": 2.6416474952384306e-05, + "normalized": [ + 1.0583395321083173, + 1.18417524025499, + 1.0887520391639638, + 1.0135132254608268, + 1.0023744568550392, + 0.9356357677270415, + 0.8663869238627898, + 0.7804236609988654, + 0.6948391179201333, + 0.645410883875558, + 0.609013089334734, + 0.5813567671594309, + 0.7649466228246298, + 0.711734220686414, + 0.6645112375219416, + 0.6291608441906736, + 0.8117379342921259, + 0.7682870001881678, + 0.7656778963007645, + 0.7856293365916892, + 3.109981047613895, + 2.532125441581352, + 1.911742908862577, + 1.3712180563471432, + 1.6504233650379605, + 1.361126344660198, + 1.0867081917704957, + 0.8610084831897756, + 0.5742988763844462, + 0.4302061202657988, + 0.3792452416796221, + 0.3700101252886388 + ], + "peak_layer": 21, + "peak_normalized": 3.109981047613895, + "population_cv": 0.5830394716350688, + "rest_mean": 2.0961481856863463e-05, + "spike_contrast": 2.6655289950267567, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.5873437668196856e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7957599741057493e-05, + 3.1281735573429614e-05, + 2.8760990971932188e-05, + 2.6773446734296158e-05, + 2.6479199732420966e-05, + 2.471619882271625e-05, + 2.2886888473294675e-05, + 2.061604209302459e-05, + 1.8355200154474005e-05, + 1.7049480447894894e-05, + 1.6087979020085186e-05, + 1.535739647806622e-05, + 2.0207193301757798e-05, + 1.880150921351742e-05, + 1.7554044461576268e-05, + 1.6620211681583896e-05, + 2.144325480912812e-05, + 2.029543429671321e-05, + 2.0226510969223455e-05, + 2.0753557691932656e-05, + 8.215473644668236e-05, + 6.688982830382884e-05, + 5.050150866736658e-05, + 3.62227474397514e-05, + 4.3598367483355105e-05, + 3.595615999074653e-05, + 2.870699972845614e-05, + 2.2744809029973112e-05, + 1.5170951883192174e-05, + 1.1364529200363904e-05, + 1.0018322427640669e-05, + 9.774363206815906e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.068014514923561e-05, + "first_to_last_ratio": 0.7913424091198598, + "imbalance_abs_log_ratio": 0.23402452356595546, + "last_quartile_mean": 0.00011459027609816985, + "mean": 0.00010754564084436424, + "normalized": [ + 0.8540109519123018, + 1.0092398375236236, + 0.8947367225413311, + 0.8160956211381821, + 0.862539829414268, + 0.8245016378814637, + 0.7800932919128317, + 0.7042081007547356, + 0.5666342926354664, + 0.5256616841881218, + 0.49571278437559146, + 0.4792103383514405, + 0.6722886591537548, + 0.6295888625242309, + 0.5934294393111791, + 0.5594389550092885, + 0.6645012760984752, + 0.6282095204823469, + 0.6412757892302924, + 0.6736811938431846, + 3.0068371075614557, + 2.616557188534586, + 2.1686834473810053, + 1.8088340274566967, + 2.0615653735311414, + 1.8302172118360363, + 1.4236627074872454, + 0.9903997896635313, + 0.8090570392266916, + 0.5559293048344423, + 0.43884492996422475, + 0.4143530842408338 + ], + "peak_layer": 21, + "peak_normalized": 3.0068371075614557, + "population_cv": 0.6573647361404942, + "rest_mean": 8.100784921057143e-05, + "spike_contrast": 3.0966099965793172, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00025084971566684543, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.184515511151403e-05, + 0.00010853934509214014, + 9.622503421269357e-05, + 8.776752656558529e-05, + 9.276239870814607e-05, + 8.867155702318996e-05, + 8.389563299715519e-05, + 7.573451148346066e-05, + 6.093904812587425e-05, + 5.653262269333936e-05, + 5.3311749070417136e-05, + 5.1536982937250286e-05, + 7.230171468108892e-05, + 6.770953768864274e-05, + 6.382074934663251e-05, + 6.016522092977539e-05, + 7.146421557990834e-05, + 6.756119546480477e-05, + 6.896641571074724e-05, + 7.245147571666166e-05, + 0.00032337222364731133, + 0.00028139931964688003, + 0.0002332324511371553, + 0.0001945322146639228, + 0.00022171236923895776, + 0.00019683188293129206, + 0.00015310871822293848, + 0.00010651318007148802, + 8.701055776327848e-05, + 5.978777335258201e-05, + 4.7195859224302694e-05, + 4.456186798051931e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16101159155368805, + 0.3334391713142395, + 0.3570481836795807, + 0.37708213925361633, + 0.11129721254110336, + 0.12943366169929504, + 0.1512180119752884, + 0.1945808082818985, + 0.07549218833446503, + 0.11765842139720917, + 0.16134388744831085, + 0.21493762731552124, + 0.08329613506793976, + 0.12312135100364685, + 0.16809259355068207, + 0.23101076483726501, + 0.11538279056549072, + 0.16158254444599152, + 0.20080390572547913, + 0.27125677466392517, + 0.0543946772813797, + 0.09335283935070038, + 0.12897421419620514, + 0.17114612460136414, + 0.06173322722315788, + 0.09693212062120438, + 0.13771499693393707, + 0.18387214839458466, + 0.07983565330505371, + 0.15526357293128967, + 0.26127612590789795, + 0.39550676941871643 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1373772622391698e-05, + "first_to_last_ratio": 1.0318485034659548, + "imbalance_abs_log_ratio": 0.03135185731624994, + "last_quartile_mean": 1.1022715625585988e-05, + "mean": 1.1832133452571725e-05, + "normalized": [ + 0.9800044623886516, + 1.1858619463319722, + 1.052769444074511, + 0.9435868051213633, + 0.9942662730454839, + 0.9213689272443336, + 0.8458140758926892, + 0.7664188972680795, + 0.6009172873544569, + 0.5652322062780974, + 0.5379371038219452, + 0.5220047919249511, + 0.7166699752090641, + 0.6623360432121947, + 0.616224251829878, + 0.5903966461838739, + 0.7324373700274538, + 0.6882103836910076, + 0.7092295284797906, + 0.8089552049077647, + 3.1154122124911208, + 2.432133583525468, + 1.964217211992137, + 1.5948629117896413, + 1.9522993684268721, + 1.5038120812358047, + 1.036848162520458, + 0.8807155414212365, + 0.6474761343085111, + 0.5016554233955102, + 0.4711561792646531, + 0.4587695653410252 + ], + "peak_layer": 21, + "peak_normalized": 3.1154122124911208, + "population_cv": 0.6032722735217898, + "rest_mean": 9.176947801110887e-06, + "spike_contrast": 2.851725490613808, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.617013597046025e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1595543583098333e-05, + 1.4031276805326343e-05, + 1.245650855707936e-05, + 1.1164645002281759e-05, + 1.1764291230065282e-05, + 1.0901760106207803e-05, + 1.0007785022025928e-05, + 9.068370673048776e-06, + 7.110133537935326e-06, + 6.687902896373998e-06, + 6.364943601511186e-06, + 6.176430360937957e-06, + 8.479734788124915e-06, + 7.836848453735001e-06, + 7.2912475843622815e-06, + 6.985651907598367e-06, + 8.66629670781549e-06, + 8.142997103277594e-06, + 8.391698429477401e-06, + 9.571665941621177e-06, + 3.686197305796668e-05, + 2.8777329134754837e-05, + 2.324088018212933e-05, + 1.8870630810852163e-05, + 2.3099866666598246e-05, + 1.7793305232771672e-05, + 1.2268125828995835e-05, + 1.0420743819850031e-05, + 7.661024028493557e-06, + 5.935653916822048e-06, + 5.574782790063182e-06, + 5.428222721093334e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.529294619715074e-05, + "first_to_last_ratio": 1.0314184626645753, + "imbalance_abs_log_ratio": 0.03093500306067748, + "last_quartile_mean": 4.391325910546584e-05, + "mean": 4.717383586694268e-05, + "normalized": [ + 0.9832121149293677, + 1.1731445898725168, + 1.0505704052806353, + 0.9501685038967237, + 0.9944626347672784, + 0.9147662213723499, + 0.8464650022543867, + 0.7682388544141363, + 0.6067490194436969, + 0.5681705700165066, + 0.53864641920515, + 0.5211305553096096, + 0.7167263481864653, + 0.6632364446817158, + 0.6172546895643319, + 0.588531029519173, + 0.7318735922946267, + 0.6893643707481226, + 0.7026758072754231, + 0.8079900073900919, + 3.115049509935762, + 2.4538624912763685, + 1.9660898437352061, + 1.5845676157597377, + 1.937541165727475, + 1.4892951790023068, + 1.0350080319051866, + 0.8840334683174624, + 0.6516976556256888, + 0.50650640646803, + 0.47764736477161285, + 0.4653240870528551 + ], + "peak_layer": 21, + "peak_normalized": 3.115049509935762, + "population_cv": 0.6027314599912318, + "rest_mean": 3.6590978736596835e-05, + "spike_contrast": 2.8510104941924466, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010432126437081024, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.638188693206757e-05, + 5.534173033083789e-05, + 4.9559435865376145e-05, + 4.482309304876253e-05, + 4.6912617108318955e-05, + 4.315303158364259e-05, + 3.9931001083459705e-05, + 3.624077362474054e-05, + 2.8622678655665368e-05, + 2.680278521438595e-05, + 2.541001776990015e-05, + 2.4583727281424217e-05, + 3.3810731110861525e-05, + 3.128740718238987e-05, + 2.9118271413608454e-05, + 2.7763266189140268e-05, + 3.4525284718256444e-05, + 3.251996167819016e-05, + 3.314791320008226e-05, + 3.811598799075e-05, + 0.00014694883429910988, + 0.00011575810640351847, + 9.27479995880276e-05, + 7.475013262592256e-05, + 9.14012489374727e-05, + 7.025576633168384e-05, + 4.882529901806265e-05, + 4.170324973529205e-05, + 3.074307824135758e-05, + 2.38938500842778e-05, + 2.2532458388013765e-05, + 2.1951122107566334e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.040166237857192755, + "first_to_last_ratio": 1.0314184457192797, + "imbalance_abs_log_ratio": 0.030934986631559365, + "last_quartile_mean": 0.038942718179896474, + "mean": 0.0418342292541638, + "normalized": [ + 0.9832121171072725, + 1.1731446022953238, + 1.0505704535174525, + 0.9501684905945749, + 0.9944626387492155, + 0.9147662270065798, + 0.8464650412985084, + 0.7682388261849044, + 0.6067490356906232, + 0.5681705774712352, + 0.5386464289336567, + 0.5211305632866429, + 0.7167263331576038, + 0.6632364212118128, + 0.6172547081682093, + 0.5885310198061827, + 0.7318735882573365, + 0.6893643353923572, + 0.7026757592562258, + 0.8079899688020947, + 3.1150494118332737, + 2.453862539114119, + 1.9660897667076618, + 1.5845675971028321, + 1.9375412374473495, + 1.4892952935873054, + 1.0350080081219681, + 0.8840335073818288, + 0.6516976484197683, + 0.5065064256973429, + 0.4776473385482515, + 0.46532408985048596 + ], + "peak_layer": 21, + "peak_normalized": 3.1150494118332737, + "population_cv": 0.6027314563638121, + "rest_mean": 0.03244924586680201, + "spike_contrast": 2.851010464947826, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09251313954591751, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.041131921112537384, + 0.0490776002407074, + 0.043949805200099945, + 0.039749566465616226, + 0.04160257801413536, + 0.03826854005455971, + 0.03541121259331703, + 0.032138679176568985, + 0.02538287825882435, + 0.02376897819340229, + 0.022533858194947243, + 0.021801095455884933, + 0.02998369373381138, + 0.027745984494686127, + 0.02582237496972084, + 0.024620741605758667, + 0.0306173674762249, + 0.028839025646448135, + 0.029395898804068565, + 0.03380163758993149, + 0.13031569123268127, + 0.10265544801950455, + 0.08224985003471375, + 0.06628916412591934, + 0.08105554431676865, + 0.06230352073907852, + 0.043298762291669846, + 0.036982860416173935, + 0.02726326882839203, + 0.021189305931329727, + 0.019982008263468742, + 0.019466474652290344 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8091637201678168e-06, + "first_to_last_ratio": 0.5997838633923696, + "imbalance_abs_log_ratio": 0.511185916342951, + "last_quartile_mean": 3.0163594431087404e-06, + "mean": 2.4871596693287756e-06, + "normalized": [ + 0.8903650291088842, + 0.8096671832364504, + 0.7756357252979299, + 0.7664205976400669, + 0.6819345308185697, + 0.6497344242040057, + 0.6267720257428505, + 0.6186826317778965, + 0.4671058445444584, + 0.45225062104576175, + 0.4492165155127133, + 0.4573049495780591, + 0.555081192151318, + 0.5467853740610928, + 0.5411086649066317, + 0.5362598920190076, + 0.601197329062199, + 0.609347333830986, + 0.6241580819811688, + 0.6880678322200046, + 2.9157464374333077, + 2.626463595143142, + 2.355941396471949, + 2.0525708846936164, + 2.4216180933419804, + 1.9231831825131547, + 1.4564932487560838, + 1.2844496192048258, + 0.8068213094456221, + 0.6295223209584038, + 0.59698327836313, + 0.5831108549347289 + ], + "peak_layer": 21, + "peak_normalized": 2.9157464374333077, + "population_cv": 0.7105915967606766, + "rest_mean": 1.8080416052894347e-06, + "spike_contrast": 3.4039024307496555, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.154397215141216e-06, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.214479991380358e-06, + 2.013771563724731e-06, + 1.9291298940515844e-06, + 1.9062104001932312e-06, + 1.6960800621745875e-06, + 1.6159932556547574e-06, + 1.558882104291115e-06, + 1.5387624898721697e-06, + 1.1617668178587337e-06, + 1.1248195050939103e-06, + 1.1172732001796248e-06, + 1.137390427174978e-06, + 1.3805755543216947e-06, + 1.3599425301435986e-06, + 1.3458236480801133e-06, + 1.3337639757082798e-06, + 1.4952737501516822e-06, + 1.5155441133174463e-06, + 1.5523808087891666e-06, + 1.711334562060074e-06, + 7.251926945173182e-06, + 6.532434326800285e-06, + 5.8596024246071465e-06, + 5.105071522848448e-06, + 6.022950856277021e-06, + 4.78326364827808e-06, + 3.622531266955775e-06, + 3.194631290170946e-06, + 2.0066934212081833e-06, + 1.5657225276299869e-06, + 1.484792733208451e-06, + 1.45028980114148e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.464059318503132e-05, + "first_to_last_ratio": 1.2085006098527633, + "imbalance_abs_log_ratio": 0.1893804257974035, + "last_quartile_mean": 2.8664108981502068e-05, + "mean": 3.421737719122575e-05, + "normalized": [ + 1.0599522534665642, + 1.2224211904533802, + 1.1066526183773722, + 1.0166167984558396, + 1.0363163370114612, + 0.9614978327769692, + 0.8912999264066335, + 0.8041906605597857, + 0.6595162801975746, + 0.6198396025381588, + 0.5858589457780455, + 0.5655951157979022, + 0.7633864100877904, + 0.7040498510939207, + 0.65484714742333, + 0.6226490455428274, + 0.7974248515656103, + 0.7481679248848441, + 0.7573152935416273, + 0.844065677484853, + 3.1218032666740525, + 2.4268896738358103, + 1.8727607046170862, + 1.4552330006230305, + 1.7605509694422596, + 1.3320662835665362, + 0.9562601026332771, + 0.8300407364235332, + 0.5336839174330044, + 0.4444323110860877, + 0.4233154787321452, + 0.421299791488687 + ], + "peak_layer": 21, + "peak_normalized": 3.1218032666740525, + "population_cv": 0.5796332934591588, + "rest_mean": 2.7073248087685056e-05, + "spike_contrast": 2.6888415499527167, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.279567435034551e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.626878606155515e-05, + 4.182804696029052e-05, + 3.786675006267615e-05, + 3.478596045169979e-05, + 3.546002699295059e-05, + 3.289993401267566e-05, + 3.049794577236753e-05, + 2.7517295166035183e-05, + 2.2566917323274538e-05, + 2.120928547810763e-05, + 2.004655652854126e-05, + 1.9353181414771825e-05, + 2.6121080736629665e-05, + 2.4090739316307008e-05, + 2.2407151845982298e-05, + 2.1305417249095626e-05, + 2.728578692767769e-05, + 2.5600344088161364e-05, + 2.591334305179771e-05, + 2.888171366066672e-05, + 0.00010681991989258677, + 8.304179937113076e-05, + 6.408095941878855e-05, + 4.9794256483437493e-05, + 6.024143658578396e-05, + 4.557981446851045e-05, + 3.272071262472309e-05, + 2.840181696228683e-05, + 1.826126390369609e-05, + 1.5207308024400845e-05, + 1.4484745406662114e-05, + 1.4415773875953164e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6039962676804862e-05, + "first_to_last_ratio": 1.3600902622782072, + "imbalance_abs_log_ratio": 0.3075510668678258, + "last_quartile_mean": 1.914576068884344e-05, + "mean": 2.476154466535263e-05, + "normalized": [ + 1.123958441324931, + 1.2626354632798173, + 1.1622790571900958, + 1.0807265666022514, + 1.0670121245383055, + 0.9899364488555507, + 0.9055408259899391, + 0.8209444362453999, + 0.7058751897560187, + 0.6697662418103106, + 0.6333373009906147, + 0.6058288961358885, + 0.7980579670309706, + 0.7373130204202144, + 0.6858704929138739, + 0.6566823825816065, + 0.8578675442054349, + 0.8063397293421843, + 0.8071587377333899, + 0.875107050101149, + 3.1369738883948477, + 2.3886515998281816, + 1.7583385106457243, + 1.2781546769066958, + 1.5599158953991201, + 1.1978164843461745, + 0.9315129197863214, + 0.8390098557963587, + 0.46047650401210477, + 0.40917670978266013, + 0.3951608602894278, + 0.39257417776443665 + ], + "peak_layer": 21, + "peak_normalized": 3.1369738883948477, + "population_cv": 0.558754889865697, + "rest_mean": 2.0064156227885246e-05, + "spike_contrast": 2.4983578506037145, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.012744222767651e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7830947146867402e-05, + 3.126480442006141e-05, + 2.87798247882165e-05, + 2.676045914995484e-05, + 2.6420868380228058e-05, + 2.4512355594197288e-05, + 2.2422589609050192e-05, + 2.0327852325863205e-05, + 1.7478560039307922e-05, + 1.6584446711931378e-05, + 1.5682409866712987e-05, + 1.5001259271230083e-05, + 1.9761147996177897e-05, + 1.8257009287481196e-05, + 1.6983212844934314e-05, + 1.6260470147244632e-05, + 2.1242125512799248e-05, + 1.996621722355485e-05, + 1.9986497136414982e-05, + 2.1669002308044583e-05, + 7.767631905153394e-05, + 5.9146703279111534e-05, + 4.353917756816372e-05, + 3.164908412145451e-05, + 3.862592711811885e-05, + 2.965978637803346e-05, + 2.306569876964204e-05, + 2.0775180018972605e-05, + 1.1402109521441162e-05, + 1.013184737530537e-05, + 9.784793292055838e-06, + 9.720743037178181e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 9.000182035379112e-05, + "first_to_last_ratio": 0.9625877483741323, + "imbalance_abs_log_ratio": 0.03813004983371345, + "last_quartile_mean": 9.349986066808924e-05, + "mean": 9.762683816916251e-05, + "normalized": [ + 0.9359774389006617, + 1.1139547805652443, + 0.9858638933617162, + 0.896277058777463, + 0.9496551618529471, + 0.8993751236921529, + 0.8464771316171845, + 0.7475897673874267, + 0.588915541149348, + 0.5540229040457487, + 0.5295015812507878, + 0.5183640792456576, + 0.7262152137199526, + 0.6676039603614504, + 0.6223324443316169, + 0.5861986137162843, + 0.7254235746253427, + 0.6817432367794596, + 0.6942948378413707, + 0.7958728955594262, + 3.0862176056110457, + 2.4751904263802063, + 2.0171676620749306, + 1.6939489172879052, + 2.0229993487511715, + 1.572447147387479, + 1.0908649591739474, + 0.9643206077313163, + 0.6147279541208738, + 0.4857837184299343, + 0.45630950884426186, + 0.45436290542568586 + ], + "peak_layer": 21, + "peak_normalized": 3.0862176056110457, + "population_cv": 0.617326253069026, + "rest_mean": 7.486342710089284e-05, + "spike_contrast": 2.946021394940767, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022054925793781876, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.137651795754209e-05, + 0.00010875188309000805, + 9.624677477404475e-05, + 8.750069537200034e-05, + 9.271183080272749e-05, + 8.780314965406433e-05, + 8.263888594228774e-05, + 7.298482523765415e-05, + 5.7493962231092155e-05, + 5.408750439528376e-05, + 5.169356518308632e-05, + 5.060624607722275e-05, + 7.089809514582157e-05, + 6.51760637992993e-05, + 6.0756348830182105e-05, + 5.72287171962671e-05, + 7.082080992404372e-05, + 6.655643664998934e-05, + 6.778180977562442e-05, + 7.769855437800288e-05, + 0.0003012976667378098, + 0.00024164501519408077, + 0.00019692970090545714, + 0.00016537487681489438, + 0.00019749903003685176, + 0.00015351304318755865, + 0.00010649769683368504, + 9.414357191417366e-05, + 6.00139464950189e-05, + 4.74255284643732e-05, + 4.454805457498878e-05, + 4.4358013838063926e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059412047266960144, + 0.08935670554637909, + 0.17747226357460022, + 0.17946027219295502, + 0.17260213196277618, + 0.12674333155155182, + 0.12412907183170319, + 0.11767174303531647, + 0.13582420349121094, + 0.10855290293693542, + 0.10545514523983002, + 0.11005345731973648, + 0.13049271702766418, + 0.09666567295789719, + 0.10358717292547226, + 0.11041814833879471, + 0.13209649920463562, + 0.10239993035793304, + 0.11153820157051086, + 0.11912636458873749, + 0.127323716878891, + 0.07468672841787338, + 0.07975661009550095, + 0.08746611326932907, + 0.09389390796422958, + 0.06575146317481995, + 0.0681111291050911, + 0.07568862289190292, + 0.08424623310565948, + 0.0719028189778328, + 0.08439797908067703, + 0.08961156010627747 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.588972493977053e-06, + "first_to_last_ratio": 1.465723686175851, + "imbalance_abs_log_ratio": 0.38234910423701196, + "last_quartile_mean": 5.1776283385152055e-06, + "mean": 8.926610817638903e-06, + "normalized": [ + 0.8001707694815087, + 1.2301759322409134, + 0.8021990606376289, + 0.7996309276786352, + 0.4189560575118035, + 0.770181199703876, + 0.9964730820564364, + 0.983425792900568, + 0.6138765442017692, + 0.5043130598626571, + 0.39700601441149197, + 0.37391212513577055, + 0.38881218172106297, + 0.7038080142822467, + 0.6284832461563316, + 0.48736568495191596, + 0.7748207724466236, + 1.0896235823984757, + 1.2330955711393075, + 1.3976034073521157, + 2.3882743999212943, + 4.5696632038041, + 2.815663441519544, + 2.192292008694111, + 1.06869084333539, + 1.167599927936825, + 0.6186038915435874, + 0.511201557417111, + 0.4679157405813128, + 0.21996309432798952, + 0.26258255269417496, + 0.3236163119534213 + ], + "peak_layer": 22, + "peak_normalized": 4.5696632038041, + "population_cv": 0.8757958168839719, + "rest_mean": 6.270255131244927e-06, + "spike_contrast": 3.711321315811602, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.3270931524166372e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 7.14281304681208e-06, + 1.0981301784340758e-05, + 7.160918812587624e-06, + 7.137994089134736e-06, + 3.739857675100211e-06, + 6.875107828818727e-06, + 8.895127393770963e-06, + 8.778659321251325e-06, + 5.479837000166299e-06, + 4.50180641564657e-06, + 3.5439181829133304e-06, + 3.3377680210833205e-06, + 3.4707750273810234e-06, + 6.2826202338328585e-06, + 5.610225343843922e-06, + 4.350523795437766e-06, + 6.916523489053361e-06, + 9.726645657792687e-06, + 1.1007364264514763e-05, + 1.2475861694838386e-05, + 2.1319196093827486e-05, + 4.079160498804413e-05, + 2.513433173589874e-05, + 1.956973756023217e-05, + 9.539787242829334e-06, + 1.0422710147395264e-05, + 5.52203619008651e-06, + 4.5632973524334375e-06, + 4.176901711616665e-06, + 1.9635249373095576e-06, + 2.3439722554030595e-06, + 2.8887968710478162e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.9898558636887174e-05, + "first_to_last_ratio": 1.4621096880940556, + "imbalance_abs_log_ratio": 0.37988038456939854, + "last_quartile_mean": 2.0448916302484577e-05, + "mean": 3.561926479278554e-05, + "normalized": [ + 0.7085385219770569, + 1.1946020815752356, + 0.7754024698062456, + 0.7950922364389753, + 0.4162410374634061, + 0.8241424849247687, + 0.9939701500428968, + 1.0071543674888268, + 0.5806040892681411, + 0.5114068129628899, + 0.4225456612843993, + 0.35601883603809836, + 0.39596565343777096, + 0.700750155357531, + 0.608338530669077, + 0.5057982656951122, + 0.7834207937068051, + 1.0599792777683275, + 1.2280812674610306, + 1.4767524648757513, + 2.409304768407655, + 4.508238395616571, + 2.902287830939038, + 2.2425871184827204, + 1.0734427327031963, + 1.1671335490266965, + 0.5965647485206973, + 0.46999739473988966, + 0.46865462397650987, + 0.22456393251833243, + 0.25925607539704565, + 0.3331636714293014 + ], + "peak_layer": 22, + "peak_normalized": 4.508238395616571, + "population_cv": 0.8793156123901573, + "rest_mean": 2.4886176578109843e-05, + "spike_contrast": 3.760237771291333, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.357794115203433e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.523762123018969e-05, + 4.255084786564112e-05, + 2.761926589300856e-05, + 2.832060090440791e-05, + 1.482619973103283e-05, + 2.9355349397519603e-05, + 3.540448597050272e-05, + 3.587409810279496e-05, + 2.0680690795416012e-05, + 1.8215934687759727e-05, + 1.5050765796331689e-05, + 1.2681129192060325e-05, + 1.4104005458648317e-05, + 2.4960205337265506e-05, + 2.1668571207555942e-05, + 1.80161623575259e-05, + 2.790487269521691e-05, + 3.775568256969564e-05, + 4.374335185275413e-05, + 5.2600837079808116e-05, + 8.581766451243311e-05, + 0.00016058013716246933, + 0.00010337735875509679, + 7.987930439412594e-05, + 3.8235240936046466e-05, + 4.157243893132545e-05, + 2.1249197743600234e-05, + 1.674096165515948e-05, + 1.6693133147782646e-05, + 7.998802175279707e-06, + 9.234510798705742e-06, + 1.1867045031976886e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.026514341006986797, + "first_to_last_ratio": 1.4621096829473716, + "imbalance_abs_log_ratio": 0.379880381049359, + "last_quartile_mean": 0.01813430368201807, + "mean": 0.03158751998853404, + "normalized": [ + 0.7085385035567081, + 1.1946021626253802, + 0.7754024540661398, + 0.7950922593029921, + 0.4162410309209195, + 0.8241424880807221, + 0.9939701540557254, + 1.0071544006568143, + 0.5806040665606002, + 0.5114068209416861, + 0.4225456864143744, + 0.3560188250781175, + 0.39596564639005655, + 0.7007501603721394, + 0.6083384888465566, + 0.5057982797995526, + 0.7834208304641355, + 1.059979249016567, + 1.2280813415682141, + 1.4767524934129506, + 2.409304781935525, + 4.508238208880784, + 2.902287717411148, + 2.2425871343428208, + 1.0734427669067585, + 1.167133566980318, + 0.5965647509522538, + 0.46999742239503073, + 0.4686546086713296, + 0.22456393938088035, + 0.25925609979218267, + 0.3331636602206165 + ], + "peak_layer": 22, + "peak_normalized": 4.508238208880784, + "population_cv": 0.8793155841158318, + "rest_mean": 0.022069310372764314, + "spike_contrast": 3.7602376563657023, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.08298585191369057, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.022380974143743515, + 0.03773451969027519, + 0.024493040516972542, + 0.025114992633461952, + 0.013148021884262562, + 0.026032617315649986, + 0.031397052109241486, + 0.03181350976228714, + 0.018339842557907104, + 0.016154073178768158, + 0.013347170315682888, + 0.01124575175344944, + 0.012507572770118713, + 0.02213495969772339, + 0.0192159041762352, + 0.015976913273334503, + 0.024746321141719818, + 0.03348211571574211, + 0.038792043924331665, + 0.04664694890379906, + 0.07610396295785904, + 0.14240406453609467, + 0.09167607128620148, + 0.07083776593208313, + 0.03390739485621452, + 0.03686685487627983, + 0.01884400099515915, + 0.01484605297446251, + 0.014803636819124222, + 0.007093417923897505, + 0.008189257234334946, + 0.010523813776671886 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.343415339112198e-06, + "first_to_last_ratio": 0.7161840718970601, + "imbalance_abs_log_ratio": 0.33381806140671977, + "last_quartile_mean": 3.272085251637691e-06, + "mean": 2.6718059498165303e-06, + "normalized": [ + 3.352170512510883, + 1.252135055595148, + 0.26582849245093404, + 0.3779079423767161, + 0.26494682358528443, + 0.3763245935115975, + 0.4665297795225812, + 0.6608799701010516, + 0.3129183029559072, + 0.27411212897478776, + 0.2550957946560799, + 0.3550642916181046, + 0.2737856172573793, + 0.47262961496261435, + 0.48685762845780345, + 0.5118283391293013, + 0.3284988072562945, + 0.6155436681481607, + 0.5758322948420541, + 0.7857703722429643, + 1.46149784943323, + 3.1539420937482103, + 2.83862849215308, + 2.483897584167724, + 1.672423695519606, + 1.9844503515036886, + 1.6695842115910071, + 1.4607872550915708, + 1.1268636557445397, + 0.7047297682619745, + 0.577264376468574, + 0.6012706361611467 + ], + "peak_layer": 1, + "peak_normalized": 3.352170512510883, + "population_cv": 0.8823003320995952, + "rest_mean": 2.0176697064387014e-06, + "spike_contrast": 3.074904502088927, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.204141664056806e-06, + "top_five_layers": [ + 1, + 22, + 23, + 24, + 26 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.956349120126106e-06, + 3.3454618915129686e-06, + 7.102421477611642e-07, + 1.0096966889250325e-06, + 7.078864996401535e-07, + 1.0054662880065734e-06, + 1.2464770406950265e-06, + 1.7657430362305604e-06, + 8.360569836440845e-07, + 7.323744171117141e-07, + 6.815664619352901e-07, + 9.486628869126434e-07, + 7.315020411624573e-07, + 1.2627746173166088e-06, + 1.300789108427125e-06, + 1.36750600177038e-06, + 8.776850677350012e-07, + 1.6446132349301479e-06, + 1.5385121514555067e-06, + 2.099425955748302e-06, + 3.904838649759768e-06, + 8.426721251453273e-06, + 7.5842644946533255e-06, + 6.636492344114231e-06, + 4.468391580303432e-06, + 5.3020662562630605e-06, + 4.4608050302485935e-06, + 3.902940079569817e-06, + 3.0107610200502677e-06, + 1.8829011878551682e-06, + 1.5423383956658654e-06, + 1.6064784631453222e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.9795748357864795e-05, + "first_to_last_ratio": 1.5829781482320753, + "imbalance_abs_log_ratio": 0.4593079767809046, + "last_quartile_mean": 1.2505383210736909e-05, + "mean": 2.345933930314459e-05, + "normalized": [ + 0.6329672888423138, + 1.1769775978635326, + 0.8122053642936898, + 0.8103566266208889, + 0.46133266366701603, + 0.8656603467447103, + 1.0301469021057994, + 0.9610116498331323, + 0.61408958600413, + 0.5510707221672605, + 0.4548082316962977, + 0.3885266091083219, + 0.4407469545604743, + 0.7576792724096253, + 0.6251552614433971, + 0.5326228510318479, + 0.8273444966654151, + 1.114509292339196, + 1.2367312992724153, + 1.4148882628934067, + 2.3782100012844283, + 4.280445457492481, + 3.0878808679778498, + 2.2801019909661657, + 1.0813850741694784, + 1.1060747126282833, + 0.5844412396792442, + 0.4054991266359997, + 0.4197995696214123, + 0.21544640490341943, + 0.19403464509867754, + 0.2578496299796893 + ], + "peak_layer": 22, + "peak_normalized": 4.280445457492481, + "population_cv": 0.8646936695537649, + "rest_mean": 1.6414566272470876e-05, + "spike_contrast": 3.746740099488776, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.150111366878263e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4848994396743365e-05, + 2.7611116820480675e-05, + 1.9053801224799827e-05, + 1.9010431060451083e-05, + 1.0822559488588013e-05, + 2.0307819795561954e-05, + 2.416656570858322e-05, + 2.2544698367710225e-05, + 1.4406135960598476e-05, + 1.2927755051350687e-05, + 1.0669500625226647e-05, + 9.11457755137235e-06, + 1.0339632353861816e-05, + 1.7774655134417117e-05, + 1.4665729395346716e-05, + 1.2494980182964355e-05, + 1.9408955267863348e-05, + 2.6145651645492762e-05, + 2.9012899176450446e-05, + 3.319234383525327e-05, + 5.5791235354263335e-05, + 0.00010041642235592008, + 7.2439645009581e-05, + 5.34896862518508e-05, + 2.5368579372297972e-05, + 2.5947781978175044e-05, + 1.371060534438584e-05, + 9.512741598882712e-06, + 9.84822054306278e-06, + 5.05423031427199e-06, + 4.551924575935118e-06, + 6.0489819588838145e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.3026621559220075e-05, + "first_to_last_ratio": 1.6815341811376112, + "imbalance_abs_log_ratio": 0.5197065797570352, + "last_quartile_mean": 7.74686694171578e-06, + "mean": 1.525919709166601e-05, + "normalized": [ + 0.5774672585272599, + 1.1989902313733303, + 0.8415468606033967, + 0.8210487753497788, + 0.5264205831559432, + 0.9142648476833113, + 1.0733362714129566, + 0.8764439105095523, + 0.6569979339342769, + 0.6167902521881127, + 0.4964350034575567, + 0.41696619601743, + 0.504954663560933, + 0.8317766688879529, + 0.6542358689500138, + 0.5551252336862755, + 0.852113468285689, + 1.140364193630489, + 1.2382881917527824, + 1.3141660941820785, + 2.300727833390314, + 4.0533116788495525, + 3.2452755011871197, + 2.231471710715914, + 1.0423588969466941, + 1.097025384808554, + 0.5832116217690613, + 0.3693409741830814, + 0.3982639506890586, + 0.21137018039235425, + 0.16048903759429983, + 0.19942072232487634 + ], + "peak_layer": 22, + "peak_normalized": 4.0533116788495525, + "population_cv": 0.8414682188802379, + "rest_mean": 1.0809645952273664e-05, + "spike_contrast": 3.6344181315319797, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.9286773244384676e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.811686711851507e-06, + 1.8295628251507878e-05, + 1.2841329407820012e-05, + 1.2528545084933285e-05, + 8.032755431486294e-06, + 1.3950947504781652e-05, + 1.6378249711124226e-05, + 1.3373830370255746e-05, + 1.0025260962720495e-05, + 9.411724022356793e-06, + 7.575199560960755e-06, + 6.362569365592208e-06, + 7.705202733632177e-06, + 1.2692244126810692e-05, + 9.983114068745635e-06, + 8.47076535137603e-06, + 1.3002567357034422e-05, + 1.7401041986886412e-05, + 1.889528357423842e-05, + 2.0053119442309253e-05, + 3.510725946398452e-05, + 6.185028178151697e-05, + 4.952029848936945e-05, + 3.4050466638291255e-05, + 1.5905559848761186e-05, + 1.6739726561354473e-05, + 8.899341082724277e-06, + 5.635846719087567e-06, + 6.077188118069898e-06, + 3.225339241907932e-06, + 2.448933855703217e-06, + 3.043000106117688e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 5.830217355651257e-05, + "first_to_last_ratio": 1.5342432776570591, + "imbalance_abs_log_ratio": 0.42803728076015524, + "last_quartile_mean": 3.8000605513843766e-05, + "mean": 7.032809671159157e-05, + "normalized": [ + 0.6819589535091998, + 1.1712319271696168, + 0.7869167841712673, + 0.779993936454377, + 0.40965657804504296, + 0.8112730725890709, + 0.9714853978881673, + 1.0195039829060222, + 0.5575391424076114, + 0.48579339303929814, + 0.4244288891192081, + 0.3806909807485322, + 0.3969007539297048, + 0.6868963514684149, + 0.5698714639132343, + 0.5145191503639089, + 0.8281057882616928, + 1.0694789461698069, + 1.2364533982124963, + 1.5208781026521505, + 2.5059295507917216, + 4.611853308457518, + 2.9277607037675337, + 2.3282138476317833, + 1.1223656127309714, + 1.0704287879823866, + 0.5800732884744664, + 0.412100266213389, + 0.4197925520111647, + 0.21365215237778729, + 0.20903494194495364, + 0.2952179945975007 + ], + "peak_layer": 22, + "peak_normalized": 4.611853308457518, + "population_cv": 0.9090106345148538, + "rest_mean": 4.819786850189471e-05, + "spike_contrast": 3.9385834881161212, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001898313290439546, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.796087523573078e-05, + 8.237051224568859e-05, + 5.534235970117152e-05, + 5.4855488997418433e-05, + 2.881036743929144e-05, + 5.705529110855423e-05, + 6.832271901657805e-05, + 7.169977470766753e-05, + 3.921066672774032e-05, + 3.4164924727519974e-05, + 2.984927596116904e-05, + 2.6773272111313418e-05, + 2.7913274607271887e-05, + 4.830811303691007e-05, + 4.007797542726621e-05, + 3.6185152566758916e-05, + 5.82391039642971e-05, + 7.521441875724122e-05, + 8.695741416886449e-05, + 0.00010696046228986233, + 0.0001762372558005154, + 0.00032434286549687386, + 0.00020590383792296052, + 0.00016373884864151478, + 7.893383735790849e-05, + 7.528121932409704e-05, + 4.0795450331643224e-05, + 2.8982227377127856e-05, + 2.9523211196647026e-05, + 1.5025749235064723e-05, + 1.4701029613206629e-05, + 2.0762119675055146e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.056020382791757584, + 0.0742802768945694, + 0.1285802572965622, + 0.10598119348287582, + 0.10133238881826401, + 0.10985831171274185, + 0.12611564993858337, + 0.1125042662024498, + 0.07848236709833145, + 0.0899009183049202, + 0.09790628403425217, + 0.09713682532310486, + 0.09147685021162033, + 0.10080859810113907, + 0.09896469116210938, + 0.10007023066282272, + 0.10974904894828796, + 0.10291008651256561, + 0.10674569010734558, + 0.12148815393447876, + 0.055056143552064896, + 0.05981758236885071, + 0.07101839780807495, + 0.08339047431945801, + 0.06143829599022865, + 0.06018613278865814, + 0.06816941499710083, + 0.08121132105588913, + 0.05916989967226982, + 0.07458337396383286, + 0.08333029597997665, + 0.09104464948177338 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.630958900657788e-06, + "first_to_last_ratio": 0.20421556767006768, + "imbalance_abs_log_ratio": 1.5885791387918327, + "last_quartile_mean": 1.778002990704408e-05, + "mean": 1.0223539980813712e-05, + "normalized": [ + 0.8429681862296823, + 0.35291594683302396, + 0.22484209501334745, + 0.36790271245902445, + 0.21868611610463093, + 0.18545641795638437, + 0.23387248761479024, + 0.41460977766546225, + 0.4054488572719161, + 0.3330083345972086, + 0.3085788428775866, + 0.3192217174198833, + 0.3816324391793382, + 0.4112131193630629, + 0.4344687728023616, + 0.46821459950254946, + 0.37185344111151275, + 0.4758162142971942, + 0.41846445085124506, + 0.6781708994429974, + 2.4552099328163637, + 2.8496156807280086, + 2.603123697672816, + 2.3316927886492507, + 2.616234924154862, + 2.65437990886044, + 2.4006671523655148, + 1.672837600494882, + 1.585060117848551, + 1.2276893902450918, + 0.8940161400914827, + 0.8621272374795348 + ], + "peak_layer": 22, + "peak_normalized": 2.8496156807280086, + "population_cv": 0.8996576825534632, + "rest_mean": 7.248915060834541e-06, + "spike_contrast": 3.626268804103626, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.6286514548701233e-05, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.618118954473175e-06, + 3.6080502923141466e-06, + 2.298682147738873e-06, + 3.7612680898746476e-06, + 2.2357462512445636e-06, + 1.8960211036755936e-06, + 2.3910047275421675e-06, + 4.238779638399137e-06, + 4.145122602494666e-06, + 3.4045240226987517e-06, + 3.1547681373922387e-06, + 3.263575990786194e-06, + 3.9016344999254216e-06, + 4.204053766443394e-06, + 4.441808869160013e-06, + 4.786810677614994e-06, + 3.801658522206708e-06, + 4.864526090386789e-06, + 4.278188043826958e-06, + 6.933307304279879e-06, + 2.5100936909439042e-05, + 2.9133159841876477e-05, + 2.661313919816166e-05, + 2.383815444773063e-05, + 2.674718234629836e-05, + 2.7137159122503363e-05, + 2.4543316612835042e-05, + 1.71023220900679e-05, + 1.6204925486817956e-05, + 1.2551331565191504e-05, + 9.140009751718026e-06, + 8.813992280920502e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.470679973181177e-05, + "first_to_last_ratio": 0.2069929445185519, + "imbalance_abs_log_ratio": 1.5750705707490962, + "last_quartile_mean": 7.104976339178393e-05, + "mean": 4.0840292868438155e-05, + "normalized": [ + 0.8622316727455452, + 0.3538749471728574, + 0.22575416255506364, + 0.3708511987720451, + 0.22255638964420116, + 0.1850393452789397, + 0.23623438814969275, + 0.4242990843945505, + 0.41091535564437, + 0.3361491695289819, + 0.31238992658037873, + 0.3212003149521768, + 0.3849834068983422, + 0.4107834308653127, + 0.4382023170687432, + 0.4728175618985744, + 0.3691518544033243, + 0.4681213159064821, + 0.4170992962583633, + 0.6272493229024287, + 2.4438193245013227, + 2.8483751664462695, + 2.6052355716893176, + 2.3350839107599644, + 2.618275725913621, + 2.653345803911492, + 2.3878946201876037, + 1.6698710725380324, + 1.59563349859018, + 1.224673847138525, + 0.9002294831629131, + 0.8676575135403847 + ], + "peak_layer": 22, + "peak_normalized": 2.8483751664462695, + "population_cv": 0.8985289334793173, + "rest_mean": 2.8965161365906373e-05, + "spike_contrast": 3.6238708169484144, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00010496600298210978, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.521379403537139e-05, + 1.4452356481342576e-05, + 9.219866115017794e-06, + 1.5145671568461694e-05, + 9.089268132811412e-06, + 7.557061053375946e-06, + 9.647881597629748e-06, + 1.73284988704836e-05, + 1.678190346865449e-05, + 1.3728430531045888e-05, + 1.2758096090692561e-05, + 1.3117914932081476e-05, + 1.572283508721739e-05, + 1.6776515622041188e-05, + 1.7896310964715667e-05, + 1.9310007701278664e-05, + 1.5076269846758805e-05, + 1.9118211639579386e-05, + 1.703445741441101e-05, + 2.5617046048864722e-05, + 9.980629693018273e-05, + 0.00011632847599685192, + 0.00010639858373906463, + 9.536551078781486e-05, + 0.00010693114745663479, + 0.0001083634197129868, + 9.752231562742963e-05, + 6.819802365498617e-05, + 6.516613939311355e-05, + 5.0016038585454226e-05, + 3.6765635741176084e-05, + 3.5435386962490156e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.013042137026786804, + "first_to_last_ratio": 0.20699294542918592, + "imbalance_abs_log_ratio": 1.5750705663497482, + "last_quartile_mean": 0.06300764018669724, + "mean": 0.03621758034569211, + "normalized": [ + 0.8622316598933797, + 0.3538749471508598, + 0.22575414844580807, + 0.37085122221171446, + 0.22255637683921708, + 0.1850393384194697, + 0.236234387422702, + 0.42429908068863703, + 0.41091534861415707, + 0.3361491670478898, + 0.3123899060706743, + 0.3212003125118064, + 0.3849834050274649, + 0.41078345814992373, + 0.43820230021845763, + 0.4728175684224875, + 0.36915184353764025, + 0.46812130102476074, + 0.41709928593336826, + 0.6272492932959962, + 2.4438193692237777, + 2.8483752780000278, + 2.605235605762955, + 2.3350840258688303, + 2.6182758157645756, + 2.6533456676353473, + 2.3878945209130196, + 1.6698711077439734, + 1.5956335166208298, + 1.2246738042807082, + 0.9002294337837535, + 0.8676575034757876 + ], + "peak_layer": 22, + "peak_normalized": 2.8483752780000278, + "population_cv": 0.8985289457588168, + "rest_mean": 0.025686594381652498, + "spike_contrast": 3.623871003234085, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.09308490455150605, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.031227944418787956, + 0.012816494330763817, + 0.008176269009709358, + 0.013431333936750889, + 0.008060453459620476, + 0.006701677106320858, + 0.008555837906897068, + 0.015367086045444012, + 0.014882359653711319, + 0.012174509465694427, + 0.01131400652229786, + 0.011633098125457764, + 0.01394316740334034, + 0.014877582900226116, + 0.015870627015829086, + 0.01712430827319622, + 0.01336978655308485, + 0.016954220831394196, + 0.015106326900422573, + 0.02271745167672634, + 0.0885092243552208, + 0.10316126048564911, + 0.09435532987117767, + 0.08457109332084656, + 0.09482761472463608, + 0.09609775990247726, + 0.08648376166820526, + 0.06047869101166725, + 0.057789985090494156, + 0.044354721903800964, + 0.03260413184762001, + 0.03142445534467697 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 8.568965839117482e-07, + "first_to_last_ratio": 0.18051220552416403, + "imbalance_abs_log_ratio": 1.7119568828628857, + "last_quartile_mean": 4.747028498286454e-06, + "mean": 2.6146394631609837e-06, + "normalized": [ + 0.8589787963589357, + 0.3082289930915967, + 0.22695141403248437, + 0.38502719312209793, + 0.20717193772164363, + 0.14969905615852777, + 0.1917600510841699, + 0.2940249836979971, + 0.3501028028074662, + 0.30509132547078094, + 0.2840337285464057, + 0.28520738633381276, + 0.33814434197742116, + 0.362621645282935, + 0.4284643797973123, + 0.4847838221845296, + 0.3528575777552836, + 0.44423254880618596, + 0.4021648360838168, + 0.39838852122464097, + 2.445175567674409, + 2.9178538331950583, + 2.621993890930227, + 2.432580720612022, + 2.6636416217847234, + 2.7243600687846947, + 2.580677484419162, + 1.8055086156001192, + 1.6378764119490026, + 1.2592445178301535, + 0.939779150965119, + 0.9133727747172655 + ], + "peak_layer": 22, + "peak_normalized": 2.9178538331950583, + "population_cv": 0.9456027936919144, + "rest_mean": 1.8320637688399681e-06, + "spike_contrast": 3.7337937296941295, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.840548212494468e-06, + "top_five_layers": [ + 22, + 26, + 25, + 23, + 27 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.2459198589785956e-06, + 8.059076890276629e-07, + 5.933961233495211e-07, + 1.0067072935271426e-06, + 5.416799240265391e-07, + 3.9140905983003904e-07, + 5.013833970224368e-07, + 7.687693255320482e-07, + 9.153926043836691e-07, + 7.977038194439956e-07, + 7.426457955261867e-07, + 7.457144874933874e-07, + 8.841255407787685e-07, + 9.481248639531259e-07, + 1.1202798759768484e-06, + 1.2675349125856883e-06, + 9.225953476743598e-07, + 1.1615079529292416e-06, + 1.0515160511204158e-06, + 1.0416423492642934e-06, + 6.39325253359857e-06, + 7.629135780007346e-06, + 6.855568699393189e-06, + 6.360321549436776e-06, + 6.964462500036461e-06, + 7.123219347704435e-06, + 6.747541192453355e-06, + 4.7207540774252266e-06, + 4.2824563024623785e-06, + 3.2924704100878444e-06, + 2.4571836547693238e-06, + 2.388140501352609e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 9.912464463468496e-06, + "first_to_last_ratio": 0.22715332429513255, + "imbalance_abs_log_ratio": 1.4821300520824365, + "last_quartile_mean": 4.363776974969369e-05, + "mean": 2.6559063869058264e-05, + "normalized": [ + 0.9366358613935047, + 0.3792603305202499, + 0.18134819814305192, + 0.34760749803128715, + 0.23747281376634713, + 0.19753839084684555, + 0.24598063870488243, + 0.4599435726461178, + 0.4623870697047853, + 0.36990754631923695, + 0.34903068824582323, + 0.37085556359303906, + 0.4380455173785828, + 0.45529019266842574, + 0.48514444968944176, + 0.5201735235841253, + 0.4152101632813542, + 0.5283175891854636, + 0.46450504110947355, + 0.6293184811026621, + 2.502663692008064, + 2.9461029153808944, + 2.6595225397265994, + 2.273367886792679, + 2.65101572499286, + 2.525147631813147, + 2.234627662919172, + 1.5898700335733806, + 1.426452730852729, + 1.112160691183748, + 0.8183556047810898, + 0.7867397560609359 + ], + "peak_layer": 22, + "peak_normalized": 2.9461029153808944, + "population_cv": 0.880175635609515, + "rest_mean": 1.8657572430432385e-05, + "spike_contrast": 3.7104032636486823, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.9227117637638e-05, + "top_five_layers": [ + 22, + 23, + 25, + 26, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.4876171664800495e-05, + 1.0072799341287464e-05, + 4.816438377019949e-06, + 9.2321297415765e-06, + 6.307055627985392e-06, + 5.246434739092365e-06, + 6.533015493914718e-06, + 1.2215670722071081e-05, + 1.2280567716516089e-05, + 9.824398148339242e-06, + 9.269928341382183e-06, + 9.849576599663123e-06, + 1.1634078873612452e-05, + 1.2092081306036562e-05, + 1.2884982425021008e-05, + 1.381532183586387e-05, + 1.1027593245671596e-05, + 1.4031620594323613e-05, + 1.2336819054326043e-05, + 1.6714109733584337e-05, + 6.646840483881533e-05, + 7.824573549441993e-05, + 7.063442899379879e-05, + 6.037852290319279e-05, + 7.040849595796317e-05, + 6.70655572321266e-05, + 5.934961882303469e-05, + 4.222545976517722e-05, + 3.788524918491021e-05, + 2.953794682980515e-05, + 2.1734758774982765e-05, + 2.0895071429549716e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 6.6299761840582505e-06, + "first_to_last_ratio": 0.25432294789989096, + "imbalance_abs_log_ratio": 1.369150371144901, + "last_quartile_mean": 2.6069122895933106e-05, + "mean": 1.693132426083821e-05, + "normalized": [ + 1.009812755573757, + 0.39968507734198944, + 0.16094253648199763, + 0.32854895069647194, + 0.25477694605768825, + 0.2155789505292242, + 0.25860705350981517, + 0.5046916184590391, + 0.515329188356828, + 0.40705935830374973, + 0.38337515923971865, + 0.42269040487394594, + 0.4945745095736154, + 0.5045501286967757, + 0.5247552857381682, + 0.5730846754557868, + 0.46495437450577576, + 0.5946150769636939, + 0.5125942030699491, + 0.6193454891464502, + 2.628840032502442, + 3.0610378618268497, + 2.7106344368298605, + 2.1323334410119363, + 2.565140847337925, + 2.330500972416668, + 2.0157738095858844, + 1.503546921244241, + 1.3029013314823592, + 1.0664512040978233, + 0.7882888849242707, + 0.7449785141652996 + ], + "peak_layer": 22, + "peak_normalized": 3.0610378618268497, + "population_cv": 0.8541047749122238, + "rest_mean": 1.1853189545473907e-05, + "spike_contrast": 3.7418832756911033, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.435325172380544e-05, + "top_five_layers": [ + 22, + 23, + 21, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.7097467207349837e-05, + 6.767197646695422e-06, + 2.724970272538485e-06, + 5.562768819800112e-06, + 4.313711087888805e-06, + 3.650037115221494e-06, + 4.3785598791146185e-06, + 8.54509744385723e-06, + 8.725205589144025e-06, + 6.892053988849511e-06, + 6.4910491346381605e-06, + 7.156708306865767e-06, + 8.373801392735913e-06, + 8.54270183481276e-06, + 8.884801900421735e-06, + 9.703082469059154e-06, + 7.872293281252496e-06, + 1.0067620678455569e-05, + 8.678898666403256e-06, + 1.0486339306226e-05, + 4.45097430201713e-05, + 5.182742461329326e-05, + 4.589463060256094e-05, + 3.610322892200202e-05, + 4.34312314609997e-05, + 3.945846765418537e-05, + 3.412972000660375e-05, + 2.5457040464971215e-05, + 2.2059844923205674e-05, + 1.8056431144941598e-05, + 1.3346774721867405e-05, + 1.2613472790690139e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.929049958311225e-05, + "first_to_last_ratio": 0.19264517754015853, + "imbalance_abs_log_ratio": 1.6469052404570816, + "last_quartile_mean": 0.00015204377268673852, + "mean": 8.56582414883178e-05, + "normalized": [ + 0.8435321808032064, + 0.34370726972583865, + 0.1808664069876229, + 0.33993318208269363, + 0.2101403998193939, + 0.17675988933980996, + 0.23091552849720237, + 0.4097137534640246, + 0.40325567762898656, + 0.3332485242329118, + 0.315649429844806, + 0.31114263500046885, + 0.3744856227345994, + 0.39208745649289284, + 0.44367077395553656, + 0.4702768538142674, + 0.35548322757045653, + 0.45205464826790226, + 0.41818583974351736, + 0.6083319800365703, + 2.3949709614831773, + 2.7856906015999057, + 2.6171687947428244, + 2.3886915605377412, + 2.7341447434319206, + 2.7222211356577324, + 2.584598405216518, + 1.8271047030330585, + 1.5687480327138839, + 1.1190385826225775, + 0.8424439925404574, + 0.8017372063774949 + ], + "peak_layer": 22, + "peak_normalized": 2.7856906015999057, + "population_cv": 0.9245587442879203, + "rest_mean": 6.052970898222325e-05, + "spike_contrast": 3.6569202255083786, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00022135231702122838, + "top_five_layers": [ + 22, + 25, + 26, + 23, + 27 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.22554832464084e-05, + 2.944136031146627e-05, + 1.5492698366870172e-05, + 2.911807860073168e-05, + 1.8000257114181295e-05, + 1.5140941286517773e-05, + 1.977981810341589e-05, + 3.509535963530652e-05, + 3.4542172215878963e-05, + 2.8545482564368285e-05, + 2.7037975087296218e-05, + 2.665193096618168e-05, + 3.207777990610339e-05, + 3.358552203280851e-05, + 3.800405829679221e-05, + 4.028308831038885e-05, + 3.04500681522768e-05, + 3.8722206227248535e-05, + 3.582106364774518e-05, + 5.2108647651039064e-05, + 0.00020514900097623467, + 0.000238617358263582, + 0.0002241820766357705, + 0.00020461111853364855, + 0.00023420203069690615, + 0.00023318067542277277, + 0.00022139215434435755, + 0.0001565065758768469, + 0.00013437619782052934, + 9.58548771450296e-05, + 7.21622709534131e-05, + 6.867539923405275e-05 + ] + } + } + } + } + } + }, + "step": 4000 + }, + { + "forward_identity_gate": { + "comparisons": { + "detached_learned": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + }, + "uniform_value_backward": { + "activations_exact": true, + "logits_exact": true, + "loss_exact": true, + "mixer_summaries_exact": true + } + }, + "passed": true + }, + "modes": { + "detached_learned": { + "bits_per_byte": 1.8740129107275643, + "forward": { + "activation_sha256": { + "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" + }, + "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", + "loss_nats": 1.2989667654037476, + "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" + }, + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22886337339878082, + "group": 1, + "latest_source_mass": 0.06350618600845337, + "layer": 1, + "max_source_mass": 0.9364938139915466, + "mixer_index": 1, + "normalized_entropy": 0.3301800538435402, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9364938139915466, + "median": 0.9452826976776123, + "p05": 0.8725647330284119, + "p95": 0.9712384939193726 + }, + { + "label": "current_group_partial", + "mean": 0.06350618600845337, + "median": 0.0547172985970974, + "p05": 0.028761589899659157, + "p95": 0.12743529677391052 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.603729784488678, + "group": 1, + "latest_source_mass": 0.4133155345916748, + "layer": 2, + "max_source_mass": 0.5866844654083252, + "mixer_index": 2, + "normalized_entropy": 0.8709979661187783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5866844654083252, + "median": 0.5734395384788513, + "p05": 0.29813364148139954, + "p95": 0.8843642473220825 + }, + { + "label": "current_group_partial", + "mean": 0.4133155345916748, + "median": 0.4265604615211487, + "p05": 0.11563576757907867, + "p95": 0.7018663287162781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3967982828617096, + "group": 1, + "latest_source_mass": 0.14110636711120605, + "layer": 2, + "max_source_mass": 0.858893632888794, + "mixer_index": 3, + "normalized_entropy": 0.5724589149178446, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.858893632888794, + "median": 0.8705973029136658, + "p05": 0.7599387168884277, + "p95": 0.9270997643470764 + }, + { + "label": "current_group_partial", + "mean": 0.14110636711120605, + "median": 0.12940272688865662, + "p05": 0.07290027290582657, + "p95": 0.2400612235069275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6852496862411499, + "group": 1, + "latest_source_mass": 0.518975019454956, + "layer": 3, + "max_source_mass": 0.518975019454956, + "mixer_index": 4, + "normalized_entropy": 0.9886063241108252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48102498054504395, + "median": 0.48102718591690063, + "p05": 0.38786420226097107, + "p95": 0.580143928527832 + }, + { + "label": "current_group_partial", + "mean": 0.518975019454956, + "median": 0.5189728140830994, + "p05": 0.41985607147216797, + "p95": 0.6121357679367065 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5890734195709229, + "group": 1, + "latest_source_mass": 0.3123423159122467, + "layer": 3, + "max_source_mass": 0.6876576542854309, + "mixer_index": 5, + "normalized_entropy": 0.8498533011344741, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6876576542854309, + "median": 0.6909762620925903, + "p05": 0.48803940415382385, + "p95": 0.8900076746940613 + }, + { + "label": "current_group_partial", + "mean": 0.3123423159122467, + "median": 0.3090237081050873, + "p05": 0.10999235510826111, + "p95": 0.5119606256484985 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6873323321342468, + "group": 1, + "latest_source_mass": 0.4831485152244568, + "layer": 4, + "max_source_mass": 0.5168514251708984, + "mixer_index": 6, + "normalized_entropy": 0.9916109470127238, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5168514251708984, + "median": 0.5219584703445435, + "p05": 0.4251725673675537, + "p95": 0.5947866439819336 + }, + { + "label": "current_group_partial", + "mean": 0.4831485152244568, + "median": 0.47804147005081177, + "p05": 0.405213326215744, + "p95": 0.5748274326324463 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.487301766872406, + "group": 1, + "latest_source_mass": 0.2060384750366211, + "layer": 4, + "max_source_mass": 0.7939615249633789, + "mixer_index": 7, + "normalized_entropy": 0.7030278424832499, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7939615249633789, + "median": 0.7953404188156128, + "p05": 0.6590759754180908, + "p95": 0.9329437017440796 + }, + { + "label": "current_group_partial", + "mean": 0.2060384750366211, + "median": 0.20465964078903198, + "p05": 0.06705628335475922, + "p95": 0.3409239649772644 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674697756767273, + "group": 2, + "latest_source_mass": 0.42952626943588257, + "layer": 5, + "max_source_mass": 0.5704737305641174, + "mixer_index": 8, + "normalized_entropy": 0.9733831077870527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5704737305641174, + "median": 0.5696132779121399, + "p05": 0.4695202708244324, + "p95": 0.6750621199607849 + }, + { + "label": "completed_group_1", + "mean": 0.42952626943588257, + "median": 0.4303867816925049, + "p05": 0.3249378800392151, + "p95": 0.5304797887802124 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9691112041473389, + "group": 2, + "latest_source_mass": 0.2681158185005188, + "layer": 5, + "max_source_mass": 0.5506307482719421, + "mixer_index": 9, + "normalized_entropy": 0.8821230329784768, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5506307482719421, + "median": 0.543993353843689, + "p05": 0.41262108087539673, + "p95": 0.7161091566085815 + }, + { + "label": "completed_group_1", + "mean": 0.18125343322753906, + "median": 0.1794438362121582, + "p05": 0.11575750261545181, + "p95": 0.25031501054763794 + }, + { + "label": "current_group_partial", + "mean": 0.2681158185005188, + "median": 0.2664111256599426, + "p05": 0.1564716249704361, + "p95": 0.3817881643772125 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0745112895965576, + "group": 2, + "latest_source_mass": 0.412794291973114, + "layer": 6, + "max_source_mass": 0.412794291973114, + "mixer_index": 10, + "normalized_entropy": 0.9780623252441762, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34262996912002563, + "median": 0.3412924110889435, + "p05": 0.29556700587272644, + "p95": 0.39125555753707886 + }, + { + "label": "completed_group_1", + "mean": 0.24457570910453796, + "median": 0.24382859468460083, + "p05": 0.22120657563209534, + "p95": 0.2734297513961792 + }, + { + "label": "current_group_partial", + "mean": 0.412794291973114, + "median": 0.4151499271392822, + "p05": 0.37735897302627563, + "p95": 0.43749260902404785 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074045658111572, + "group": 2, + "latest_source_mass": 0.3775783181190491, + "layer": 6, + "max_source_mass": 0.4399106502532959, + "mixer_index": 11, + "normalized_entropy": 0.9169791528842925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4399106502532959, + "median": 0.44856905937194824, + "p05": 0.22063353657722473, + "p95": 0.6359403133392334 + }, + { + "label": "completed_group_1", + "mean": 0.18251106142997742, + "median": 0.18562838435173035, + "p05": 0.1196163073182106, + "p95": 0.23045453429222107 + }, + { + "label": "current_group_partial", + "mean": 0.3775783181190491, + "median": 0.3675365447998047, + "p05": 0.23817461729049683, + "p95": 0.5557451844215393 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.060497522354126, + "group": 2, + "latest_source_mass": 0.4025024175643921, + "layer": 7, + "max_source_mass": 0.4025024175643921, + "mixer_index": 12, + "normalized_entropy": 0.9653064445872968, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3786153197288513, + "median": 0.3907395303249359, + "p05": 0.28039106726646423, + "p95": 0.4436405599117279 + }, + { + "label": "completed_group_1", + "mean": 0.2188822478055954, + "median": 0.21226292848587036, + "p05": 0.1885872334241867, + "p95": 0.2672030031681061 + }, + { + "label": "current_group_partial", + "mean": 0.4025024175643921, + "median": 0.39815032482147217, + "p05": 0.3533596992492676, + "p95": 0.460100919008255 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9941019415855408, + "group": 2, + "latest_source_mass": 0.4801652431488037, + "layer": 7, + "max_source_mass": 0.4801652431488037, + "mixer_index": 13, + "normalized_entropy": 0.9048705824970601, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32296955585479736, + "median": 0.2803739905357361, + "p05": 0.1784922033548355, + "p95": 0.6440245509147644 + }, + { + "label": "completed_group_1", + "mean": 0.19686520099639893, + "median": 0.2053825855255127, + "p05": 0.09813719242811203, + "p95": 0.2504696547985077 + }, + { + "label": "current_group_partial", + "mean": 0.4801652431488037, + "median": 0.5057140588760376, + "p05": 0.2529003322124481, + "p95": 0.598870575428009 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0503522157669067, + "group": 2, + "latest_source_mass": 0.4001518487930298, + "layer": 8, + "max_source_mass": 0.40130966901779175, + "mixer_index": 14, + "normalized_entropy": 0.9560717885654542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40130966901779175, + "median": 0.396759569644928, + "p05": 0.3479611873626709, + "p95": 0.4733521044254303 + }, + { + "label": "completed_group_1", + "mean": 0.19853851199150085, + "median": 0.19769755005836487, + "p05": 0.17136667668819427, + "p95": 0.22788912057876587 + }, + { + "label": "current_group_partial", + "mean": 0.4001518487930298, + "median": 0.40291696786880493, + "p05": 0.3506356179714203, + "p95": 0.43941178917884827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0129402875900269, + "group": 2, + "latest_source_mass": 0.42161279916763306, + "layer": 8, + "max_source_mass": 0.42161279916763306, + "mixer_index": 15, + "normalized_entropy": 0.9220179839951123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4093267023563385, + "median": 0.4046899676322937, + "p05": 0.2895161807537079, + "p95": 0.5762131810188293 + }, + { + "label": "completed_group_1", + "mean": 0.16906049847602844, + "median": 0.16888955235481262, + "p05": 0.11233551055192947, + "p95": 0.22477403283119202 + }, + { + "label": "current_group_partial", + "mean": 0.42161279916763306, + "median": 0.4227708578109741, + "p05": 0.3131049573421478, + "p95": 0.519075334072113 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0774019956588745, + "group": 3, + "latest_source_mass": 0.3560434877872467, + "layer": 9, + "max_source_mass": 0.38510820269584656, + "mixer_index": 16, + "normalized_entropy": 0.9806935592947451, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38510820269584656, + "median": 0.3892287313938141, + "p05": 0.2934730052947998, + "p95": 0.4718902111053467 + }, + { + "label": "completed_group_1", + "mean": 0.25884830951690674, + "median": 0.25734007358551025, + "p05": 0.22217319905757904, + "p95": 0.29551801085472107 + }, + { + "label": "completed_group_2", + "mean": 0.3560434877872467, + "median": 0.35412120819091797, + "p05": 0.29444968700408936, + "p95": 0.41908568143844604 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2740377187728882, + "group": 3, + "latest_source_mass": 0.3633861243724823, + "layer": 9, + "max_source_mass": 0.3633861243724823, + "mixer_index": 17, + "normalized_entropy": 0.9190239493895668, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29550763964653015, + "median": 0.30360478162765503, + "p05": 0.16904757916927338, + "p95": 0.45146965980529785 + }, + { + "label": "completed_group_1", + "mean": 0.09541244804859161, + "median": 0.09486871212720871, + "p05": 0.061360105872154236, + "p95": 0.13037073612213135 + }, + { + "label": "completed_group_2", + "mean": 0.24569380283355713, + "median": 0.24333015084266663, + "p05": 0.1698218435049057, + "p95": 0.31589236855506897 + }, + { + "label": "current_group_partial", + "mean": 0.3633861243724823, + "median": 0.36472851037979126, + "p05": 0.2795379161834717, + "p95": 0.4399876892566681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.360302448272705, + "group": 3, + "latest_source_mass": 0.28569501638412476, + "layer": 10, + "max_source_mass": 0.3109527826309204, + "mixer_index": 18, + "normalized_entropy": 0.9812507981160736, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3109527826309204, + "median": 0.32172977924346924, + "p05": 0.23520725965499878, + "p95": 0.37052470445632935 + }, + { + "label": "completed_group_1", + "mean": 0.20984819531440735, + "median": 0.20958667993545532, + "p05": 0.1891367882490158, + "p95": 0.23184512555599213 + }, + { + "label": "completed_group_2", + "mean": 0.19350400567054749, + "median": 0.19153150916099548, + "p05": 0.16607457399368286, + "p95": 0.22877082228660583 + }, + { + "label": "current_group_partial", + "mean": 0.28569501638412476, + "median": 0.28060320019721985, + "p05": 0.23824527859687805, + "p95": 0.3439165949821472 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2722704410552979, + "group": 3, + "latest_source_mass": 0.29912811517715454, + "layer": 10, + "max_source_mass": 0.35036465525627136, + "mixer_index": 19, + "normalized_entropy": 0.9177491279900463, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35036465525627136, + "median": 0.36210548877716064, + "p05": 0.12807878851890564, + "p95": 0.5797606706619263 + }, + { + "label": "completed_group_1", + "mean": 0.12631990015506744, + "median": 0.1264108121395111, + "p05": 0.06977735459804535, + "p95": 0.1755298376083374 + }, + { + "label": "completed_group_2", + "mean": 0.22418735921382904, + "median": 0.22563719749450684, + "p05": 0.1176326721906662, + "p95": 0.32001155614852905 + }, + { + "label": "current_group_partial", + "mean": 0.29912811517715454, + "median": 0.28589946031570435, + "p05": 0.193109393119812, + "p95": 0.44119665026664734 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3558036088943481, + "group": 3, + "latest_source_mass": 0.314197838306427, + "layer": 11, + "max_source_mass": 0.314197838306427, + "mixer_index": 20, + "normalized_entropy": 0.978005571485618, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298976868391037, + "median": 0.2961939573287964, + "p05": 0.24950163066387177, + "p95": 0.35835936665534973 + }, + { + "label": "completed_group_1", + "mean": 0.1941717267036438, + "median": 0.1918032318353653, + "p05": 0.16809208691120148, + "p95": 0.22895419597625732 + }, + { + "label": "completed_group_2", + "mean": 0.1926535815000534, + "median": 0.19306811690330505, + "p05": 0.16983944177627563, + "p95": 0.2155935913324356 + }, + { + "label": "current_group_partial", + "mean": 0.314197838306427, + "median": 0.31541913747787476, + "p05": 0.2673022747039795, + "p95": 0.35941022634506226 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2696614265441895, + "group": 3, + "latest_source_mass": 0.2862822115421295, + "layer": 11, + "max_source_mass": 0.3667730689048767, + "mixer_index": 21, + "normalized_entropy": 0.9158671218416545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3667730689048767, + "median": 0.4067659378051758, + "p05": 0.10593366622924805, + "p95": 0.5672274827957153 + }, + { + "label": "completed_group_1", + "mean": 0.15310388803482056, + "median": 0.15050218999385834, + "p05": 0.08651046454906464, + "p95": 0.22459912300109863 + }, + { + "label": "completed_group_2", + "mean": 0.19384080171585083, + "median": 0.18732509016990662, + "p05": 0.1267009675502777, + "p95": 0.281107634305954 + }, + { + "label": "current_group_partial", + "mean": 0.2862822115421295, + "median": 0.26271265745162964, + "p05": 0.17024393379688263, + "p95": 0.475044310092926 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3606936931610107, + "group": 3, + "latest_source_mass": 0.3028184771537781, + "layer": 12, + "max_source_mass": 0.3028184771537781, + "mixer_index": 22, + "normalized_entropy": 0.9815330216461395, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298227459192276, + "median": 0.2954302132129669, + "p05": 0.25287362933158875, + "p95": 0.35405808687210083 + }, + { + "label": "completed_group_1", + "mean": 0.18611131608486176, + "median": 0.1850002110004425, + "p05": 0.15901757776737213, + "p95": 0.21725836396217346 + }, + { + "label": "completed_group_2", + "mean": 0.21284276247024536, + "median": 0.21402311325073242, + "p05": 0.18819917738437653, + "p95": 0.23418807983398438 + }, + { + "label": "current_group_partial", + "mean": 0.3028184771537781, + "median": 0.3041122555732727, + "p05": 0.25661808252334595, + "p95": 0.34458184242248535 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.269316554069519, + "group": 3, + "latest_source_mass": 0.2653314769268036, + "layer": 12, + "max_source_mass": 0.4081852436065674, + "mixer_index": 23, + "normalized_entropy": 0.9156183489371815, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4081852436065674, + "median": 0.4254342317581177, + "p05": 0.20765496790409088, + "p95": 0.587238073348999 + }, + { + "label": "completed_group_1", + "mean": 0.14420127868652344, + "median": 0.1438048928976059, + "p05": 0.1064920499920845, + "p95": 0.1871720254421234 + }, + { + "label": "completed_group_2", + "mean": 0.18228203058242798, + "median": 0.17507661879062653, + "p05": 0.12183455377817154, + "p95": 0.26505547761917114 + }, + { + "label": "current_group_partial", + "mean": 0.2653314769268036, + "median": 0.24924281239509583, + "p05": 0.15609969198703766, + "p95": 0.4061311185359955 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3636136054992676, + "group": 4, + "latest_source_mass": 0.3136237859725952, + "layer": 13, + "max_source_mass": 0.3136237859725952, + "mixer_index": 24, + "normalized_entropy": 0.9836392931712563, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27427661418914795, + "median": 0.27403637766838074, + "p05": 0.22928935289382935, + "p95": 0.319773405790329 + }, + { + "label": "completed_group_1", + "mean": 0.21341991424560547, + "median": 0.20697325468063354, + "p05": 0.18552586436271667, + "p95": 0.26688331365585327 + }, + { + "label": "completed_group_2", + "mean": 0.19867971539497375, + "median": 0.197545126080513, + "p05": 0.1794913411140442, + "p95": 0.222100168466568 + }, + { + "label": "completed_group_3", + "mean": 0.3136237859725952, + "median": 0.31875866651535034, + "p05": 0.24156072735786438, + "p95": 0.3695686459541321 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.570773959159851, + "group": 4, + "latest_source_mass": 0.2294243425130844, + "layer": 13, + "max_source_mass": 0.2658740282058716, + "mixer_index": 25, + "normalized_entropy": 0.9759767351225285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2658740282058716, + "median": 0.2747163474559784, + "p05": 0.17927880585193634, + "p95": 0.3512483835220337 + }, + { + "label": "completed_group_1", + "mean": 0.13419361412525177, + "median": 0.13374894857406616, + "p05": 0.11072897166013718, + "p95": 0.1586270034313202 + }, + { + "label": "completed_group_2", + "mean": 0.1907704919576645, + "median": 0.19074805080890656, + "p05": 0.14699944853782654, + "p95": 0.23316305875778198 + }, + { + "label": "completed_group_3", + "mean": 0.17973749339580536, + "median": 0.17616114020347595, + "p05": 0.1363743096590042, + "p95": 0.23202598094940186 + }, + { + "label": "current_group_partial", + "mean": 0.2294243425130844, + "median": 0.22995705902576447, + "p05": 0.17975632846355438, + "p95": 0.2789157032966614 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5560038089752197, + "group": 4, + "latest_source_mass": 0.32468360662460327, + "layer": 14, + "max_source_mass": 0.32468360662460327, + "mixer_index": 26, + "normalized_entropy": 0.9667995248241249, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20477107167243958, + "median": 0.206403911113739, + "p05": 0.16619175672531128, + "p95": 0.2431035190820694 + }, + { + "label": "completed_group_1", + "mean": 0.15721669793128967, + "median": 0.15658551454544067, + "p05": 0.13878680765628815, + "p95": 0.17778633534908295 + }, + { + "label": "completed_group_2", + "mean": 0.13811248540878296, + "median": 0.13743364810943604, + "p05": 0.12352626770734787, + "p95": 0.15553249418735504 + }, + { + "label": "completed_group_3", + "mean": 0.1752161681652069, + "median": 0.17309701442718506, + "p05": 0.15223677456378937, + "p95": 0.20474471151828766 + }, + { + "label": "current_group_partial", + "mean": 0.32468360662460327, + "median": 0.32497870922088623, + "p05": 0.2946181893348694, + "p95": 0.35203880071640015 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5659973621368408, + "group": 4, + "latest_source_mass": 0.26703086495399475, + "layer": 14, + "max_source_mass": 0.26703086495399475, + "mixer_index": 27, + "normalized_entropy": 0.9730088685238187, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20531430840492249, + "median": 0.20229148864746094, + "p05": 0.10698921978473663, + "p95": 0.3393132984638214 + }, + { + "label": "completed_group_1", + "mean": 0.15986156463623047, + "median": 0.15952220559120178, + "p05": 0.11969029903411865, + "p95": 0.20180393755435944 + }, + { + "label": "completed_group_2", + "mean": 0.21557100117206573, + "median": 0.21654027700424194, + "p05": 0.1673085242509842, + "p95": 0.2610440254211426 + }, + { + "label": "completed_group_3", + "mean": 0.15222224593162537, + "median": 0.1510905921459198, + "p05": 0.11800260841846466, + "p95": 0.18893754482269287 + }, + { + "label": "current_group_partial", + "mean": 0.26703086495399475, + "median": 0.258539617061615, + "p05": 0.19330690801143646, + "p95": 0.36521556973457336 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5552496910095215, + "group": 4, + "latest_source_mass": 0.31774693727493286, + "layer": 15, + "max_source_mass": 0.31774693727493286, + "mixer_index": 28, + "normalized_entropy": 0.9663309649872576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2162160575389862, + "median": 0.2145705670118332, + "p05": 0.17727568745613098, + "p95": 0.26878347992897034 + }, + { + "label": "completed_group_1", + "mean": 0.17982497811317444, + "median": 0.18024154007434845, + "p05": 0.15453973412513733, + "p95": 0.2030555009841919 + }, + { + "label": "completed_group_2", + "mean": 0.1307673156261444, + "median": 0.13107559084892273, + "p05": 0.11463287472724915, + "p95": 0.14630235731601715 + }, + { + "label": "completed_group_3", + "mean": 0.1554446816444397, + "median": 0.15516987442970276, + "p05": 0.13627474009990692, + "p95": 0.1754152774810791 + }, + { + "label": "current_group_partial", + "mean": 0.31774693727493286, + "median": 0.319039523601532, + "p05": 0.27606821060180664, + "p95": 0.35650834441185 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5068919658660889, + "group": 4, + "latest_source_mass": 0.3298327624797821, + "layer": 15, + "max_source_mass": 0.3298327624797821, + "mixer_index": 29, + "normalized_entropy": 0.9362846209998111, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20646697282791138, + "median": 0.18775638937950134, + "p05": 0.0806753933429718, + "p95": 0.4274584949016571 + }, + { + "label": "completed_group_1", + "mean": 0.1197461187839508, + "median": 0.11989086866378784, + "p05": 0.0809398666024208, + "p95": 0.15743419528007507 + }, + { + "label": "completed_group_2", + "mean": 0.19215428829193115, + "median": 0.19620640575885773, + "p05": 0.1148109883069992, + "p95": 0.2491212636232376 + }, + { + "label": "completed_group_3", + "mean": 0.15179988741874695, + "median": 0.15212899446487427, + "p05": 0.10697861760854721, + "p95": 0.19542087614536285 + }, + { + "label": "current_group_partial", + "mean": 0.3298327624797821, + "median": 0.32837340235710144, + "p05": 0.22746729850769043, + "p95": 0.4339301884174347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5435223579406738, + "group": 4, + "latest_source_mass": 0.3224271535873413, + "layer": 16, + "max_source_mass": 0.3224271535873413, + "mixer_index": 30, + "normalized_entropy": 0.9590443632623663, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22410975396633148, + "median": 0.22760415077209473, + "p05": 0.14593973755836487, + "p95": 0.30093055963516235 + }, + { + "label": "completed_group_1", + "mean": 0.16986176371574402, + "median": 0.16963240504264832, + "p05": 0.13569067418575287, + "p95": 0.20651353895664215 + }, + { + "label": "completed_group_2", + "mean": 0.13080403208732605, + "median": 0.13075244426727295, + "p05": 0.10945598781108856, + "p95": 0.15187400579452515 + }, + { + "label": "completed_group_3", + "mean": 0.15279728174209595, + "median": 0.15036612749099731, + "p05": 0.12810495495796204, + "p95": 0.18468116223812103 + }, + { + "label": "current_group_partial", + "mean": 0.3224271535873413, + "median": 0.3164973258972168, + "p05": 0.2600978910923004, + "p95": 0.40530598163604736 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4949920177459717, + "group": 4, + "latest_source_mass": 0.31847020983695984, + "layer": 16, + "max_source_mass": 0.31847020983695984, + "mixer_index": 31, + "normalized_entropy": 0.9288907675133354, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27174079418182373, + "median": 0.2585451304912567, + "p05": 0.13639721274375916, + "p95": 0.4698348045349121 + }, + { + "label": "completed_group_1", + "mean": 0.12178683280944824, + "median": 0.11714950203895569, + "p05": 0.08594125509262085, + "p95": 0.1723237782716751 + }, + { + "label": "completed_group_2", + "mean": 0.1529826819896698, + "median": 0.15223106741905212, + "p05": 0.10371473431587219, + "p95": 0.203575000166893 + }, + { + "label": "completed_group_3", + "mean": 0.1350194811820984, + "median": 0.13363957405090332, + "p05": 0.08964549005031586, + "p95": 0.18468207120895386 + }, + { + "label": "current_group_partial", + "mean": 0.31847020983695984, + "median": 0.3185216188430786, + "p05": 0.21444711089134216, + "p95": 0.4301546812057495 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.544264793395996, + "group": 5, + "latest_source_mass": 0.2861037850379944, + "layer": 17, + "max_source_mass": 0.2861037850379944, + "mixer_index": 32, + "normalized_entropy": 0.9595056643474137, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24016115069389343, + "median": 0.23868846893310547, + "p05": 0.18435582518577576, + "p95": 0.2983453571796417 + }, + { + "label": "completed_group_1", + "mean": 0.20730282366275787, + "median": 0.1998077780008316, + "p05": 0.16255050897598267, + "p95": 0.28107208013534546 + }, + { + "label": "completed_group_2", + "mean": 0.09692573547363281, + "median": 0.0967404842376709, + "p05": 0.08098706603050232, + "p95": 0.11221462488174438 + }, + { + "label": "completed_group_3", + "mean": 0.1695065200328827, + "median": 0.1696585714817047, + "p05": 0.136396586894989, + "p95": 0.20130634307861328 + }, + { + "label": "completed_group_4", + "mean": 0.2861037850379944, + "median": 0.28883200883865356, + "p05": 0.22401469945907593, + "p95": 0.3409370183944702 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.686682939529419, + "group": 5, + "latest_source_mass": 0.12281867861747742, + "layer": 17, + "max_source_mass": 0.2885604202747345, + "mixer_index": 33, + "normalized_entropy": 0.9413556721740635, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16755425930023193, + "median": 0.1327776312828064, + "p05": 0.0790192037820816, + "p95": 0.3667412996292114 + }, + { + "label": "completed_group_1", + "mean": 0.10962115228176117, + "median": 0.10525335371494293, + "p05": 0.07840172946453094, + "p95": 0.15405339002609253 + }, + { + "label": "completed_group_2", + "mean": 0.1783454418182373, + "median": 0.17896884679794312, + "p05": 0.10802385956048965, + "p95": 0.24048911035060883 + }, + { + "label": "completed_group_3", + "mean": 0.13310004770755768, + "median": 0.13033035397529602, + "p05": 0.0904831737279892, + "p95": 0.17934636771678925 + }, + { + "label": "completed_group_4", + "mean": 0.2885604202747345, + "median": 0.2831825315952301, + "p05": 0.16967125236988068, + "p95": 0.4189499616622925 + }, + { + "label": "current_group_partial", + "mean": 0.12281867861747742, + "median": 0.12062528729438782, + "p05": 0.07461707293987274, + "p95": 0.1782219111919403 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7070834636688232, + "group": 5, + "latest_source_mass": 0.2988377809524536, + "layer": 18, + "max_source_mass": 0.2988377809524536, + "mixer_index": 34, + "normalized_entropy": 0.9527414214834803, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.181177020072937, + "median": 0.18419909477233887, + "p05": 0.13624237477779388, + "p95": 0.22192169725894928 + }, + { + "label": "completed_group_1", + "mean": 0.14557164907455444, + "median": 0.144476979970932, + "p05": 0.1293315887451172, + "p95": 0.16486899554729462 + }, + { + "label": "completed_group_2", + "mean": 0.08318156749010086, + "median": 0.0820261612534523, + "p05": 0.07088497281074524, + "p95": 0.0990293100476265 + }, + { + "label": "completed_group_3", + "mean": 0.11424209177494049, + "median": 0.11267061531543732, + "p05": 0.0979941189289093, + "p95": 0.1353992223739624 + }, + { + "label": "completed_group_4", + "mean": 0.17698991298675537, + "median": 0.17597860097885132, + "p05": 0.14837387204170227, + "p95": 0.21050144731998444 + }, + { + "label": "current_group_partial", + "mean": 0.2988377809524536, + "median": 0.2989804744720459, + "p05": 0.2721370458602905, + "p95": 0.32572072744369507 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7419235706329346, + "group": 5, + "latest_source_mass": 0.26016175746917725, + "layer": 18, + "max_source_mass": 0.26016175746917725, + "mixer_index": 35, + "normalized_entropy": 0.972186055410333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1532188355922699, + "median": 0.1451343297958374, + "p05": 0.11514341086149216, + "p95": 0.22103679180145264 + }, + { + "label": "completed_group_1", + "mean": 0.11420352011919022, + "median": 0.11261284351348877, + "p05": 0.0928458571434021, + "p95": 0.14123988151550293 + }, + { + "label": "completed_group_2", + "mean": 0.16597424447536469, + "median": 0.1655328869819641, + "p05": 0.13871800899505615, + "p95": 0.19611573219299316 + }, + { + "label": "completed_group_3", + "mean": 0.1250365972518921, + "median": 0.12369821965694427, + "p05": 0.10375828295946121, + "p95": 0.14912742376327515 + }, + { + "label": "completed_group_4", + "mean": 0.18140508234500885, + "median": 0.1796584278345108, + "p05": 0.14511024951934814, + "p95": 0.22286373376846313 + }, + { + "label": "current_group_partial", + "mean": 0.26016175746917725, + "median": 0.26266711950302124, + "p05": 0.1889389455318451, + "p95": 0.3234381675720215 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7127439975738525, + "group": 5, + "latest_source_mass": 0.2907913327217102, + "layer": 19, + "max_source_mass": 0.2907913327217102, + "mixer_index": 36, + "normalized_entropy": 0.9559006256078307, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18731768429279327, + "median": 0.19042012095451355, + "p05": 0.13879702985286713, + "p95": 0.23792000114917755 + }, + { + "label": "completed_group_1", + "mean": 0.16715484857559204, + "median": 0.1667211651802063, + "p05": 0.14574480056762695, + "p95": 0.19026139378547668 + }, + { + "label": "completed_group_2", + "mean": 0.08496320247650146, + "median": 0.08452782034873962, + "p05": 0.07410907745361328, + "p95": 0.097386933863163 + }, + { + "label": "completed_group_3", + "mean": 0.11362075805664062, + "median": 0.11248823255300522, + "p05": 0.0977015346288681, + "p95": 0.13219426572322845 + }, + { + "label": "completed_group_4", + "mean": 0.15615218877792358, + "median": 0.15540224313735962, + "p05": 0.1289147436618805, + "p95": 0.18598109483718872 + }, + { + "label": "current_group_partial", + "mean": 0.2907913327217102, + "median": 0.2900415062904358, + "p05": 0.25822532176971436, + "p95": 0.32475030422210693 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076834201812744, + "group": 5, + "latest_source_mass": 0.3738824725151062, + "layer": 19, + "max_source_mass": 0.3738824725151062, + "mixer_index": 37, + "normalized_entropy": 0.8972652009334232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15032559633255005, + "median": 0.12544110417366028, + "p05": 0.06976518034934998, + "p95": 0.35840943455696106 + }, + { + "label": "completed_group_1", + "mean": 0.08624927699565887, + "median": 0.08602266013622284, + "p05": 0.061998721212148666, + "p95": 0.11138717085123062 + }, + { + "label": "completed_group_2", + "mean": 0.15736976265907288, + "median": 0.15802210569381714, + "p05": 0.10121746361255646, + "p95": 0.20687000453472137 + }, + { + "label": "completed_group_3", + "mean": 0.08584291487932205, + "median": 0.0827617198228836, + "p05": 0.060561805963516235, + "p95": 0.11351694166660309 + }, + { + "label": "completed_group_4", + "mean": 0.14632999897003174, + "median": 0.1426895558834076, + "p05": 0.10106660425662994, + "p95": 0.20179018378257751 + }, + { + "label": "current_group_partial", + "mean": 0.3738824725151062, + "median": 0.3799295425415039, + "p05": 0.23631010949611664, + "p95": 0.4884527325630188 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.709290862083435, + "group": 5, + "latest_source_mass": 0.2901708781719208, + "layer": 20, + "max_source_mass": 0.2901708781719208, + "mixer_index": 38, + "normalized_entropy": 0.9539733939957076, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18991519510746002, + "median": 0.1924753338098526, + "p05": 0.14201581478118896, + "p95": 0.23709386587142944 + }, + { + "label": "completed_group_1", + "mean": 0.17091524600982666, + "median": 0.17006167769432068, + "p05": 0.15297070145606995, + "p95": 0.19153237342834473 + }, + { + "label": "completed_group_2", + "mean": 0.07822999358177185, + "median": 0.07778562605381012, + "p05": 0.06707640737295151, + "p95": 0.09089364111423492 + }, + { + "label": "completed_group_3", + "mean": 0.116765096783638, + "median": 0.11516137421131134, + "p05": 0.10162825882434845, + "p95": 0.13653048872947693 + }, + { + "label": "completed_group_4", + "mean": 0.1540035903453827, + "median": 0.15287795662879944, + "p05": 0.12593276798725128, + "p95": 0.18590238690376282 + }, + { + "label": "current_group_partial", + "mean": 0.2901708781719208, + "median": 0.29104921221733093, + "p05": 0.25643545389175415, + "p95": 0.3205650746822357 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6690887212753296, + "group": 5, + "latest_source_mass": 0.3342471718788147, + "layer": 20, + "max_source_mass": 0.3342471718788147, + "mixer_index": 39, + "normalized_entropy": 0.9315361520005944, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19029226899147034, + "median": 0.19644704461097717, + "p05": 0.09456793963909149, + "p95": 0.30468082427978516 + }, + { + "label": "completed_group_1", + "mean": 0.1100241169333458, + "median": 0.10808776319026947, + "p05": 0.09035106748342514, + "p95": 0.1359245330095291 + }, + { + "label": "completed_group_2", + "mean": 0.11471512168645859, + "median": 0.11494918912649155, + "p05": 0.08799737691879272, + "p95": 0.14024731516838074 + }, + { + "label": "completed_group_3", + "mean": 0.09999909996986389, + "median": 0.09682698547840118, + "p05": 0.07921088486909866, + "p95": 0.1264391839504242 + }, + { + "label": "completed_group_4", + "mean": 0.1507222205400467, + "median": 0.1476232260465622, + "p05": 0.11397356539964676, + "p95": 0.19991227984428406 + }, + { + "label": "current_group_partial", + "mean": 0.3342471718788147, + "median": 0.33802270889282227, + "p05": 0.2548266053199768, + "p95": 0.40130624175071716 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7458693981170654, + "group": 6, + "latest_source_mass": 0.2361394464969635, + "layer": 21, + "max_source_mass": 0.2361394464969635, + "mixer_index": 40, + "normalized_entropy": 0.9743882636597644, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20528709888458252, + "median": 0.2140858769416809, + "p05": 0.13168928027153015, + "p95": 0.27098023891448975 + }, + { + "label": "completed_group_1", + "mean": 0.1722964197397232, + "median": 0.17238952219486237, + "p05": 0.15053534507751465, + "p95": 0.19365043938159943 + }, + { + "label": "completed_group_2", + "mean": 0.10435444861650467, + "median": 0.10361101478338242, + "p05": 0.08888213336467743, + "p95": 0.12212536484003067 + }, + { + "label": "completed_group_3", + "mean": 0.12075909972190857, + "median": 0.1180175170302391, + "p05": 0.10358668118715286, + "p95": 0.14458929002285004 + }, + { + "label": "completed_group_4", + "mean": 0.16116347908973694, + "median": 0.15841388702392578, + "p05": 0.1337081342935562, + "p95": 0.19791515171527863 + }, + { + "label": "completed_group_5", + "mean": 0.2361394464969635, + "median": 0.23709020018577576, + "p05": 0.20446214079856873, + "p95": 0.26509594917297363 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4612125158309937, + "group": 6, + "latest_source_mass": 0.5543455481529236, + "layer": 21, + "max_source_mass": 0.5543455481529236, + "mixer_index": 41, + "normalized_entropy": 0.7509146897354808, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08934466540813446, + "median": 0.07911523431539536, + "p05": 0.04488275945186615, + "p95": 0.1737232506275177 + }, + { + "label": "completed_group_1", + "mean": 0.054379627108573914, + "median": 0.05344422906637192, + "p05": 0.04221750423312187, + "p95": 0.06924191862344742 + }, + { + "label": "completed_group_2", + "mean": 0.07567165046930313, + "median": 0.07438549399375916, + "p05": 0.05811355262994766, + "p95": 0.09632256627082825 + }, + { + "label": "completed_group_3", + "mean": 0.05758609622716904, + "median": 0.05612001568078995, + "p05": 0.044792283326387405, + "p95": 0.07347827404737473 + }, + { + "label": "completed_group_4", + "mean": 0.07884950935840607, + "median": 0.07757881283760071, + "p05": 0.05758974328637123, + "p95": 0.10452556610107422 + }, + { + "label": "completed_group_5", + "mean": 0.08982288837432861, + "median": 0.08912864327430725, + "p05": 0.0663854256272316, + "p95": 0.11604885756969452 + }, + { + "label": "current_group_partial", + "mean": 0.5543455481529236, + "median": 0.5596089363098145, + "p05": 0.45455610752105713, + "p95": 0.6367484331130981 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7241814136505127, + "group": 6, + "latest_source_mass": 0.39484071731567383, + "layer": 22, + "max_source_mass": 0.39484071731567383, + "mixer_index": 42, + "normalized_entropy": 0.886053970419732, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13360010087490082, + "median": 0.14072942733764648, + "p05": 0.07699814438819885, + "p95": 0.17506597936153412 + }, + { + "label": "completed_group_1", + "mean": 0.10609637200832367, + "median": 0.10586297512054443, + "p05": 0.08648421615362167, + "p95": 0.12602579593658447 + }, + { + "label": "completed_group_2", + "mean": 0.06363911926746368, + "median": 0.06324908882379532, + "p05": 0.054980404675006866, + "p95": 0.07316673547029495 + }, + { + "label": "completed_group_3", + "mean": 0.07426095008850098, + "median": 0.07362888753414154, + "p05": 0.06318790465593338, + "p95": 0.08665015548467636 + }, + { + "label": "completed_group_4", + "mean": 0.09383076429367065, + "median": 0.09294568747282028, + "p05": 0.07638302445411682, + "p95": 0.11445100605487823 + }, + { + "label": "completed_group_5", + "mean": 0.13373197615146637, + "median": 0.13327741622924805, + "p05": 0.11399447172880173, + "p95": 0.15641780197620392 + }, + { + "label": "current_group_partial", + "mean": 0.39484071731567383, + "median": 0.3879151940345764, + "p05": 0.3302172124385834, + "p95": 0.4852263331413269 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3979300260543823, + "group": 6, + "latest_source_mass": 0.5830056071281433, + "layer": 22, + "max_source_mass": 0.5830056071281433, + "mixer_index": 43, + "normalized_entropy": 0.7183939231382495, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08959861099720001, + "median": 0.07544538378715515, + "p05": 0.04165133833885193, + "p95": 0.20099139213562012 + }, + { + "label": "completed_group_1", + "mean": 0.05414731800556183, + "median": 0.05336771905422211, + "p05": 0.04069708660244942, + "p95": 0.07046990841627121 + }, + { + "label": "completed_group_2", + "mean": 0.07668881118297577, + "median": 0.07607124000787735, + "p05": 0.0555989071726799, + "p95": 0.09980960935354233 + }, + { + "label": "completed_group_3", + "mean": 0.05603286623954773, + "median": 0.054573021829128265, + "p05": 0.04162490367889404, + "p95": 0.0730118378996849 + }, + { + "label": "completed_group_4", + "mean": 0.06979228556156158, + "median": 0.06884932518005371, + "p05": 0.04576495662331581, + "p95": 0.09806846082210541 + }, + { + "label": "completed_group_5", + "mean": 0.07073447108268738, + "median": 0.07023533433675766, + "p05": 0.04956750199198723, + "p95": 0.09379896521568298 + }, + { + "label": "current_group_partial", + "mean": 0.5830056071281433, + "median": 0.5862616896629333, + "p05": 0.4658474922180176, + "p95": 0.6849814057350159 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7459560632705688, + "group": 6, + "latest_source_mass": 0.37691548466682434, + "layer": 23, + "max_source_mass": 0.37691548466682434, + "mixer_index": 44, + "normalized_entropy": 0.897243926765161, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14318783581256866, + "median": 0.1493408977985382, + "p05": 0.08292607218027115, + "p95": 0.19464610517024994 + }, + { + "label": "completed_group_1", + "mean": 0.1094653457403183, + "median": 0.10858799517154694, + "p05": 0.09432064741849899, + "p95": 0.1279931515455246 + }, + { + "label": "completed_group_2", + "mean": 0.06349228322505951, + "median": 0.06325754523277283, + "p05": 0.05439673364162445, + "p95": 0.0737837627530098 + }, + { + "label": "completed_group_3", + "mean": 0.07859592139720917, + "median": 0.0775163471698761, + "p05": 0.06603655964136124, + "p95": 0.09357087314128876 + }, + { + "label": "completed_group_4", + "mean": 0.09567482024431229, + "median": 0.09468592703342438, + "p05": 0.07671141624450684, + "p95": 0.11774884164333344 + }, + { + "label": "completed_group_5", + "mean": 0.13266831636428833, + "median": 0.13199329376220703, + "p05": 0.11244359612464905, + "p95": 0.15504513680934906 + }, + { + "label": "current_group_partial", + "mean": 0.37691548466682434, + "median": 0.37180638313293457, + "p05": 0.31207239627838135, + "p95": 0.4595233201980591 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4055393934249878, + "group": 6, + "latest_source_mass": 0.5772827863693237, + "layer": 23, + "max_source_mass": 0.5772827863693237, + "mixer_index": 45, + "normalized_entropy": 0.7223043644164862, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10154955089092255, + "median": 0.08766824007034302, + "p05": 0.04901552200317383, + "p95": 0.20287738740444183 + }, + { + "label": "completed_group_1", + "mean": 0.05619276314973831, + "median": 0.055477917194366455, + "p05": 0.041608311235904694, + "p95": 0.07371112704277039 + }, + { + "label": "completed_group_2", + "mean": 0.07791627198457718, + "median": 0.07736755907535553, + "p05": 0.055920060724020004, + "p95": 0.10158214718103409 + }, + { + "label": "completed_group_3", + "mean": 0.053915880620479584, + "median": 0.05215863138437271, + "p05": 0.03798118978738785, + "p95": 0.07156119495630264 + }, + { + "label": "completed_group_4", + "mean": 0.06398355215787888, + "median": 0.06229078769683838, + "p05": 0.040616974234580994, + "p95": 0.09270931780338287 + }, + { + "label": "completed_group_5", + "mean": 0.06915920227766037, + "median": 0.0684593990445137, + "p05": 0.046986356377601624, + "p95": 0.09408729523420334 + }, + { + "label": "current_group_partial", + "mean": 0.5772827863693237, + "median": 0.5824689269065857, + "p05": 0.4518115520477295, + "p95": 0.6926202178001404 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7853620052337646, + "group": 6, + "latest_source_mass": 0.3407441973686218, + "layer": 24, + "max_source_mass": 0.3407441973686218, + "mixer_index": 46, + "normalized_entropy": 0.9174945750195659, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15284113585948944, + "median": 0.1593906283378601, + "p05": 0.08678612112998962, + "p95": 0.211342453956604 + }, + { + "label": "completed_group_1", + "mean": 0.12206703424453735, + "median": 0.12153112143278122, + "p05": 0.1060115247964859, + "p95": 0.1399766057729721 + }, + { + "label": "completed_group_2", + "mean": 0.06508444249629974, + "median": 0.06482555717229843, + "p05": 0.055350616574287415, + "p95": 0.07557765394449234 + }, + { + "label": "completed_group_3", + "mean": 0.08347313106060028, + "median": 0.08190013468265533, + "p05": 0.06966532766819, + "p95": 0.10050469636917114 + }, + { + "label": "completed_group_4", + "mean": 0.09811970591545105, + "median": 0.09664031863212585, + "p05": 0.07688035070896149, + "p95": 0.12369741499423981 + }, + { + "label": "completed_group_5", + "mean": 0.13767032325267792, + "median": 0.1373325139284134, + "p05": 0.11642646044492722, + "p95": 0.1606694608926773 + }, + { + "label": "current_group_partial", + "mean": 0.3407441973686218, + "median": 0.33805108070373535, + "p05": 0.271301805973053, + "p95": 0.41781002283096313 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.509842872619629, + "group": 6, + "latest_source_mass": 0.5227814316749573, + "layer": 24, + "max_source_mass": 0.5227814316749573, + "mixer_index": 47, + "normalized_entropy": 0.7759057494780099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11788781732320786, + "median": 0.10097194463014603, + "p05": 0.06265684217214584, + "p95": 0.2408325970172882 + }, + { + "label": "completed_group_1", + "mean": 0.06822571158409119, + "median": 0.0673636943101883, + "p05": 0.05209728330373764, + "p95": 0.08648915588855743 + }, + { + "label": "completed_group_2", + "mean": 0.08556114137172699, + "median": 0.08463940769433975, + "p05": 0.06234566867351532, + "p95": 0.1125786080956459 + }, + { + "label": "completed_group_3", + "mean": 0.06067090481519699, + "median": 0.058974385261535645, + "p05": 0.043267179280519485, + "p95": 0.07997901737689972 + }, + { + "label": "completed_group_4", + "mean": 0.07228855788707733, + "median": 0.06972275674343109, + "p05": 0.04689513146877289, + "p95": 0.106512151658535 + }, + { + "label": "completed_group_5", + "mean": 0.07258446514606476, + "median": 0.07082539796829224, + "p05": 0.050949547439813614, + "p95": 0.09970614314079285 + }, + { + "label": "current_group_partial", + "mean": 0.5227814316749573, + "median": 0.5290273427963257, + "p05": 0.38773614168167114, + "p95": 0.6392589807510376 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.839367389678955, + "group": 7, + "latest_source_mass": 0.29583215713500977, + "layer": 25, + "max_source_mass": 0.29583215713500977, + "mixer_index": 48, + "normalized_entropy": 0.9452478525649903, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15906387567520142, + "median": 0.15846745669841766, + "p05": 0.09519976377487183, + "p95": 0.24517810344696045 + }, + { + "label": "completed_group_1", + "mean": 0.13139380514621735, + "median": 0.1308426856994629, + "p05": 0.11478274315595627, + "p95": 0.15000829100608826 + }, + { + "label": "completed_group_2", + "mean": 0.07717745751142502, + "median": 0.07715500891208649, + "p05": 0.06616044044494629, + "p95": 0.08791667222976685 + }, + { + "label": "completed_group_3", + "mean": 0.09607303142547607, + "median": 0.09436552226543427, + "p05": 0.08152313530445099, + "p95": 0.11384639143943787 + }, + { + "label": "completed_group_4", + "mean": 0.1107850894331932, + "median": 0.1089654266834259, + "p05": 0.08814431726932526, + "p95": 0.139452263712883 + }, + { + "label": "completed_group_5", + "mean": 0.12967455387115479, + "median": 0.1299237608909607, + "p05": 0.10832332074642181, + "p95": 0.15138982236385345 + }, + { + "label": "completed_group_6", + "mean": 0.29583215713500977, + "median": 0.29647016525268555, + "p05": 0.2276897132396698, + "p95": 0.3604946434497833 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5841834545135498, + "group": 7, + "latest_source_mass": 0.40767741203308105, + "layer": 25, + "max_source_mass": 0.40767741203308105, + "mixer_index": 49, + "normalized_entropy": 0.7618312045616817, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07896879315376282, + "median": 0.06809917092323303, + "p05": 0.025932887569069862, + "p95": 0.19368338584899902 + }, + { + "label": "completed_group_1", + "mean": 0.048752427101135254, + "median": 0.04767637699842453, + "p05": 0.034443892538547516, + "p95": 0.0667153000831604 + }, + { + "label": "completed_group_2", + "mean": 0.055707164108753204, + "median": 0.0549345538020134, + "p05": 0.038696639239788055, + "p95": 0.0750277042388916 + }, + { + "label": "completed_group_3", + "mean": 0.04139826446771622, + "median": 0.03878612071275711, + "p05": 0.026673054322600365, + "p95": 0.05746251344680786 + }, + { + "label": "completed_group_4", + "mean": 0.04449018836021423, + "median": 0.04196460545063019, + "p05": 0.026162121444940567, + "p95": 0.07164917886257172 + }, + { + "label": "completed_group_5", + "mean": 0.04285799711942673, + "median": 0.04109475016593933, + "p05": 0.029393533244729042, + "p95": 0.06277710199356079 + }, + { + "label": "completed_group_6", + "mean": 0.2801477909088135, + "median": 0.2763082981109619, + "p05": 0.1455708146095276, + "p95": 0.4254714846611023 + }, + { + "label": "current_group_partial", + "mean": 0.40767741203308105, + "median": 0.4120897948741913, + "p05": 0.21399420499801636, + "p95": 0.5853031277656555 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8125314712524414, + "group": 7, + "latest_source_mass": 0.3700707256793976, + "layer": 26, + "max_source_mass": 0.3700707256793976, + "mixer_index": 50, + "normalized_entropy": 0.8716433883436914, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10889513790607452, + "median": 0.11333495378494263, + "p05": 0.04850132763385773, + "p95": 0.1620434820652008 + }, + { + "label": "completed_group_1", + "mean": 0.08656036853790283, + "median": 0.08640051633119583, + "p05": 0.0650845319032669, + "p95": 0.1084962785243988 + }, + { + "label": "completed_group_2", + "mean": 0.048123255372047424, + "median": 0.04800213500857353, + "p05": 0.03576180338859558, + "p95": 0.06107594817876816 + }, + { + "label": "completed_group_3", + "mean": 0.06561170518398285, + "median": 0.06379580497741699, + "p05": 0.050968848168849945, + "p95": 0.08332888036966324 + }, + { + "label": "completed_group_4", + "mean": 0.06917259097099304, + "median": 0.06724178791046143, + "p05": 0.050207968801259995, + "p95": 0.09373342245817184 + }, + { + "label": "completed_group_5", + "mean": 0.08286689221858978, + "median": 0.08183546364307404, + "p05": 0.06535022705793381, + "p95": 0.10340933501720428 + }, + { + "label": "completed_group_6", + "mean": 0.16869932413101196, + "median": 0.16509854793548584, + "p05": 0.11059704422950745, + "p95": 0.23654375970363617 + }, + { + "label": "current_group_partial", + "mean": 0.3700707256793976, + "median": 0.3694133758544922, + "p05": 0.23441238701343536, + "p95": 0.5025027990341187 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.50089693069458, + "group": 7, + "latest_source_mass": 0.49096518754959106, + "layer": 26, + "max_source_mass": 0.49096518754959106, + "mixer_index": 51, + "normalized_entropy": 0.7217788529328457, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07974998652935028, + "median": 0.07402504235506058, + "p05": 0.04579866677522659, + "p95": 0.1359037309885025 + }, + { + "label": "completed_group_1", + "mean": 0.04166398197412491, + "median": 0.04038450866937637, + "p05": 0.027992289513349533, + "p95": 0.05969971418380737 + }, + { + "label": "completed_group_2", + "mean": 0.04607272148132324, + "median": 0.044438350945711136, + "p05": 0.030038591474294662, + "p95": 0.06808309257030487 + }, + { + "label": "completed_group_3", + "mean": 0.03586764633655548, + "median": 0.03317485377192497, + "p05": 0.02203454077243805, + "p95": 0.05572832003235817 + }, + { + "label": "completed_group_4", + "mean": 0.04223340004682541, + "median": 0.03930673003196716, + "p05": 0.02465122565627098, + "p95": 0.06968094408512115 + }, + { + "label": "completed_group_5", + "mean": 0.04095568507909775, + "median": 0.03919431194663048, + "p05": 0.026239857077598572, + "p95": 0.06058088690042496 + }, + { + "label": "completed_group_6", + "mean": 0.22249138355255127, + "median": 0.2125401496887207, + "p05": 0.10647299885749817, + "p95": 0.3711881935596466 + }, + { + "label": "current_group_partial", + "mean": 0.49096518754959106, + "median": 0.4988659620285034, + "p05": 0.2878011167049408, + "p95": 0.6676807403564453 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8059929609298706, + "group": 7, + "latest_source_mass": 0.37139976024627686, + "layer": 27, + "max_source_mass": 0.37139976024627686, + "mixer_index": 52, + "normalized_entropy": 0.8684990295379666, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10711923241615295, + "median": 0.10972198843955994, + "p05": 0.04896187782287598, + "p95": 0.16201043128967285 + }, + { + "label": "completed_group_1", + "mean": 0.07651707530021667, + "median": 0.07666032761335373, + "p05": 0.05141128972172737, + "p95": 0.10152198374271393 + }, + { + "label": "completed_group_2", + "mean": 0.058284737169742584, + "median": 0.05812416970729828, + "p05": 0.039273619651794434, + "p95": 0.07745087146759033 + }, + { + "label": "completed_group_3", + "mean": 0.06609587371349335, + "median": 0.06383530795574188, + "p05": 0.04481930285692215, + "p95": 0.09336265176534653 + }, + { + "label": "completed_group_4", + "mean": 0.07139429450035095, + "median": 0.0694514587521553, + "p05": 0.0485839769244194, + "p95": 0.10034126043319702 + }, + { + "label": "completed_group_5", + "mean": 0.07493151724338531, + "median": 0.07319007813930511, + "p05": 0.051876723766326904, + "p95": 0.10402432829141617 + }, + { + "label": "completed_group_6", + "mean": 0.17425751686096191, + "median": 0.16937679052352905, + "p05": 0.10449051856994629, + "p95": 0.2624811828136444 + }, + { + "label": "current_group_partial", + "mean": 0.37139976024627686, + "median": 0.3732905089855194, + "p05": 0.23830299079418182, + "p95": 0.5014542937278748 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5622732639312744, + "group": 7, + "latest_source_mass": 0.4768889844417572, + "layer": 27, + "max_source_mass": 0.4768889844417572, + "mixer_index": 53, + "normalized_entropy": 0.7512946301290215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08775827288627625, + "median": 0.08504067361354828, + "p05": 0.0428270623087883, + "p95": 0.144285649061203 + }, + { + "label": "completed_group_1", + "mean": 0.047765426337718964, + "median": 0.04646281152963638, + "p05": 0.031814731657505035, + "p95": 0.06807165592908859 + }, + { + "label": "completed_group_2", + "mean": 0.053928617388010025, + "median": 0.052262187004089355, + "p05": 0.035437777638435364, + "p95": 0.0784272700548172 + }, + { + "label": "completed_group_3", + "mean": 0.04293380305171013, + "median": 0.03996016085147858, + "p05": 0.02556057646870613, + "p95": 0.06623870134353638 + }, + { + "label": "completed_group_4", + "mean": 0.04174989461898804, + "median": 0.03903091698884964, + "p05": 0.025013400241732597, + "p95": 0.06869364529848099 + }, + { + "label": "completed_group_5", + "mean": 0.04734005779027939, + "median": 0.04609068110585213, + "p05": 0.029884062707424164, + "p95": 0.06911227852106094 + }, + { + "label": "completed_group_6", + "mean": 0.20163494348526, + "median": 0.19222964346408844, + "p05": 0.09963390231132507, + "p95": 0.3307305574417114 + }, + { + "label": "current_group_partial", + "mean": 0.4768889844417572, + "median": 0.48383551836013794, + "p05": 0.276212215423584, + "p95": 0.6487575173377991 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8757455348968506, + "group": 7, + "latest_source_mass": 0.3109493851661682, + "layer": 28, + "max_source_mass": 0.3109493851661682, + "mixer_index": 54, + "normalized_entropy": 0.9020429270551009, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11563669890165329, + "median": 0.12087491154670715, + "p05": 0.05733206868171692, + "p95": 0.1688953936100006 + }, + { + "label": "completed_group_1", + "mean": 0.08108697831630707, + "median": 0.08185897022485733, + "p05": 0.05175650864839554, + "p95": 0.10753051191568375 + }, + { + "label": "completed_group_2", + "mean": 0.07280858606100082, + "median": 0.07357072830200195, + "p05": 0.04870978742837906, + "p95": 0.0948689877986908 + }, + { + "label": "completed_group_3", + "mean": 0.07553227245807648, + "median": 0.07362619787454605, + "p05": 0.056317005306482315, + "p95": 0.09756095707416534 + }, + { + "label": "completed_group_4", + "mean": 0.07606014609336853, + "median": 0.07456053793430328, + "p05": 0.05528895929455757, + "p95": 0.1022825762629509 + }, + { + "label": "completed_group_5", + "mean": 0.06935927271842957, + "median": 0.06843917071819305, + "p05": 0.04978285729885101, + "p95": 0.09167985618114471 + }, + { + "label": "completed_group_6", + "mean": 0.19856664538383484, + "median": 0.19508783519268036, + "p05": 0.1281973123550415, + "p95": 0.27796900272369385 + }, + { + "label": "current_group_partial", + "mean": 0.3109493851661682, + "median": 0.3105033040046692, + "p05": 0.18172834813594818, + "p95": 0.43916669487953186 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6924852132797241, + "group": 7, + "latest_source_mass": 0.3315581679344177, + "layer": 28, + "max_source_mass": 0.3315581679344177, + "mixer_index": 55, + "normalized_entropy": 0.8139133413255193, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08638577908277512, + "median": 0.058608733117580414, + "p05": 0.026544082909822464, + "p95": 0.2740764319896698 + }, + { + "label": "completed_group_1", + "mean": 0.059912845492362976, + "median": 0.05901291221380234, + "p05": 0.03629481792449951, + "p95": 0.08707985281944275 + }, + { + "label": "completed_group_2", + "mean": 0.06609142571687698, + "median": 0.0647939145565033, + "p05": 0.038691695779561996, + "p95": 0.09703366458415985 + }, + { + "label": "completed_group_3", + "mean": 0.056745611131191254, + "median": 0.05418796092271805, + "p05": 0.032174739986658096, + "p95": 0.08370660990476608 + }, + { + "label": "completed_group_4", + "mean": 0.06277773529291153, + "median": 0.0600711889564991, + "p05": 0.03501531109213829, + "p95": 0.10094858705997467 + }, + { + "label": "completed_group_5", + "mean": 0.05352145433425903, + "median": 0.05179818347096443, + "p05": 0.032271772623062134, + "p95": 0.08084158599376678 + }, + { + "label": "completed_group_6", + "mean": 0.2830069661140442, + "median": 0.2759752869606018, + "p05": 0.12981083989143372, + "p95": 0.46204638481140137 + }, + { + "label": "current_group_partial", + "mean": 0.3315581679344177, + "median": 0.3204721510410309, + "p05": 0.13446375727653503, + "p95": 0.5667697787284851 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8980522155761719, + "group": 8, + "latest_source_mass": 0.20123262703418732, + "layer": 29, + "max_source_mass": 0.26602888107299805, + "mixer_index": 56, + "normalized_entropy": 0.9127701729200177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11603894829750061, + "median": 0.09991352260112762, + "p05": 0.030035395175218582, + "p95": 0.2941743731498718 + }, + { + "label": "completed_group_1", + "mean": 0.08804173022508621, + "median": 0.08880250155925751, + "p05": 0.05664174258708954, + "p95": 0.11779069900512695 + }, + { + "label": "completed_group_2", + "mean": 0.07975024729967117, + "median": 0.07984748482704163, + "p05": 0.05428915470838547, + "p95": 0.10499341040849686 + }, + { + "label": "completed_group_3", + "mean": 0.09027000516653061, + "median": 0.08898856490850449, + "p05": 0.06189651042222977, + "p95": 0.11996553838253021 + }, + { + "label": "completed_group_4", + "mean": 0.08430293202400208, + "median": 0.0819794088602066, + "p05": 0.05464346706867218, + "p95": 0.12159879505634308 + }, + { + "label": "completed_group_5", + "mean": 0.07433462142944336, + "median": 0.07287181913852692, + "p05": 0.052520088851451874, + "p95": 0.1010315865278244 + }, + { + "label": "completed_group_6", + "mean": 0.26602888107299805, + "median": 0.2639757990837097, + "p05": 0.15474572777748108, + "p95": 0.3846370279788971 + }, + { + "label": "completed_group_7", + "mean": 0.20123262703418732, + "median": 0.18140269815921783, + "p05": 0.08419936150312424, + "p95": 0.3782624900341034 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7512003183364868, + "group": 8, + "latest_source_mass": 0.33482813835144043, + "layer": 29, + "max_source_mass": 0.33482813835144043, + "mixer_index": 57, + "normalized_entropy": 0.7970056117156376, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07744461297988892, + "median": 0.06970229744911194, + "p05": 0.014401955530047417, + "p95": 0.17144526541233063 + }, + { + "label": "completed_group_1", + "mean": 0.046086348593235016, + "median": 0.04394831880927086, + "p05": 0.028066720813512802, + "p95": 0.0718703418970108 + }, + { + "label": "completed_group_2", + "mean": 0.05025433003902435, + "median": 0.04861236736178398, + "p05": 0.029644077643752098, + "p95": 0.07761462777853012 + }, + { + "label": "completed_group_3", + "mean": 0.03748488053679466, + "median": 0.03434944897890091, + "p05": 0.020774798467755318, + "p95": 0.05938909202814102 + }, + { + "label": "completed_group_4", + "mean": 0.04005061462521553, + "median": 0.037498839199543, + "p05": 0.019541850313544273, + "p95": 0.06922487914562225 + }, + { + "label": "completed_group_5", + "mean": 0.041623204946517944, + "median": 0.039552345871925354, + "p05": 0.022081367671489716, + "p95": 0.06821337342262268 + }, + { + "label": "completed_group_6", + "mean": 0.198423832654953, + "median": 0.18318486213684082, + "p05": 0.07548617571592331, + "p95": 0.3721975088119507 + }, + { + "label": "completed_group_7", + "mean": 0.17380405962467194, + "median": 0.1514919549226761, + "p05": 0.05777408182621002, + "p95": 0.36326974630355835 + }, + { + "label": "current_group_partial", + "mean": 0.33482813835144043, + "median": 0.33602219820022583, + "p05": 0.0795319601893425, + "p95": 0.5979493856430054 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0160694122314453, + "group": 8, + "latest_source_mass": 0.21711516380310059, + "layer": 30, + "max_source_mass": 0.21711516380310059, + "mixer_index": 58, + "normalized_entropy": 0.9175527313077866, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09385271370410919, + "median": 0.09211117774248123, + "p05": 0.032377734780311584, + "p95": 0.16496515274047852 + }, + { + "label": "completed_group_1", + "mean": 0.06208505108952522, + "median": 0.06140740588307381, + "p05": 0.0407925546169281, + "p95": 0.0863042101264 + }, + { + "label": "completed_group_2", + "mean": 0.06560768187046051, + "median": 0.06516212224960327, + "p05": 0.041897669434547424, + "p95": 0.09085705131292343 + }, + { + "label": "completed_group_3", + "mean": 0.07029052078723907, + "median": 0.06780634075403214, + "p05": 0.04846132919192314, + "p95": 0.09917552024126053 + }, + { + "label": "completed_group_4", + "mean": 0.06935097277164459, + "median": 0.06708167493343353, + "p05": 0.04668639227747917, + "p95": 0.09815650433301926 + }, + { + "label": "completed_group_5", + "mean": 0.056832700967788696, + "median": 0.05595892667770386, + "p05": 0.038822539150714874, + "p95": 0.0771557092666626 + }, + { + "label": "completed_group_6", + "mean": 0.19425815343856812, + "median": 0.19145764410495758, + "p05": 0.12249615788459778, + "p95": 0.27496030926704407 + }, + { + "label": "completed_group_7", + "mean": 0.17060703039169312, + "median": 0.16317546367645264, + "p05": 0.09239593893289566, + "p95": 0.2726203501224518 + }, + { + "label": "current_group_partial", + "mean": 0.21711516380310059, + "median": 0.21198657155036926, + "p05": 0.11019522696733475, + "p95": 0.3391069769859314 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7837426662445068, + "group": 8, + "latest_source_mass": 0.29026859998703003, + "layer": 30, + "max_source_mass": 0.29026859998703003, + "mixer_index": 59, + "normalized_entropy": 0.8118162725118464, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08836327493190765, + "median": 0.07904411852359772, + "p05": 0.02631576545536518, + "p95": 0.17534467577934265 + }, + { + "label": "completed_group_1", + "mean": 0.04953558370471001, + "median": 0.04658038914203644, + "p05": 0.025181813165545464, + "p95": 0.08318150043487549 + }, + { + "label": "completed_group_2", + "mean": 0.04813014343380928, + "median": 0.045740917325019836, + "p05": 0.026714935898780823, + "p95": 0.07843323051929474 + }, + { + "label": "completed_group_3", + "mean": 0.0373263955116272, + "median": 0.03401809185743332, + "p05": 0.01990434154868126, + "p95": 0.06290259957313538 + }, + { + "label": "completed_group_4", + "mean": 0.037015400826931, + "median": 0.03405017405748367, + "p05": 0.018646355718374252, + "p95": 0.06544196605682373 + }, + { + "label": "completed_group_5", + "mean": 0.03390160948038101, + "median": 0.03198609501123428, + "p05": 0.018189728260040283, + "p95": 0.05570102855563164 + }, + { + "label": "completed_group_6", + "mean": 0.1840541511774063, + "median": 0.17327937483787537, + "p05": 0.07966600358486176, + "p95": 0.32982927560806274 + }, + { + "label": "completed_group_7", + "mean": 0.23140482604503632, + "median": 0.22406929731369019, + "p05": 0.08831030130386353, + "p95": 0.40350067615509033 + }, + { + "label": "current_group_partial", + "mean": 0.29026859998703003, + "median": 0.2745037376880646, + "p05": 0.10280418395996094, + "p95": 0.5399085879325867 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9973840713500977, + "group": 8, + "latest_source_mass": 0.19101998209953308, + "layer": 31, + "max_source_mass": 0.20831164717674255, + "mixer_index": 60, + "normalized_entropy": 0.9090486661912383, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10261696577072144, + "median": 0.10389091819524765, + "p05": 0.01371697522699833, + "p95": 0.21163305640220642 + }, + { + "label": "completed_group_1", + "mean": 0.06039813905954361, + "median": 0.06085003539919853, + "p05": 0.026107093319296837, + "p95": 0.09348798543214798 + }, + { + "label": "completed_group_2", + "mean": 0.06671924144029617, + "median": 0.06656201928853989, + "p05": 0.034684643149375916, + "p95": 0.09819839894771576 + }, + { + "label": "completed_group_3", + "mean": 0.08377155661582947, + "median": 0.0804532840847969, + "p05": 0.05448959022760391, + "p95": 0.12039553374052048 + }, + { + "label": "completed_group_4", + "mean": 0.07002593576908112, + "median": 0.06737451255321503, + "p05": 0.044841229915618896, + "p95": 0.10655929893255234 + }, + { + "label": "completed_group_5", + "mean": 0.05768897384405136, + "median": 0.056077782064676285, + "p05": 0.039495617151260376, + "p95": 0.08242568373680115 + }, + { + "label": "completed_group_6", + "mean": 0.20831164717674255, + "median": 0.2021285891532898, + "p05": 0.11652229726314545, + "p95": 0.31706833839416504 + }, + { + "label": "completed_group_7", + "mean": 0.1594475358724594, + "median": 0.1546279788017273, + "p05": 0.06593266129493713, + "p95": 0.2735573351383209 + }, + { + "label": "current_group_partial", + "mean": 0.19101998209953308, + "median": 0.1713225245475769, + "p05": 0.06408411264419556, + "p95": 0.3850628137588501 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.969987154006958, + "group": 8, + "latest_source_mass": 0.16864418983459473, + "layer": 31, + "max_source_mass": 0.24503237009048462, + "mixer_index": 61, + "normalized_entropy": 0.8965797917640489, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08120635151863098, + "median": 0.061164144426584244, + "p05": 0.0259944386780262, + "p95": 0.23622018098831177 + }, + { + "label": "completed_group_1", + "mean": 0.07894632965326309, + "median": 0.07658770680427551, + "p05": 0.0496862456202507, + "p95": 0.11596276611089706 + }, + { + "label": "completed_group_2", + "mean": 0.08615730702877045, + "median": 0.08467075228691101, + "p05": 0.050409119576215744, + "p95": 0.12811316549777985 + }, + { + "label": "completed_group_3", + "mean": 0.07609779387712479, + "median": 0.0722871869802475, + "p05": 0.04099677503108978, + "p95": 0.11950768530368805 + }, + { + "label": "completed_group_4", + "mean": 0.058728985488414764, + "median": 0.056080836802721024, + "p05": 0.03323862701654434, + "p95": 0.09344407916069031 + }, + { + "label": "completed_group_5", + "mean": 0.05076971277594566, + "median": 0.04946017265319824, + "p05": 0.03047419898211956, + "p95": 0.07563400268554688 + }, + { + "label": "completed_group_6", + "mean": 0.24503237009048462, + "median": 0.24110376834869385, + "p05": 0.10613486170768738, + "p95": 0.3991503119468689 + }, + { + "label": "completed_group_7", + "mean": 0.1544169783592224, + "median": 0.1400495171546936, + "p05": 0.06199025735259056, + "p95": 0.2995939254760742 + }, + { + "label": "current_group_partial", + "mean": 0.16864418983459473, + "median": 0.15222924947738647, + "p05": 0.0543956533074379, + "p95": 0.3329208791255951 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.071648359298706, + "group": 8, + "latest_source_mass": 0.11520501971244812, + "layer": 32, + "max_source_mass": 0.20955611765384674, + "mixer_index": 62, + "normalized_entropy": 0.9428478002054053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10569609701633453, + "median": 0.08689188957214355, + "p05": 0.03173588588833809, + "p95": 0.24571751058101654 + }, + { + "label": "completed_group_1", + "mean": 0.09922641515731812, + "median": 0.09795533865690231, + "p05": 0.06407676637172699, + "p95": 0.14120742678642273 + }, + { + "label": "completed_group_2", + "mean": 0.08180510997772217, + "median": 0.080217644572258, + "p05": 0.054607540369033813, + "p95": 0.11442367732524872 + }, + { + "label": "completed_group_3", + "mean": 0.10646799206733704, + "median": 0.10510502755641937, + "p05": 0.07415887713432312, + "p95": 0.14216329157352448 + }, + { + "label": "completed_group_4", + "mean": 0.0826798751950264, + "median": 0.08022409677505493, + "p05": 0.05491747707128525, + "p95": 0.11860857903957367 + }, + { + "label": "completed_group_5", + "mean": 0.06615597754716873, + "median": 0.06485182791948318, + "p05": 0.04477996751666069, + "p95": 0.09238749742507935 + }, + { + "label": "completed_group_6", + "mean": 0.20955611765384674, + "median": 0.206287682056427, + "p05": 0.129981130361557, + "p95": 0.2996031939983368 + }, + { + "label": "completed_group_7", + "mean": 0.13320739567279816, + "median": 0.12465103715658188, + "p05": 0.06303662061691284, + "p95": 0.23065781593322754 + }, + { + "label": "current_group_partial", + "mean": 0.11520501971244812, + "median": 0.10386130958795547, + "p05": 0.03912060335278511, + "p95": 0.22735844552516937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781701564788818, + "group": 8, + "latest_source_mass": 0.16112206876277924, + "layer": 32, + "max_source_mass": 0.22251944243907928, + "mixer_index": 63, + "normalized_entropy": 0.9003040366848136, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09446848928928375, + "median": 0.074150450527668, + "p05": 0.042592622339725494, + "p95": 0.19443637132644653 + }, + { + "label": "completed_group_1", + "mean": 0.07056106626987457, + "median": 0.06868506222963333, + "p05": 0.04818308353424072, + "p95": 0.0982733890414238 + }, + { + "label": "completed_group_2", + "mean": 0.08033448457717896, + "median": 0.08047192543745041, + "p05": 0.048296794295310974, + "p95": 0.11253741383552551 + }, + { + "label": "completed_group_3", + "mean": 0.0658978745341301, + "median": 0.06256154924631119, + "p05": 0.04223886504769325, + "p95": 0.0967884361743927 + }, + { + "label": "completed_group_4", + "mean": 0.058234501630067825, + "median": 0.0563921295106411, + "p05": 0.03529864549636841, + "p95": 0.08846980333328247 + }, + { + "label": "completed_group_5", + "mean": 0.04316519945859909, + "median": 0.042268507182598114, + "p05": 0.029443470761179924, + "p95": 0.05996086448431015 + }, + { + "label": "completed_group_6", + "mean": 0.2036968618631363, + "median": 0.19645483791828156, + "p05": 0.09398645162582397, + "p95": 0.3395216464996338 + }, + { + "label": "completed_group_7", + "mean": 0.22251944243907928, + "median": 0.2130982130765915, + "p05": 0.11867167800664902, + "p95": 0.35465875267982483 + }, + { + "label": "current_group_partial", + "mean": 0.16112206876277924, + "median": 0.14023703336715698, + "p05": 0.06008770316839218, + "p95": 0.3247341811656952 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.6036605834960938, + "group": null, + "latest_source_mass": 0.31639498472213745, + "layer": null, + "max_source_mass": 0.3412485122680664, + "mixer_index": null, + "normalized_entropy": 0.7298573846467136, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09759747982025146, + "median": 0.09915056824684143, + "p05": 0.025461740791797638, + "p95": 0.16398483514785767 + }, + { + "label": "completed_group_1", + "mean": 0.04215940088033676, + "median": 0.04050712287425995, + "p05": 0.025720715522766113, + "p95": 0.06398092210292816 + }, + { + "label": "completed_group_2", + "mean": 0.041145481169223785, + "median": 0.03881259262561798, + "p05": 0.023510465398430824, + "p95": 0.0662468671798706 + }, + { + "label": "completed_group_3", + "mean": 0.029554028064012527, + "median": 0.027420252561569214, + "p05": 0.01627083122730255, + "p95": 0.04924973472952843 + }, + { + "label": "completed_group_4", + "mean": 0.0289996936917305, + "median": 0.027226522564888, + "p05": 0.01560397818684578, + "p95": 0.048351090401411057 + }, + { + "label": "completed_group_5", + "mean": 0.03647579252719879, + "median": 0.0347352996468544, + "p05": 0.020744025707244873, + "p95": 0.058257561177015305 + }, + { + "label": "completed_group_6", + "mean": 0.06642461568117142, + "median": 0.057809896767139435, + "p05": 0.022837381809949875, + "p95": 0.14078694581985474 + }, + { + "label": "completed_group_7", + "mean": 0.3412485122680664, + "median": 0.3336990475654602, + "p05": 0.0819513276219368, + "p95": 0.6423859596252441 + }, + { + "label": "completed_group_8", + "mean": 0.31639498472213745, + "median": 0.30713334679603577, + "p05": 0.05841828137636185, + "p95": 0.6156357526779175 + } + ], + "sources": 9 + } + ], + "mode": "detached_learned", + "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.24537226557731628, + 0.07698935270309448, + 0.0523640513420105, + 0.09380833059549332, + 0.03647654876112938, + 0.039120711386203766, + 0.06371714919805527, + 0.05156813934445381, + 0.03728010877966881, + 0.03983496502041817, + 0.040284499526023865, + 0.046871740370988846, + 0.04314620420336723, + 0.04033593460917473, + 0.04471700266003609, + 0.08397205173969269, + 0.0528092160820961, + 0.04443390294909477, + 0.07560808956623077, + 0.03708760067820549, + 0.03599191829562187, + 0.033304885029792786, + 0.04061548039317131, + 0.025487789884209633, + 0.021565141156315804, + 0.02342291921377182, + 0.017244853079319, + 0.024166326969861984, + 0.016819356009364128, + 0.021590016782283783, + 0.026201345026493073 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4657019391961512e-05, + "first_to_last_ratio": 0.6094218601945491, + "imbalance_abs_log_ratio": 0.4952445414057865, + "last_quartile_mean": 2.4050695174082648e-05, + "mean": 2.0123495630741672e-05, + "normalized": [ + 0.6757922451357102, + 0.7862872769192628, + 0.6956720089134658, + 0.6457028717597475, + 0.8486081172555344, + 0.7866590564328257, + 0.7355608412572463, + 0.6525459469054529, + 0.5676581895263636, + 0.5065196225058339, + 0.47654698952417385, + 0.4567643060278592, + 0.6388028952758977, + 0.6012584522984605, + 0.5677051478189228, + 0.5320958926863361, + 0.6601484899512206, + 0.6199678744151763, + 0.6252065487643861, + 0.6703348689792878, + 3.1476439082057013, + 2.6843722856032843, + 2.147025515739066, + 1.709881143629556, + 2.310023308278431, + 2.0253173950794507, + 1.5260326037153789, + 0.9852764539113186, + 1.0403760218043532, + 0.7424001651630263, + 0.4909889920627068, + 0.44082456445456286 + ], + "peak_layer": 21, + "peak_normalized": 3.1476439082057013, + "population_cv": 0.7023423019155828, + "rest_mean": 1.4907078501117346e-05, + "spike_contrast": 3.2395447657361793, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.829214813071303e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.359930229227757e-05, + 1.5822848581592552e-05, + 1.3999352631799411e-05, + 1.299379891861463e-05, + 1.7076961739803664e-05, + 1.5830330085009336e-05, + 1.4802055375184864e-05, + 1.3131505511410069e-05, + 1.1423267096688505e-05, + 1.0192945410381071e-05, + 9.58979126153281e-06, + 9.191694516630378e-06, + 1.2854947271989658e-05, + 1.2099421837774571e-05, + 1.1424212061683647e-05, + 1.0707629371609073e-05, + 1.32844952531741e-05, + 1.2475920811994001e-05, + 1.2581341252371203e-05, + 1.348948080703849e-05, + 6.334159843390808e-05, + 5.401895396062173e-05, + 4.3205658585065976e-05, + 3.4408785722916946e-05, + 4.648574395105243e-05, + 4.075646575074643e-05, + 3.0709110433235765e-05, + 1.9827206415357068e-05, + 2.0936002329108305e-05, + 1.4939686479920056e-05, + 9.880414836516138e-06, + 8.870931196724996e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.829688234371133e-05, + "first_to_last_ratio": 0.6072503732166264, + "imbalance_abs_log_ratio": 0.4988140964987654, + "last_quartile_mean": 9.600139401300112e-05, + "mean": 8.040016996346822e-05, + "normalized": [ + 0.6783743745296411, + 0.7863508052165757, + 0.7000215696280339, + 0.647124428880743, + 0.8353484843612504, + 0.7743411566859596, + 0.7293747929726763, + 0.6497369355766927, + 0.5707307458024328, + 0.510545581500116, + 0.4799754930157162, + 0.4590012320770852, + 0.6477982227600472, + 0.6029757489491335, + 0.5684984161378385, + 0.5312301622013348, + 0.6575876681964574, + 0.6204772859169876, + 0.6292313575338936, + 0.6709435017853759, + 3.152621969125565, + 2.693161601564409, + 2.150493956812164, + 1.7016971884685128, + 2.3039790532500866, + 2.0188035652467615, + 1.519776732418517, + 0.9811964658615112, + 1.039948884922119, + 0.7451655332994308, + 0.496455864160715, + 0.4470312211422162 + ], + "peak_layer": 21, + "peak_normalized": 3.152621969125565, + "population_cv": 0.7020159094649734, + "rest_mean": 5.954986355155362e-05, + "spike_contrast": 3.24084411076324, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019299182458780705, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.454141501104459e-05, + 6.322273839032277e-05, + 5.6281853176187724e-05, + 5.2028914069524035e-05, + 6.716216012137011e-05, + 6.225716060725972e-05, + 5.864185732207261e-05, + 5.223896005190909e-05, + 4.588684896589257e-05, + 4.104795152670704e-05, + 3.8590111216763034e-05, + 3.690377707243897e-05, + 5.208308721194044e-05, + 4.847935269935988e-05, + 4.57073692814447e-05, + 4.27109953307081e-05, + 5.287016028887592e-05, + 4.988647924619727e-05, + 5.059030809206888e-05, + 5.394397157942876e-05, + 0.0002534713421482593, + 0.00021653065050486475, + 0.00017290007963310927, + 0.00013681674317922443, + 0.00018524030747357756, + 0.00016231214976869524, + 0.0001221903075929731, + 7.888836262281984e-05, + 8.361206710105762e-05, + 5.991143552819267e-05, + 3.991513585788198e-05, + 3.594138615881093e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05169825907796621, + "first_to_last_ratio": 0.607250394007124, + "imbalance_abs_log_ratio": 0.4988140622616565, + "last_quartile_mean": 0.08513499470427632, + "mean": 0.07129967457149178, + "normalized": [ + 0.6783743842436116, + 0.7863508228047997, + 0.700021554566655, + 0.6471244375634602, + 0.8353485410067275, + 0.7743411217427667, + 0.7293748339206622, + 0.6497369595226863, + 0.5707307154188002, + 0.5105455806753446, + 0.47997548951564134, + 0.4590012256825957, + 0.6477982322753953, + 0.6029757319104283, + 0.5684984046970705, + 0.5312301796601451, + 0.6575876919843755, + 0.6204773092105361, + 0.6292313633824181, + 0.6709435195164998, + 3.152621923151655, + 2.693161643472544, + 2.150493986671358, + 1.7016971770874902, + 2.30397915541657, + 2.0188034853391237, + 1.5197766802362542, + 0.9811964117969386, + 1.0399488445555949, + 0.7451654867762114, + 0.49645587282964704, + 0.44703123336599304 + ], + "peak_layer": 21, + "peak_normalized": 3.152621923151655, + "population_cv": 0.7020159090531873, + "rest_mean": 0.052809414074376775, + "spike_contrast": 3.2408441611351573, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.17114708125591277, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.04836787283420563, + 0.05606655776500702, + 0.04991130903363228, + 0.04613976180553436, + 0.059560079127550125, + 0.05521026998758316, + 0.05200418829917908, + 0.046326033771038055, + 0.04069291427731514, + 0.03640173375606537, + 0.03422209620475769, + 0.03272663801908493, + 0.04618780314922333, + 0.04299197345972061, + 0.040533751249313354, + 0.037876538932323456, + 0.046885788440704346, + 0.0442398302257061, + 0.0448639914393425, + 0.04783805459737778, + 0.22478091716766357, + 0.19202154874801636, + 0.1533295214176178, + 0.12133045494556427, + 0.1642729640007019, + 0.1439400315284729, + 0.10835958272218704, + 0.06995898485183716, + 0.0741480141878128, + 0.05313005670905113, + 0.03539714217185974, + 0.0318731814622879 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.317203296797743e-06, + "first_to_last_ratio": 0.35664923571220175, + "imbalance_abs_log_ratio": 1.0310025132235103, + "last_quartile_mean": 6.49714920086808e-06, + "mean": 4.492464356076198e-06, + "normalized": [ + 0.5694178936246215, + 0.527096091999763, + 0.5069565208451643, + 0.5168980803173514, + 0.5167362729922896, + 0.5011287746943238, + 0.4906181312174465, + 0.49753075069099545, + 0.4121289001968579, + 0.3863295396034041, + 0.3787441822616284, + 0.3878491467448858, + 0.49538413383836966, + 0.4937248622259753, + 0.4965317157240282, + 0.4922453146709141, + 0.5531637227610273, + 0.5731353625097597, + 0.5949767701689571, + 0.6269395119699129, + 2.991445440442326, + 2.8607213177085975, + 2.466076440453951, + 2.094358907038209, + 2.7670265254785567, + 2.47746135973099, + 1.9032918373610526, + 1.2284996183916332, + 1.2122250507961028, + 0.87482980171944, + 0.5899517341136686, + 0.5165762877077978 + ], + "peak_layer": 21, + "peak_normalized": 2.991445440442326, + "population_cv": 0.8381751118691627, + "rest_mean": 3.131475834918631e-06, + "spike_contrast": 3.7815403964740346, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.184180237032706e-05, + "top_five_layers": [ + 21, + 22, + 25, + 26, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.5580895908206003e-06, + 2.367960405535996e-06, + 2.2774840999773005e-06, + 2.322146201549913e-06, + 2.321419287909521e-06, + 2.2513031581183895e-06, + 2.2040844669390935e-06, + 2.23513916353113e-06, + 1.8514743942432688e-06, + 1.7355716863676207e-06, + 1.7014947388815926e-06, + 1.742398467285966e-06, + 2.2254955638345564e-06, + 2.2180413452588255e-06, + 2.2306510345515562e-06, + 2.2113945306045935e-06, + 2.485068307578331e-06, + 2.5747901872819057e-06, + 2.67291193267738e-06, + 2.8165034109406406e-06, + 1.343896201433381e-05, + 1.2851688552473206e-05, + 1.107876050809864e-05, + 9.408832738699857e-06, + 1.2430768038029782e-05, + 1.1129906852147542e-05, + 8.550470738555305e-06, + 5.518990747077623e-06, + 5.44587783224415e-06, + 3.930141701857792e-06, + 2.6503371373109985e-06, + 2.3207005597214447e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.480986444832524e-05, + "first_to_last_ratio": 0.7542988096897616, + "imbalance_abs_log_ratio": 0.28196669015672604, + "last_quartile_mean": 5.940598589404544e-05, + "mean": 5.624602624720865e-05, + "normalized": [ + 0.7593253658919704, + 0.8584935198542845, + 0.7758342247870141, + 0.7257052209284447, + 0.9115911301096014, + 0.8478599752338415, + 0.7912991525483561, + 0.7033001562001767, + 0.6342038809025199, + 0.5708934677878957, + 0.5360030772522035, + 0.5119164360939763, + 0.7122797103032084, + 0.65763851786042, + 0.6170495828015774, + 0.5742094044317299, + 0.7380398429228872, + 0.6958734981683372, + 0.7020379955965178, + 0.7406705623157649, + 3.198556354491257, + 2.6580706472996987, + 2.0667008648707084, + 1.5629991771988188, + 2.1587737798302467, + 1.7912836314300635, + 1.3377158632775958, + 0.8975280712077295, + 0.8407131217921131, + 0.5976858891866252, + 0.431587196610723, + 0.394160680813692 + ], + "peak_layer": 21, + "peak_normalized": 3.198556354491257, + "population_cv": 0.6612253154131089, + "rest_mean": 4.240304419740655e-05, + "spike_contrast": 3.089356714727384, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013099812931613997, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.270903446013108e-05, + 4.828684905078262e-05, + 4.363759217085317e-05, + 4.081803490407765e-05, + 5.127337863086723e-05, + 4.768875442096032e-05, + 4.4507432903628796e-05, + 3.955783904530108e-05, + 3.567144813132472e-05, + 3.211048897355795e-05, + 3.0148043151712045e-05, + 2.8793265300919302e-05, + 4.006290328106843e-05, + 3.698955333675258e-05, + 3.4706587030086666e-05, + 3.229699723306112e-05, + 4.151180837652646e-05, + 3.9140119042713195e-05, + 3.948684752685949e-05, + 4.16597758885473e-05, + 0.00017990608466789126, + 0.00014950591139495373, + 0.00011624371109064668, + 8.791249274509028e-05, + 0.00012142244668211788, + 0.00010075258614961058, + 7.524120155721903e-05, + 5.0482387450756505e-05, + 4.7286772314691916e-05, + 3.3617456210777164e-05, + 2.4275064788525924e-05, + 2.216997199866455e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.384813680895604e-05, + "first_to_last_ratio": 0.9740578591651334, + "imbalance_abs_log_ratio": 0.026284573443628244, + "last_quartile_mean": 3.474961624760908e-05, + "mean": 3.869269224310301e-05, + "normalized": [ + 0.8541900771208318, + 0.9413918806879924, + 0.859570413488977, + 0.8162196112220202, + 0.9909206111274242, + 0.9266644039643532, + 0.8522558488291101, + 0.7571395238293102, + 0.7081640210424396, + 0.6442698940442247, + 0.603596002806512, + 0.575557073332959, + 0.7887193829441419, + 0.7247641884135786, + 0.6750858213648586, + 0.6336813762759561, + 0.8435498470464944, + 0.7939958245882761, + 0.7970984689300282, + 0.8220042438012486, + 3.299009695161972, + 2.613224785716232, + 1.9509379481481457, + 1.3432491513722262, + 1.9254306181237832, + 1.4942554786145408, + 1.1183073895840108, + 0.8225722329666332, + 0.6567052933181462, + 0.46462952270788027, + 0.36224409286636666, + 0.34059527655932637 + ], + "peak_layer": 21, + "peak_normalized": 3.299009695161972, + "population_cv": 0.6253906355726564, + "rest_mean": 2.9905363724672633e-05, + "spike_contrast": 2.880562397960604, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.614426624262706e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.305091377114877e-05, + 3.642498631961644e-05, + 3.325909347040579e-05, + 3.158173421979882e-05, + 3.834138624370098e-05, + 3.5855140595231205e-05, + 3.297607327112928e-05, + 2.9295766580617055e-05, + 2.740077252383344e-05, + 2.4928536731749773e-05, + 2.3354754375759512e-05, + 2.2269852706813253e-05, + 3.051767635042779e-05, + 2.804307769110892e-05, + 2.612088792375289e-05, + 2.4518838472431526e-05, + 3.263921462348662e-05, + 3.072183608310297e-05, + 3.084188574575819e-05, + 3.180555722792633e-05, + 0.00012764756684191525, + 0.00010111270239576697, + 7.548704161308706e-05, + 5.1973926019854844e-05, + 7.450009434251115e-05, + 5.781676736660302e-05, + 4.327032365836203e-05, + 3.182753425789997e-05, + 2.5409695808775723e-05, + 1.7977767129195854e-05, + 1.4016199202160351e-05, + 1.3178548215364572e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001188372980323038, + "first_to_last_ratio": 0.585909807801185, + "imbalance_abs_log_ratio": 0.534589412852804, + "last_quartile_mean": 0.00020282524110371014, + "mean": 0.00016756316449573205, + "normalized": [ + 0.6676704870164084, + 0.7483084683488639, + 0.6769676096236417, + 0.6368287558643128, + 0.8100066941936953, + 0.7564679298832396, + 0.7242231413041909, + 0.6531986131482728, + 0.5690738743162227, + 0.5079946750459964, + 0.47515152884923867, + 0.45873382032473325, + 0.6302192489862594, + 0.5851397763517093, + 0.5571686419883745, + 0.5132471085839729, + 0.6453685522588855, + 0.614712668030142, + 0.6206621143750464, + 0.6587669430337698, + 3.1211444947467313, + 2.6936917077524942, + 2.215351775191127, + 1.776377347392615, + 2.3967757154563434, + 2.05632794722869, + 1.5725614581941492, + 1.0240120317645556, + 1.032250695352248, + 0.7057034519699571, + 0.4689146276836353, + 0.42697809574047824 + ], + "peak_layer": 21, + "peak_normalized": 3.1211444947467313, + "population_cv": 0.718404168086637, + "rest_mean": 0.00012285891932185255, + "spike_contrast": 3.3287456107546904, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004089660884346813, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011187697964487597, + 0.00012538893497548997, + 0.00011343483492964879, + 0.00010670904157450423, + 0.0001357272849418223, + 0.00012675616017077118, + 0.00012135312135796994, + 0.00010945202666334808, + 9.535581921227276e-05, + 8.512119529768825e-05, + 7.961789378896356e-05, + 7.686689059482887e-05, + 0.0001056015316862613, + 9.804787259781733e-05, + 9.336094080936164e-05, + 8.60013096826151e-05, + 0.00010813999688252807, + 0.00010300319991074502, + 0.00010400010796729475, + 0.00011038507363991812, + 0.000522988848388195, + 0.0004513635067269206, + 0.0003712113539222628, + 0.0002976554096676409, + 0.0004016113234683871, + 0.000344564818078652, + 0.0002635033742990345, + 0.00017158669652417302, + 0.0001729671930661425, + 0.00011824990360764787, + 7.857281889300793e-05, + 7.154580089263618e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13583384454250336, + 0.0415157787501812, + 0.018140379339456558, + 0.039518751204013824, + 0.022877270355820656, + 0.015532293356955051, + 0.02314678393304348, + 0.05432315170764923, + 0.03743875026702881, + 0.04006272554397583, + 0.045583780854940414, + 0.05995568633079529, + 0.038651175796985626, + 0.04559725895524025, + 0.05632574483752251, + 0.07952121645212173, + 0.042337141931056976, + 0.05044656619429588, + 0.04418695345520973, + 0.06310147047042847, + 0.029010163620114326, + 0.04586305841803551, + 0.06033914536237717, + 0.07656749337911606, + 0.045884523540735245, + 0.051604680716991425, + 0.06502768397331238, + 0.06792232394218445, + 0.060306549072265625, + 0.08025261014699936, + 0.09663844108581543, + 0.12230555713176727 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4509995708067436e-05, + "first_to_last_ratio": 0.7743031141222962, + "imbalance_abs_log_ratio": 0.2557918617425859, + "last_quartile_mean": 1.8739425740932347e-05, + "mean": 1.7722179109114222e-05, + "normalized": [ + 0.7655666384169513, + 0.8917483357831588, + 0.7902102983812471, + 0.7316333655654478, + 0.9592978291511814, + 0.8828085684394261, + 0.8112146819967133, + 0.7175029311175319, + 0.593548092754179, + 0.5514020456598471, + 0.522638501525885, + 0.5037614169375741, + 0.7045039393419045, + 0.6588959250307225, + 0.6114268576480177, + 0.579302599818883, + 0.7435448955446501, + 0.6887296875838754, + 0.6921332531242588, + 0.8093892960298869, + 3.2325455196933226, + 2.5123032009763855, + 1.9971252826810495, + 1.5895697479726458, + 2.322878133149869, + 1.7452609194969326, + 1.1261946298991334, + 0.8913309023592738, + 0.8429252014113704, + 0.5609036083922285, + 0.5024807882604625, + 0.46722290585598436 + ], + "peak_layer": 21, + "peak_normalized": 3.2325455196933226, + "population_cv": 0.6537085087930902, + "rest_mean": 1.335436960895701e-05, + "spike_contrast": 3.09324600258607, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.130835040996317e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.3567509085987695e-05, + 1.580372372700367e-05, + 1.4004248441779055e-05, + 1.2966137546754908e-05, + 1.700084794720169e-05, + 1.564529156894423e-05, + 1.437649189028889e-05, + 1.2715715456579346e-05, + 1.05189656096627e-05, + 9.772045814315788e-06, + 9.2622931333608e-06, + 8.927750059228856e-06, + 1.2485344996093772e-05, + 1.167707159765996e-05, + 1.0835816283361055e-05, + 1.0266504432365764e-05, + 1.3177235814509913e-05, + 1.2205790881125722e-05, + 1.2266109479242004e-05, + 1.4344142073241528e-05, + 5.7287750678369775e-05, + 4.452348730410449e-05, + 3.539341196301393e-05, + 2.8170639780000784e-05, + 4.116646232432686e-05, + 3.092982660746202e-05, + 1.9958622942795046e-05, + 1.579632589709945e-05, + 1.4938471394998487e-05, + 9.940434210875537e-06, + 8.905054528440814e-06, + 8.280208021460567e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.7723796544451034e-05, + "first_to_last_ratio": 0.7709532556520535, + "imbalance_abs_log_ratio": 0.2601275354642249, + "last_quartile_mean": 7.487327684430056e-05, + "mean": 7.085550987540046e-05, + "normalized": [ + 0.767459364760998, + 0.8911005669273344, + 0.7949447373790853, + 0.7325731165004719, + 0.9435545502986998, + 0.8683878924182112, + 0.8054854811328411, + 0.7138473259026528, + 0.5978031833741329, + 0.5556350739964981, + 0.526183189154655, + 0.5057617239608265, + 0.7151088387962502, + 0.660810541031278, + 0.6122576977156078, + 0.5785452674352782, + 0.739958743085955, + 0.6892518806239643, + 0.6970577547891826, + 0.8113719765230756, + 3.2405178430845596, + 2.524955075289844, + 1.989505542525185, + 1.5842933275736053, + 2.315101352895563, + 1.736925913599177, + 1.1174905309838439, + 0.8878674587909959, + 0.8457916652531124, + 0.5666462793963278, + 0.5093376021450956, + 0.4744685026556913 + ], + "peak_layer": 21, + "peak_normalized": 3.2405178430845596, + "population_cv": 0.6532804064480114, + "rest_mean": 5.3392583881591066e-05, + "spike_contrast": 3.093225654863168, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016515531024197116, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.437872459879145e-05, + 6.313938501989469e-05, + 5.63262146897614e-05, + 5.190684169065207e-05, + 6.685603875666857e-05, + 6.153006688691676e-05, + 5.7073084462899715e-05, + 5.058001625002362e-05, + 4.2357649363111705e-05, + 3.936980647267774e-05, + 3.728297815541737e-05, + 3.58360048267059e-05, + 5.066940138931386e-05, + 4.682206781581044e-05, + 4.33818313467782e-05, + 4.099311991012655e-05, + 5.2430154028115794e-05, + 4.883729343418963e-05, + 4.93903826281894e-05, + 5.749017509515397e-05, + 0.0002296085440320894, + 0.00017890697927214205, + 0.0001409674296155572, + 0.00011225591151742265, + 0.00016403768677264452, + 0.00012307077122386545, + 7.918036135379225e-05, + 6.291030149441212e-05, + 5.992899968987331e-05, + 4.015001104562543e-05, + 3.608937549870461e-05, + 3.36187076754868e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05119003914296627, + "first_to_last_ratio": 0.7709532409156976, + "imbalance_abs_log_ratio": 0.260127554578686, + "last_quartile_mean": 0.06639837077818811, + "mean": 0.06283537502167746, + "normalized": [ + 0.7674593499219178, + 0.8911005434486201, + 0.7949446960721035, + 0.7325731497404293, + 0.9435544639718605, + 0.8683878801940695, + 0.8054854824220893, + 0.7138473266473999, + 0.5978031662245904, + 0.5556351000440413, + 0.5261832202468875, + 0.5057617435862689, + 0.715108825077915, + 0.6608105011251547, + 0.6122576924651686, + 0.5785453059609885, + 0.7399587078512944, + 0.6892519015892086, + 0.6970578010795431, + 0.8113719810435487, + 3.2405178171891893, + 2.52495521313829, + 1.9895055108588098, + 1.5842933381515236, + 2.315101350530748, + 1.7369258738398763, + 1.1174905410660712, + 0.8878674720175045, + 0.8457916556516879, + 0.5666462655907423, + 0.5093376096374259, + 0.4744685136150315 + ], + "peak_layer": 21, + "peak_normalized": 3.2405178171891893, + "population_cv": 0.6532804095990112, + "rest_mean": 0.04734907733897368, + "spike_contrast": 3.0932256918071666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1464613825082779, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.048223596066236496, + 0.05599263682961464, + 0.049950648099184036, + 0.04603150859475136, + 0.05928859859704971, + 0.05456547811627388, + 0.050612982362508774, + 0.04485486447811127, + 0.03756318613886833, + 0.03491353988647461, + 0.03306291997432709, + 0.03177972882986069, + 0.04493413120508194, + 0.041522275656461716, + 0.038471441715955734, + 0.036353111267089844, + 0.04649558290839195, + 0.04330940172076225, + 0.04379988834261894, + 0.05098286271095276, + 0.20361915230751038, + 0.158656507730484, + 0.12501132488250732, + 0.09954966604709625, + 0.1454702615737915, + 0.10914038866758347, + 0.07021793723106384, + 0.055789485573768616, + 0.0531456358730793, + 0.035605430603027344, + 0.03200441971421242, + 0.029813406988978386 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.293479383297381e-06, + "first_to_last_ratio": 0.445969741543177, + "imbalance_abs_log_ratio": 0.8075041733371805, + "last_quartile_mean": 5.142679356140434e-06, + "mean": 3.873292421019414e-06, + "normalized": [ + 0.6548345314656568, + 0.6098593868708077, + 0.5890922218088536, + 0.6012559418227537, + 0.5982935563608919, + 0.5743111722490878, + 0.5597275993993273, + 0.5496382066817245, + 0.4494881925876058, + 0.4317042818847807, + 0.42858151997472205, + 0.44091762118925537, + 0.5606916778798313, + 0.5514610505577893, + 0.5418155695170769, + 0.543856847054833, + 0.6362195338062625, + 0.6499443412604623, + 0.6710047275371834, + 0.7226720079645516, + 3.0150009122712964, + 2.639960940380615, + 2.3465929682489137, + 2.0112500416813845, + 2.8998499180306, + 2.237539331592885, + 1.506545124394083, + 1.1285810413622663, + 1.0054446853908898, + 0.6799751247000538, + 0.607691428375801, + 0.5561984956977537 + ], + "peak_layer": 21, + "peak_normalized": 3.0150009122712964, + "population_cv": 0.7767874334961873, + "rest_mean": 2.7381803546530946e-06, + "spike_contrast": 3.653118598414905, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.000289757939754e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.5363656277477276e-06, + 2.362163741054246e-06, + 2.2817264380137203e-06, + 2.3288400825549616e-06, + 2.3173658973973943e-06, + 2.224475110779167e-06, + 2.167988668588805e-06, + 2.128909500243026e-06, + 1.7409992096872884e-06, + 1.6721169231459498e-06, + 1.6600215531070717e-06, + 1.707802880446252e-06, + 2.1717228264606092e-06, + 2.135969907612889e-06, + 2.0986101390008116e-06, + 2.106516603816999e-06, + 2.4642642983963015e-06, + 2.5174244910886046e-06, + 2.5989975256379694e-06, + 2.799120011331979e-06, + 1.1677980182867032e-05, + 1.0225340702163521e-05, + 9.089040759135969e-06, + 7.790159543219488e-06, + 1.1231966709601693e-05, + 8.666644134791568e-06, + 5.835289812239353e-06, + 4.371324394014664e-06, + 3.8943812796787824e-06, + 2.6337424969824497e-06, + 2.353766603846452e-06, + 2.1543194179685088e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.442123326953151e-05, + "first_to_last_ratio": 0.9390502622041328, + "imbalance_abs_log_ratio": 0.06288627383246957, + "last_quartile_mean": 4.730442560685333e-05, + "mean": 5.047756872045284e-05, + "normalized": [ + 0.8442457511608136, + 0.9554189601166778, + 0.8651399119276457, + 0.8073037852085534, + 1.011773735449473, + 0.9346086902233326, + 0.859731545150837, + 0.7619317954676181, + 0.657244201472419, + 0.6123980431859172, + 0.5782989976765822, + 0.5555348776283999, + 0.7735150062029652, + 0.7093353168527977, + 0.6548347172091762, + 0.6154299343077795, + 0.8164721063408712, + 0.7613444152108153, + 0.7654949233837004, + 0.8721362954625393, + 3.2652345954629065, + 2.4905637834711873, + 1.8895133502771215, + 1.4453947777525853, + 2.0912180007544503, + 1.5360217136102785, + 1.0169235828420913, + 0.8253134404517026, + 0.6717824033622655, + 0.48834596354787496, + 0.4437661443704546, + 0.4237292344581673 + ], + "peak_layer": 21, + "peak_normalized": 3.2652345954629065, + "population_cv": 0.6182834768826335, + "rest_mean": 3.892021615886026e-05, + "spike_contrast": 2.900479074737969, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011288727255305275, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.2615472921170294e-05, + 4.8227226216113195e-05, + 4.367015935713425e-05, + 4.075073229614645e-05, + 5.107187826070003e-05, + 4.717677438748069e-05, + 4.3397158151492476e-05, + 3.846046456601471e-05, + 3.317608934594318e-05, + 3.091236430918798e-05, + 2.9191127396188676e-05, + 2.8042049962095916e-05, + 3.904515688191168e-05, + 3.580552220228128e-05, + 3.305446443846449e-05, + 3.106540680164471e-05, + 4.1213526856154203e-05, + 3.843081503873691e-05, + 3.8640322600258514e-05, + 4.402331978781149e-05, + 0.00016482110368087888, + 0.00012571760453283787, + 9.537803998682648e-05, + 7.296001422218978e-05, + 0.00010555960034253076, + 7.753464160487056e-05, + 5.133183003636077e-05, + 4.165981590631418e-05, + 3.390994243090972e-05, + 2.4650516934343614e-05, + 2.2400236048270017e-05, + 2.138882155122701e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.35970580636058e-05, + "first_to_last_ratio": 1.183082469649485, + "imbalance_abs_log_ratio": 0.16812329486526428, + "last_quartile_mean": 2.8397900337040483e-05, + "mean": 3.5426135951865945e-05, + "normalized": [ + 0.9325726638234952, + 1.0264228324198814, + 0.9395008785893599, + 0.8896754656866502, + 1.0823546164520876, + 1.002292895319797, + 0.9051263930494292, + 0.8090082728462467, + 0.7240290452324956, + 0.6814307833477395, + 0.6400266234002182, + 0.6142285583444715, + 0.8434981841324323, + 0.7676858556869983, + 0.7064344235164509, + 0.6656552980925776, + 0.9160791051424548, + 0.8560305037244728, + 0.8526061894702537, + 0.9377555262910077, + 3.3385841431029775, + 2.450559245049383, + 1.764118224393788, + 1.2414543227163002, + 1.7869665660194411, + 1.2750853185739486, + 0.921819584871679, + 0.7867281257526472, + 0.5090598376007598, + 0.4052896271109379, + 0.3729632040504296, + 0.3549576861891871 + ], + "peak_layer": 21, + "peak_normalized": 3.3385841431029775, + "population_cv": 0.5922680449883523, + "rest_mean": 2.8102526947030264e-05, + "spike_contrast": 2.6678606062470647, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.497362457797863e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.303744597360492e-05, + 3.6362194805406034e-05, + 3.328288585180417e-05, + 3.151776400045492e-05, + 3.834364179056138e-05, + 3.5507364373188466e-05, + 3.206513065379113e-05, + 2.8660037060035393e-05, + 2.5649551389506087e-05, + 2.414045957266353e-05, + 2.2673670173389837e-05, + 2.175974441342987e-05, + 2.98818813462276e-05, + 2.7196143491892144e-05, + 2.5026241928571835e-05, + 2.3581595087307505e-05, + 3.24531429214403e-05, + 3.032585300388746e-05, + 3.0204542781575583e-05, + 3.3221054763998836e-05, + 0.00011827313574030995, + 8.681384497322142e-05, + 6.249589205253869e-05, + 4.3979929614579305e-05, + 6.330532050924376e-05, + 4.5171345846029e-05, + 3.265650593675673e-05, + 2.7870737540069968e-05, + 1.8034023014479317e-05, + 1.435784542991314e-05, + 1.3212645171734039e-05, + 1.2574779248097911e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.0001175618963316083, + "first_to_last_ratio": 0.7472280359888956, + "imbalance_abs_log_ratio": 0.2913848713729993, + "last_quartile_mean": 0.00015733068175904918, + "mean": 0.0001475783419664367, + "normalized": [ + 0.7539335055517485, + 0.8485908925894718, + 0.7671380034067055, + 0.7221187547562506, + 0.9161421981299216, + 0.8510423035463365, + 0.8040139845348296, + 0.7098739762546249, + 0.5921519684486223, + 0.5526890935329054, + 0.5267158319052243, + 0.5064890538358937, + 0.699155203785276, + 0.6460998215976334, + 0.6066243745851786, + 0.5643569853524535, + 0.729020737260662, + 0.6850322033333521, + 0.6876316692176827, + 0.7819456161539314, + 3.243720072824733, + 2.5794511063238676, + 2.043978587180185, + 1.6534243806222502, + 2.3371192579790736, + 1.7969709048957057, + 1.167004731785933, + 0.9411888715768515, + 0.8017105792030909, + 0.5375092481175581, + 0.48493150700127435, + 0.4622245747107726 + ], + "peak_layer": 21, + "peak_normalized": 3.243720072824733, + "population_cv": 0.6705093936976806, + "rest_mean": 0.00011009511891407547, + "spike_contrast": 3.178957885701889, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003499877464491874, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011126425670227036, + 0.00012523363693617284, + 0.00011321295460220426, + 0.00010656908852979541, + 0.00013520274660550058, + 0.00012559541210066527, + 0.00011865505075547844, + 0.0001047620244207792, + 8.738880569580942e-05, + 8.156494004651904e-05, + 7.773184916004539e-05, + 7.474681478925049e-05, + 0.0001031801657518372, + 9.535034041618928e-05, + 8.952461939770728e-05, + 8.328686817549169e-05, + 0.00010758767166407779, + 0.00010109591676155105, + 0.00010147954162675887, + 0.00011539823753992096, + 0.00047870283015072346, + 0.00038067111745476723, + 0.0003016469709109515, + 0.00024400962865911424, + 0.00034490818507038057, + 0.0002651939867064357, + 0.00017222462338395417, + 0.00013889909314457327, + 0.00011831511801574379, + 7.932472362881526e-05, + 7.156538777053356e-05, + 6.821433635195717e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.35791492462158203, + 0.3940892517566681, + 0.4057151973247528, + 0.09380833059549332, + 0.11286329478025436, + 0.13231655955314636, + 0.1621972620487213, + 0.05156813934445381, + 0.08533531427383423, + 0.12003090232610703, + 0.16063299775123596, + 0.046871740370988846, + 0.079384446144104, + 0.11459959298372269, + 0.15625107288360596, + 0.08397205173969269, + 0.11654024571180344, + 0.15294702351093292, + 0.21572710573673248, + 0.03708760067820549, + 0.06553087383508682, + 0.09390194714069366, + 0.12852425873279572, + 0.025487789884209633, + 0.06059965491294861, + 0.08860157430171967, + 0.12100546807050705, + 0.024166326969861984, + 0.06877554953098297, + 0.12682415544986725, + 0.2025238573551178 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4657116139460413e-05, + "first_to_last_ratio": 0.6094207983868245, + "imbalance_abs_log_ratio": 0.49524628372702406, + "last_quartile_mean": 2.4050895831351227e-05, + "mean": 2.012364140568934e-05, + "normalized": [ + 0.6757810223737426, + 0.7862782366272546, + 0.6956742911341667, + 0.645696070130233, + 0.8486116417698539, + 0.7866567023571445, + 0.7355711504661757, + 0.6525555016113147, + 0.5676629357160193, + 0.5065196141176592, + 0.4765477406007062, + 0.4567702622869, + 0.6387850255773179, + 0.6012516562554105, + 0.5676978717094758, + 0.5320979135959467, + 0.660140815357474, + 0.6199672249932086, + 0.6251969579127784, + 0.6703296515344952, + 3.1476456930462797, + 2.6843935159061667, + 2.147025871519746, + 1.7098926204474763, + 2.310003501268429, + 2.02536057377974, + 1.5260208260637402, + 0.9852869427838804, + 1.0403527573808369, + 0.7424129105715245, + 0.4909974573147262, + 0.4408150437901761 + ], + "peak_layer": 21, + "peak_normalized": 3.1476456930462797, + "population_cv": 0.7023455047176551, + "rest_mean": 1.4907175278090614e-05, + "spike_contrast": 3.239551262652627, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.829255849472247e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.3599174963019323e-05, + 1.582278127898462e-05, + 1.3999499969941098e-05, + 1.2993756172363646e-05, + 1.7077156371669844e-05, + 1.5830397387617268e-05, + 1.4802370060351677e-05, + 1.3131792911735829e-05, + 1.1423445357650053e-05, + 1.0193019079451915e-05, + 9.589875844540074e-06, + 9.191880963044241e-06, + 1.2854680790042039e-05, + 1.2099372725060675e-05, + 1.1424148397054523e-05, + 1.0707747605920304e-05, + 1.3284437045513187e-05, + 1.2475998119043652e-05, + 1.2581239388964605e-05, + 1.3489473531080876e-05, + 6.334209319902584e-05, + 5.4019772505853325e-05, + 4.3205978727201e-05, + 3.440926593611948e-05, + 4.648568210541271e-05, + 4.07576299039647e-05, + 3.0709095881320536e-05, + 1.982756111829076e-05, + 2.0935685824952088e-05, + 1.4940051187295467e-05, + 9.88065676210681e-06, + 8.870803867466748e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.829710562466062e-05, + "first_to_last_ratio": 0.6072508868460227, + "imbalance_abs_log_ratio": 0.49881325067107773, + "last_quartile_mean": 9.600168050383218e-05, + "mean": 8.04003983603252e-05, + "normalized": [ + 0.6783620403407459, + 0.7863497478511254, + 0.7000374993610626, + 0.6471153960919661, + 0.8353499122055624, + 0.7743388664853165, + 0.7293878791763712, + 0.6497369450201427, + 0.570732789612889, + 0.5105438596817938, + 0.4799740390322282, + 0.4590044982477276, + 0.6477850704647679, + 0.6029712759057101, + 0.5684996518209374, + 0.5312286531130485, + 0.6575848951936205, + 0.6204730346458087, + 0.629222556567805, + 0.6709457586481146, + 3.1526166331987144, + 2.693185986758889, + 2.1505035942092414, + 1.701690725446724, + 2.3039587527644554, + 2.018829142146718, + 1.519765537386825, + 0.9812068005325093, + 1.0399315417453956, + 0.7451740498188497, + 0.49646309627653795, + 0.44702977024839613 + ], + "peak_layer": 21, + "peak_normalized": 3.1526166331987144, + "population_cv": 0.7020174312406612, + "rest_mean": 5.955002699112002e-05, + "spike_contrast": 3.2408449417296112, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00019299240375403315, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.454057827591896e-05, + 6.322283297777176e-05, + 5.628329381579533e-05, + 5.202833563089371e-05, + 6.71624657115899e-05, + 6.22571533313021e-05, + 5.8643076044972986e-05, + 5.223910920904018e-05, + 4.588714364217594e-05, + 4.1047929698834196e-05, + 3.859010394080542e-05, + 3.6904144508298486e-05, + 5.2082177717238665e-05, + 4.8479130782652646e-05, + 4.5707598474109545e-05, + 4.27109953307081e-05, + 5.287008752929978e-05, + 4.988627915736288e-05, + 5.058974420535378e-05, + 5.3944306273479015e-05, + 0.00025347163318656385, + 0.00021653322619386017, + 0.00017290134564973414, + 0.00013681661221198738, + 0.0001852392015280202, + 0.00016231466725002974, + 0.00012218975462019444, + 7.88894176366739e-05, + 8.361091022379696e-05, + 5.9912290453212336e-05, + 3.991583071183413e-05, + 3.59413716068957e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05169845558702946, + "first_to_last_ratio": 0.6072508754048147, + "imbalance_abs_log_ratio": 0.49881326951206806, + "last_quartile_mean": 0.08513525081798434, + "mean": 0.0712998784147203, + "normalized": [ + 0.6783619951609938, + 0.786349724122647, + 0.7000374743682577, + 0.6471153772114631, + 0.8353498624040391, + 0.774338855687226, + 0.729387848399732, + 0.6497369828883814, + 0.5707327933468814, + 0.5105438598068986, + 0.4799740650385461, + 0.45900445901159087, + 0.6477850423892585, + 0.6029712911228355, + 0.5684996530360322, + 0.5312286608963362, + 0.6575848715023773, + 0.6204730273795, + 0.6292225631787356, + 0.6709457289261637, + 3.152616671814094, + 2.693186128732524, + 2.150503721956032, + 1.7016907445638214, + 2.303958774916342, + 2.0188290625883343, + 1.5197655429953278, + 0.981206773145755, + 1.039931555380646, + 0.7451740672617071, + 0.49646307443703724, + 0.4470297463304832 + ], + "peak_layer": 21, + "peak_normalized": 3.152616671814094, + "population_cv": 0.70201745433721, + "rest_mean": 0.052809559360698415, + "spike_contrast": 3.2408450927884243, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.17114760130643844, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.048367127776145935, + 0.056066639721393585, + 0.04991258680820465, + 0.046139247715473175, + 0.059560343623161316, + 0.05521026626229286, + 0.05200526490807533, + 0.0463261678814888, + 0.04069317877292633, + 0.036401715129613876, + 0.03422209247946739, + 0.0327269621193409, + 0.04618699476122856, + 0.04299177974462509, + 0.04053395614027977, + 0.037876538932323456, + 0.04688572138547897, + 0.044239651411771774, + 0.044863492250442505, + 0.047838348895311356, + 0.22478118538856506, + 0.1920238435268402, + 0.15333065390586853, + 0.12133034318685532, + 0.1642719805240631, + 0.14394226670265198, + 0.10835909843444824, + 0.06995992362499237, + 0.07414699345827103, + 0.05313082039356232, + 0.03539775684475899, + 0.03187316656112671 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.317259230721902e-06, + "first_to_last_ratio": 0.35665874650259244, + "imbalance_abs_log_ratio": 1.0309758465083512, + "last_quartile_mean": 6.497132773120029e-06, + "mean": 4.492468697492313e-06, + "normalized": [ + 0.5694283261976025, + 0.527162238879597, + 0.5069677223499952, + 0.5169219758743683, + 0.5167285360874581, + 0.5011532928366966, + 0.49061467097646116, + 0.4975013703289752, + 0.41211554520475835, + 0.3863198030086292, + 0.3787459166580058, + 0.38785476948100367, + 0.4953856795983567, + 0.493696852070983, + 0.4965364995551244, + 0.49228294995504224, + 0.553135351492583, + 0.5731144625451857, + 0.5949797886628256, + 0.6269543934401883, + 2.991422102255377, + 2.8607693678126824, + 2.46602465979906, + 2.0943619443179213, + 2.767059684919099, + 2.477440137836224, + 1.9032189385510152, + 1.2285251544343567, + 1.2121681046978803, + 0.874835232215944, + 0.589970244793492, + 0.5166042831631082 + ], + "peak_layer": 21, + "peak_normalized": 2.991422102255377, + "population_cv": 0.8381704476164953, + "rest_mean": 3.1314773423218867e-06, + "spike_contrast": 3.7815448495732364, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1841822015412617e-05, + "top_five_layers": [ + 21, + 22, + 25, + 26, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.5581389309081715e-06, + 2.3682598566665547e-06, + 2.277536623296328e-06, + 2.3222557956614764e-06, + 2.3213867734739324e-06, + 2.2514154807140585e-06, + 2.2040710518922424e-06, + 2.235009333162452e-06, + 1.8514161865823553e-06, + 1.7355296222376637e-06, + 1.7015041748891235e-06, + 1.742425411066506e-06, + 2.225504658781574e-06, + 2.2179176539793843e-06, + 2.2306746814138023e-06, + 2.211565742982202e-06, + 2.484943252056837e-06, + 2.5746987830643775e-06, + 2.6729280762083363e-06, + 2.8165729872853262e-06, + 1.3438870155368932e-05, + 1.2851916835643351e-05, + 1.1078538591391407e-05, + 9.408855476067401e-06, + 1.2430929018591996e-05, + 1.1129822269140277e-05, + 8.550151505914982e-06, + 5.519110800378257e-06, + 5.445627266453812e-06, + 3.930169896193547e-06, + 2.6504228571866406e-06, + 2.3208285711007193e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.480992902244907e-05, + "first_to_last_ratio": 0.7542989526231301, + "imbalance_abs_log_ratio": 0.28196650066503776, + "last_quartile_mean": 5.940606024523731e-05, + "mean": 5.624609735832564e-05, + "normalized": [ + 0.7593189081163986, + 0.8584899766461837, + 0.7758411348304551, + 0.7257032038756371, + 0.9115949579325926, + 0.8478585798985628, + 0.791306819195711, + 0.703296291763442, + 0.6342085768594776, + 0.5708926166563143, + 0.5360034021261918, + 0.5119200900833328, + 0.7122730532881045, + 0.6576365221836818, + 0.6170512605021473, + 0.5742091312248548, + 0.7380382630332617, + 0.6958730711433307, + 0.702031804287528, + 0.7406728598801368, + 3.1985536041966367, + 2.658074789581204, + 2.066704719937707, + 1.5629982359965144, + 2.1587687220561924, + 1.7912910686863504, + 1.3377069279004006, + 0.8975311406544447, + 0.8407126410078246, + 0.5976870739310991, + 0.4315898849446666, + 0.3941606675796141 + ], + "peak_layer": 21, + "peak_normalized": 3.1985536041966367, + "population_cv": 0.6612253265050619, + "rest_mean": 4.240309937341846e-05, + "spike_contrast": 3.089356401125345, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013099828647682442, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.2708725231932476e-05, + 4.828671080758795e-05, + 4.363803600426763e-05, + 4.081797305843793e-05, + 5.1273658755235374e-05, + 4.7688736231066287e-05, + 4.450792039278895e-05, + 3.9557671698275954e-05, + 3.5671757359523326e-05, + 3.2110481697600335e-05, + 3.0148099540383555e-05, + 2.8793507226509973e-05, + 4.00625795009546e-05, + 3.698948785313405e-05, + 3.4706725273281336e-05, + 3.229702269891277e-05, + 4.151177199673839e-05, + 3.9140144508564845e-05, + 3.948654921259731e-05, + 4.1659957787487656e-05, + 0.0001799061574274674, + 0.00014950633340049535, + 0.0001162440748885274, + 8.791255095275119e-05, + 0.00012142231571488082, + 0.00010075313184643164, + 7.524079410359263e-05, + 5.0482623919378966e-05, + 4.728680505650118e-05, + 3.3617565350141376e-05, + 2.427524668746628e-05, + 2.2169999283505604e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.384801061656617e-05, + "first_to_last_ratio": 0.9740047819331958, + "imbalance_abs_log_ratio": 0.02633906576931732, + "last_quartile_mean": 3.475138032627001e-05, + "mean": 3.8690275459885015e-05, + "normalized": [ + 0.8541538250770527, + 0.9415230863648691, + 0.8597053468534278, + 0.816066837065049, + 0.9910309334070734, + 0.9267294340872885, + 0.8523049476412223, + 0.7572490181408758, + 0.7081191176405685, + 0.6443517927963444, + 0.6037734323315225, + 0.5756116900407472, + 0.7887443909138365, + 0.7248706260931408, + 0.6750345264379841, + 0.633794207108887, + 0.8434851920289833, + 0.7938278400728305, + 0.7971348135467743, + 0.8220444008320537, + 3.298671908004925, + 2.612487605975051, + 1.950851070533051, + 1.3428804993059804, + 1.9255864325842058, + 1.4942946568254274, + 1.1186305566147283, + 0.822664893239064, + 0.6568214429021311, + 0.4646495190211241, + 0.3623372415841501, + 0.34056871492963064 + ], + "peak_layer": 21, + "peak_normalized": 3.298671908004925, + "population_cv": 0.6252780463659825, + "rest_mean": 2.9905465689134196e-05, + "spike_contrast": 2.8800169546676924, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.612824822193944e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.304744677734561e-05, + 3.64277875632979e-05, + 3.3262236684095114e-05, + 3.157385071972385e-05, + 3.834325980278663e-05, + 3.5855417081620544e-05, + 3.297591320006177e-05, + 2.9298173103597946e-05, + 2.7397323719924316e-05, + 2.4930148356361315e-05, + 2.336016041226685e-05, + 2.227057484560646e-05, + 3.0516737751895562e-05, + 2.8045444196322933e-05, + 2.611727177281864e-05, + 2.452167245792225e-05, + 3.263467442593537e-05, + 3.071341780014336e-05, + 3.084136551478878e-05, + 3.1805124308448285e-05, + 0.00012762652477249503, + 0.00010107786511071026, + 7.547896530013531e-05, + 5.195641642785631e-05, + 7.450146949850023e-05, + 5.781467189081013e-05, + 4.3280124373268336e-05, + 3.182913133059628e-05, + 2.541260255384259e-05, + 1.7977417883230373e-05, + 1.401892768626567e-05, + 1.3176697393646464e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011881819409609307, + "first_to_last_ratio": 0.5858660432005869, + "imbalance_abs_log_ratio": 0.5346641107578359, + "last_quartile_mean": 0.0002028077842624043, + "mean": 0.0001675532937497337, + "normalized": [ + 0.6676405577533395, + 0.7480364199301687, + 0.6769763115579581, + 0.6368334430352999, + 0.8101874659576359, + 0.7560101568775733, + 0.7242722329347805, + 0.6531372168922783, + 0.569150085590359, + 0.5080318969181871, + 0.47519098467081156, + 0.45888126162494614, + 0.6302390928396696, + 0.5851986956962459, + 0.5573344319393583, + 0.5133335361358992, + 0.6455392341889507, + 0.6148024241191441, + 0.6209170612517536, + 0.6589570869141987, + 3.121371094689797, + 2.6936365726462816, + 2.2146820529210776, + 1.7763796870270414, + 2.3963701078287363, + 2.056340004797266, + 1.572482658633456, + 1.024072444271845, + 1.0322195327669619, + 0.7057443743535575, + 0.46898398312387723, + 0.42704789011154415 + ], + "peak_layer": 21, + "peak_normalized": 3.121371094689797, + "population_cv": 0.7183385489551041, + "rest_mean": 0.00012285727657231123, + "spike_contrast": 3.328348128139876, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004089117865078151, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011186537449248135, + 0.00012533596600405872, + 0.0001134296107920818, + 0.0001067035409505479, + 0.00013574957847595215, + 0.0001266719918930903, + 0.00012135419819969684, + 0.00010943529196083546, + 9.53629714786075e-05, + 8.512241765856743e-05, + 7.96198146417737e-05, + 7.6887066825293e-05, + 0.00010559863585513085, + 9.805196896195412e-05, + 9.338321979157627e-05, + 8.601072477176785e-05, + 0.0001081622249330394, + 0.00010301217116648331, + 0.00010403669875813648, + 0.00011041043035220355, + 0.0005229960079304874, + 0.00045132767991162837, + 0.0003710772725753486, + 0.0002976382675115019, + 0.00040151970461010933, + 0.0003445465408731252, + 0.0002634746488183737, + 0.00017158671107608825, + 0.00017295178258791566, + 0.00011824979446828365, + 7.857981108827516e-05, + 7.155328057706356e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.4510050277749542e-05, + "first_to_last_ratio": 0.7743096996079971, + "imbalance_abs_log_ratio": 0.255783356730045, + "last_quartile_mean": 1.873933683782525e-05, + "mean": 1.772222358908948e-05, + "normalized": [ + 0.7655738518273417, + 0.8917418894414295, + 0.7902270979958571, + 0.7316384060873433, + 0.9592931634098782, + 0.8828162060633833, + 0.8112185990320713, + 0.7174816289091832, + 0.5935539930427093, + 0.5514073845754808, + 0.522640268952773, + 0.50377149417381, + 0.7044978089956251, + 0.6589104369318574, + 0.6114200884798895, + 0.5793087411391302, + 0.7435486231843338, + 0.6887380175928803, + 0.692133312159999, + 0.8093969639662545, + 3.2325450017939272, + 2.512318244382783, + 1.9971229388240013, + 1.5895601132651613, + 2.322861628648705, + 1.745234574445335, + 1.1261863634676146, + 0.8913245597041289, + 0.8429168761495727, + 0.5609154923495704, + 0.5024744978075042, + 0.4672217332004658 + ], + "peak_layer": 21, + "peak_normalized": 3.2325450017939272, + "population_cv": 0.6537061924560114, + "rest_mean": 1.3354412287838016e-05, + "spike_contrast": 3.0932401760178796, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1308404615847397e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.356767097604461e-05, + 1.5803649148438126e-05, + 1.4004581316839904e-05, + 1.2966259419044945e-05, + 1.7000807929434814e-05, + 1.564546619192697e-05, + 1.4376597391674295e-05, + 1.2715369848592672e-05, + 1.0519096576899756e-05, + 9.77216495812172e-06, + 9.262347703042906e-06, + 8.927951057557948e-06, + 1.2485267689044122e-05, + 1.1677358088491019e-05, + 1.0835723514901474e-05, + 1.0266639037581626e-05, + 1.3177334949432407e-05, + 1.220596914208727e-05, + 1.2266141311556567e-05, + 1.4344313967740163e-05, + 5.728788528358564e-05, + 4.452386565390043e-05, + 3.5393459256738424e-05, + 2.817053973558359e-05, + 4.116627314942889e-05, + 3.092943734372966e-05, + 1.9958526536356658e-05, + 1.579625313752331e-05, + 1.4938361346139573e-05, + 9.940669770003296e-06, + 8.90496539796004e-06, + 8.280208021460567e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 5.772373378931661e-05, + "first_to_last_ratio": 0.7709541968266947, + "imbalance_abs_log_ratio": 0.26012631467157654, + "last_quartile_mean": 7.487310404030723e-05, + "mean": 7.085559036568156e-05, + "normalized": [ + 0.767456541889667, + 0.8910969874796977, + 0.7949526654342541, + 0.7325741840266811, + 0.943544133913626, + 0.868390294625501, + 0.8054913948148298, + 0.7138323441642012, + 0.5978064063949543, + 0.5556378314836203, + 0.5261854153191788, + 0.505768953649729, + 0.715108334510866, + 0.6608161569684796, + 0.612256437426044, + 0.5785464585899268, + 0.739960161627241, + 0.6892568994732284, + 0.697060043562065, + 0.8113764458982461, + 3.2405349047340315, + 2.5249733605503017, + 1.9894958890200054, + 1.5842775624032772, + 2.3150907133984013, + 1.7369124395330813, + 1.1174777605817472, + 0.8878712764898554, + 0.8457898829566253, + 0.5666580094993431, + 0.5093321459106411, + 0.474467963670652 + ], + "peak_layer": 21, + "peak_normalized": 3.2405349047340315, + "population_cv": 0.6532808754860918, + "rest_mean": 5.3392646400930564e-05, + "spike_contrast": 3.093225357948515, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016515548777533694, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.437858635559678e-05, + 6.313920312095433e-05, + 5.632684042211622e-05, + 5.1906976295867935e-05, + 6.685537664452568e-05, + 6.153030699351802e-05, + 5.707356831408106e-05, + 5.057901216787286e-05, + 4.2357925849501044e-05, + 3.9370046579279006e-05, + 3.728317824425176e-05, + 3.583655779948458e-05, + 5.0669423217186704e-05, + 4.682251892518252e-05, + 4.338179132901132e-05, + 4.099325087736361e-05, + 5.2430314099183306e-05, + 4.8837704525794834e-05, + 4.9390600906917825e-05, + 5.749055708292872e-05, + 0.00022961001377552748, + 0.00017890847811941057, + 0.00014096690574660897, + 0.00011225492198718712, + 0.0001640371192479506, + 0.00012306995631661266, + 7.917954644653946e-05, + 6.291064346441999e-05, + 5.9928941482212394e-05, + 4.015088779851794e-05, + 3.608902989071794e-05, + 3.36187076754868e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.051189984660595655, + "first_to_last_ratio": 0.7709541992191667, + "imbalance_abs_log_ratio": 0.26012631156831567, + "last_quartile_mean": 0.06639821757562459, + "mean": 0.06283544522011653, + "normalized": [ + 0.7674565953661515, + 0.8910969393261454, + 0.7949526416572095, + 0.7325742284914066, + 0.9435441611668938, + 0.8683903486633082, + 0.8054914004920491, + 0.7138323596913118, + 0.5978064112769625, + 0.5556377993412199, + 0.5261854188690636, + 0.5057690043708923, + 0.7151083226055974, + 0.6608161658178341, + 0.6122564748857159, + 0.5785464975021892, + 0.7399601340699639, + 0.6892568823568349, + 0.6970600459500164, + 0.8113764696625058, + 3.2405348286343636, + 2.524973261140758, + 1.9894956995598303, + 1.584277576610783, + 2.3150909383371494, + 1.736912431812052, + 1.1174776724854902, + 0.8878712823128355, + 0.8457898807407633, + 0.5666580234128831, + 0.5093321198410424, + 0.47446798354877795 + ], + "peak_layer": 21, + "peak_normalized": 3.2405348286343636, + "population_cv": 0.6532808615381831, + "rest_mean": 0.04734913239048587, + "spike_contrast": 3.0932253054239998, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14646153450012206, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.048223476856946945, + 0.05599247291684151, + 0.04995120316743851, + 0.04603162780404091, + 0.05928801745176315, + 0.05456569418311119, + 0.0506134107708931, + 0.044853974133729935, + 0.03756343200802803, + 0.03491374850273132, + 0.033063095062971115, + 0.031780220568180084, + 0.04493414983153343, + 0.04152267798781395, + 0.03847140818834305, + 0.036353226751089096, + 0.046495724469423294, + 0.043309763073921204, + 0.043800078332424164, + 0.05098320171236992, + 0.20362044870853424, + 0.15865781903266907, + 0.12501084804534912, + 0.09954878687858582, + 0.1454697698354721, + 0.10913966596126556, + 0.07021720707416534, + 0.05578978732228279, + 0.05314558371901512, + 0.03560620918869972, + 0.032004110515117645, + 0.029813406988978386 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.293416855536634e-06, + "first_to_last_ratio": 0.44595751640176395, + "imbalance_abs_log_ratio": 0.8075315862036556, + "last_quartile_mean": 5.1426801235265884e-06, + "mean": 3.873313932700739e-06, + "normalized": [ + 0.6547872198845468, + 0.6098622810029859, + 0.5891095547173398, + 0.6012100431571663, + 0.5982908205734621, + 0.5743244780654446, + 0.5596766481388143, + 0.5495961168488454, + 0.44946743969713066, + 0.4316777868522346, + 0.4285841881361952, + 0.44095385743945303, + 0.560667078741325, + 0.5514542308757108, + 0.5417913687261439, + 0.5438534743570708, + 0.636286678312979, + 0.6499450755816364, + 0.6710355180394164, + 0.7226628285450687, + 3.0150865448734856, + 2.639875835373709, + 2.3467426593605225, + 2.0112905298698336, + 2.8997859114535975, + 2.237494500860103, + 1.506591468158109, + 1.1286195048247083, + 1.0054186728222039, + 0.6799993493915261, + 0.607662928646794, + 0.5561954066724383 + ], + "peak_layer": 21, + "peak_normalized": 3.0150865448734856, + "population_cv": 0.7767952241449113, + "rest_mean": 2.738177386163443e-06, + "spike_contrast": 3.6531786927134653, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.000305128400214e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.536196461733198e-06, + 2.3621880700375186e-06, + 2.281806246173801e-06, + 2.328675236640265e-06, + 2.3173681711341487e-06, + 2.2245390027819667e-06, + 2.1678033590433188e-06, + 2.128758296748856e-06, + 1.7409284964742255e-06, + 1.6720235862521804e-06, + 1.6600411072431598e-06, + 1.707952719698369e-06, + 2.171639607695397e-06, + 2.1359553556976607e-06, + 2.0985280571039766e-06, + 2.1065152395749465e-06, + 2.464538056301535e-06, + 2.5174413167405874e-06, + 2.59913122135913e-06, + 2.79910000244854e-06, + 1.1678376722557005e-05, + 1.022506785375299e-05, + 9.089671038964298e-06, + 7.790359632053878e-06, + 1.1231781172682531e-05, + 8.666518624522723e-06, + 5.835501724504866e-06, + 4.371497652755352e-06, + 3.894302153639728e-06, + 2.633850954225636e-06, + 2.3536692879133625e-06, + 2.1543194179685088e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.4421208258427214e-05, + "first_to_last_ratio": 0.9390471381412793, + "imbalance_abs_log_ratio": 0.06288960067042051, + "last_quartile_mean": 4.730455634671671e-05, + "mean": 5.047763789889359e-05, + "normalized": [ + 0.8442417833686091, + 0.9554185876612655, + 0.8651397352686891, + 0.8073036157415787, + 1.0117688173530386, + 0.9346106525630644, + 0.8597336821790709, + 0.7619236882899906, + 0.6572438052320593, + 0.6123997263969396, + 0.5783007276195616, + 0.5555384405465414, + 0.7735135857629635, + 0.7093370113557148, + 0.654834684626451, + 0.6154293791596152, + 0.8164712035979397, + 0.7613467591474443, + 0.7654961084945793, + 0.8721378389215614, + 3.265225507983762, + 2.4905834329623366, + 1.889506148192275, + 1.4453841483405094, + 2.0912250805998966, + 1.5360203292388628, + 1.0169178649058275, + 0.8253190119907806, + 0.6717866718190295, + 0.48834958251132726, + 0.44376373442170963, + 0.42372865374700563 + ], + "peak_layer": 21, + "peak_normalized": 3.265225507983762, + "population_cv": 0.6182834965857263, + "rest_mean": 3.8920269852732655e-05, + "spike_contrast": 2.9004789991258666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011288742534816265, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.261533103999682e-05, + 4.822727350983769e-05, + 4.367021028883755e-05, + 4.0750779589870945e-05, + 5.1071699999738485e-05, + 4.717693809652701e-05, + 4.33973254985176e-05, + 3.846010804409161e-05, + 3.317611481179483e-05, + 3.091249163844623e-05, + 2.9191254725446925e-05, + 2.8042268240824342e-05, + 3.9045138692017645e-05, + 3.5805656807497144e-05, + 3.305450809421018e-05, + 3.106542135355994e-05, + 4.1213537770090625e-05, + 3.843098602374084e-05, + 3.8640435377601534e-05, + 4.402345803100616e-05, + 0.00016482087085023522, + 0.00012571876868605614, + 9.537780715618283e-05, + 7.295957766473293e-05, + 0.00010556010238360614, + 7.753467798465863e-05, + 5.1331611757632345e-05, + 4.166015423834324e-05, + 3.3910204365383834e-05, + 2.4650733394082636e-05, + 2.240014509879984e-05, + 2.138882155122701e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.359962329341215e-05, + "first_to_last_ratio": 1.183161693959634, + "imbalance_abs_log_ratio": 0.1681902569407737, + "last_quartile_mean": 2.839816693267494e-05, + "mean": 3.542719008464701e-05, + "normalized": [ + 0.9327605618775979, + 1.0266348424598963, + 0.9395266301163642, + 0.8894102418069995, + 1.082225986222851, + 1.0025232858328599, + 0.9050487327854893, + 0.8091772559917478, + 0.7238157816582237, + 0.6814271430375796, + 0.6400264742225962, + 0.6141081065852076, + 0.843581628077757, + 0.7675711067089205, + 0.7064606405143428, + 0.6655561644133133, + 0.9162025945827288, + 0.8560421033047051, + 0.8525829253737746, + 0.9377293691727334, + 3.3385973508618734, + 2.4504448425731913, + 1.7642485194416198, + 1.2415583752024149, + 1.7870101279275883, + 1.2751219306613844, + 0.9216663623251447, + 0.7867121103297002, + 0.5090691818722745, + 0.4053411321779632, + 0.37287136742526056, + 0.3549471244558967 + ], + "peak_layer": 21, + "peak_normalized": 3.3385973508618734, + "population_cv": 0.592278758078665, + "rest_mean": 2.8103131289417097e-05, + "spike_contrast": 2.6679271717711734, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.497710757888853e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.304508572909981e-05, + 3.6370787711348385e-05, + 3.3284788514720276e-05, + 3.150930569972843e-05, + 3.834022572846152e-05, + 3.5516583011485636e-05, + 3.2063333492260426e-05, + 2.8666876460192725e-05, + 2.5642759283073246e-05, + 2.414104892523028e-05, + 2.2674339561490342e-05, + 2.1756124624516815e-05, + 2.9885726689826697e-05, + 2.7192887500859797e-05, + 2.5027915398823097e-05, + 2.3578784748679027e-05, + 3.2458483474329114e-05, + 3.032716631423682e-05, + 3.0204617360141128e-05, + 3.322111660963856e-05, + 0.00011827712296508253, + 8.681237522978336e-05, + 6.250236765481532e-05, + 4.398492455948144e-05, + 6.330874748528004e-05, + 4.517398701864295e-05, + 3.265204941271804e-05, + 2.7870999474544078e-05, + 1.8034890672424808e-05, + 1.436009733879473e-05, + 1.3209784810896963e-05, + 1.2574779248097911e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00011756234653148567, + "first_to_last_ratio": 0.7472422235887175, + "imbalance_abs_log_ratio": 0.29136588457853974, + "last_quartile_mean": 0.00015732829706394114, + "mean": 0.0001475833446420438, + "normalized": [ + 0.7536685945005097, + 0.8489437148344299, + 0.7668873856587688, + 0.7219862591209203, + 0.9160304874966925, + 0.8511301995015967, + 0.8040925298742102, + 0.7099228291391, + 0.592249280993652, + 0.5528729353184129, + 0.5267845496399454, + 0.5066519805725661, + 0.6991026141257565, + 0.6459439706463571, + 0.6067103504076782, + 0.5642338800686106, + 0.7290006596953322, + 0.6851008790441766, + 0.687775193830433, + 0.7817329016891325, + 3.243635755848753, + 2.5800278485304857, + 2.0438475774053395, + 1.653426311601236, + 2.3369623381128184, + 1.7968342666222143, + 1.1670758041878126, + 0.9412043950671267, + 0.8016088114740478, + 0.5374717514485083, + 0.4848750369936043, + 0.46220890654977165 + ], + "peak_layer": 21, + "peak_normalized": 3.243635755848753, + "population_cv": 0.6705219269582756, + "rest_mean": 0.0001100977226289817, + "spike_contrast": 3.179045807260368, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003500057035125792, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0001112289319280535, + 0.00012528995284810662, + 0.00011317980533931404, + 0.00010655314690666273, + 0.00013519084313884377, + 0.00012561264156829566, + 0.00011867066496051848, + 0.00010477278556209058, + 8.74061297508888e-05, + 8.159483695635572e-05, + 7.774462574161589e-05, + 7.47733938624151e-05, + 0.00010317590204067528, + 9.533057163935155e-05, + 8.954034274211153e-05, + 8.327152318088338e-05, + 0.00010758835560409352, + 0.00010110947914654389, + 0.00010150416346732527, + 0.00011537075624801219, + 0.00047870661364868283, + 0.0003807691391557455, + 0.00030163786141201854, + 0.00024401818518526852, + 0.0003448967181611806, + 0.0002651828108355403, + 0.0001722409506328404, + 0.00013890609261579812, + 0.00011830410949187353, + 7.932187872938812e-05, + 7.155947969295084e-05, + 6.821433635195717e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0626930519938469, + 0.09030644595623016, + 0.20229491591453552, + 0.20300470292568207, + 0.19049881398677826, + 0.12695962190628052, + 0.12132706493139267, + 0.11585728824138641, + 0.13427142798900604, + 0.10485932976007462, + 0.10419609397649765, + 0.10852954536676407, + 0.1286265105009079, + 0.09096142649650574, + 0.10080574452877045, + 0.10566592961549759, + 0.1287592649459839, + 0.09650235623121262, + 0.1065765842795372, + 0.11446042358875275, + 0.12395420670509338, + 0.07221997529268265, + 0.07677769660949707, + 0.08416415005922318, + 0.09094838052988052, + 0.059439923614263535, + 0.062182098627090454, + 0.06896103918552399, + 0.0776243805885315, + 0.06285123527050018, + 0.07231374084949493, + 0.08016443997621536 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 9.220272659149487e-06, + "first_to_last_ratio": 1.069429605014154, + "imbalance_abs_log_ratio": 0.06712542691043744, + "last_quartile_mean": 8.621673288189413e-06, + "mean": 1.2944014976312701e-05, + "normalized": [ + 0.6303729714265094, + 0.8682643012395536, + 0.6280898219637274, + 0.5502831812812704, + 0.33676078665107495, + 0.7338236582315528, + 0.9360409006501473, + 1.0149193793174633, + 0.6122272943195698, + 0.5131421981111545, + 0.43080038533307174, + 0.37792854873844645, + 0.39300605492101337, + 0.6863011395079349, + 0.6243318369244146, + 0.5414566873457767, + 0.793386920098201, + 1.135440497453951, + 1.3017177042809938, + 1.597588130028887, + 2.6171146314958285, + 4.210609056409907, + 2.860451118900051, + 2.2773498949645425, + 1.2385768946194602, + 1.256602631548737, + 0.8025963162001695, + 0.5822644883727114, + 0.47699344412607797, + 0.2914573895713198, + 0.32576029794875644, + 0.3543414380177253 + ], + "peak_layer": 22, + "peak_normalized": 4.210609056409907, + "population_cv": 0.8557638034672043, + "rest_mean": 9.010903349243799e-06, + "spike_contrast": 3.7934951067201736, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.4182817762484775e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 8.159557182807475e-06, + 1.1238826118642464e-05, + 8.130004061968066e-06, + 7.122873739717761e-06, + 4.359036665846361e-06, + 9.498624422121793e-06, + 1.2116127436456736e-05, + 1.3137131645635236e-05, + 7.924679266579915e-06, + 6.642120297328802e-06, + 5.5762866395525634e-06, + 4.891912794846576e-06, + 5.087076260679169e-06, + 8.883492228051182e-06, + 8.081360647338443e-06, + 7.008623470028397e-06, + 1.0269612175761722e-05, + 1.4697158803755883e-05, + 1.684945345914457e-05, + 2.0679204681073315e-05, + 3.38759709848091e-05, + 5.450218668556772e-05, + 3.7025722122052684e-05, + 2.9478051146725193e-05, + 1.603215787326917e-05, + 1.6265483282040805e-05, + 1.0388818736828398e-05, + 7.536840257671429e-06, + 6.174210284370929e-06, + 3.7726288155681686e-06, + 4.216646175336791e-06, + 4.586600880429614e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.628570084401872e-05, + "first_to_last_ratio": 1.065353016422061, + "imbalance_abs_log_ratio": 0.0633062150559101, + "last_quartile_mean": 3.405979077797383e-05, + "mean": 5.163632124549622e-05, + "normalized": [ + 0.5687040132298282, + 0.8764593520729407, + 0.6051388108071001, + 0.5571670858837222, + 0.3412536542304683, + 0.7524665950392966, + 0.8942377527803768, + 1.0263056508948738, + 0.5946205407803693, + 0.5239558700278845, + 0.42939602856142756, + 0.3883393864768877, + 0.40120729316033693, + 0.7381389545660257, + 0.6270734263766993, + 0.5557805536677036, + 0.7656375942833815, + 1.1752771944325544, + 1.2670292024807646, + 1.589140645161745, + 2.652708081192408, + 4.256645166896803, + 2.872494573593182, + 2.263949247584013, + 1.2185560925718533, + 1.2799157794404965, + 0.7966972545457802, + 0.5211011847842488, + 0.47241308488259476, + 0.29979401599371164, + 0.32815719518376885, + 0.36023871841675353 + ], + "peak_layer": 22, + "peak_normalized": 4.256645166896803, + "population_cv": 0.8639609553478637, + "rest_mean": 3.5831106625092046e-05, + "spike_contrast": 3.823060270757878, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013698448019567878, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.9365783120738342e-05, + 4.5257136662257835e-05, + 3.124714203295298e-05, + 2.8770058634108864e-05, + 1.762108331604395e-05, + 3.885460682795383e-05, + 4.6175147872418165e-05, + 5.2994648285675794e-05, + 3.0704017262905836e-05, + 2.7055153623223305e-05, + 2.2172431272338144e-05, + 2.0052417312399484e-05, + 2.0716868675663136e-05, + 3.8114780181786045e-05, + 3.2379764888901263e-05, + 2.86984632111853e-05, + 3.953470877604559e-05, + 6.06869907642249e-05, + 6.542472692672163e-05, + 8.205737685784698e-05, + 0.00013697608665097505, + 0.0002197974972659722, + 0.00014832505257800221, + 0.00011690201063174754, + 6.292175385169685e-05, + 6.609014235436916e-05, + 4.113851537113078e-05, + 2.690774817892816e-05, + 2.4393673811573535e-05, + 1.5480260117328726e-05, + 1.6944830349530093e-05, + 1.860140218923334e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.032178521854802966, + "first_to_last_ratio": 1.0653530079886349, + "imbalance_abs_log_ratio": 0.06330620713982399, + "last_quartile_mean": 0.030204562819562852, + "mean": 0.04579160569119267, + "normalized": [ + 0.5687040352411169, + 0.8764592932397164, + 0.6051388058773625, + 0.5571671042879097, + 0.3412536282040628, + 0.7524665522800605, + 0.8942377987777075, + 1.0263056510911557, + 0.5946205787763836, + 0.523955869943047, + 0.42939603938950854, + 0.388339353567458, + 0.4012073030262001, + 0.7381389650225099, + 0.627073439274379, + 0.5557805622734189, + 0.7656375421356683, + 1.1752771374183286, + 1.2670292275109216, + 1.589140749744225, + 2.6527080853277174, + 4.256645179045396, + 2.872494621066323, + 2.2639491530094316, + 1.2185561059252055, + 1.2799157254825457, + 0.796697271015912, + 0.52110118872801, + 0.47241309150505434, + 0.299794020683871, + 0.32815719848905306, + 0.3602387226403389 + ], + "peak_layer": 22, + "peak_normalized": 4.256645179045396, + "population_cv": 0.8639609572615631, + "rest_mean": 0.031775383451194676, + "spike_contrast": 3.823060262160724, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12147920578718185, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.026041870936751366, + 0.040134478360414505, + 0.02771027758717537, + 0.025513576343655586, + 0.01562655158340931, + 0.03445665165781975, + 0.04094858467578888, + 0.046996183693408966, + 0.02722863107919693, + 0.023992780596017838, + 0.019662734121084213, + 0.017782682552933693, + 0.018371926620602608, + 0.03380056843161583, + 0.028714699670672417, + 0.02545008435845375, + 0.03505977243185043, + 0.05381782725453377, + 0.058019302785396576, + 0.07276930660009384, + 0.12147176265716553, + 0.19491861760616302, + 0.13153614103794098, + 0.10366986691951752, + 0.05579964071512222, + 0.05860939621925354, + 0.03648204728960991, + 0.02386206015944481, + 0.02163255400955677, + 0.013728049583733082, + 0.015026845037937164, + 0.01649590954184532 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.4475698694459425e-06, + "first_to_last_ratio": 0.4032796404419227, + "imbalance_abs_log_ratio": 0.9081250607923619, + "last_quartile_mean": 6.069162992616839e-06, + "mean": 4.194480464292383e-06, + "normalized": [ + 1.9615731362727682, + 0.7747288452238285, + 0.1785692992504359, + 0.2633308680375469, + 0.18148950204723951, + 0.349408887478932, + 0.383871487008665, + 0.575200617814069, + 0.3462103003815746, + 0.3364194451166286, + 0.2854708899046967, + 0.31108734109147806, + 0.30067908576983693, + 0.513261777831553, + 0.5156365685672093, + 0.5885731468966555, + 0.3799425036372034, + 0.7326093628890369, + 0.5694277551741158, + 0.8619428981160375, + 1.5998405056629517, + 3.0497744670188736, + 2.8846131894263896, + 2.4808153927254097, + 1.8266821603141778, + 2.5325708585606295, + 2.0509024328203003, + 1.5887038326529273, + 1.3182729046192574, + 0.967953515357782, + 0.6305408473131235, + 0.6598961750186662 + ], + "peak_layer": 22, + "peak_normalized": 3.0497744670188736, + "population_cv": 0.8409982900394547, + "rest_mean": 3.131610654876298e-06, + "spike_contrast": 3.1721623566680712, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.93397743513924e-06, + "top_five_layers": [ + 22, + 23, + 26, + 24, + 27 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.227780199376866e-06, + 3.2495850064151455e-06, + 7.490054372283339e-07, + 1.1045361816286459e-06, + 7.612541708112985e-07, + 1.4655887525805156e-06, + 1.6101414530567126e-06, + 2.4126677544700215e-06, + 1.4521723414873122e-06, + 1.411104790349782e-06, + 1.197402070829412e-06, + 1.3048497748968657e-06, + 1.2611925512828748e-06, + 2.1528665001824265e-06, + 2.162827513529919e-06, + 2.4687585664651124e-06, + 1.5936614090605872e-06, + 3.072915660595754e-06, + 2.3884535949036945e-06, + 3.615402647483279e-06, + 6.710499746986898e-06, + 1.279221942240838e-05, + 1.2099453670089133e-05, + 1.0405731700302567e-05, + 7.661982635909226e-06, + 1.0622818990668748e-05, + 8.60247018863447e-06, + 6.663787189609138e-06, + 5.529469945031451e-06, + 4.060062110511353e-06, + 2.6447912659932626e-06, + 2.7679216145770624e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.4055149197010905e-05, + "first_to_last_ratio": 1.0862063963884478, + "imbalance_abs_log_ratio": 0.08269125537722917, + "last_quartile_mean": 2.2146020569380198e-05, + "mean": 3.472467025744663e-05, + "normalized": [ + 0.4666666997881285, + 0.8663177887052829, + 0.6219999695375312, + 0.5679169854003736, + 0.3844149632749681, + 0.7928386669686874, + 0.9321622159542684, + 0.9095983613860097, + 0.6255219017157577, + 0.5547521439337422, + 0.4549635591797121, + 0.4053293755903786, + 0.4456534403712468, + 0.7958744306526087, + 0.6527431991239231, + 0.5563097060509357, + 0.7807692106312896, + 1.1853454205486673, + 1.2646459398653336, + 1.548550635017449, + 2.506691252154866, + 4.290967865992318, + 3.0302171410000085, + 2.2576657022736453, + 1.197190310100688, + 1.343054897090299, + 0.7711430639674054, + 0.472191241764576, + 0.47986836664550414, + 0.29867429600060663, + 0.25046027339059757, + 0.2895009759231917 + ], + "peak_layer": 22, + "peak_normalized": 4.290967865992318, + "population_cv": 0.8713516434982468, + "rest_mean": 2.4072257407102302e-05, + "spike_contrast": 3.832116701364666, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.224769964930602e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.6204847270273603e-05, + 3.0082599550951272e-05, + 2.1598743842332624e-05, + 1.9720730051631108e-05, + 1.3348682841751724e-05, + 2.7531061277841218e-05, + 3.236902557546273e-05, + 3.1585503165842965e-05, + 2.1721041775890626e-05, + 1.926358527271077e-05, + 1.579845957166981e-05, + 1.4074928913032636e-05, + 1.54751687659882e-05, + 2.763647717074491e-05, + 2.2666292352369055e-05, + 1.9317671103635803e-05, + 2.7111953386338428e-05, + 4.116072886972688e-05, + 4.391441325424239e-05, + 5.37729101779405e-05, + 8.704402716830373e-05, + 0.0001490024442318827, + 0.00010522329102968797, + 7.839669706299901e-05, + 4.157203875365667e-05, + 4.663713843910955e-05, + 2.6777688617585227e-05, + 1.6396685168729164e-05, + 1.6663270798744634e-05, + 1.0371366442996077e-05, + 8.697150406078435e-06, + 1.0052825928141829e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.579084118930041e-05, + "first_to_last_ratio": 1.101662799461071, + "imbalance_abs_log_ratio": 0.09682067430659903, + "last_quartile_mean": 1.433364292324768e-05, + "mean": 2.2897426177337366e-05, + "normalized": [ + 0.39886794777854795, + 0.877049729540605, + 0.6305457825207785, + 0.5772129763101231, + 0.4367770087210306, + 0.8436499656131583, + 0.972408138490726, + 0.7805596072336106, + 0.6568359938107564, + 0.6066017752376727, + 0.49315422395705855, + 0.4249778682175203, + 0.4986390140344086, + 0.857095556009823, + 0.6846582999905741, + 0.5629018786115396, + 0.7936422327048538, + 1.202661838363613, + 1.2736141330113293, + 1.5255025761233132, + 2.344832382358406, + 4.19579128393446, + 3.181636935928761, + 2.172433813999188, + 1.1390303293883177, + 1.4232014401744595, + 0.759652938746426, + 0.4324960642291122, + 0.4879317473364259, + 0.30796555094004274, + 0.21544260190277736, + 0.24222836478058168 + ], + "peak_layer": 22, + "peak_normalized": 4.19579128393446, + "population_cv": 0.858855507029887, + "rest_mean": 1.6084403240610198e-05, + "spike_contrast": 3.7109085828539374, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.968775003566407e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.133049388765357e-06, + 2.0082181436009705e-05, + 1.4437875506700948e-05, + 1.3216691513662226e-05, + 1.0001069313148037e-05, + 1.93174128071405e-05, + 2.226564356533345e-05, + 1.7872805983643048e-05, + 1.5039853678899817e-05, + 1.3889619367546402e-05, + 1.1291962437098846e-05, + 9.73089936451288e-06, + 1.141755001299316e-05, + 1.9625282220658846e-05, + 1.5676912880735472e-05, + 1.2889004210592248e-05, + 1.8172364434576593e-05, + 2.7537860660231672e-05, + 2.9162485589040443e-05, + 3.493008262012154e-05, + 5.36906263732817e-05, + 9.607282117940485e-05, + 7.285129686351866e-05, + 4.9743142881197855e-05, + 2.6080862880917266e-05, + 3.2587649911874905e-05, + 1.7394097085343674e-05, + 9.903046702675056e-06, + 1.117238116421504e-05, + 7.051618467812659e-06, + 4.933081072522327e-06, + 5.546406100620516e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.034897362245829e-05, + "first_to_last_ratio": 1.0677221298454265, + "imbalance_abs_log_ratio": 0.06552752863846958, + "last_quartile_mean": 6.588696783182968e-05, + "mean": 0.00010326150248829435, + "normalized": [ + 0.5211538788724873, + 0.8562183017948943, + 0.6056449715109208, + 0.5367927676701035, + 0.33719464941795635, + 0.7400076028920698, + 0.8752614309891937, + 0.977887157814224, + 0.5891182329173178, + 0.4959165872409433, + 0.4220040517626212, + 0.3715438426401133, + 0.3950700830815582, + 0.7426266558595574, + 0.622546917450552, + 0.5356415681171627, + 0.7637846157174611, + 1.2023092054640232, + 1.2927045115765263, + 1.5352424858917262, + 2.711590866084263, + 4.42744668313384, + 2.973944015958163, + 2.3638740639156786, + 1.2453125473146385, + 1.2805717504661769, + 0.7715376325621908, + 0.48453681266643056, + 0.47249674362438154, + 0.27056316221994425, + 0.2589436600974457, + 0.3205125432754346 + ], + "peak_layer": 22, + "peak_normalized": 4.42744668313384, + "population_cv": 0.9059813522703718, + "rest_mean": 6.990356949380064e-05, + "spike_contrast": 4.054075388577877, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002833943406585604, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.381513255997561e-05, + 8.841438830131665e-05, + 6.25398097326979e-05, + 5.5430027714464813e-05, + 3.481922612991184e-05, + 7.641429692739621e-05, + 9.03808104339987e-05, + 0.00010097809717990458, + 6.083323387429118e-05, + 5.120909190736711e-05, + 4.357677244115621e-05, + 3.8366175431292504e-05, + 4.079553036717698e-05, + 7.66847442719154e-05, + 6.428513006540015e-05, + 5.5311153118964285e-05, + 7.886954699642956e-05, + 0.00012415225501172245, + 0.00013348661013878882, + 0.0001585314457770437, + 0.0002800029469653964, + 0.00045718479668721557, + 0.00030709392740391195, + 0.00024409718753304332, + 0.00012859284470323473, + 0.00013223376299720258, + 7.96701351646334e-05, + 5.003399928682484e-05, + 4.879072366748005e-05, + 2.7938758648815565e-05, + 2.6738911401480436e-05, + 3.309660678496584e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059514161199331284, + 0.08157797157764435, + 0.14292451739311218, + 0.10997853428125381, + 0.0991024449467659, + 0.1065012738108635, + 0.12155740708112717, + 0.1095810979604721, + 0.06964563578367233, + 0.08311934769153595, + 0.09363401681184769, + 0.0941167026758194, + 0.08564268052577972, + 0.09493856877088547, + 0.09063401818275452, + 0.09473779052495956, + 0.10505300015211105, + 0.09355910867452621, + 0.09618847072124481, + 0.11416846513748169, + 0.047205954790115356, + 0.053352247923612595, + 0.06512638926506042, + 0.07829347252845764, + 0.05368548259139061, + 0.05310077592730522, + 0.061235878616571426, + 0.07320034503936768, + 0.05304349213838577, + 0.0639304369688034, + 0.07404901832342148, + 0.08157160878181458 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.406091136388568e-06, + "first_to_last_ratio": 0.13952873923075965, + "imbalance_abs_log_ratio": 1.9694846829479884, + "last_quartile_mean": 3.15783770474809e-05, + "mean": 1.6862885999557875e-05, + "normalized": [ + 0.5744948983086973, + 0.20976707009476936, + 0.1380798037056023, + 0.25679297795080097, + 0.1911977573081938, + 0.15397727234373496, + 0.20783011320461262, + 0.35817424975363493, + 0.40845309421862647, + 0.30185320868879156, + 0.2734882217077138, + 0.2725610842103788, + 0.35283552344917235, + 0.37101154440150896, + 0.4205580365391262, + 0.41850822128869175, + 0.3521537888379847, + 0.4454210973675298, + 0.41314047811549354, + 0.5286573606706721, + 2.6023136902657806, + 2.9811111131589443, + 2.583962732113176, + 2.2024122663803394, + 2.8672118131663793, + 2.8994466418579012, + 2.457231935453014, + 1.624795250078144, + 1.8440638614795903, + 1.4598750180798175, + 0.9499358030010668, + 0.8786840728001112 + ], + "peak_layer": 22, + "peak_normalized": 2.9811111131589443, + "population_cv": 0.9711949879643771, + "rest_mean": 1.1718449412809503e-05, + "spike_contrast": 3.8096203683057026, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.464284356799908e-05, + "top_five_layers": [ + 22, + 26, + 25, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.687641977507155e-06, + 3.5372781894693617e-06, + 2.3284239887289004e-06, + 4.330270712671336e-06, + 3.2241459848592058e-06, + 2.596501190055278e-06, + 3.5046155062445905e-06, + 6.039851541572716e-06, + 6.88769796397537e-06, + 5.090116246719845e-06, + 4.611800704878988e-06, + 4.596166490955511e-06, + 5.949825208517723e-06, + 6.2563253777625505e-06, + 7.091822226357181e-06, + 7.05725642546895e-06, + 5.938329195487313e-06, + 7.511085186706623e-06, + 6.9667407842644025e-06, + 8.914688805816695e-06, + 4.388251909404062e-05, + 5.027013685321435e-05, + 4.357306897873059e-05, + 3.7139026971999556e-05, + 4.8349465942010283e-05, + 4.88930381834507e-05, + 4.1436022002017125e-05, + 2.739873707469087e-05, + 3.1096238672034815e-05, + 2.4617706003482454e-05, + 1.6018659152905457e-05, + 1.4817149349255487e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.77339601350468e-05, + "first_to_last_ratio": 0.14031202708197554, + "imbalance_abs_log_ratio": 1.9638865715137788, + "last_quartile_mean": 0.00012638945145226899, + "mean": 6.748338435613732e-05, + "normalized": [ + 0.581715983352747, + 0.21329408694826651, + 0.1383654282103214, + 0.26055913189981234, + 0.18941324497814624, + 0.15268445345034073, + 0.20588194209370914, + 0.3604059140126238, + 0.4112482421872158, + 0.3050474729826253, + 0.2735060049358427, + 0.27320166025083037, + 0.35640352937895586, + 0.3762475521803712, + 0.424911703962158, + 0.41981156521974744, + 0.35303414718819204, + 0.4487530978367584, + 0.4155766160707765, + 0.5338541629759799, + 2.603208152672671, + 2.9606390037219725, + 2.573692405652612, + 2.185365730719144, + 2.873997131282303, + 2.8916679373421434, + 2.4621472618371403, + 1.629468290031214, + 1.8469331471633454, + 1.452438714650874, + 0.9530063641283487, + 0.8735199206828114 + ], + "peak_layer": 22, + "peak_normalized": 2.9606390037219725, + "population_cv": 0.9681787165444765, + "rest_mean": 4.699617262236576e-05, + "spike_contrast": 3.7899751783986364, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017811432771850378, + "top_five_layers": [ + 22, + 26, + 25, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.9256163290701807e-05, + 1.4393806850421242e-05, + 9.337367373518646e-06, + 1.7583412045496516e-05, + 1.278224681300344e-05, + 1.03036636573961e-05, + 1.3893610230297782e-05, + 2.432141081953887e-05, + 2.775242319330573e-05, + 2.0585635866154917e-05, + 1.8457110854797065e-05, + 1.843657264544163e-05, + 2.405131635896396e-05, + 2.5390458176843822e-05, + 2.8674479835899547e-05, + 2.8330305212875828e-05, + 2.382393904554192e-05, + 3.028337778232526e-05, + 2.8044516511727124e-05, + 3.602628567023203e-05, + 0.00017567329632584006, + 0.00019979393982794136, + 0.00017368147382512689, + 0.00014747587556485087, + 0.0001939470530487597, + 0.00019513953884597868, + 0.00016615403001196682, + 0.00010996203491231427, + 0.00012463729945011437, + 9.801548003451899e-05, + 6.431209476431832e-05, + 5.894808055018075e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.015726652927696705, + "first_to_last_ratio": 0.14031202298106044, + "imbalance_abs_log_ratio": 1.9638866007408895, + "last_quartile_mean": 0.11208343086764216, + "mean": 0.05984494066797197, + "normalized": [ + 0.5817159394623853, + 0.21329409008400638, + 0.13836543030820245, + 0.26055913499009653, + 0.18941324063527437, + 0.1526844532232541, + 0.20588194088455028, + 0.36040590002189726, + 0.41124824181954295, + 0.3050474835118139, + 0.2735060178075916, + 0.273201651110741, + 0.35640353553867715, + 0.37624753453424215, + 0.42491172006038797, + 0.41981156257536634, + 0.35303415045149444, + 0.4487531036157151, + 0.4155766045421345, + 0.5338542063524581, + 2.6032081764056025, + 2.9606389456555853, + 2.5736924190660213, + 2.185365706690064, + 2.873996992351884, + 2.891668001746537, + 2.4621473625257404, + 1.6294681803958442, + 1.8469332209343858, + 1.4524387488575696, + 0.9530063959374594, + 0.8735199079034743 + ], + "peak_layer": 22, + "peak_normalized": 2.9606389456555853, + "population_cv": 0.9681787143090018, + "rest_mean": 0.04167667665967235, + "spike_contrast": 3.789975088527889, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15795356631278992, + "top_five_layers": [ + 22, + 26, + 25, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.03481275588274002, + 0.01276457216590643, + 0.008280470967292786, + 0.015593145973980427, + 0.011335424147546291, + 0.009137392044067383, + 0.012320992536842823, + 0.02156846970319748, + 0.024611126631498337, + 0.018255548551678658, + 0.016367951408028603, + 0.016349736601114273, + 0.021328948438167572, + 0.022516511380672455, + 0.02542881667613983, + 0.0251235980540514, + 0.021127307787537575, + 0.026855602860450745, + 0.02487015724182129, + 0.03194847330451012, + 0.1557888388633728, + 0.1771792620420456, + 0.1540224701166153, + 0.1307830810546875, + 0.1719941794872284, + 0.17305169999599457, + 0.14734706282615662, + 0.09751542657613754, + 0.11052960902452469, + 0.08692111074924469, + 0.05703261122107506, + 0.05227574706077576 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0061295867558329e-06, + "first_to_last_ratio": 0.11842320932790659, + "imbalance_abs_log_ratio": 2.1334905510136277, + "last_quartile_mean": 8.49605066832737e-06, + "mean": 4.365163350783519e-06, + "normalized": [ + 0.5368020226986673, + 0.16971060838105761, + 0.13410619991648343, + 0.2737672720355794, + 0.17254528945791953, + 0.13905042866169556, + 0.15902329960072473, + 0.2589205349972531, + 0.33151883176756186, + 0.2629812299966789, + 0.24045074835041885, + 0.24352968420582896, + 0.32204843448988535, + 0.36030617882440874, + 0.42811660272336727, + 0.41111875160131917, + 0.35123001138976506, + 0.43809006822116814, + 0.40879993942951587, + 0.3687203828633752, + 2.5744168736304367, + 2.993182569498261, + 2.6417671760629977, + 2.209152675699325, + 2.926027285435564, + 2.960732429505928, + 2.7232927720785614, + 1.7159123604866093, + 1.8688437451101783, + 1.456254776895987, + 1.0064626168064281, + 0.9131181991770502 + ], + "peak_layer": 22, + "peak_normalized": 2.993182569498261, + "population_cv": 1.010043630070163, + "rest_mean": 3.0160778355484246e-06, + "spike_contrast": 3.8627070547516635, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1650225133053027e-05, + "top_five_layers": [ + 22, + 26, + 25, + 27, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.3432285161106847e-06, + 7.408145279441669e-07, + 5.853954689882812e-07, + 1.1950388625336927e-06, + 7.531883738920442e-07, + 6.069778351047717e-07, + 6.941626793377509e-07, + 1.1302304301352706e-06, + 1.447133854526328e-06, + 1.147956027125474e-06, + 1.049606794367719e-06, + 1.0630468523231684e-06, + 1.4057940234124544e-06, + 1.5727953268651618e-06, + 1.8687989040699904e-06, + 1.7946005073099514e-06, + 1.5331763734138804e-06, + 1.9123347101412946e-06, + 1.784478513400245e-06, + 1.6095247019620729e-06, + 1.1237750186410267e-05, + 1.306573085457785e-05, + 1.1531745258253068e-05, + 9.643312296248041e-06, + 1.277258706977591e-05, + 1.2924080692755524e-05, + 1.188761780213099e-05, + 7.490237749152584e-06, + 8.157808224495966e-06, + 6.356789981509792e-06, + 4.393373728817096e-06, + 3.985910097981105e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.197550119513835e-05, + "first_to_last_ratio": 0.1587346511851631, + "imbalance_abs_log_ratio": 1.8405213313314666, + "last_quartile_mean": 7.544352229160722e-05, + "mean": 4.3035416283032646e-05, + "normalized": [ + 0.6336653627326162, + 0.22557503269458085, + 0.11384714614846705, + 0.24237874371328197, + 0.21264877275758737, + 0.17043176216342354, + 0.2212795583484355, + 0.40634029614109113, + 0.4665961306783005, + 0.344071525310092, + 0.31263674643247324, + 0.325989461953546, + 0.4165298351149085, + 0.4247779117423742, + 0.4717272077470243, + 0.4737094580054267, + 0.40283409959357847, + 0.521705164723695, + 0.4749365612827932, + 0.5725415698804192, + 2.7082874349482955, + 3.047902384226458, + 2.6438885206938516, + 2.1412461889756425, + 2.9349243481759872, + 2.7222959905080586, + 2.283569880862303, + 1.5378599969243498, + 1.615015848446693, + 1.3002882037476036, + 0.8505045545457767, + 0.7799943007808661 + ], + "peak_layer": 22, + "peak_normalized": 3.047902384226458, + "population_cv": 0.9449871736776022, + "rest_mean": 2.9525086692619544e-05, + "spike_contrast": 3.928564114944936, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001159911960712634, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.727005266933702e-05, + 9.707715435069986e-06, + 4.899459327134537e-06, + 1.0430870133859571e-05, + 9.151428457698785e-06, + 7.334601832553744e-06, + 9.522857908450533e-06, + 1.748702379700262e-05, + 2.0080158719792962e-05, + 1.4807261322857812e-05, + 1.3454452528094407e-05, + 1.4029092199052684e-05, + 1.7925534848473035e-05, + 1.8280494259670377e-05, + 2.0300976757425815e-05, + 2.038628372247331e-05, + 1.733613316901028e-05, + 2.245179894089233e-05, + 2.0439092622837052e-05, + 2.4639564799144864e-05, + 0.00011655227717710659, + 0.00013116774789523333, + 0.00011378084309399128, + 9.214942110702395e-05, + 0.00012630569108296186, + 0.00011715514119714499, + 9.827438043430448e-05, + 6.61824451526627e-05, + 6.95028793415986e-05, + 5.5958444136194885e-05, + 3.6601817555492744e-05, + 3.3567379432497546e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 7.998359336625072e-06, + "first_to_last_ratio": 0.19394161321534342, + "imbalance_abs_log_ratio": 1.6401981280247118, + "last_quartile_mean": 4.124106840208697e-05, + "mean": 2.6285412580762113e-05, + "normalized": [ + 0.6933384670594045, + 0.2456346256669455, + 0.10397585233667712, + 0.2374486790180797, + 0.24369297059389394, + 0.19793868329883177, + 0.24449766268182652, + 0.46778427650330523, + 0.5456273930041772, + 0.3924025527768945, + 0.36245298346490396, + 0.39321573934626614, + 0.49513567631933314, + 0.492561727340712, + 0.5399874725375117, + 0.5520792544136082, + 0.47500593515532746, + 0.6277771942942818, + 0.5578424227128029, + 0.6222043300599132, + 2.928340543497061, + 3.220069339518848, + 2.7546600577445735, + 2.05455257890152, + 2.8717816218189083, + 2.429966999218037, + 1.946404647300736, + 1.3608722565402098, + 1.3761877892744612, + 1.13757832466242, + 0.7380055991332236, + 0.6909763438053044 + ], + "peak_layer": 22, + "peak_normalized": 3.220069339518848, + "population_cv": 0.9165336328191448, + "rest_mean": 1.7689689221461542e-05, + "spike_contrast": 4.109869755811259, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.27023187209852e-05, + "top_five_layers": [ + 22, + 21, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.822468766476959e-05, + 6.4566074797767214e-06, + 2.7330481771059567e-06, + 6.241436494747177e-06, + 6.405570275092032e-06, + 5.2028999562026e-06, + 6.426721938623814e-06, + 1.2295902706682682e-05, + 1.4342041140480433e-05, + 1.0314462997484952e-05, + 9.527226211503148e-06, + 1.033583794196602e-05, + 1.3014845535508357e-05, + 1.2947188224643469e-05, + 1.4193793504091445e-05, + 1.4511630979541223e-05, + 1.2485726983868517e-05, + 1.6501382560818456e-05, + 1.4663118236057926e-05, + 1.6354897525161505e-05, + 7.697263936279342e-05, + 8.464085112791508e-05, + 7.24073761375621e-05, + 5.4004762205295265e-05, + 7.548596477136016e-05, + 6.387268513208255e-05, + 5.116204920341261e-05, + 3.577108873287216e-05, + 3.617366382968612e-05, + 2.9901715606683865e-05, + 1.9398781660129316e-05, + 1.8162598280468956e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.53738644207624e-05, + "first_to_last_ratio": 0.131222819995581, + "imbalance_abs_log_ratio": 2.0308584847035727, + "last_quartile_mean": 0.0002695709817999159, + "mean": 0.00014080404997685037, + "normalized": [ + 0.577470254414939, + 0.20295174191935292, + 0.1151700578824124, + 0.23719265118106814, + 0.1794139596985285, + 0.14982370308789797, + 0.19985925174978317, + 0.3479392297690331, + 0.40530357048439714, + 0.29950920312948914, + 0.2746918219238327, + 0.2680798564391957, + 0.33955736615799725, + 0.36796996435571633, + 0.4082064803862017, + 0.40241709302556344, + 0.34764889743937344, + 0.4446227376348821, + 0.404675572121426, + 0.502974600996141, + 2.462986532813244, + 2.8953303082983743, + 2.595561209030766, + 2.254551431946411, + 3.008292708807131, + 2.942171245358548, + 2.6598889524780778, + 1.7951160354833269, + 1.7746811041390398, + 1.3931209151914112, + 0.9062455092896525, + 0.836576033366786 + ], + "peak_layer": 25, + "peak_normalized": 3.008292708807131, + "population_cv": 0.9883689549295365, + "rest_mean": 9.795413286526498e-05, + "spike_contrast": 3.799672269993553, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003721936023794115, + "top_five_layers": [ + 25, + 26, + 22, + 27, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 8.131015056278557e-05, + 2.8576427212101407e-05, + 1.6216410585911945e-05, + 3.339768591104075e-05, + 2.5262212147936225e-05, + 2.1095784177305177e-05, + 2.814099207171239e-05, + 4.8991252697305754e-05, + 5.706838419428095e-05, + 4.217210880597122e-05, + 3.867772102239542e-05, + 3.7746729503851384e-05, + 4.7811052354518324e-05, + 5.181166125112213e-05, + 5.7477125665172935e-05, + 5.666195647791028e-05, + 4.8950372729450464e-05, + 6.260468217078596e-05, + 5.697995948139578e-05, + 7.082086085574701e-05, + 0.0003467984788585454, + 0.0004076742334291339, + 0.00036546553019434214, + 0.00031744997249916196, + 0.0004235797969158739, + 0.00041426962707191706, + 0.00037452313699759543, + 0.00025275960797443986, + 0.00024988228688016534, + 0.000196157066966407, + 0.00012760303798131645, + 0.00011779329361161217 + ] + } + } + } + } + }, + "learned": { + "bits_per_byte": 1.8740129107275643, + "forward": { + "activation_sha256": { + "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" + }, + "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", + "loss_nats": 1.2989667654037476, + "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" + }, + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22886337339878082, + "group": 1, + "latest_source_mass": 0.06350618600845337, + "layer": 1, + "max_source_mass": 0.9364938139915466, + "mixer_index": 1, + "normalized_entropy": 0.3301800538435402, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9364938139915466, + "median": 0.9452826976776123, + "p05": 0.8725647330284119, + "p95": 0.9712384939193726 + }, + { + "label": "current_group_partial", + "mean": 0.06350618600845337, + "median": 0.0547172985970974, + "p05": 0.028761589899659157, + "p95": 0.12743529677391052 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.603729784488678, + "group": 1, + "latest_source_mass": 0.4133155345916748, + "layer": 2, + "max_source_mass": 0.5866844654083252, + "mixer_index": 2, + "normalized_entropy": 0.8709979661187783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5866844654083252, + "median": 0.5734395384788513, + "p05": 0.29813364148139954, + "p95": 0.8843642473220825 + }, + { + "label": "current_group_partial", + "mean": 0.4133155345916748, + "median": 0.4265604615211487, + "p05": 0.11563576757907867, + "p95": 0.7018663287162781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3967982828617096, + "group": 1, + "latest_source_mass": 0.14110636711120605, + "layer": 2, + "max_source_mass": 0.858893632888794, + "mixer_index": 3, + "normalized_entropy": 0.5724589149178446, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.858893632888794, + "median": 0.8705973029136658, + "p05": 0.7599387168884277, + "p95": 0.9270997643470764 + }, + { + "label": "current_group_partial", + "mean": 0.14110636711120605, + "median": 0.12940272688865662, + "p05": 0.07290027290582657, + "p95": 0.2400612235069275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6852496862411499, + "group": 1, + "latest_source_mass": 0.518975019454956, + "layer": 3, + "max_source_mass": 0.518975019454956, + "mixer_index": 4, + "normalized_entropy": 0.9886063241108252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48102498054504395, + "median": 0.48102718591690063, + "p05": 0.38786420226097107, + "p95": 0.580143928527832 + }, + { + "label": "current_group_partial", + "mean": 0.518975019454956, + "median": 0.5189728140830994, + "p05": 0.41985607147216797, + "p95": 0.6121357679367065 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5890734195709229, + "group": 1, + "latest_source_mass": 0.3123423159122467, + "layer": 3, + "max_source_mass": 0.6876576542854309, + "mixer_index": 5, + "normalized_entropy": 0.8498533011344741, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6876576542854309, + "median": 0.6909762620925903, + "p05": 0.48803940415382385, + "p95": 0.8900076746940613 + }, + { + "label": "current_group_partial", + "mean": 0.3123423159122467, + "median": 0.3090237081050873, + "p05": 0.10999235510826111, + "p95": 0.5119606256484985 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6873323321342468, + "group": 1, + "latest_source_mass": 0.4831485152244568, + "layer": 4, + "max_source_mass": 0.5168514251708984, + "mixer_index": 6, + "normalized_entropy": 0.9916109470127238, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5168514251708984, + "median": 0.5219584703445435, + "p05": 0.4251725673675537, + "p95": 0.5947866439819336 + }, + { + "label": "current_group_partial", + "mean": 0.4831485152244568, + "median": 0.47804147005081177, + "p05": 0.405213326215744, + "p95": 0.5748274326324463 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.487301766872406, + "group": 1, + "latest_source_mass": 0.2060384750366211, + "layer": 4, + "max_source_mass": 0.7939615249633789, + "mixer_index": 7, + "normalized_entropy": 0.7030278424832499, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7939615249633789, + "median": 0.7953404188156128, + "p05": 0.6590759754180908, + "p95": 0.9329437017440796 + }, + { + "label": "current_group_partial", + "mean": 0.2060384750366211, + "median": 0.20465964078903198, + "p05": 0.06705628335475922, + "p95": 0.3409239649772644 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674697756767273, + "group": 2, + "latest_source_mass": 0.42952626943588257, + "layer": 5, + "max_source_mass": 0.5704737305641174, + "mixer_index": 8, + "normalized_entropy": 0.9733831077870527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5704737305641174, + "median": 0.5696132779121399, + "p05": 0.4695202708244324, + "p95": 0.6750621199607849 + }, + { + "label": "completed_group_1", + "mean": 0.42952626943588257, + "median": 0.4303867816925049, + "p05": 0.3249378800392151, + "p95": 0.5304797887802124 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9691112041473389, + "group": 2, + "latest_source_mass": 0.2681158185005188, + "layer": 5, + "max_source_mass": 0.5506307482719421, + "mixer_index": 9, + "normalized_entropy": 0.8821230329784768, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5506307482719421, + "median": 0.543993353843689, + "p05": 0.41262108087539673, + "p95": 0.7161091566085815 + }, + { + "label": "completed_group_1", + "mean": 0.18125343322753906, + "median": 0.1794438362121582, + "p05": 0.11575750261545181, + "p95": 0.25031501054763794 + }, + { + "label": "current_group_partial", + "mean": 0.2681158185005188, + "median": 0.2664111256599426, + "p05": 0.1564716249704361, + "p95": 0.3817881643772125 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0745112895965576, + "group": 2, + "latest_source_mass": 0.412794291973114, + "layer": 6, + "max_source_mass": 0.412794291973114, + "mixer_index": 10, + "normalized_entropy": 0.9780623252441762, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34262996912002563, + "median": 0.3412924110889435, + "p05": 0.29556700587272644, + "p95": 0.39125555753707886 + }, + { + "label": "completed_group_1", + "mean": 0.24457570910453796, + "median": 0.24382859468460083, + "p05": 0.22120657563209534, + "p95": 0.2734297513961792 + }, + { + "label": "current_group_partial", + "mean": 0.412794291973114, + "median": 0.4151499271392822, + "p05": 0.37735897302627563, + "p95": 0.43749260902404785 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074045658111572, + "group": 2, + "latest_source_mass": 0.3775783181190491, + "layer": 6, + "max_source_mass": 0.4399106502532959, + "mixer_index": 11, + "normalized_entropy": 0.9169791528842925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4399106502532959, + "median": 0.44856905937194824, + "p05": 0.22063353657722473, + "p95": 0.6359403133392334 + }, + { + "label": "completed_group_1", + "mean": 0.18251106142997742, + "median": 0.18562838435173035, + "p05": 0.1196163073182106, + "p95": 0.23045453429222107 + }, + { + "label": "current_group_partial", + "mean": 0.3775783181190491, + "median": 0.3675365447998047, + "p05": 0.23817461729049683, + "p95": 0.5557451844215393 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.060497522354126, + "group": 2, + "latest_source_mass": 0.4025024175643921, + "layer": 7, + "max_source_mass": 0.4025024175643921, + "mixer_index": 12, + "normalized_entropy": 0.9653064445872968, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3786153197288513, + "median": 0.3907395303249359, + "p05": 0.28039106726646423, + "p95": 0.4436405599117279 + }, + { + "label": "completed_group_1", + "mean": 0.2188822478055954, + "median": 0.21226292848587036, + "p05": 0.1885872334241867, + "p95": 0.2672030031681061 + }, + { + "label": "current_group_partial", + "mean": 0.4025024175643921, + "median": 0.39815032482147217, + "p05": 0.3533596992492676, + "p95": 0.460100919008255 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9941019415855408, + "group": 2, + "latest_source_mass": 0.4801652431488037, + "layer": 7, + "max_source_mass": 0.4801652431488037, + "mixer_index": 13, + "normalized_entropy": 0.9048705824970601, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32296955585479736, + "median": 0.2803739905357361, + "p05": 0.1784922033548355, + "p95": 0.6440245509147644 + }, + { + "label": "completed_group_1", + "mean": 0.19686520099639893, + "median": 0.2053825855255127, + "p05": 0.09813719242811203, + "p95": 0.2504696547985077 + }, + { + "label": "current_group_partial", + "mean": 0.4801652431488037, + "median": 0.5057140588760376, + "p05": 0.2529003322124481, + "p95": 0.598870575428009 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0503522157669067, + "group": 2, + "latest_source_mass": 0.4001518487930298, + "layer": 8, + "max_source_mass": 0.40130966901779175, + "mixer_index": 14, + "normalized_entropy": 0.9560717885654542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40130966901779175, + "median": 0.396759569644928, + "p05": 0.3479611873626709, + "p95": 0.4733521044254303 + }, + { + "label": "completed_group_1", + "mean": 0.19853851199150085, + "median": 0.19769755005836487, + "p05": 0.17136667668819427, + "p95": 0.22788912057876587 + }, + { + "label": "current_group_partial", + "mean": 0.4001518487930298, + "median": 0.40291696786880493, + "p05": 0.3506356179714203, + "p95": 0.43941178917884827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0129402875900269, + "group": 2, + "latest_source_mass": 0.42161279916763306, + "layer": 8, + "max_source_mass": 0.42161279916763306, + "mixer_index": 15, + "normalized_entropy": 0.9220179839951123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4093267023563385, + "median": 0.4046899676322937, + "p05": 0.2895161807537079, + "p95": 0.5762131810188293 + }, + { + "label": "completed_group_1", + "mean": 0.16906049847602844, + "median": 0.16888955235481262, + "p05": 0.11233551055192947, + "p95": 0.22477403283119202 + }, + { + "label": "current_group_partial", + "mean": 0.42161279916763306, + "median": 0.4227708578109741, + "p05": 0.3131049573421478, + "p95": 0.519075334072113 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0774019956588745, + "group": 3, + "latest_source_mass": 0.3560434877872467, + "layer": 9, + "max_source_mass": 0.38510820269584656, + "mixer_index": 16, + "normalized_entropy": 0.9806935592947451, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38510820269584656, + "median": 0.3892287313938141, + "p05": 0.2934730052947998, + "p95": 0.4718902111053467 + }, + { + "label": "completed_group_1", + "mean": 0.25884830951690674, + "median": 0.25734007358551025, + "p05": 0.22217319905757904, + "p95": 0.29551801085472107 + }, + { + "label": "completed_group_2", + "mean": 0.3560434877872467, + "median": 0.35412120819091797, + "p05": 0.29444968700408936, + "p95": 0.41908568143844604 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2740377187728882, + "group": 3, + "latest_source_mass": 0.3633861243724823, + "layer": 9, + "max_source_mass": 0.3633861243724823, + "mixer_index": 17, + "normalized_entropy": 0.9190239493895668, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29550763964653015, + "median": 0.30360478162765503, + "p05": 0.16904757916927338, + "p95": 0.45146965980529785 + }, + { + "label": "completed_group_1", + "mean": 0.09541244804859161, + "median": 0.09486871212720871, + "p05": 0.061360105872154236, + "p95": 0.13037073612213135 + }, + { + "label": "completed_group_2", + "mean": 0.24569380283355713, + "median": 0.24333015084266663, + "p05": 0.1698218435049057, + "p95": 0.31589236855506897 + }, + { + "label": "current_group_partial", + "mean": 0.3633861243724823, + "median": 0.36472851037979126, + "p05": 0.2795379161834717, + "p95": 0.4399876892566681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.360302448272705, + "group": 3, + "latest_source_mass": 0.28569501638412476, + "layer": 10, + "max_source_mass": 0.3109527826309204, + "mixer_index": 18, + "normalized_entropy": 0.9812507981160736, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3109527826309204, + "median": 0.32172977924346924, + "p05": 0.23520725965499878, + "p95": 0.37052470445632935 + }, + { + "label": "completed_group_1", + "mean": 0.20984819531440735, + "median": 0.20958667993545532, + "p05": 0.1891367882490158, + "p95": 0.23184512555599213 + }, + { + "label": "completed_group_2", + "mean": 0.19350400567054749, + "median": 0.19153150916099548, + "p05": 0.16607457399368286, + "p95": 0.22877082228660583 + }, + { + "label": "current_group_partial", + "mean": 0.28569501638412476, + "median": 0.28060320019721985, + "p05": 0.23824527859687805, + "p95": 0.3439165949821472 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2722704410552979, + "group": 3, + "latest_source_mass": 0.29912811517715454, + "layer": 10, + "max_source_mass": 0.35036465525627136, + "mixer_index": 19, + "normalized_entropy": 0.9177491279900463, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35036465525627136, + "median": 0.36210548877716064, + "p05": 0.12807878851890564, + "p95": 0.5797606706619263 + }, + { + "label": "completed_group_1", + "mean": 0.12631990015506744, + "median": 0.1264108121395111, + "p05": 0.06977735459804535, + "p95": 0.1755298376083374 + }, + { + "label": "completed_group_2", + "mean": 0.22418735921382904, + "median": 0.22563719749450684, + "p05": 0.1176326721906662, + "p95": 0.32001155614852905 + }, + { + "label": "current_group_partial", + "mean": 0.29912811517715454, + "median": 0.28589946031570435, + "p05": 0.193109393119812, + "p95": 0.44119665026664734 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3558036088943481, + "group": 3, + "latest_source_mass": 0.314197838306427, + "layer": 11, + "max_source_mass": 0.314197838306427, + "mixer_index": 20, + "normalized_entropy": 0.978005571485618, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298976868391037, + "median": 0.2961939573287964, + "p05": 0.24950163066387177, + "p95": 0.35835936665534973 + }, + { + "label": "completed_group_1", + "mean": 0.1941717267036438, + "median": 0.1918032318353653, + "p05": 0.16809208691120148, + "p95": 0.22895419597625732 + }, + { + "label": "completed_group_2", + "mean": 0.1926535815000534, + "median": 0.19306811690330505, + "p05": 0.16983944177627563, + "p95": 0.2155935913324356 + }, + { + "label": "current_group_partial", + "mean": 0.314197838306427, + "median": 0.31541913747787476, + "p05": 0.2673022747039795, + "p95": 0.35941022634506226 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2696614265441895, + "group": 3, + "latest_source_mass": 0.2862822115421295, + "layer": 11, + "max_source_mass": 0.3667730689048767, + "mixer_index": 21, + "normalized_entropy": 0.9158671218416545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3667730689048767, + "median": 0.4067659378051758, + "p05": 0.10593366622924805, + "p95": 0.5672274827957153 + }, + { + "label": "completed_group_1", + "mean": 0.15310388803482056, + "median": 0.15050218999385834, + "p05": 0.08651046454906464, + "p95": 0.22459912300109863 + }, + { + "label": "completed_group_2", + "mean": 0.19384080171585083, + "median": 0.18732509016990662, + "p05": 0.1267009675502777, + "p95": 0.281107634305954 + }, + { + "label": "current_group_partial", + "mean": 0.2862822115421295, + "median": 0.26271265745162964, + "p05": 0.17024393379688263, + "p95": 0.475044310092926 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3606936931610107, + "group": 3, + "latest_source_mass": 0.3028184771537781, + "layer": 12, + "max_source_mass": 0.3028184771537781, + "mixer_index": 22, + "normalized_entropy": 0.9815330216461395, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298227459192276, + "median": 0.2954302132129669, + "p05": 0.25287362933158875, + "p95": 0.35405808687210083 + }, + { + "label": "completed_group_1", + "mean": 0.18611131608486176, + "median": 0.1850002110004425, + "p05": 0.15901757776737213, + "p95": 0.21725836396217346 + }, + { + "label": "completed_group_2", + "mean": 0.21284276247024536, + "median": 0.21402311325073242, + "p05": 0.18819917738437653, + "p95": 0.23418807983398438 + }, + { + "label": "current_group_partial", + "mean": 0.3028184771537781, + "median": 0.3041122555732727, + "p05": 0.25661808252334595, + "p95": 0.34458184242248535 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.269316554069519, + "group": 3, + "latest_source_mass": 0.2653314769268036, + "layer": 12, + "max_source_mass": 0.4081852436065674, + "mixer_index": 23, + "normalized_entropy": 0.9156183489371815, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4081852436065674, + "median": 0.4254342317581177, + "p05": 0.20765496790409088, + "p95": 0.587238073348999 + }, + { + "label": "completed_group_1", + "mean": 0.14420127868652344, + "median": 0.1438048928976059, + "p05": 0.1064920499920845, + "p95": 0.1871720254421234 + }, + { + "label": "completed_group_2", + "mean": 0.18228203058242798, + "median": 0.17507661879062653, + "p05": 0.12183455377817154, + "p95": 0.26505547761917114 + }, + { + "label": "current_group_partial", + "mean": 0.2653314769268036, + "median": 0.24924281239509583, + "p05": 0.15609969198703766, + "p95": 0.4061311185359955 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3636136054992676, + "group": 4, + "latest_source_mass": 0.3136237859725952, + "layer": 13, + "max_source_mass": 0.3136237859725952, + "mixer_index": 24, + "normalized_entropy": 0.9836392931712563, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27427661418914795, + "median": 0.27403637766838074, + "p05": 0.22928935289382935, + "p95": 0.319773405790329 + }, + { + "label": "completed_group_1", + "mean": 0.21341991424560547, + "median": 0.20697325468063354, + "p05": 0.18552586436271667, + "p95": 0.26688331365585327 + }, + { + "label": "completed_group_2", + "mean": 0.19867971539497375, + "median": 0.197545126080513, + "p05": 0.1794913411140442, + "p95": 0.222100168466568 + }, + { + "label": "completed_group_3", + "mean": 0.3136237859725952, + "median": 0.31875866651535034, + "p05": 0.24156072735786438, + "p95": 0.3695686459541321 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.570773959159851, + "group": 4, + "latest_source_mass": 0.2294243425130844, + "layer": 13, + "max_source_mass": 0.2658740282058716, + "mixer_index": 25, + "normalized_entropy": 0.9759767351225285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2658740282058716, + "median": 0.2747163474559784, + "p05": 0.17927880585193634, + "p95": 0.3512483835220337 + }, + { + "label": "completed_group_1", + "mean": 0.13419361412525177, + "median": 0.13374894857406616, + "p05": 0.11072897166013718, + "p95": 0.1586270034313202 + }, + { + "label": "completed_group_2", + "mean": 0.1907704919576645, + "median": 0.19074805080890656, + "p05": 0.14699944853782654, + "p95": 0.23316305875778198 + }, + { + "label": "completed_group_3", + "mean": 0.17973749339580536, + "median": 0.17616114020347595, + "p05": 0.1363743096590042, + "p95": 0.23202598094940186 + }, + { + "label": "current_group_partial", + "mean": 0.2294243425130844, + "median": 0.22995705902576447, + "p05": 0.17975632846355438, + "p95": 0.2789157032966614 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5560038089752197, + "group": 4, + "latest_source_mass": 0.32468360662460327, + "layer": 14, + "max_source_mass": 0.32468360662460327, + "mixer_index": 26, + "normalized_entropy": 0.9667995248241249, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20477107167243958, + "median": 0.206403911113739, + "p05": 0.16619175672531128, + "p95": 0.2431035190820694 + }, + { + "label": "completed_group_1", + "mean": 0.15721669793128967, + "median": 0.15658551454544067, + "p05": 0.13878680765628815, + "p95": 0.17778633534908295 + }, + { + "label": "completed_group_2", + "mean": 0.13811248540878296, + "median": 0.13743364810943604, + "p05": 0.12352626770734787, + "p95": 0.15553249418735504 + }, + { + "label": "completed_group_3", + "mean": 0.1752161681652069, + "median": 0.17309701442718506, + "p05": 0.15223677456378937, + "p95": 0.20474471151828766 + }, + { + "label": "current_group_partial", + "mean": 0.32468360662460327, + "median": 0.32497870922088623, + "p05": 0.2946181893348694, + "p95": 0.35203880071640015 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5659973621368408, + "group": 4, + "latest_source_mass": 0.26703086495399475, + "layer": 14, + "max_source_mass": 0.26703086495399475, + "mixer_index": 27, + "normalized_entropy": 0.9730088685238187, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20531430840492249, + "median": 0.20229148864746094, + "p05": 0.10698921978473663, + "p95": 0.3393132984638214 + }, + { + "label": "completed_group_1", + "mean": 0.15986156463623047, + "median": 0.15952220559120178, + "p05": 0.11969029903411865, + "p95": 0.20180393755435944 + }, + { + "label": "completed_group_2", + "mean": 0.21557100117206573, + "median": 0.21654027700424194, + "p05": 0.1673085242509842, + "p95": 0.2610440254211426 + }, + { + "label": "completed_group_3", + "mean": 0.15222224593162537, + "median": 0.1510905921459198, + "p05": 0.11800260841846466, + "p95": 0.18893754482269287 + }, + { + "label": "current_group_partial", + "mean": 0.26703086495399475, + "median": 0.258539617061615, + "p05": 0.19330690801143646, + "p95": 0.36521556973457336 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5552496910095215, + "group": 4, + "latest_source_mass": 0.31774693727493286, + "layer": 15, + "max_source_mass": 0.31774693727493286, + "mixer_index": 28, + "normalized_entropy": 0.9663309649872576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2162160575389862, + "median": 0.2145705670118332, + "p05": 0.17727568745613098, + "p95": 0.26878347992897034 + }, + { + "label": "completed_group_1", + "mean": 0.17982497811317444, + "median": 0.18024154007434845, + "p05": 0.15453973412513733, + "p95": 0.2030555009841919 + }, + { + "label": "completed_group_2", + "mean": 0.1307673156261444, + "median": 0.13107559084892273, + "p05": 0.11463287472724915, + "p95": 0.14630235731601715 + }, + { + "label": "completed_group_3", + "mean": 0.1554446816444397, + "median": 0.15516987442970276, + "p05": 0.13627474009990692, + "p95": 0.1754152774810791 + }, + { + "label": "current_group_partial", + "mean": 0.31774693727493286, + "median": 0.319039523601532, + "p05": 0.27606821060180664, + "p95": 0.35650834441185 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5068919658660889, + "group": 4, + "latest_source_mass": 0.3298327624797821, + "layer": 15, + "max_source_mass": 0.3298327624797821, + "mixer_index": 29, + "normalized_entropy": 0.9362846209998111, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20646697282791138, + "median": 0.18775638937950134, + "p05": 0.0806753933429718, + "p95": 0.4274584949016571 + }, + { + "label": "completed_group_1", + "mean": 0.1197461187839508, + "median": 0.11989086866378784, + "p05": 0.0809398666024208, + "p95": 0.15743419528007507 + }, + { + "label": "completed_group_2", + "mean": 0.19215428829193115, + "median": 0.19620640575885773, + "p05": 0.1148109883069992, + "p95": 0.2491212636232376 + }, + { + "label": "completed_group_3", + "mean": 0.15179988741874695, + "median": 0.15212899446487427, + "p05": 0.10697861760854721, + "p95": 0.19542087614536285 + }, + { + "label": "current_group_partial", + "mean": 0.3298327624797821, + "median": 0.32837340235710144, + "p05": 0.22746729850769043, + "p95": 0.4339301884174347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5435223579406738, + "group": 4, + "latest_source_mass": 0.3224271535873413, + "layer": 16, + "max_source_mass": 0.3224271535873413, + "mixer_index": 30, + "normalized_entropy": 0.9590443632623663, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22410975396633148, + "median": 0.22760415077209473, + "p05": 0.14593973755836487, + "p95": 0.30093055963516235 + }, + { + "label": "completed_group_1", + "mean": 0.16986176371574402, + "median": 0.16963240504264832, + "p05": 0.13569067418575287, + "p95": 0.20651353895664215 + }, + { + "label": "completed_group_2", + "mean": 0.13080403208732605, + "median": 0.13075244426727295, + "p05": 0.10945598781108856, + "p95": 0.15187400579452515 + }, + { + "label": "completed_group_3", + "mean": 0.15279728174209595, + "median": 0.15036612749099731, + "p05": 0.12810495495796204, + "p95": 0.18468116223812103 + }, + { + "label": "current_group_partial", + "mean": 0.3224271535873413, + "median": 0.3164973258972168, + "p05": 0.2600978910923004, + "p95": 0.40530598163604736 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4949920177459717, + "group": 4, + "latest_source_mass": 0.31847020983695984, + "layer": 16, + "max_source_mass": 0.31847020983695984, + "mixer_index": 31, + "normalized_entropy": 0.9288907675133354, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27174079418182373, + "median": 0.2585451304912567, + "p05": 0.13639721274375916, + "p95": 0.4698348045349121 + }, + { + "label": "completed_group_1", + "mean": 0.12178683280944824, + "median": 0.11714950203895569, + "p05": 0.08594125509262085, + "p95": 0.1723237782716751 + }, + { + "label": "completed_group_2", + "mean": 0.1529826819896698, + "median": 0.15223106741905212, + "p05": 0.10371473431587219, + "p95": 0.203575000166893 + }, + { + "label": "completed_group_3", + "mean": 0.1350194811820984, + "median": 0.13363957405090332, + "p05": 0.08964549005031586, + "p95": 0.18468207120895386 + }, + { + "label": "current_group_partial", + "mean": 0.31847020983695984, + "median": 0.3185216188430786, + "p05": 0.21444711089134216, + "p95": 0.4301546812057495 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.544264793395996, + "group": 5, + "latest_source_mass": 0.2861037850379944, + "layer": 17, + "max_source_mass": 0.2861037850379944, + "mixer_index": 32, + "normalized_entropy": 0.9595056643474137, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24016115069389343, + "median": 0.23868846893310547, + "p05": 0.18435582518577576, + "p95": 0.2983453571796417 + }, + { + "label": "completed_group_1", + "mean": 0.20730282366275787, + "median": 0.1998077780008316, + "p05": 0.16255050897598267, + "p95": 0.28107208013534546 + }, + { + "label": "completed_group_2", + "mean": 0.09692573547363281, + "median": 0.0967404842376709, + "p05": 0.08098706603050232, + "p95": 0.11221462488174438 + }, + { + "label": "completed_group_3", + "mean": 0.1695065200328827, + "median": 0.1696585714817047, + "p05": 0.136396586894989, + "p95": 0.20130634307861328 + }, + { + "label": "completed_group_4", + "mean": 0.2861037850379944, + "median": 0.28883200883865356, + "p05": 0.22401469945907593, + "p95": 0.3409370183944702 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.686682939529419, + "group": 5, + "latest_source_mass": 0.12281867861747742, + "layer": 17, + "max_source_mass": 0.2885604202747345, + "mixer_index": 33, + "normalized_entropy": 0.9413556721740635, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16755425930023193, + "median": 0.1327776312828064, + "p05": 0.0790192037820816, + "p95": 0.3667412996292114 + }, + { + "label": "completed_group_1", + "mean": 0.10962115228176117, + "median": 0.10525335371494293, + "p05": 0.07840172946453094, + "p95": 0.15405339002609253 + }, + { + "label": "completed_group_2", + "mean": 0.1783454418182373, + "median": 0.17896884679794312, + "p05": 0.10802385956048965, + "p95": 0.24048911035060883 + }, + { + "label": "completed_group_3", + "mean": 0.13310004770755768, + "median": 0.13033035397529602, + "p05": 0.0904831737279892, + "p95": 0.17934636771678925 + }, + { + "label": "completed_group_4", + "mean": 0.2885604202747345, + "median": 0.2831825315952301, + "p05": 0.16967125236988068, + "p95": 0.4189499616622925 + }, + { + "label": "current_group_partial", + "mean": 0.12281867861747742, + "median": 0.12062528729438782, + "p05": 0.07461707293987274, + "p95": 0.1782219111919403 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7070834636688232, + "group": 5, + "latest_source_mass": 0.2988377809524536, + "layer": 18, + "max_source_mass": 0.2988377809524536, + "mixer_index": 34, + "normalized_entropy": 0.9527414214834803, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.181177020072937, + "median": 0.18419909477233887, + "p05": 0.13624237477779388, + "p95": 0.22192169725894928 + }, + { + "label": "completed_group_1", + "mean": 0.14557164907455444, + "median": 0.144476979970932, + "p05": 0.1293315887451172, + "p95": 0.16486899554729462 + }, + { + "label": "completed_group_2", + "mean": 0.08318156749010086, + "median": 0.0820261612534523, + "p05": 0.07088497281074524, + "p95": 0.0990293100476265 + }, + { + "label": "completed_group_3", + "mean": 0.11424209177494049, + "median": 0.11267061531543732, + "p05": 0.0979941189289093, + "p95": 0.1353992223739624 + }, + { + "label": "completed_group_4", + "mean": 0.17698991298675537, + "median": 0.17597860097885132, + "p05": 0.14837387204170227, + "p95": 0.21050144731998444 + }, + { + "label": "current_group_partial", + "mean": 0.2988377809524536, + "median": 0.2989804744720459, + "p05": 0.2721370458602905, + "p95": 0.32572072744369507 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7419235706329346, + "group": 5, + "latest_source_mass": 0.26016175746917725, + "layer": 18, + "max_source_mass": 0.26016175746917725, + "mixer_index": 35, + "normalized_entropy": 0.972186055410333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1532188355922699, + "median": 0.1451343297958374, + "p05": 0.11514341086149216, + "p95": 0.22103679180145264 + }, + { + "label": "completed_group_1", + "mean": 0.11420352011919022, + "median": 0.11261284351348877, + "p05": 0.0928458571434021, + "p95": 0.14123988151550293 + }, + { + "label": "completed_group_2", + "mean": 0.16597424447536469, + "median": 0.1655328869819641, + "p05": 0.13871800899505615, + "p95": 0.19611573219299316 + }, + { + "label": "completed_group_3", + "mean": 0.1250365972518921, + "median": 0.12369821965694427, + "p05": 0.10375828295946121, + "p95": 0.14912742376327515 + }, + { + "label": "completed_group_4", + "mean": 0.18140508234500885, + "median": 0.1796584278345108, + "p05": 0.14511024951934814, + "p95": 0.22286373376846313 + }, + { + "label": "current_group_partial", + "mean": 0.26016175746917725, + "median": 0.26266711950302124, + "p05": 0.1889389455318451, + "p95": 0.3234381675720215 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7127439975738525, + "group": 5, + "latest_source_mass": 0.2907913327217102, + "layer": 19, + "max_source_mass": 0.2907913327217102, + "mixer_index": 36, + "normalized_entropy": 0.9559006256078307, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18731768429279327, + "median": 0.19042012095451355, + "p05": 0.13879702985286713, + "p95": 0.23792000114917755 + }, + { + "label": "completed_group_1", + "mean": 0.16715484857559204, + "median": 0.1667211651802063, + "p05": 0.14574480056762695, + "p95": 0.19026139378547668 + }, + { + "label": "completed_group_2", + "mean": 0.08496320247650146, + "median": 0.08452782034873962, + "p05": 0.07410907745361328, + "p95": 0.097386933863163 + }, + { + "label": "completed_group_3", + "mean": 0.11362075805664062, + "median": 0.11248823255300522, + "p05": 0.0977015346288681, + "p95": 0.13219426572322845 + }, + { + "label": "completed_group_4", + "mean": 0.15615218877792358, + "median": 0.15540224313735962, + "p05": 0.1289147436618805, + "p95": 0.18598109483718872 + }, + { + "label": "current_group_partial", + "mean": 0.2907913327217102, + "median": 0.2900415062904358, + "p05": 0.25822532176971436, + "p95": 0.32475030422210693 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076834201812744, + "group": 5, + "latest_source_mass": 0.3738824725151062, + "layer": 19, + "max_source_mass": 0.3738824725151062, + "mixer_index": 37, + "normalized_entropy": 0.8972652009334232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15032559633255005, + "median": 0.12544110417366028, + "p05": 0.06976518034934998, + "p95": 0.35840943455696106 + }, + { + "label": "completed_group_1", + "mean": 0.08624927699565887, + "median": 0.08602266013622284, + "p05": 0.061998721212148666, + "p95": 0.11138717085123062 + }, + { + "label": "completed_group_2", + "mean": 0.15736976265907288, + "median": 0.15802210569381714, + "p05": 0.10121746361255646, + "p95": 0.20687000453472137 + }, + { + "label": "completed_group_3", + "mean": 0.08584291487932205, + "median": 0.0827617198228836, + "p05": 0.060561805963516235, + "p95": 0.11351694166660309 + }, + { + "label": "completed_group_4", + "mean": 0.14632999897003174, + "median": 0.1426895558834076, + "p05": 0.10106660425662994, + "p95": 0.20179018378257751 + }, + { + "label": "current_group_partial", + "mean": 0.3738824725151062, + "median": 0.3799295425415039, + "p05": 0.23631010949611664, + "p95": 0.4884527325630188 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.709290862083435, + "group": 5, + "latest_source_mass": 0.2901708781719208, + "layer": 20, + "max_source_mass": 0.2901708781719208, + "mixer_index": 38, + "normalized_entropy": 0.9539733939957076, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18991519510746002, + "median": 0.1924753338098526, + "p05": 0.14201581478118896, + "p95": 0.23709386587142944 + }, + { + "label": "completed_group_1", + "mean": 0.17091524600982666, + "median": 0.17006167769432068, + "p05": 0.15297070145606995, + "p95": 0.19153237342834473 + }, + { + "label": "completed_group_2", + "mean": 0.07822999358177185, + "median": 0.07778562605381012, + "p05": 0.06707640737295151, + "p95": 0.09089364111423492 + }, + { + "label": "completed_group_3", + "mean": 0.116765096783638, + "median": 0.11516137421131134, + "p05": 0.10162825882434845, + "p95": 0.13653048872947693 + }, + { + "label": "completed_group_4", + "mean": 0.1540035903453827, + "median": 0.15287795662879944, + "p05": 0.12593276798725128, + "p95": 0.18590238690376282 + }, + { + "label": "current_group_partial", + "mean": 0.2901708781719208, + "median": 0.29104921221733093, + "p05": 0.25643545389175415, + "p95": 0.3205650746822357 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6690887212753296, + "group": 5, + "latest_source_mass": 0.3342471718788147, + "layer": 20, + "max_source_mass": 0.3342471718788147, + "mixer_index": 39, + "normalized_entropy": 0.9315361520005944, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19029226899147034, + "median": 0.19644704461097717, + "p05": 0.09456793963909149, + "p95": 0.30468082427978516 + }, + { + "label": "completed_group_1", + "mean": 0.1100241169333458, + "median": 0.10808776319026947, + "p05": 0.09035106748342514, + "p95": 0.1359245330095291 + }, + { + "label": "completed_group_2", + "mean": 0.11471512168645859, + "median": 0.11494918912649155, + "p05": 0.08799737691879272, + "p95": 0.14024731516838074 + }, + { + "label": "completed_group_3", + "mean": 0.09999909996986389, + "median": 0.09682698547840118, + "p05": 0.07921088486909866, + "p95": 0.1264391839504242 + }, + { + "label": "completed_group_4", + "mean": 0.1507222205400467, + "median": 0.1476232260465622, + "p05": 0.11397356539964676, + "p95": 0.19991227984428406 + }, + { + "label": "current_group_partial", + "mean": 0.3342471718788147, + "median": 0.33802270889282227, + "p05": 0.2548266053199768, + "p95": 0.40130624175071716 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7458693981170654, + "group": 6, + "latest_source_mass": 0.2361394464969635, + "layer": 21, + "max_source_mass": 0.2361394464969635, + "mixer_index": 40, + "normalized_entropy": 0.9743882636597644, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20528709888458252, + "median": 0.2140858769416809, + "p05": 0.13168928027153015, + "p95": 0.27098023891448975 + }, + { + "label": "completed_group_1", + "mean": 0.1722964197397232, + "median": 0.17238952219486237, + "p05": 0.15053534507751465, + "p95": 0.19365043938159943 + }, + { + "label": "completed_group_2", + "mean": 0.10435444861650467, + "median": 0.10361101478338242, + "p05": 0.08888213336467743, + "p95": 0.12212536484003067 + }, + { + "label": "completed_group_3", + "mean": 0.12075909972190857, + "median": 0.1180175170302391, + "p05": 0.10358668118715286, + "p95": 0.14458929002285004 + }, + { + "label": "completed_group_4", + "mean": 0.16116347908973694, + "median": 0.15841388702392578, + "p05": 0.1337081342935562, + "p95": 0.19791515171527863 + }, + { + "label": "completed_group_5", + "mean": 0.2361394464969635, + "median": 0.23709020018577576, + "p05": 0.20446214079856873, + "p95": 0.26509594917297363 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4612125158309937, + "group": 6, + "latest_source_mass": 0.5543455481529236, + "layer": 21, + "max_source_mass": 0.5543455481529236, + "mixer_index": 41, + "normalized_entropy": 0.7509146897354808, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08934466540813446, + "median": 0.07911523431539536, + "p05": 0.04488275945186615, + "p95": 0.1737232506275177 + }, + { + "label": "completed_group_1", + "mean": 0.054379627108573914, + "median": 0.05344422906637192, + "p05": 0.04221750423312187, + "p95": 0.06924191862344742 + }, + { + "label": "completed_group_2", + "mean": 0.07567165046930313, + "median": 0.07438549399375916, + "p05": 0.05811355262994766, + "p95": 0.09632256627082825 + }, + { + "label": "completed_group_3", + "mean": 0.05758609622716904, + "median": 0.05612001568078995, + "p05": 0.044792283326387405, + "p95": 0.07347827404737473 + }, + { + "label": "completed_group_4", + "mean": 0.07884950935840607, + "median": 0.07757881283760071, + "p05": 0.05758974328637123, + "p95": 0.10452556610107422 + }, + { + "label": "completed_group_5", + "mean": 0.08982288837432861, + "median": 0.08912864327430725, + "p05": 0.0663854256272316, + "p95": 0.11604885756969452 + }, + { + "label": "current_group_partial", + "mean": 0.5543455481529236, + "median": 0.5596089363098145, + "p05": 0.45455610752105713, + "p95": 0.6367484331130981 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7241814136505127, + "group": 6, + "latest_source_mass": 0.39484071731567383, + "layer": 22, + "max_source_mass": 0.39484071731567383, + "mixer_index": 42, + "normalized_entropy": 0.886053970419732, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13360010087490082, + "median": 0.14072942733764648, + "p05": 0.07699814438819885, + "p95": 0.17506597936153412 + }, + { + "label": "completed_group_1", + "mean": 0.10609637200832367, + "median": 0.10586297512054443, + "p05": 0.08648421615362167, + "p95": 0.12602579593658447 + }, + { + "label": "completed_group_2", + "mean": 0.06363911926746368, + "median": 0.06324908882379532, + "p05": 0.054980404675006866, + "p95": 0.07316673547029495 + }, + { + "label": "completed_group_3", + "mean": 0.07426095008850098, + "median": 0.07362888753414154, + "p05": 0.06318790465593338, + "p95": 0.08665015548467636 + }, + { + "label": "completed_group_4", + "mean": 0.09383076429367065, + "median": 0.09294568747282028, + "p05": 0.07638302445411682, + "p95": 0.11445100605487823 + }, + { + "label": "completed_group_5", + "mean": 0.13373197615146637, + "median": 0.13327741622924805, + "p05": 0.11399447172880173, + "p95": 0.15641780197620392 + }, + { + "label": "current_group_partial", + "mean": 0.39484071731567383, + "median": 0.3879151940345764, + "p05": 0.3302172124385834, + "p95": 0.4852263331413269 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3979300260543823, + "group": 6, + "latest_source_mass": 0.5830056071281433, + "layer": 22, + "max_source_mass": 0.5830056071281433, + "mixer_index": 43, + "normalized_entropy": 0.7183939231382495, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08959861099720001, + "median": 0.07544538378715515, + "p05": 0.04165133833885193, + "p95": 0.20099139213562012 + }, + { + "label": "completed_group_1", + "mean": 0.05414731800556183, + "median": 0.05336771905422211, + "p05": 0.04069708660244942, + "p95": 0.07046990841627121 + }, + { + "label": "completed_group_2", + "mean": 0.07668881118297577, + "median": 0.07607124000787735, + "p05": 0.0555989071726799, + "p95": 0.09980960935354233 + }, + { + "label": "completed_group_3", + "mean": 0.05603286623954773, + "median": 0.054573021829128265, + "p05": 0.04162490367889404, + "p95": 0.0730118378996849 + }, + { + "label": "completed_group_4", + "mean": 0.06979228556156158, + "median": 0.06884932518005371, + "p05": 0.04576495662331581, + "p95": 0.09806846082210541 + }, + { + "label": "completed_group_5", + "mean": 0.07073447108268738, + "median": 0.07023533433675766, + "p05": 0.04956750199198723, + "p95": 0.09379896521568298 + }, + { + "label": "current_group_partial", + "mean": 0.5830056071281433, + "median": 0.5862616896629333, + "p05": 0.4658474922180176, + "p95": 0.6849814057350159 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7459560632705688, + "group": 6, + "latest_source_mass": 0.37691548466682434, + "layer": 23, + "max_source_mass": 0.37691548466682434, + "mixer_index": 44, + "normalized_entropy": 0.897243926765161, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14318783581256866, + "median": 0.1493408977985382, + "p05": 0.08292607218027115, + "p95": 0.19464610517024994 + }, + { + "label": "completed_group_1", + "mean": 0.1094653457403183, + "median": 0.10858799517154694, + "p05": 0.09432064741849899, + "p95": 0.1279931515455246 + }, + { + "label": "completed_group_2", + "mean": 0.06349228322505951, + "median": 0.06325754523277283, + "p05": 0.05439673364162445, + "p95": 0.0737837627530098 + }, + { + "label": "completed_group_3", + "mean": 0.07859592139720917, + "median": 0.0775163471698761, + "p05": 0.06603655964136124, + "p95": 0.09357087314128876 + }, + { + "label": "completed_group_4", + "mean": 0.09567482024431229, + "median": 0.09468592703342438, + "p05": 0.07671141624450684, + "p95": 0.11774884164333344 + }, + { + "label": "completed_group_5", + "mean": 0.13266831636428833, + "median": 0.13199329376220703, + "p05": 0.11244359612464905, + "p95": 0.15504513680934906 + }, + { + "label": "current_group_partial", + "mean": 0.37691548466682434, + "median": 0.37180638313293457, + "p05": 0.31207239627838135, + "p95": 0.4595233201980591 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4055393934249878, + "group": 6, + "latest_source_mass": 0.5772827863693237, + "layer": 23, + "max_source_mass": 0.5772827863693237, + "mixer_index": 45, + "normalized_entropy": 0.7223043644164862, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10154955089092255, + "median": 0.08766824007034302, + "p05": 0.04901552200317383, + "p95": 0.20287738740444183 + }, + { + "label": "completed_group_1", + "mean": 0.05619276314973831, + "median": 0.055477917194366455, + "p05": 0.041608311235904694, + "p95": 0.07371112704277039 + }, + { + "label": "completed_group_2", + "mean": 0.07791627198457718, + "median": 0.07736755907535553, + "p05": 0.055920060724020004, + "p95": 0.10158214718103409 + }, + { + "label": "completed_group_3", + "mean": 0.053915880620479584, + "median": 0.05215863138437271, + "p05": 0.03798118978738785, + "p95": 0.07156119495630264 + }, + { + "label": "completed_group_4", + "mean": 0.06398355215787888, + "median": 0.06229078769683838, + "p05": 0.040616974234580994, + "p95": 0.09270931780338287 + }, + { + "label": "completed_group_5", + "mean": 0.06915920227766037, + "median": 0.0684593990445137, + "p05": 0.046986356377601624, + "p95": 0.09408729523420334 + }, + { + "label": "current_group_partial", + "mean": 0.5772827863693237, + "median": 0.5824689269065857, + "p05": 0.4518115520477295, + "p95": 0.6926202178001404 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7853620052337646, + "group": 6, + "latest_source_mass": 0.3407441973686218, + "layer": 24, + "max_source_mass": 0.3407441973686218, + "mixer_index": 46, + "normalized_entropy": 0.9174945750195659, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15284113585948944, + "median": 0.1593906283378601, + "p05": 0.08678612112998962, + "p95": 0.211342453956604 + }, + { + "label": "completed_group_1", + "mean": 0.12206703424453735, + "median": 0.12153112143278122, + "p05": 0.1060115247964859, + "p95": 0.1399766057729721 + }, + { + "label": "completed_group_2", + "mean": 0.06508444249629974, + "median": 0.06482555717229843, + "p05": 0.055350616574287415, + "p95": 0.07557765394449234 + }, + { + "label": "completed_group_3", + "mean": 0.08347313106060028, + "median": 0.08190013468265533, + "p05": 0.06966532766819, + "p95": 0.10050469636917114 + }, + { + "label": "completed_group_4", + "mean": 0.09811970591545105, + "median": 0.09664031863212585, + "p05": 0.07688035070896149, + "p95": 0.12369741499423981 + }, + { + "label": "completed_group_5", + "mean": 0.13767032325267792, + "median": 0.1373325139284134, + "p05": 0.11642646044492722, + "p95": 0.1606694608926773 + }, + { + "label": "current_group_partial", + "mean": 0.3407441973686218, + "median": 0.33805108070373535, + "p05": 0.271301805973053, + "p95": 0.41781002283096313 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.509842872619629, + "group": 6, + "latest_source_mass": 0.5227814316749573, + "layer": 24, + "max_source_mass": 0.5227814316749573, + "mixer_index": 47, + "normalized_entropy": 0.7759057494780099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11788781732320786, + "median": 0.10097194463014603, + "p05": 0.06265684217214584, + "p95": 0.2408325970172882 + }, + { + "label": "completed_group_1", + "mean": 0.06822571158409119, + "median": 0.0673636943101883, + "p05": 0.05209728330373764, + "p95": 0.08648915588855743 + }, + { + "label": "completed_group_2", + "mean": 0.08556114137172699, + "median": 0.08463940769433975, + "p05": 0.06234566867351532, + "p95": 0.1125786080956459 + }, + { + "label": "completed_group_3", + "mean": 0.06067090481519699, + "median": 0.058974385261535645, + "p05": 0.043267179280519485, + "p95": 0.07997901737689972 + }, + { + "label": "completed_group_4", + "mean": 0.07228855788707733, + "median": 0.06972275674343109, + "p05": 0.04689513146877289, + "p95": 0.106512151658535 + }, + { + "label": "completed_group_5", + "mean": 0.07258446514606476, + "median": 0.07082539796829224, + "p05": 0.050949547439813614, + "p95": 0.09970614314079285 + }, + { + "label": "current_group_partial", + "mean": 0.5227814316749573, + "median": 0.5290273427963257, + "p05": 0.38773614168167114, + "p95": 0.6392589807510376 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.839367389678955, + "group": 7, + "latest_source_mass": 0.29583215713500977, + "layer": 25, + "max_source_mass": 0.29583215713500977, + "mixer_index": 48, + "normalized_entropy": 0.9452478525649903, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15906387567520142, + "median": 0.15846745669841766, + "p05": 0.09519976377487183, + "p95": 0.24517810344696045 + }, + { + "label": "completed_group_1", + "mean": 0.13139380514621735, + "median": 0.1308426856994629, + "p05": 0.11478274315595627, + "p95": 0.15000829100608826 + }, + { + "label": "completed_group_2", + "mean": 0.07717745751142502, + "median": 0.07715500891208649, + "p05": 0.06616044044494629, + "p95": 0.08791667222976685 + }, + { + "label": "completed_group_3", + "mean": 0.09607303142547607, + "median": 0.09436552226543427, + "p05": 0.08152313530445099, + "p95": 0.11384639143943787 + }, + { + "label": "completed_group_4", + "mean": 0.1107850894331932, + "median": 0.1089654266834259, + "p05": 0.08814431726932526, + "p95": 0.139452263712883 + }, + { + "label": "completed_group_5", + "mean": 0.12967455387115479, + "median": 0.1299237608909607, + "p05": 0.10832332074642181, + "p95": 0.15138982236385345 + }, + { + "label": "completed_group_6", + "mean": 0.29583215713500977, + "median": 0.29647016525268555, + "p05": 0.2276897132396698, + "p95": 0.3604946434497833 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5841834545135498, + "group": 7, + "latest_source_mass": 0.40767741203308105, + "layer": 25, + "max_source_mass": 0.40767741203308105, + "mixer_index": 49, + "normalized_entropy": 0.7618312045616817, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07896879315376282, + "median": 0.06809917092323303, + "p05": 0.025932887569069862, + "p95": 0.19368338584899902 + }, + { + "label": "completed_group_1", + "mean": 0.048752427101135254, + "median": 0.04767637699842453, + "p05": 0.034443892538547516, + "p95": 0.0667153000831604 + }, + { + "label": "completed_group_2", + "mean": 0.055707164108753204, + "median": 0.0549345538020134, + "p05": 0.038696639239788055, + "p95": 0.0750277042388916 + }, + { + "label": "completed_group_3", + "mean": 0.04139826446771622, + "median": 0.03878612071275711, + "p05": 0.026673054322600365, + "p95": 0.05746251344680786 + }, + { + "label": "completed_group_4", + "mean": 0.04449018836021423, + "median": 0.04196460545063019, + "p05": 0.026162121444940567, + "p95": 0.07164917886257172 + }, + { + "label": "completed_group_5", + "mean": 0.04285799711942673, + "median": 0.04109475016593933, + "p05": 0.029393533244729042, + "p95": 0.06277710199356079 + }, + { + "label": "completed_group_6", + "mean": 0.2801477909088135, + "median": 0.2763082981109619, + "p05": 0.1455708146095276, + "p95": 0.4254714846611023 + }, + { + "label": "current_group_partial", + "mean": 0.40767741203308105, + "median": 0.4120897948741913, + "p05": 0.21399420499801636, + "p95": 0.5853031277656555 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8125314712524414, + "group": 7, + "latest_source_mass": 0.3700707256793976, + "layer": 26, + "max_source_mass": 0.3700707256793976, + "mixer_index": 50, + "normalized_entropy": 0.8716433883436914, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10889513790607452, + "median": 0.11333495378494263, + "p05": 0.04850132763385773, + "p95": 0.1620434820652008 + }, + { + "label": "completed_group_1", + "mean": 0.08656036853790283, + "median": 0.08640051633119583, + "p05": 0.0650845319032669, + "p95": 0.1084962785243988 + }, + { + "label": "completed_group_2", + "mean": 0.048123255372047424, + "median": 0.04800213500857353, + "p05": 0.03576180338859558, + "p95": 0.06107594817876816 + }, + { + "label": "completed_group_3", + "mean": 0.06561170518398285, + "median": 0.06379580497741699, + "p05": 0.050968848168849945, + "p95": 0.08332888036966324 + }, + { + "label": "completed_group_4", + "mean": 0.06917259097099304, + "median": 0.06724178791046143, + "p05": 0.050207968801259995, + "p95": 0.09373342245817184 + }, + { + "label": "completed_group_5", + "mean": 0.08286689221858978, + "median": 0.08183546364307404, + "p05": 0.06535022705793381, + "p95": 0.10340933501720428 + }, + { + "label": "completed_group_6", + "mean": 0.16869932413101196, + "median": 0.16509854793548584, + "p05": 0.11059704422950745, + "p95": 0.23654375970363617 + }, + { + "label": "current_group_partial", + "mean": 0.3700707256793976, + "median": 0.3694133758544922, + "p05": 0.23441238701343536, + "p95": 0.5025027990341187 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.50089693069458, + "group": 7, + "latest_source_mass": 0.49096518754959106, + "layer": 26, + "max_source_mass": 0.49096518754959106, + "mixer_index": 51, + "normalized_entropy": 0.7217788529328457, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07974998652935028, + "median": 0.07402504235506058, + "p05": 0.04579866677522659, + "p95": 0.1359037309885025 + }, + { + "label": "completed_group_1", + "mean": 0.04166398197412491, + "median": 0.04038450866937637, + "p05": 0.027992289513349533, + "p95": 0.05969971418380737 + }, + { + "label": "completed_group_2", + "mean": 0.04607272148132324, + "median": 0.044438350945711136, + "p05": 0.030038591474294662, + "p95": 0.06808309257030487 + }, + { + "label": "completed_group_3", + "mean": 0.03586764633655548, + "median": 0.03317485377192497, + "p05": 0.02203454077243805, + "p95": 0.05572832003235817 + }, + { + "label": "completed_group_4", + "mean": 0.04223340004682541, + "median": 0.03930673003196716, + "p05": 0.02465122565627098, + "p95": 0.06968094408512115 + }, + { + "label": "completed_group_5", + "mean": 0.04095568507909775, + "median": 0.03919431194663048, + "p05": 0.026239857077598572, + "p95": 0.06058088690042496 + }, + { + "label": "completed_group_6", + "mean": 0.22249138355255127, + "median": 0.2125401496887207, + "p05": 0.10647299885749817, + "p95": 0.3711881935596466 + }, + { + "label": "current_group_partial", + "mean": 0.49096518754959106, + "median": 0.4988659620285034, + "p05": 0.2878011167049408, + "p95": 0.6676807403564453 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8059929609298706, + "group": 7, + "latest_source_mass": 0.37139976024627686, + "layer": 27, + "max_source_mass": 0.37139976024627686, + "mixer_index": 52, + "normalized_entropy": 0.8684990295379666, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10711923241615295, + "median": 0.10972198843955994, + "p05": 0.04896187782287598, + "p95": 0.16201043128967285 + }, + { + "label": "completed_group_1", + "mean": 0.07651707530021667, + "median": 0.07666032761335373, + "p05": 0.05141128972172737, + "p95": 0.10152198374271393 + }, + { + "label": "completed_group_2", + "mean": 0.058284737169742584, + "median": 0.05812416970729828, + "p05": 0.039273619651794434, + "p95": 0.07745087146759033 + }, + { + "label": "completed_group_3", + "mean": 0.06609587371349335, + "median": 0.06383530795574188, + "p05": 0.04481930285692215, + "p95": 0.09336265176534653 + }, + { + "label": "completed_group_4", + "mean": 0.07139429450035095, + "median": 0.0694514587521553, + "p05": 0.0485839769244194, + "p95": 0.10034126043319702 + }, + { + "label": "completed_group_5", + "mean": 0.07493151724338531, + "median": 0.07319007813930511, + "p05": 0.051876723766326904, + "p95": 0.10402432829141617 + }, + { + "label": "completed_group_6", + "mean": 0.17425751686096191, + "median": 0.16937679052352905, + "p05": 0.10449051856994629, + "p95": 0.2624811828136444 + }, + { + "label": "current_group_partial", + "mean": 0.37139976024627686, + "median": 0.3732905089855194, + "p05": 0.23830299079418182, + "p95": 0.5014542937278748 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5622732639312744, + "group": 7, + "latest_source_mass": 0.4768889844417572, + "layer": 27, + "max_source_mass": 0.4768889844417572, + "mixer_index": 53, + "normalized_entropy": 0.7512946301290215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08775827288627625, + "median": 0.08504067361354828, + "p05": 0.0428270623087883, + "p95": 0.144285649061203 + }, + { + "label": "completed_group_1", + "mean": 0.047765426337718964, + "median": 0.04646281152963638, + "p05": 0.031814731657505035, + "p95": 0.06807165592908859 + }, + { + "label": "completed_group_2", + "mean": 0.053928617388010025, + "median": 0.052262187004089355, + "p05": 0.035437777638435364, + "p95": 0.0784272700548172 + }, + { + "label": "completed_group_3", + "mean": 0.04293380305171013, + "median": 0.03996016085147858, + "p05": 0.02556057646870613, + "p95": 0.06623870134353638 + }, + { + "label": "completed_group_4", + "mean": 0.04174989461898804, + "median": 0.03903091698884964, + "p05": 0.025013400241732597, + "p95": 0.06869364529848099 + }, + { + "label": "completed_group_5", + "mean": 0.04734005779027939, + "median": 0.04609068110585213, + "p05": 0.029884062707424164, + "p95": 0.06911227852106094 + }, + { + "label": "completed_group_6", + "mean": 0.20163494348526, + "median": 0.19222964346408844, + "p05": 0.09963390231132507, + "p95": 0.3307305574417114 + }, + { + "label": "current_group_partial", + "mean": 0.4768889844417572, + "median": 0.48383551836013794, + "p05": 0.276212215423584, + "p95": 0.6487575173377991 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8757455348968506, + "group": 7, + "latest_source_mass": 0.3109493851661682, + "layer": 28, + "max_source_mass": 0.3109493851661682, + "mixer_index": 54, + "normalized_entropy": 0.9020429270551009, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11563669890165329, + "median": 0.12087491154670715, + "p05": 0.05733206868171692, + "p95": 0.1688953936100006 + }, + { + "label": "completed_group_1", + "mean": 0.08108697831630707, + "median": 0.08185897022485733, + "p05": 0.05175650864839554, + "p95": 0.10753051191568375 + }, + { + "label": "completed_group_2", + "mean": 0.07280858606100082, + "median": 0.07357072830200195, + "p05": 0.04870978742837906, + "p95": 0.0948689877986908 + }, + { + "label": "completed_group_3", + "mean": 0.07553227245807648, + "median": 0.07362619787454605, + "p05": 0.056317005306482315, + "p95": 0.09756095707416534 + }, + { + "label": "completed_group_4", + "mean": 0.07606014609336853, + "median": 0.07456053793430328, + "p05": 0.05528895929455757, + "p95": 0.1022825762629509 + }, + { + "label": "completed_group_5", + "mean": 0.06935927271842957, + "median": 0.06843917071819305, + "p05": 0.04978285729885101, + "p95": 0.09167985618114471 + }, + { + "label": "completed_group_6", + "mean": 0.19856664538383484, + "median": 0.19508783519268036, + "p05": 0.1281973123550415, + "p95": 0.27796900272369385 + }, + { + "label": "current_group_partial", + "mean": 0.3109493851661682, + "median": 0.3105033040046692, + "p05": 0.18172834813594818, + "p95": 0.43916669487953186 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6924852132797241, + "group": 7, + "latest_source_mass": 0.3315581679344177, + "layer": 28, + "max_source_mass": 0.3315581679344177, + "mixer_index": 55, + "normalized_entropy": 0.8139133413255193, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08638577908277512, + "median": 0.058608733117580414, + "p05": 0.026544082909822464, + "p95": 0.2740764319896698 + }, + { + "label": "completed_group_1", + "mean": 0.059912845492362976, + "median": 0.05901291221380234, + "p05": 0.03629481792449951, + "p95": 0.08707985281944275 + }, + { + "label": "completed_group_2", + "mean": 0.06609142571687698, + "median": 0.0647939145565033, + "p05": 0.038691695779561996, + "p95": 0.09703366458415985 + }, + { + "label": "completed_group_3", + "mean": 0.056745611131191254, + "median": 0.05418796092271805, + "p05": 0.032174739986658096, + "p95": 0.08370660990476608 + }, + { + "label": "completed_group_4", + "mean": 0.06277773529291153, + "median": 0.0600711889564991, + "p05": 0.03501531109213829, + "p95": 0.10094858705997467 + }, + { + "label": "completed_group_5", + "mean": 0.05352145433425903, + "median": 0.05179818347096443, + "p05": 0.032271772623062134, + "p95": 0.08084158599376678 + }, + { + "label": "completed_group_6", + "mean": 0.2830069661140442, + "median": 0.2759752869606018, + "p05": 0.12981083989143372, + "p95": 0.46204638481140137 + }, + { + "label": "current_group_partial", + "mean": 0.3315581679344177, + "median": 0.3204721510410309, + "p05": 0.13446375727653503, + "p95": 0.5667697787284851 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8980522155761719, + "group": 8, + "latest_source_mass": 0.20123262703418732, + "layer": 29, + "max_source_mass": 0.26602888107299805, + "mixer_index": 56, + "normalized_entropy": 0.9127701729200177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11603894829750061, + "median": 0.09991352260112762, + "p05": 0.030035395175218582, + "p95": 0.2941743731498718 + }, + { + "label": "completed_group_1", + "mean": 0.08804173022508621, + "median": 0.08880250155925751, + "p05": 0.05664174258708954, + "p95": 0.11779069900512695 + }, + { + "label": "completed_group_2", + "mean": 0.07975024729967117, + "median": 0.07984748482704163, + "p05": 0.05428915470838547, + "p95": 0.10499341040849686 + }, + { + "label": "completed_group_3", + "mean": 0.09027000516653061, + "median": 0.08898856490850449, + "p05": 0.06189651042222977, + "p95": 0.11996553838253021 + }, + { + "label": "completed_group_4", + "mean": 0.08430293202400208, + "median": 0.0819794088602066, + "p05": 0.05464346706867218, + "p95": 0.12159879505634308 + }, + { + "label": "completed_group_5", + "mean": 0.07433462142944336, + "median": 0.07287181913852692, + "p05": 0.052520088851451874, + "p95": 0.1010315865278244 + }, + { + "label": "completed_group_6", + "mean": 0.26602888107299805, + "median": 0.2639757990837097, + "p05": 0.15474572777748108, + "p95": 0.3846370279788971 + }, + { + "label": "completed_group_7", + "mean": 0.20123262703418732, + "median": 0.18140269815921783, + "p05": 0.08419936150312424, + "p95": 0.3782624900341034 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7512003183364868, + "group": 8, + "latest_source_mass": 0.33482813835144043, + "layer": 29, + "max_source_mass": 0.33482813835144043, + "mixer_index": 57, + "normalized_entropy": 0.7970056117156376, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07744461297988892, + "median": 0.06970229744911194, + "p05": 0.014401955530047417, + "p95": 0.17144526541233063 + }, + { + "label": "completed_group_1", + "mean": 0.046086348593235016, + "median": 0.04394831880927086, + "p05": 0.028066720813512802, + "p95": 0.0718703418970108 + }, + { + "label": "completed_group_2", + "mean": 0.05025433003902435, + "median": 0.04861236736178398, + "p05": 0.029644077643752098, + "p95": 0.07761462777853012 + }, + { + "label": "completed_group_3", + "mean": 0.03748488053679466, + "median": 0.03434944897890091, + "p05": 0.020774798467755318, + "p95": 0.05938909202814102 + }, + { + "label": "completed_group_4", + "mean": 0.04005061462521553, + "median": 0.037498839199543, + "p05": 0.019541850313544273, + "p95": 0.06922487914562225 + }, + { + "label": "completed_group_5", + "mean": 0.041623204946517944, + "median": 0.039552345871925354, + "p05": 0.022081367671489716, + "p95": 0.06821337342262268 + }, + { + "label": "completed_group_6", + "mean": 0.198423832654953, + "median": 0.18318486213684082, + "p05": 0.07548617571592331, + "p95": 0.3721975088119507 + }, + { + "label": "completed_group_7", + "mean": 0.17380405962467194, + "median": 0.1514919549226761, + "p05": 0.05777408182621002, + "p95": 0.36326974630355835 + }, + { + "label": "current_group_partial", + "mean": 0.33482813835144043, + "median": 0.33602219820022583, + "p05": 0.0795319601893425, + "p95": 0.5979493856430054 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0160694122314453, + "group": 8, + "latest_source_mass": 0.21711516380310059, + "layer": 30, + "max_source_mass": 0.21711516380310059, + "mixer_index": 58, + "normalized_entropy": 0.9175527313077866, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09385271370410919, + "median": 0.09211117774248123, + "p05": 0.032377734780311584, + "p95": 0.16496515274047852 + }, + { + "label": "completed_group_1", + "mean": 0.06208505108952522, + "median": 0.06140740588307381, + "p05": 0.0407925546169281, + "p95": 0.0863042101264 + }, + { + "label": "completed_group_2", + "mean": 0.06560768187046051, + "median": 0.06516212224960327, + "p05": 0.041897669434547424, + "p95": 0.09085705131292343 + }, + { + "label": "completed_group_3", + "mean": 0.07029052078723907, + "median": 0.06780634075403214, + "p05": 0.04846132919192314, + "p95": 0.09917552024126053 + }, + { + "label": "completed_group_4", + "mean": 0.06935097277164459, + "median": 0.06708167493343353, + "p05": 0.04668639227747917, + "p95": 0.09815650433301926 + }, + { + "label": "completed_group_5", + "mean": 0.056832700967788696, + "median": 0.05595892667770386, + "p05": 0.038822539150714874, + "p95": 0.0771557092666626 + }, + { + "label": "completed_group_6", + "mean": 0.19425815343856812, + "median": 0.19145764410495758, + "p05": 0.12249615788459778, + "p95": 0.27496030926704407 + }, + { + "label": "completed_group_7", + "mean": 0.17060703039169312, + "median": 0.16317546367645264, + "p05": 0.09239593893289566, + "p95": 0.2726203501224518 + }, + { + "label": "current_group_partial", + "mean": 0.21711516380310059, + "median": 0.21198657155036926, + "p05": 0.11019522696733475, + "p95": 0.3391069769859314 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7837426662445068, + "group": 8, + "latest_source_mass": 0.29026859998703003, + "layer": 30, + "max_source_mass": 0.29026859998703003, + "mixer_index": 59, + "normalized_entropy": 0.8118162725118464, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08836327493190765, + "median": 0.07904411852359772, + "p05": 0.02631576545536518, + "p95": 0.17534467577934265 + }, + { + "label": "completed_group_1", + "mean": 0.04953558370471001, + "median": 0.04658038914203644, + "p05": 0.025181813165545464, + "p95": 0.08318150043487549 + }, + { + "label": "completed_group_2", + "mean": 0.04813014343380928, + "median": 0.045740917325019836, + "p05": 0.026714935898780823, + "p95": 0.07843323051929474 + }, + { + "label": "completed_group_3", + "mean": 0.0373263955116272, + "median": 0.03401809185743332, + "p05": 0.01990434154868126, + "p95": 0.06290259957313538 + }, + { + "label": "completed_group_4", + "mean": 0.037015400826931, + "median": 0.03405017405748367, + "p05": 0.018646355718374252, + "p95": 0.06544196605682373 + }, + { + "label": "completed_group_5", + "mean": 0.03390160948038101, + "median": 0.03198609501123428, + "p05": 0.018189728260040283, + "p95": 0.05570102855563164 + }, + { + "label": "completed_group_6", + "mean": 0.1840541511774063, + "median": 0.17327937483787537, + "p05": 0.07966600358486176, + "p95": 0.32982927560806274 + }, + { + "label": "completed_group_7", + "mean": 0.23140482604503632, + "median": 0.22406929731369019, + "p05": 0.08831030130386353, + "p95": 0.40350067615509033 + }, + { + "label": "current_group_partial", + "mean": 0.29026859998703003, + "median": 0.2745037376880646, + "p05": 0.10280418395996094, + "p95": 0.5399085879325867 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9973840713500977, + "group": 8, + "latest_source_mass": 0.19101998209953308, + "layer": 31, + "max_source_mass": 0.20831164717674255, + "mixer_index": 60, + "normalized_entropy": 0.9090486661912383, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10261696577072144, + "median": 0.10389091819524765, + "p05": 0.01371697522699833, + "p95": 0.21163305640220642 + }, + { + "label": "completed_group_1", + "mean": 0.06039813905954361, + "median": 0.06085003539919853, + "p05": 0.026107093319296837, + "p95": 0.09348798543214798 + }, + { + "label": "completed_group_2", + "mean": 0.06671924144029617, + "median": 0.06656201928853989, + "p05": 0.034684643149375916, + "p95": 0.09819839894771576 + }, + { + "label": "completed_group_3", + "mean": 0.08377155661582947, + "median": 0.0804532840847969, + "p05": 0.05448959022760391, + "p95": 0.12039553374052048 + }, + { + "label": "completed_group_4", + "mean": 0.07002593576908112, + "median": 0.06737451255321503, + "p05": 0.044841229915618896, + "p95": 0.10655929893255234 + }, + { + "label": "completed_group_5", + "mean": 0.05768897384405136, + "median": 0.056077782064676285, + "p05": 0.039495617151260376, + "p95": 0.08242568373680115 + }, + { + "label": "completed_group_6", + "mean": 0.20831164717674255, + "median": 0.2021285891532898, + "p05": 0.11652229726314545, + "p95": 0.31706833839416504 + }, + { + "label": "completed_group_7", + "mean": 0.1594475358724594, + "median": 0.1546279788017273, + "p05": 0.06593266129493713, + "p95": 0.2735573351383209 + }, + { + "label": "current_group_partial", + "mean": 0.19101998209953308, + "median": 0.1713225245475769, + "p05": 0.06408411264419556, + "p95": 0.3850628137588501 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.969987154006958, + "group": 8, + "latest_source_mass": 0.16864418983459473, + "layer": 31, + "max_source_mass": 0.24503237009048462, + "mixer_index": 61, + "normalized_entropy": 0.8965797917640489, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08120635151863098, + "median": 0.061164144426584244, + "p05": 0.0259944386780262, + "p95": 0.23622018098831177 + }, + { + "label": "completed_group_1", + "mean": 0.07894632965326309, + "median": 0.07658770680427551, + "p05": 0.0496862456202507, + "p95": 0.11596276611089706 + }, + { + "label": "completed_group_2", + "mean": 0.08615730702877045, + "median": 0.08467075228691101, + "p05": 0.050409119576215744, + "p95": 0.12811316549777985 + }, + { + "label": "completed_group_3", + "mean": 0.07609779387712479, + "median": 0.0722871869802475, + "p05": 0.04099677503108978, + "p95": 0.11950768530368805 + }, + { + "label": "completed_group_4", + "mean": 0.058728985488414764, + "median": 0.056080836802721024, + "p05": 0.03323862701654434, + "p95": 0.09344407916069031 + }, + { + "label": "completed_group_5", + "mean": 0.05076971277594566, + "median": 0.04946017265319824, + "p05": 0.03047419898211956, + "p95": 0.07563400268554688 + }, + { + "label": "completed_group_6", + "mean": 0.24503237009048462, + "median": 0.24110376834869385, + "p05": 0.10613486170768738, + "p95": 0.3991503119468689 + }, + { + "label": "completed_group_7", + "mean": 0.1544169783592224, + "median": 0.1400495171546936, + "p05": 0.06199025735259056, + "p95": 0.2995939254760742 + }, + { + "label": "current_group_partial", + "mean": 0.16864418983459473, + "median": 0.15222924947738647, + "p05": 0.0543956533074379, + "p95": 0.3329208791255951 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.071648359298706, + "group": 8, + "latest_source_mass": 0.11520501971244812, + "layer": 32, + "max_source_mass": 0.20955611765384674, + "mixer_index": 62, + "normalized_entropy": 0.9428478002054053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10569609701633453, + "median": 0.08689188957214355, + "p05": 0.03173588588833809, + "p95": 0.24571751058101654 + }, + { + "label": "completed_group_1", + "mean": 0.09922641515731812, + "median": 0.09795533865690231, + "p05": 0.06407676637172699, + "p95": 0.14120742678642273 + }, + { + "label": "completed_group_2", + "mean": 0.08180510997772217, + "median": 0.080217644572258, + "p05": 0.054607540369033813, + "p95": 0.11442367732524872 + }, + { + "label": "completed_group_3", + "mean": 0.10646799206733704, + "median": 0.10510502755641937, + "p05": 0.07415887713432312, + "p95": 0.14216329157352448 + }, + { + "label": "completed_group_4", + "mean": 0.0826798751950264, + "median": 0.08022409677505493, + "p05": 0.05491747707128525, + "p95": 0.11860857903957367 + }, + { + "label": "completed_group_5", + "mean": 0.06615597754716873, + "median": 0.06485182791948318, + "p05": 0.04477996751666069, + "p95": 0.09238749742507935 + }, + { + "label": "completed_group_6", + "mean": 0.20955611765384674, + "median": 0.206287682056427, + "p05": 0.129981130361557, + "p95": 0.2996031939983368 + }, + { + "label": "completed_group_7", + "mean": 0.13320739567279816, + "median": 0.12465103715658188, + "p05": 0.06303662061691284, + "p95": 0.23065781593322754 + }, + { + "label": "current_group_partial", + "mean": 0.11520501971244812, + "median": 0.10386130958795547, + "p05": 0.03912060335278511, + "p95": 0.22735844552516937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781701564788818, + "group": 8, + "latest_source_mass": 0.16112206876277924, + "layer": 32, + "max_source_mass": 0.22251944243907928, + "mixer_index": 63, + "normalized_entropy": 0.9003040366848136, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09446848928928375, + "median": 0.074150450527668, + "p05": 0.042592622339725494, + "p95": 0.19443637132644653 + }, + { + "label": "completed_group_1", + "mean": 0.07056106626987457, + "median": 0.06868506222963333, + "p05": 0.04818308353424072, + "p95": 0.0982733890414238 + }, + { + "label": "completed_group_2", + "mean": 0.08033448457717896, + "median": 0.08047192543745041, + "p05": 0.048296794295310974, + "p95": 0.11253741383552551 + }, + { + "label": "completed_group_3", + "mean": 0.0658978745341301, + "median": 0.06256154924631119, + "p05": 0.04223886504769325, + "p95": 0.0967884361743927 + }, + { + "label": "completed_group_4", + "mean": 0.058234501630067825, + "median": 0.0563921295106411, + "p05": 0.03529864549636841, + "p95": 0.08846980333328247 + }, + { + "label": "completed_group_5", + "mean": 0.04316519945859909, + "median": 0.042268507182598114, + "p05": 0.029443470761179924, + "p95": 0.05996086448431015 + }, + { + "label": "completed_group_6", + "mean": 0.2036968618631363, + "median": 0.19645483791828156, + "p05": 0.09398645162582397, + "p95": 0.3395216464996338 + }, + { + "label": "completed_group_7", + "mean": 0.22251944243907928, + "median": 0.2130982130765915, + "p05": 0.11867167800664902, + "p95": 0.35465875267982483 + }, + { + "label": "current_group_partial", + "mean": 0.16112206876277924, + "median": 0.14023703336715698, + "p05": 0.06008770316839218, + "p95": 0.3247341811656952 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.6036605834960938, + "group": null, + "latest_source_mass": 0.31639498472213745, + "layer": null, + "max_source_mass": 0.3412485122680664, + "mixer_index": null, + "normalized_entropy": 0.7298573846467136, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09759747982025146, + "median": 0.09915056824684143, + "p05": 0.025461740791797638, + "p95": 0.16398483514785767 + }, + { + "label": "completed_group_1", + "mean": 0.04215940088033676, + "median": 0.04050712287425995, + "p05": 0.025720715522766113, + "p95": 0.06398092210292816 + }, + { + "label": "completed_group_2", + "mean": 0.041145481169223785, + "median": 0.03881259262561798, + "p05": 0.023510465398430824, + "p95": 0.0662468671798706 + }, + { + "label": "completed_group_3", + "mean": 0.029554028064012527, + "median": 0.027420252561569214, + "p05": 0.01627083122730255, + "p95": 0.04924973472952843 + }, + { + "label": "completed_group_4", + "mean": 0.0289996936917305, + "median": 0.027226522564888, + "p05": 0.01560397818684578, + "p95": 0.048351090401411057 + }, + { + "label": "completed_group_5", + "mean": 0.03647579252719879, + "median": 0.0347352996468544, + "p05": 0.020744025707244873, + "p95": 0.058257561177015305 + }, + { + "label": "completed_group_6", + "mean": 0.06642461568117142, + "median": 0.057809896767139435, + "p05": 0.022837381809949875, + "p95": 0.14078694581985474 + }, + { + "label": "completed_group_7", + "mean": 0.3412485122680664, + "median": 0.3336990475654602, + "p05": 0.0819513276219368, + "p95": 0.6423859596252441 + }, + { + "label": "completed_group_8", + "mean": 0.31639498472213745, + "median": 0.30713334679603577, + "p05": 0.05841828137636185, + "p95": 0.6156357526779175 + } + ], + "sources": 9 + } + ], + "mode": "learned", + "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.24537226557731628, + 0.07698935270309448, + 0.0523640513420105, + 0.09380833059549332, + 0.03647654876112938, + 0.039120711386203766, + 0.06371714919805527, + 0.05156813934445381, + 0.03728010877966881, + 0.03983496502041817, + 0.040284499526023865, + 0.046871740370988846, + 0.04314620420336723, + 0.04033593460917473, + 0.04471700266003609, + 0.08397205173969269, + 0.0528092160820961, + 0.04443390294909477, + 0.07560808956623077, + 0.03708760067820549, + 0.03599191829562187, + 0.033304885029792786, + 0.04061548039317131, + 0.025487789884209633, + 0.021565141156315804, + 0.02342291921377182, + 0.017244853079319, + 0.024166326969861984, + 0.016819356009364128, + 0.021590016782283783, + 0.026201345026493073 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6127158914969186e-05, + "first_to_last_ratio": 0.6403133863364645, + "imbalance_abs_log_ratio": 0.4457975563250414, + "last_quartile_mean": 2.518635290016391e-05, + "mean": 2.1593947650444534e-05, + "normalized": [ + 0.6929919556166857, + 0.8032314127318161, + 0.7123629256452019, + 0.6640523991876173, + 0.8725736234454132, + 0.8071370183700546, + 0.7516577913014878, + 0.6706888120378312, + 0.5882487760705537, + 0.5190639183458972, + 0.4863138588077732, + 0.4652918617004374, + 0.6603036411218667, + 0.6189718629251246, + 0.5802646281311779, + 0.5410038780854491, + 0.6697462103535458, + 0.6259079034957141, + 0.6246747715872017, + 0.6676260304525075, + 3.1377243999857436, + 2.6752853614525707, + 2.136977079899559, + 1.6970064982433113, + 2.2713553531497497, + 1.9850638882393974, + 1.4820693911808296, + 0.9615863897898643, + 1.0163024380148542, + 0.7191699586662397, + 0.4720839007000451, + 0.42326206126447913 + ], + "peak_layer": 21, + "peak_normalized": 3.1377243999857436, + "population_cv": 0.6918392109604528, + "rest_mean": 1.6060819520875897e-05, + "spike_contrast": 3.2048700555541787, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.147283955011517e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4964432011765894e-05, + 1.7344937077723444e-05, + 1.53827277245e-05, + 1.4339512745209504e-05, + 1.8842309145838954e-05, + 1.7429274521418847e-05, + 1.623125899641309e-05, + 1.4482819096883759e-05, + 1.2702613275905605e-05, + 1.1208639079995919e-05, + 1.0501436008780729e-05, + 1.0047488103737123e-05, + 1.4258562259783503e-05, + 1.3366046005103271e-05, + 1.2530204003269318e-05, + 1.1682409422064666e-05, + 1.446246460545808e-05, + 1.3515822502085939e-05, + 1.3489194316207431e-05, + 1.4416681551665533e-05, + 6.775585643481463e-05, + 5.7769972045207396e-05, + 4.61457711935509e-05, + 3.664506948553026e-05, + 4.9047528591472656e-05, + 4.2865365685429424e-05, + 3.2003728847485036e-05, + 2.076444616250228e-05, + 2.1945981643511914e-05, + 1.5529718439211138e-05, + 1.019415503833443e-05, + 9.13989879336441e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.422064188882359e-05, + "first_to_last_ratio": 0.6376814356469364, + "imbalance_abs_log_ratio": 0.4499164375054873, + "last_quartile_mean": 0.00010070959933727863, + "mean": 8.629182207187114e-05, + "normalized": [ + 0.6940773401510585, + 0.8062528427202595, + 0.7180374505283312, + 0.6647119372719499, + 0.8588689882759897, + 0.7941431696191276, + 0.7473948010342778, + 0.6703237246625169, + 0.591781846497542, + 0.522694550559361, + 0.4893812082436792, + 0.4670959071660659, + 0.6619658672285874, + 0.6164719403371215, + 0.5787012511245688, + 0.5393739243504363, + 0.6707710740427385, + 0.62668399370843, + 0.6317390712162502, + 0.6712190136492653, + 3.138964807406113, + 2.6795820619127526, + 2.136435916928567, + 1.6866743661161174, + 2.264472697470994, + 1.978410075097681, + 1.4778942897678682, + 0.9605324338748193, + 1.022040338716437, + 0.7239267286003249, + 0.47869752189559756, + 0.4306788598251707 + ], + "peak_layer": 21, + "peak_normalized": 3.138964807406113, + "population_cv": 0.6905024564046187, + "rest_mean": 6.421987658271703e-05, + "spike_contrast": 3.1996375366532934, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020548032771330326, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.989319834043272e-05, + 6.957302684895694e-05, + 6.196075992193073e-05, + 5.7359204220119864e-05, + 7.411336991935968e-05, + 6.852806109236553e-05, + 6.449405918829143e-05, + 5.784345557913184e-05, + 5.1065933803329244e-05, + 4.5104265154805034e-05, + 4.222959614708088e-05, + 4.03065569116734e-05, + 5.712224083254114e-05, + 5.319648698787205e-05, + 4.993718539481051e-05, + 4.654355871025473e-05, + 5.788205817225389e-05, + 5.4077703680377454e-05, + 5.4513915529241785e-05, + 5.792071169707924e-05, + 0.0002708669926505536, + 0.00023122601851355284, + 0.00018435694801155478, + 0.00014554620429407805, + 0.00019540547509677708, + 0.0001707206101855263, + 0.00012753019109368324, + 8.288609387818724e-05, + 8.81937230587937e-05, + 6.246895645745099e-05, + 4.1307681385660544e-05, + 3.716406354214996e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05695150885730982, + "first_to_last_ratio": 0.6376814663645546, + "imbalance_abs_log_ratio": 0.4499163893346989, + "last_quartile_mean": 0.08931027771905065, + "mean": 0.07652445149142295, + "normalized": [ + 0.6940773558515339, + 0.8062528453975966, + 0.7180374361894372, + 0.6647119756359035, + 0.8588690718004696, + 0.7941431432641803, + 0.7473947840325739, + 0.6703237318272384, + 0.5917818305157142, + 0.5226945671206367, + 0.4893812028028422, + 0.46709594515141906, + 0.6619658291829611, + 0.6164719318106706, + 0.5787012336125282, + 0.5393739103061749, + 0.6707710619535627, + 0.6266839981127393, + 0.6317390378687008, + 0.6712190249334776, + 3.138964907679676, + 2.679582133407049, + 2.136436026253706, + 1.6866743790728473, + 2.2644726279739604, + 1.9784099533155926, + 1.4778942270811004, + 0.9605324250805215, + 1.0220403417906543, + 0.7239267230206866, + 0.4786975145327867, + 0.43067882342105884 + ], + "peak_layer": 21, + "peak_normalized": 3.138964907679676, + "population_cv": 0.6905024682030625, + "rest_mean": 0.05695082868138949, + "spike_contrast": 3.1996376327558256, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18222201466560364, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05311388894915581, + 0.061698056757450104, + 0.054947420954704285, + 0.05086671933531761, + 0.06572448462247849, + 0.06077136844396591, + 0.057193975895643234, + 0.05129615589976311, + 0.04528577998280525, + 0.039998915046453476, + 0.03744962811470032, + 0.035744260996580124, + 0.05065657198429108, + 0.047175176441669464, + 0.044284794479608536, + 0.04127529263496399, + 0.051330387592315674, + 0.04795664921402931, + 0.0483434833586216, + 0.05136466771364212, + 0.24020756781101227, + 0.20505355298519135, + 0.16348959505558014, + 0.12907183170318604, + 0.1732875257730484, + 0.1513967365026474, + 0.11309504508972168, + 0.07350421696901321, + 0.07821107655763626, + 0.05539809539914131, + 0.03663206472992897, + 0.03295746073126793 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.592033183645981e-06, + "first_to_last_ratio": 0.3776007228014791, + "imbalance_abs_log_ratio": 0.9739179305137912, + "last_quartile_mean": 6.864481520096888e-06, + "mean": 4.875440396290287e-06, + "normalized": [ + 0.58913988046903, + 0.5446521181917551, + 0.5278862817384906, + 0.53898932284961, + 0.5219441801529001, + 0.509889287242028, + 0.5048315077449682, + 0.5158761132703409, + 0.4368296481828547, + 0.4047199700448834, + 0.3956716882892422, + 0.40126275663148436, + 0.5110508169296636, + 0.5085656955747676, + 0.5112234187675316, + 0.511163584085558, + 0.5662253891905324, + 0.5872486754624854, + 0.609842074156892, + 0.6494088419115644, + 3.025689231381945, + 2.8686186674565652, + 2.4420116557657345, + 2.053486551003887, + 2.698068277749679, + 2.429082514259166, + 1.8298867229318267, + 1.2191127574146867, + 1.182073363720965, + 0.8592876866491284, + 0.5581944369380665, + 0.48806688384176716 + ], + "peak_layer": 21, + "peak_normalized": 3.025689231381945, + "population_cv": 0.8235639638153561, + "rest_mean": 3.4149978226330994e-06, + "spike_contrast": 3.736995148125518, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2761830294039101e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.87231637230434e-06, + 2.6554189389571548e-06, + 2.573678102635313e-06, + 2.6278103177901357e-06, + 2.5447077405260643e-06, + 2.485934828655445e-06, + 2.461275926179951e-06, + 2.5151232421194436e-06, + 2.1297369130479638e-06, + 1.9731880911422195e-06, + 1.92907373275375e-06, + 1.9563326532079373e-06, + 2.491597797416034e-06, + 2.4794817363726906e-06, + 2.4924393073888496e-06, + 2.492147586963256e-06, + 2.7605981358647114e-06, + 2.863095915017766e-06, + 2.973248683701968e-06, + 3.166154101563734e-06, + 1.4751567505300045e-05, + 1.3985779332870152e-05, + 1.1905882274731994e-05, + 1.0011651284003165e-05, + 1.3154271073290147e-05, + 1.1842847015941516e-05, + 8.92150364961708e-06, + 5.943711585132405e-06, + 5.763128228863934e-06, + 4.18940589952399e-06, + 2.7214437068323605e-06, + 2.3795410015736707e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.8627718115312746e-05, + "first_to_last_ratio": 0.778294860341054, + "imbalance_abs_log_ratio": 0.2506498287376175, + "last_quartile_mean": 6.247981400520075e-05, + "mean": 6.0048676687074476e-05, + "normalized": [ + 0.768607661628769, + 0.8690595287737466, + 0.7879569251847977, + 0.7385864958726717, + 0.9274786983184008, + 0.8617431831538573, + 0.8054883103443636, + 0.7195191288766225, + 0.6496482220749353, + 0.5799654886485536, + 0.5429742744436054, + 0.5176928240267351, + 0.720980532018643, + 0.6651070950014618, + 0.6225467122437304, + 0.5784265987882409, + 0.7443177261701239, + 0.697888823545442, + 0.701818835999622, + 0.7379727210931422, + 3.1748914800668615, + 2.647277084571918, + 2.0581907003495648, + 1.5579720703470805, + 2.1379505239352077, + 1.7650981274587072, + 1.310550617537982, + 0.8840683850126465, + 0.8342765345805514, + 0.5872947992080164, + 0.42062138312406294, + 0.3840285075999366 + ], + "peak_layer": 21, + "peak_normalized": 3.1748914800668615, + "population_cv": 0.6535432926484233, + "rest_mean": 4.5422861014096134e-05, + "spike_contrast": 3.0607513093024403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013902808132115752, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.615387297235429e-05, + 5.218587466515601e-05, + 4.7315770643763244e-05, + 4.435114169609733e-05, + 5.569386848947033e-05, + 5.174653779249638e-05, + 4.83685071230866e-05, + 4.3206171540077776e-05, + 3.901051604771055e-05, + 3.482616011751816e-05, + 3.2604886655462906e-05, + 3.1086769013199955e-05, + 4.329392686486244e-05, + 3.993880091002211e-05, + 3.738310624612495e-05, + 3.473375181783922e-05, + 4.4695294491248205e-05, + 4.1907300328603014e-05, + 4.214329237584025e-05, + 4.431428533280268e-05, + 0.00019064803200308233, + 0.0001589654857525602, + 0.00012359162792563438, + 9.355416113976389e-05, + 0.00012838109978474677, + 0.00010599180677672848, + 7.869683031458408e-05, + 5.3087136620888487e-05, + 5.009720189264044e-05, + 3.52662755176425e-05, + 2.525775744288694e-05, + 2.3060403691488318e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.645518927442026e-05, + "first_to_last_ratio": 0.9865830246987611, + "imbalance_abs_log_ratio": 0.013507796190281324, + "last_quartile_mean": 3.6950959383830195e-05, + "mean": 4.1335310356771515e-05, + "normalized": [ + 0.8560051014711698, + 0.9467981965967722, + 0.8689980494059457, + 0.8159268191754907, + 1.0103083717536356, + 0.9426554996877177, + 0.8489560918930947, + 0.765857482001633, + 0.7201445334320749, + 0.6498760503731635, + 0.6109444686941845, + 0.5791405787907885, + 0.7989198470081217, + 0.7291371869136885, + 0.6790968935040115, + 0.6275849613644364, + 0.8454614229708973, + 0.7893993011096712, + 0.7856135784658201, + 0.8181223517247423, + 3.2763150867214206, + 2.615409622894913, + 1.9276349807221327, + 1.3402369957405367, + 1.940332562376331, + 1.4667183642001942, + 1.0945246384976166, + 0.8103162680270223, + 0.6598082696260246, + 0.4694457579306494, + 0.3666062775483438, + 0.3437043893777548 + ], + "peak_layer": 21, + "peak_normalized": 3.2763150867214206, + "population_cv": 0.6207682663855968, + "rest_mean": 3.199670040716969e-05, + "spike_contrast": 2.867914594845514, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.176380408462136e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5383236536290497e-05, + 3.913619730155915e-05, + 3.592030407162383e-05, + 3.37265882990323e-05, + 4.176141010248102e-05, + 3.896495763910934e-05, + 3.509186353767291e-05, + 3.1656956707593054e-05, + 2.9767397791147232e-05, + 2.6862828235607594e-05, + 2.5253579224226996e-05, + 2.3938955564517528e-05, + 3.3023599826265126e-05, + 3.0139111913740635e-05, + 2.8070680855307728e-05, + 2.594141915324144e-05, + 3.4947410313179716e-05, + 3.263006510678679e-05, + 3.2473581086378545e-05, + 3.381734131835401e-05, + 0.0001354275009362027, + 0.00010810876847244799, + 7.967939018271863e-05, + 5.539911217056215e-05, + 8.020424866117537e-05, + 6.062725879019126e-05, + 4.5242515625432134e-05, + 3.349467442603782e-05, + 2.7273379600956105e-05, + 1.9404686099733226e-05, + 1.5153784261201508e-05, + 1.4207127605914138e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012982768839719938, + "first_to_last_ratio": 0.6037915882942751, + "imbalance_abs_log_ratio": 0.5045261930845532, + "last_quartile_mean": 0.00021502069739653962, + "mean": 0.0001790727353636612, + "normalized": [ + 0.6739765257680471, + 0.7636067305383979, + 0.68849644503889, + 0.6447045606228033, + 0.8477445590156324, + 0.7818709864142286, + 0.7391626168129651, + 0.6604355769831363, + 0.5800901925673246, + 0.5130947085748045, + 0.4841531977734674, + 0.46136480942786895, + 0.6373637447050293, + 0.5885397553357715, + 0.5617314689244236, + 0.5244758601010534, + 0.6356987553893274, + 0.6101287890369049, + 0.6118132813754859, + 0.6643470301154188, + 3.09815074914609, + 2.693324715468958, + 2.18351992306606, + 1.7462447599227975, + 2.367437166894702, + 2.0445672235354038, + 1.5627333414952298, + 1.026078331181794, + 1.036030398706582, + 0.6947075889876938, + 0.46018158540857673, + 0.4142246216651318 + ], + "peak_layer": 21, + "peak_normalized": 3.09815074914609, + "population_cv": 0.7095514290759102, + "rest_mean": 0.00013205833400004648, + "spike_contrast": 3.278479211520481, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004329505027271807, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012069082004018128, + 0.00013674114597961307, + 0.00012329094170127064, + 0.00011544900917215273, + 0.00015180793707259, + 0.00014001177623867989, + 0.0001323638716712594, + 0.00011826600530184805, + 0.00010387833754066378, + 9.188127296511084e-05, + 8.669863746035844e-05, + 8.261785842478275e-05, + 0.00011413446918595582, + 0.00010539142385823652, + 0.00010059079068014398, + 9.391932690050453e-05, + 0.00011383631499484181, + 0.00010925743117695674, + 0.00010955907782772556, + 0.00011896643991349265, + 0.0005547943292185664, + 0.0004823010240215808, + 0.0003910088853444904, + 0.0003127048257738352, + 0.00042394344927743077, + 0.0003661262453533709, + 0.0002798429341055453, + 0.0001837426534621045, + 0.00018552479741629213, + 0.0001244031882379204, + 8.240597526310012e-05, + 7.417633605655283e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13583384454250336, + 0.0415157787501812, + 0.018140379339456558, + 0.039518751204013824, + 0.022877270355820656, + 0.015532293356955051, + 0.02314678393304348, + 0.05432315170764923, + 0.03743875026702881, + 0.04006272554397583, + 0.045583780854940414, + 0.05995568633079529, + 0.038651175796985626, + 0.04559725895524025, + 0.05632574483752251, + 0.07952121645212173, + 0.042337141931056976, + 0.05044656619429588, + 0.04418695345520973, + 0.06310147047042847, + 0.029010163620114326, + 0.04586305841803551, + 0.06033914536237717, + 0.07656749337911606, + 0.045884523540735245, + 0.051604680716991425, + 0.06502768397331238, + 0.06792232394218445, + 0.060306549072265625, + 0.08025261014699936, + 0.09663844108581543, + 0.12230555713176727 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5986858215910615e-05, + "first_to_last_ratio": 0.8170320074925627, + "imbalance_abs_log_ratio": 0.20207700803204537, + "last_quartile_mean": 1.9566991340980167e-05, + "mean": 1.9039267499465495e-05, + "normalized": [ + 0.7853354674160987, + 0.9097466615865485, + 0.8087937381555731, + 0.7523818308832718, + 0.9858416226009721, + 0.9045115156777315, + 0.8288509697382657, + 0.741963942115696, + 0.6090758738841542, + 0.5628736284922787, + 0.5329013290216021, + 0.5131548852635935, + 0.7265824288035061, + 0.6757039396775449, + 0.6225875966900762, + 0.5898833165121294, + 0.7532717273459274, + 0.6927780667296713, + 0.6897910451069404, + 0.809449516707258, + 3.207633782072864, + 2.5002059817245676, + 1.9824571594317655, + 1.5924827370882133, + 2.273874867822024, + 1.6940739554643818, + 1.0968135124011316, + 0.8719494928225496, + 0.815651324531277, + 0.538892521487639, + 0.48196570767333513, + 0.44851985507141146 + ], + "peak_layer": 21, + "peak_normalized": 3.207633782072864, + "population_cv": 0.6428607369119902, + "rest_mean": 1.4415789741582961e-05, + "spike_contrast": 3.0526282764165082, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4006047392031175e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4952212040952872e-05, + 1.7320910046692006e-05, + 1.539884033263661e-05, + 1.4324798939924221e-05, + 1.8769702364807017e-05, + 1.7221236703335308e-05, + 1.578071533003822e-05, + 1.4126449968898669e-05, + 1.1596358490351122e-05, + 1.0716701581259258e-05, + 1.0146050954062957e-05, + 9.770093129191082e-06, + 1.3833597222401295e-05, + 1.2864908057963476e-05, + 1.18536117952317e-05, + 1.1230946256546304e-05, + 1.434174191672355e-05, + 1.318998693022877e-05, + 1.3133116226526909e-05, + 1.541132587590255e-05, + 6.107099761720747e-05, + 4.760209048981778e-05, + 3.77445321646519e-05, + 3.0319704819703475e-05, + 4.3292911868775263e-05, + 3.225392720196396e-05, + 2.088252585963346e-05, + 1.660127963987179e-05, + 1.5529403754044324e-05, + 1.0260118870064616e-05, + 9.176274033961818e-06, + 8.539489499526098e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.365280069076107e-05, + "first_to_last_ratio": 0.8123684077728273, + "imbalance_abs_log_ratio": 0.2078013375593398, + "last_quartile_mean": 7.83545988269907e-05, + "mean": 7.614439766712167e-05, + "normalized": [ + 0.7854660713099145, + 0.9130496523819581, + 0.8147298517264512, + 0.752236196131495, + 0.9689924576782674, + 0.8892579571020611, + 0.8242433765322099, + 0.7396130336387524, + 0.6125904066287632, + 0.5670744382794471, + 0.5359292329408729, + 0.5145351492524995, + 0.7290151059417195, + 0.673146260009013, + 0.621425875046763, + 0.5883337897938153, + 0.7535116607120764, + 0.6934400787203782, + 0.6978162466203766, + 0.8168476312813977, + 3.2190719364475022, + 2.5076839554965473, + 1.973830744692153, + 1.575947332799018, + 2.2647184639164535, + 1.6880790923847124, + 1.0924366554795215, + 0.8732736568641918, + 0.8206255787733402, + 0.5459821164389519, + 0.48994786526283923, + 0.4571481297165359 + ], + "peak_layer": 21, + "peak_normalized": 3.2190719364475022, + "population_cv": 0.642032487569562, + "rest_mean": 5.76970003883751e-05, + "spike_contrast": 3.0462648281411457, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001757603429723531, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.9808840887853876e-05, + 6.952361582079902e-05, + 6.203711382113397e-05, + 5.727857205783948e-05, + 7.378334703389555e-05, + 6.771201151423156e-05, + 6.276151543715969e-05, + 5.6317388953175396e-05, + 4.664532752940431e-05, + 4.3179541535209864e-05, + 4.0808008634485304e-05, + 3.917896901839413e-05, + 5.551041613216512e-05, + 5.125631651026197e-05, + 4.731809895019978e-05, + 4.479832205106504e-05, + 5.7375691540073603e-05, + 5.280157711240463e-05, + 5.3134797781240195e-05, + 6.219837086973712e-05, + 0.00024511429364793, + 0.00019094608433078974, + 0.0001502961531514302, + 0.00011999956041108817, + 0.00017244562332052737, + 0.00012853776570409536, + 8.318293112097308e-05, + 6.649489660048857e-05, + 6.248604040592909e-05, + 4.157347939326428e-05, + 3.7306785088730976e-05, + 3.480926898191683e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05644794087857008, + "first_to_last_ratio": 0.8123684042388034, + "imbalance_abs_log_ratio": 0.2078013419096122, + "last_quartile_mean": 0.06948564294725657, + "mean": 0.06752561370376498, + "normalized": [ + 0.7854660588804813, + 0.9130496786513765, + 0.814729886022376, + 0.7522361645698925, + 0.968992465622299, + 0.8892579649660906, + 0.8242433717205022, + 0.7396130482968516, + 0.6125904375263521, + 0.5670744516239514, + 0.535929266538159, + 0.5145351229333333, + 0.7290151143815181, + 0.673146253269631, + 0.6214258465577596, + 0.5883338184574164, + 0.7535116614810096, + 0.6934400601144122, + 0.6978162502950979, + 0.8168476412095322, + 3.21907174067581, + 2.5076838612014067, + 1.9738308079311102, + 1.5759473804424433, + 2.264718470873259, + 1.6880792267982687, + 1.0924366234670762, + 0.8732736495232089, + 0.8206255836175469, + 0.5459820799959341, + 0.4899478787351448, + 0.45714813362074885 + ], + "peak_layer": 21, + "peak_normalized": 3.21907174067581, + "population_cv": 0.6420324679566953, + "rest_mean": 0.05116627765474496, + "spike_contrast": 3.0462647570380494, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15586602836847305, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.053039077669382095, + 0.061654239892959595, + 0.05501513555645943, + 0.05079520866274834, + 0.06543181091547012, + 0.0600476898252964, + 0.05565753951668739, + 0.049942824989557266, + 0.041365545243024826, + 0.0382920503616333, + 0.03618895262479782, + 0.034744299948215485, + 0.049227192997932434, + 0.045454613864421844, + 0.0419621616601944, + 0.039727602154016495, + 0.050881337374448776, + 0.04682496562600136, + 0.04712047055363655, + 0.055158138275146484, + 0.21736979484558105, + 0.16933289170265198, + 0.13328413665294647, + 0.10641681402921677, + 0.152926504611969, + 0.11398858577013016, + 0.07376745343208313, + 0.05896833911538124, + 0.055413246154785156, + 0.03686777502298355, + 0.03308403119444847, + 0.03086920827627182 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.57929028180115e-06, + "first_to_last_ratio": 0.4759084167214945, + "imbalance_abs_log_ratio": 0.7425298451079264, + "last_quartile_mean": 5.41971982670475e-06, + "mean": 4.213722270662856e-06, + "normalized": [ + 0.6771451422985035, + 0.6338625145334242, + 0.6120949335724296, + 0.6258725064817227, + 0.6050121598754559, + 0.5852465596314514, + 0.5785033580390577, + 0.5791973952882404, + 0.4696864977471604, + 0.4522091645384124, + 0.4457992216966758, + 0.45590579514105095, + 0.5773211960070957, + 0.5667516162949074, + 0.5644132471304436, + 0.568160519465189, + 0.6503432281671621, + 0.663170344529487, + 0.6837490184222774, + 0.7583004470800568, + 3.021070651777523, + 2.6274098317695387, + 2.3058345630897463, + 2.003282670865861, + 2.819487095674051, + 2.140274896548892, + 1.4825743986898414, + 1.122684410298403, + 0.9855482185618173, + 0.6429507763662341, + 0.5708772581826982, + 0.5252603622351902 + ], + "peak_layer": 21, + "peak_normalized": 3.021070651777523, + "population_cv": 0.7577432386283482, + "rest_mean": 3.000000956658537e-06, + "spike_contrast": 3.5892713108597136, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0767817366286181e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8533015665743733e-06, + 2.670920594027848e-06, + 2.579198053354048e-06, + 2.6372529191576177e-06, + 2.549353212089045e-06, + 2.4660664621478645e-06, + 2.4376524834224256e-06, + 2.4405769636359764e-06, + 1.9791284557868494e-06, + 1.9054838276133523e-06, + 1.8784741087074508e-06, + 1.921060402310104e-06, + 2.4326711809408152e-06, + 2.3881339075160213e-06, + 2.378280669290689e-06, + 2.394070634181844e-06, + 2.7403657441027462e-06, + 2.7944156499870587e-06, + 2.881128466469818e-06, + 3.1952674817148363e-06, + 1.27299526866409e-05, + 1.1071175322285853e-05, + 9.71614645095542e-06, + 8.441276804660447e-06, + 1.1880535566888284e-05, + 9.018523996928707e-06, + 6.247156761673978e-06, + 4.730680302600376e-06, + 4.152826477366034e-06, + 2.7092160053143743e-06, + 2.4055182166193845e-06, + 2.2133012862468604e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.826902750210138e-05, + "first_to_last_ratio": 0.9723604213825404, + "imbalance_abs_log_ratio": 0.028028739360968564, + "last_quartile_mean": 4.964108620697516e-05, + "mean": 5.395586174472555e-05, + "normalized": [ + 0.8546738942457572, + 0.9667007478721396, + 0.8780654782701063, + 0.82151592085777, + 1.0287569123437954, + 0.9492731152891766, + 0.8750422046820956, + 0.7827885676521403, + 0.6672237173045179, + 0.6202571530123847, + 0.5854234638750982, + 0.5620770546607131, + 0.7815896821357725, + 0.715521927114241, + 0.659703864497871, + 0.6215694475292887, + 0.8222045332968153, + 0.7616919988241317, + 0.764120987673305, + 0.8753289635877315, + 3.236716069528407, + 2.479479992593944, + 1.8829977352397056, + 1.4470254883420661, + 2.056662650357765, + 1.5065673712263334, + 1.0009483781315074, + 0.8155586445002443, + 0.6595748803016609, + 0.47597993512775083, + 0.43187920683965, + 0.4130800130861142 + ], + "peak_layer": 21, + "peak_normalized": 3.236716069528407, + "population_cv": 0.610766278714797, + "rest_mean": 4.176007456003895e-05, + "spike_contrast": 2.8690828214345374, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001198131125420332, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.6114666474750265e-05, + 5.2159171900711954e-05, + 4.737677954835817e-05, + 4.432559944689274e-05, + 5.550746573135257e-05, + 5.121884896652773e-05, + 4.721365621662699e-05, + 4.223603173159063e-05, + 3.600063064368442e-05, + 3.3466509194113314e-05, + 3.158702747896314e-05, + 3.032735185115598e-05, + 4.217134483042173e-05, + 3.860660217469558e-05, + 3.5594890505308285e-05, + 3.353731517563574e-05, + 4.4362754124449566e-05, + 4.109774818061851e-05, + 4.122880636714399e-05, + 4.722912854049355e-05, + 0.00017463980475440621, + 0.00013378247967921197, + 0.00010159876546822488, + 7.80755071900785e-05, + 0.00011096900561824441, + 8.128814079100266e-05, + 5.400703230407089e-05, + 4.400416946737096e-05, + 3.558793105185032e-05, + 2.568190757301636e-05, + 2.3302414774661884e-05, + 2.22880880755838e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.626883517426904e-05, + "first_to_last_ratio": 1.1972713068748915, + "imbalance_abs_log_ratio": 0.1800450565949769, + "last_quartile_mean": 3.0292912697404972e-05, + "mean": 3.794855416572318e-05, + "normalized": [ + 0.934241229967689, + 1.0326674049103126, + 0.9483113974161322, + 0.8884949951731808, + 1.1009339220259078, + 1.0148817561286225, + 0.9027362835592571, + 0.8236285600155873, + 0.7333418938739371, + 0.6823440677494514, + 0.6463725772705814, + 0.6166567793546124, + 0.850307320786337, + 0.7701976529963521, + 0.7042915982263429, + 0.6605966097637871, + 0.9142336932467707, + 0.8480352948917127, + 0.839474071242197, + 0.9443144532969522, + 3.2998505290664992, + 2.4485943259044243, + 1.7642227386757172, + 1.2451697951023746, + 1.763789999228952, + 1.2716404306318911, + 0.9094502638769789, + 0.7804639985266583, + 0.5161011792987255, + 0.40746526774101827, + 0.37633786588868356, + 0.36085204416235356 + ], + "peak_layer": 21, + "peak_normalized": 3.2998505290664992, + "population_cv": 0.5857907121700514, + "rest_mean": 3.01878957947742e-05, + "spike_contrast": 2.6453022731935993, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.985610936884768e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.545310391928069e-05, + 3.918823495041579e-05, + 3.5987046430818737e-05, + 3.3717100450303406e-05, + 4.177885057288222e-05, + 3.851329529425129e-05, + 3.425753675401211e-05, + 3.12555130221881e-05, + 2.7829264581669122e-05, + 2.5893970814649947e-05, + 2.452890475979075e-05, + 2.3401233192998916e-05, + 3.2267933420371264e-05, + 2.9227887353044935e-05, + 2.672684786375612e-05, + 2.5068686227314174e-05, + 3.469384682830423e-05, + 3.218171332264319e-05, + 3.185682726325467e-05, + 3.583536818041466e-05, + 0.00012522455654107034, + 9.292061440646648e-05, + 6.694970215903595e-05, + 4.72523934149649e-05, + 6.693328032270074e-05, + 4.825691576115787e-05, + 3.4512322599766776e-05, + 2.961748032248579e-05, + 1.9585293557611294e-05, + 1.546271778352093e-05, + 1.4281477888289373e-05, + 1.3693813343707006e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012846768913732376, + "first_to_last_ratio": 0.7741677488258334, + "imbalance_abs_log_ratio": 0.2559666991257704, + "last_quartile_mean": 0.00016594296175753698, + "mean": 0.0001574651118971815, + "normalized": [ + 0.7618335990450072, + 0.8698560675479529, + 0.7822046486498666, + 0.7351395196601278, + 0.9594102821252986, + 0.8815927774487453, + 0.812858049389013, + 0.7238937569203876, + 0.5978223149190788, + 0.5606129559633132, + 0.5319350341866579, + 0.513003926829316, + 0.6981154037320007, + 0.6408130676433959, + 0.6075107239537738, + 0.5706379352682298, + 0.7181374534396903, + 0.6783796513052409, + 0.6819318446184696, + 0.7908903240994326, + 3.2180960304949577, + 2.5452212715171636, + 2.028532306848687, + 1.6608547003669625, + 2.321647122642583, + 1.7738587692821628, + 1.1712480907534195, + 0.9285924776215382, + 0.7896984661161683, + 0.5235677697382035, + 0.4733567899608167, + 0.4487468679123384 + ], + "peak_layer": 21, + "peak_normalized": 3.2180960304949577, + "population_cv": 0.6626251267593958, + "rest_mean": 0.00011795681536939271, + "spike_contrast": 3.1436073616095452, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037080991314724087, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011996221292065457, + 0.00013697198301088065, + 0.00012316994252614677, + 0.00011575882672332227, + 0.0001510736474301666, + 0.0001388201053487137, + 0.0001279967837035656, + 0.00011398801143513992, + 9.413615771336481e-05, + 8.82769818417728e-05, + 8.376120968023315e-05, + 8.078022074187174e-05, + 0.00010992882016580552, + 0.00010090570140164346, + 9.566174412611872e-05, + 8.985556632978842e-05, + 0.0001130815944634378, + 0.00010682112770155072, + 0.00010738047421909869, + 0.00012453763338271528, + 0.0005067378515377641, + 0.0004007835523225367, + 0.0003194230666849762, + 0.0002615266712382436, + 0.0003655784239526838, + 0.0002793208695948124, + 0.00018443071166984737, + 0.0001462209183955565, + 0.00012434995733201504, + 8.244365744758397e-05, + 7.453717989847064e-05, + 7.066197576932609e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.35791492462158203, + 0.3940892517566681, + 0.4057151973247528, + 0.09380833059549332, + 0.11286329478025436, + 0.13231655955314636, + 0.1621972620487213, + 0.05156813934445381, + 0.08533531427383423, + 0.12003090232610703, + 0.16063299775123596, + 0.046871740370988846, + 0.079384446144104, + 0.11459959298372269, + 0.15625107288360596, + 0.08397205173969269, + 0.11654024571180344, + 0.15294702351093292, + 0.21572710573673248, + 0.03708760067820549, + 0.06553087383508682, + 0.09390194714069366, + 0.12852425873279572, + 0.025487789884209633, + 0.06059965491294861, + 0.08860157430171967, + 0.12100546807050705, + 0.024166326969861984, + 0.06877554953098297, + 0.12682415544986725, + 0.2025238573551178 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6127134472299076e-05, + "first_to_last_ratio": 0.6403082943802734, + "imbalance_abs_log_ratio": 0.4458055086442437, + "last_quartile_mean": 2.5186515017594502e-05, + "mean": 2.159394537670778e-05, + "normalized": [ + 0.6929830574423667, + 0.8032565154251953, + 0.7123500282965459, + 0.6640666207712503, + 0.8725494553308608, + 0.8071350816916083, + 0.7516540798235798, + 0.6706926732818747, + 0.5882490907185235, + 0.5190611932100045, + 0.48631129869562834, + 0.46529523801872363, + 0.6602888008616835, + 0.6189890701429912, + 0.5802627939183058, + 0.5409897412703016, + 0.6697392892795451, + 0.6259125602673482, + 0.6246778698614331, + 0.6676379780381497, + 3.137658015393984, + 2.675256160517513, + 2.137000554072058, + 1.6970284098395438, + 2.2713857488305376, + 1.9850977916906365, + 1.4820653354312663, + 0.9615821107638199, + 1.0163133272453195, + 0.7191715506408339, + 0.4720786856528992, + 0.423259873575668 + ], + "peak_layer": 21, + "peak_normalized": 3.137658015393984, + "population_cv": 0.691835904675886, + "rest_mean": 1.6060833668571258e-05, + "spike_contrast": 3.20486156963133, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.147274860064499e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4964238289394416e-05, + 1.7345477317576297e-05, + 1.5382447600131854e-05, + 1.43398183354293e-05, + 1.8841785276890732e-05, + 1.7429230865673162e-05, + 1.623117714188993e-05, + 1.4482900951406918e-05, + 1.2702618732873816e-05, + 1.1208579053345602e-05, + 1.0501379620109219e-05, + 1.0047559953818563e-05, + 1.4258240298659075e-05, + 1.3366416169446893e-05, + 1.2530163076007739e-05, + 1.1682102922350168e-05, + 1.4462313629337586e-05, + 1.3515921637008432e-05, + 1.3489259799825959e-05, + 1.4416938029171433e-05, + 6.775441579520702e-05, + 5.7769335398916155e-05, + 4.614627323462628e-05, + 3.664553878479637e-05, + 4.9048179789679125e-05, + 4.286609328119084e-05, + 3.200363789801486e-05, + 2.0764351575053297e-05, + 2.1946214474155568e-05, + 1.55297511810204e-05, + 1.0194041351496708e-05, + 9.139850590145215e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.422088154067751e-05, + "first_to_last_ratio": 0.637682021407296, + "imbalance_abs_log_ratio": 0.44991551892756126, + "last_quartile_mean": 0.00010070988264487823, + "mean": 8.629204910448607e-05, + "normalized": [ + 0.6940783808537224, + 0.8062729813904731, + 0.7180303336815479, + 0.6647219507645815, + 0.8588589713696609, + 0.794136864353616, + 0.7473889560340541, + 0.6703283692089702, + 0.5917748510321142, + 0.5226947352403994, + 0.4893764215021349, + 0.4670989784552037, + 0.661952700545094, + 0.6164859593679979, + 0.5786990118719374, + 0.539368710966703, + 0.6707742418503443, + 0.6266861813765591, + 0.6317438172781369, + 0.6712283354793382, + 3.1389231589965445, + 2.679577878793134, + 2.1364533990925567, + 1.686690164787602, + 2.264487313238182, + 1.9784244316759128, + 1.477878596963198, + 0.9605191983702851, + 1.022047261980904, + 0.7239260887312458, + 0.4786954192739982, + 0.4306763354738471 + ], + "peak_layer": 21, + "peak_normalized": 3.1389231589965445, + "population_cv": 0.690500620274875, + "rest_mean": 6.42200389443638e-05, + "spike_contrast": 3.1996384204494492, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002054809039691463, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.98934457229916e-05, + 6.957494770176709e-05, + 6.196030881255865e-05, + 5.736021921620704e-05, + 7.411270053125918e-05, + 6.852769729448482e-05, + 6.449372449424118e-05, + 5.784400855191052e-05, + 5.106546450406313e-05, + 4.51043997600209e-05, + 4.2229294194839895e-05, + 4.030692798551172e-05, + 5.7121254940284416e-05, + 5.319783667800948e-05, + 4.993712354917079e-05, + 4.654323129216209e-05, + 5.788248381577432e-05, + 5.40780347364489e-05, + 5.451446850202046e-05, + 5.7921668485505506e-05, + 0.00027086411137133837, + 0.00023122626589611173, + 0.00018435894162394106, + 0.00014554795052390546, + 0.00019540725043043494, + 0.0001707222982076928, + 0.00012752917245961726, + 8.288516983157024e-05, + 8.819455251796171e-05, + 6.24690655968152e-05, + 4.13076086260844e-05, + 3.716394348884933e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.056951718870550394, + "first_to_last_ratio": 0.6376820191187264, + "imbalance_abs_log_ratio": 0.4499155225164498, + "last_quartile_mean": 0.08931052964180708, + "mean": 0.07652465114369988, + "normalized": [ + 0.6940783685018588, + 0.8062729890693022, + 0.7180303052956253, + 0.6647219248274491, + 0.8588589447083164, + 0.7941368847913186, + 0.7473889396113633, + 0.6703284088385635, + 0.5917748342982165, + 0.5226947125226558, + 0.4893764209845987, + 0.46709901042220126, + 0.6619527107838585, + 0.616485998695461, + 0.5786990422522198, + 0.5393687059717984, + 0.6707742773702546, + 0.6266862088911475, + 0.6317438155472213, + 0.671228324296238, + 3.138923030946758, + 2.6795778685275287, + 2.136453429700619, + 1.6866902298133055, + 2.2644871659665227, + 1.9784244587489672, + 1.4778786878450325, + 0.9605192092561265, + 1.0220472167545944, + 0.7239261000048334, + 0.4786954380387232, + 0.4306763367173204 + ], + "peak_layer": 21, + "peak_normalized": 3.138923030946758, + "population_cv": 0.6905006031140464, + "rest_mean": 0.056950972587973984, + "spike_contrast": 3.199638339154521, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18222251534461975, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05311410501599312, + 0.0616997592151165, + 0.05494701862335205, + 0.05086761340498924, + 0.06572388112545013, + 0.060771048069000244, + 0.05719367787241936, + 0.051296647638082504, + 0.045285362750291824, + 0.03999903053045273, + 0.03744935989379883, + 0.03574458882212639, + 0.05065570026636124, + 0.04717637598514557, + 0.04428474232554436, + 0.04127500206232071, + 0.05133076757192612, + 0.04795694351196289, + 0.048343975096940994, + 0.05136551335453987, + 0.24020498991012573, + 0.20505376160144806, + 0.163491353392601, + 0.1290733814239502, + 0.17328909039497375, + 0.15139824151992798, + 0.11309415102005005, + 0.07350339740514755, + 0.07821180671453476, + 0.05539819225668907, + 0.036632001399993896, + 0.03295735642313957 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.591977136034984e-06, + "first_to_last_ratio": 0.377584641115206, + "imbalance_abs_log_ratio": 0.9739605205507381, + "last_quartile_mean": 6.864625447633443e-06, + "mean": 4.875532418679995e-06, + "normalized": [ + 0.5891470886653372, + 0.5445974877366685, + 0.5278757119769764, + 0.5390247128297342, + 0.5219320902964267, + 0.5098223481982903, + 0.5047854170324074, + 0.5158515929518226, + 0.43682532071303765, + 0.4047462819890774, + 0.3956760190748421, + 0.40124851416781243, + 0.5110306315304836, + 0.5086020328656453, + 0.5111981468235955, + 0.5112038363741968, + 0.5662160078662817, + 0.5872378713533353, + 0.609816433193417, + 0.6494206487580901, + 3.0257058078799917, + 2.8686033250815077, + 2.442085511360172, + 2.0535509509100836, + 2.6979608311079244, + 2.429139638501957, + 1.8298842703872822, + 1.2190711864897739, + 1.1821681083908706, + 0.8593060718331675, + 0.5582216762792698, + 0.48804442738052095 + ], + "peak_layer": 21, + "peak_normalized": 3.0257058078799917, + "population_cv": 0.8235699242655958, + "rest_mean": 3.415056493462611e-06, + "spike_contrast": 3.7370106290435205, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2762102414853871e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.872405730158789e-06, + 2.6552027065918082e-06, + 2.573675146777532e-06, + 2.6280324618710438e-06, + 2.544696826589643e-06, + 2.4856553864083253e-06, + 2.4610976652184036e-06, + 2.515051164664328e-06, + 2.129756012436701e-06, + 1.973353619177942e-06, + 1.929131258293637e-06, + 1.956300138772349e-06, + 2.491546410965384e-06, + 2.4797056994430022e-06, + 2.492363137207576e-06, + 2.49239087679598e-06, + 2.7606045023276238e-06, + 2.8630972792598186e-06, + 2.9731797894783085e-06, + 3.1662714263802627e-06, + 1.4751926755707245e-05, + 1.398596850776812e-05, + 1.1906467079825234e-05, + 1.0012154234573245e-05, + 1.315399549639551e-05, + 1.1843349057016894e-05, + 8.921660082705785e-06, + 5.943621090409579e-06, + 5.7636989367892966e-06, + 4.189574610791169e-06, + 2.7216278795094695e-06, + 2.3794764274498448e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.862782361669815e-05, + "first_to_last_ratio": 0.7782948608369358, + "imbalance_abs_log_ratio": 0.25064982810047876, + "last_quartile_mean": 6.247994951991132e-05, + "mean": 6.004880788168521e-05, + "normalized": [ + 0.7686076787185943, + 0.8690658694356951, + 0.7879558094938062, + 0.7385895471539009, + 0.9274752179545845, + 0.8617380288980451, + 0.805485823509466, + 0.7195218583140428, + 0.6496432282856196, + 0.579963373368429, + 0.542973633406911, + 0.5176967820022496, + 0.7209744736270646, + 0.6651136389238139, + 0.6225458367735092, + 0.578423456947426, + 0.7443181598307745, + 0.6978871776308329, + 0.7018192413444355, + 0.7379772277240845, + 3.17488720924795, + 2.647274451154299, + 2.058197351015695, + 1.557976178868823, + 2.137956515672529, + 1.7650930593990968, + 1.3105561147977765, + 0.8840594863751997, + 0.8342799220515003, + 0.5872955759322932, + 0.4206210094039151, + 0.3840270627376372 + ], + "peak_layer": 21, + "peak_normalized": 3.17488720924795, + "population_cv": 0.6535433957622422, + "rest_mean": 4.5422938354830774e-05, + "spike_contrast": 3.060755388404179, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013902850332669914, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.615397483576089e-05, + 5.218636943027377e-05, + 4.7315807023551315e-05, + 4.4351421820465475e-05, + 5.569378117797896e-05, + 5.17463413416408e-05, + 4.8368463467340916e-05, + 4.320642983657308e-05, + 3.9010301406960934e-05, + 3.482610918581486e-05, + 3.260491939727217e-05, + 3.108707460341975e-05, + 4.329365765443072e-05, + 3.9939281123224646e-05, + 3.738313534995541e-05, + 3.47336390404962e-05, + 4.4695418182527646e-05, + 4.19072930526454e-05, + 4.2143408791162074e-05, + 4.43146527686622e-05, + 0.00019064819207414985, + 0.00015896567492745817, + 0.0001235922973137349, + 9.355461224913597e-05, + 0.00012838174006901681, + 0.00010599173401715234, + 7.869733235565946e-05, + 5.308671825332567e-05, + 5.009751475881785e-05, + 3.526639920892194e-05, + 2.5257790184696205e-05, + 2.3060367311700247e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.645344395408756e-05, + "first_to_last_ratio": 0.9865245275167812, + "imbalance_abs_log_ratio": 0.01356709065898008, + "last_quartile_mean": 3.695138127568498e-05, + "mean": 4.1334421752026174e-05, + "normalized": [ + 0.8559009893021382, + 0.9469134291395908, + 0.8691128415923758, + 0.8158459610499553, + 1.0100395593992384, + 0.9426509450849102, + 0.8490005706596633, + 0.7658551995239993, + 0.720151565776352, + 0.6499584076998435, + 0.6111014164691825, + 0.5791973878203037, + 0.798669989787535, + 0.7292203680582068, + 0.6791147051708346, + 0.6276623067938342, + 0.8453470506184937, + 0.7894265691211152, + 0.7857081832788244, + 0.8180697050541997, + 3.276793902438668, + 2.6146895715088023, + 1.9277214837041228, + 1.3401559674726915, + 1.9403857172500238, + 1.4668533992566692, + 1.0945373428822949, + 0.8102509556533821, + 0.6598620161196013, + 0.46946557551492035, + 0.3666267887349235, + 0.3437101280633033 + ], + "peak_layer": 21, + "peak_normalized": 3.276793902438668, + "population_cv": 0.6207652283001258, + "rest_mean": 3.1996292111418574e-05, + "spike_contrast": 2.867842357851226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.176032181130722e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5378172469791025e-05, + 3.9140119042713195e-05, + 3.5924276744481176e-05, + 3.372252103872597e-05, + 4.174940113443881e-05, + 3.8963931729085743e-05, + 3.509294765535742e-05, + 3.165618181810714e-05, + 2.976704854518175e-05, + 2.6865654945140705e-05, + 2.5259523681597784e-05, + 2.3940789105836302e-05, + 3.301256219856441e-05, + 3.0141902243485674e-05, + 2.8070813641534187e-05, + 2.5944058506865986e-05, + 3.4941931517096236e-05, + 3.263049075030722e-05, + 3.2476793421665207e-05, + 3.381443821126595e-05, + 0.0001354443811578676, + 0.00010807668149936944, + 7.968125282786787e-05, + 5.53945719730109e-05, + 8.020472159842029e-05, + 6.06315370532684e-05, + 4.524206815403886e-05, + 3.349125472595915e-05, + 2.7275014872429892e-05, + 1.940508809639141e-05, + 1.5154306311160326e-05, + 1.4207059393811505e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012982848875253694, + "first_to_last_ratio": 0.6036924070795879, + "imbalance_abs_log_ratio": 0.5046904705661434, + "last_quartile_mean": 0.00021505734912352636, + "mean": 0.00017907720302900998, + "normalized": [ + 0.6739415088564852, + 0.7635979187076084, + 0.6884132846665824, + 0.6448880523181134, + 0.847636297931654, + 0.7820437425942102, + 0.7390348805248977, + 0.6603333703627584, + 0.580097091853396, + 0.5130765445550399, + 0.4840585582545864, + 0.4614791311662735, + 0.637447225260648, + 0.5884103323685723, + 0.5619029319457929, + 0.5245290839622635, + 0.6356966288449003, + 0.6100272686595811, + 0.6118819192326126, + 0.6643605628507145, + 3.097964891544288, + 2.6928767346712488, + 2.1827809090098653, + 1.7461631642015993, + 2.367937988760414, + 2.0449314566347367, + 1.563314534665903, + 1.0260809297257236, + 1.0359670091642164, + 0.6947146354097079, + 0.460142110411202, + 0.41426930088440395 + ], + "peak_layer": 21, + "peak_normalized": 3.097964891544288, + "population_cv": 0.7095182139952283, + "rest_mean": 0.00013206795362445214, + "spike_contrast": 3.278063587209748, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004329271498136222, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012068756041117013, + 0.00013674297952093184, + 0.0001232791255461052, + 0.0001154847486759536, + 0.00015179233741946518, + 0.0001400462060701102, + 0.00013234429934527725, + 0.00011825065303128213, + 0.00010388216469436884, + 9.188031253870577e-05, + 8.668385271448642e-05, + 8.264039206551388e-05, + 0.00011415226617828012, + 0.00010537087655393407, + 0.00010062400542665273, + 9.393120126333088e-05, + 0.00011383877426851541, + 0.00010924197704298422, + 0.00010957410268019885, + 0.00011897183139808476, + 0.0005547748878598213, + 0.0004822328337468207, + 0.00039088630001060665, + 0.0003126980154775083, + 0.0004240437119733542, + 0.00036620060564018786, + 0.0002799539943225682, + 0.00018374770297668874, + 0.00018551807443145663, + 0.0001244075538124889, + 8.240096212830395e-05, + 7.418618770316243e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5986864127626177e-05, + "first_to_last_ratio": 0.8170274391577745, + "imbalance_abs_log_ratio": 0.20208259942567244, + "last_quartile_mean": 1.956710798367567e-05, + "mean": 1.9039307829871177e-05, + "normalized": [ + 0.7853167979867789, + 0.9097665173048055, + 0.808777694116954, + 0.752402497635975, + 0.9858540561935626, + 0.9044938358034963, + 0.8288447236919866, + 0.7419578801196516, + 0.6090850929460484, + 0.5628647453138417, + 0.5328979550345672, + 0.5131403750850531, + 0.7265824660927316, + 0.6757007408890954, + 0.6225826951824261, + 0.5898791530514104, + 0.7532772493405897, + 0.6927950381940383, + 0.6897848547806317, + 0.8094505726944553, + 3.2076147584918284, + 2.5002121502474965, + 1.982484105644761, + 1.5924612114390058, + 2.273889158865966, + 1.6940625327896233, + 1.0968336406315982, + 0.8719471681044353, + 0.815653609383569, + 0.5388945327422838, + 0.4819664542042268, + 0.448525735997107 + ], + "peak_layer": 21, + "peak_normalized": 3.2076147584918284, + "population_cv": 0.6428608089466793, + "rest_mean": 1.4415815443229534e-05, + "spike_contrast": 3.052631111367605, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.400616671773605e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4951888260839041e-05, + 1.7321324776276015e-05, + 1.5398567484226078e-05, + 1.4325222764455248e-05, + 1.8769978851196356e-05, + 1.7220936570083722e-05, + 1.5780629837536253e-05, + 1.4126364476396702e-05, + 1.1596558579185512e-05, + 1.0716555152612273e-05, + 1.0146008207811974e-05, + 9.769837561179884e-06, + 1.3833627235726453e-05, + 1.286487440665951e-05, + 1.1853543583129067e-05, + 1.1230890777369495e-05, + 1.4341877431434114e-05, + 1.3190337995183654e-05, + 1.3133026186551433e-05, + 1.541137862659525e-05, + 6.107076478656381e-05, + 4.760230876854621e-05, + 3.7745125155197456e-05, + 3.03193592117168e-05, + 4.329327566665597e-05, + 3.225377804483287e-05, + 2.0882953322143294e-05, + 1.6601270544924773e-05, + 1.5529480151599273e-05, + 1.0260178896714933e-05, + 9.176307685265783e-06, + 8.539619557268452e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "rest_mean": 5.7697108584559625e-05, + "spike_contrast": 3.04626897714722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017576091195223854, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05644787335768342, + "first_to_last_ratio": 0.8123632868395458, + "imbalance_abs_log_ratio": 0.2078076412872703, + "last_quartile_mean": 0.06948599754832685, + "mean": 0.06752577290171757, + "normalized": [ + 0.7854618348329956, + 0.9130540359301327, + 0.8147131249195937, + 0.7522410113175043, + 0.9690071730110875, + 0.88924395208314, + 0.8242353599678512, + 0.7396083806676593, + 0.6126026750590019, + 0.5670643980854334, + 0.5359283892166958, + 0.5145269586503932, + 0.7290174229589911, + 0.6731434525628229, + 0.6214223402600404, + 0.5883307763560162, + 0.7535191533057549, + 0.6934459281737582, + 0.6978155981634587, + 0.8168473704719704, + 3.2190670201887848, + 2.507709505453073, + 1.9738347607250095, + 1.5759251284181852, + 2.264726151347608, + 1.68807248858185, + 1.0924542395938426, + 0.8732750663142743, + 0.820625855657368, + 0.5459870819951741, + 0.48994655813463567, + 0.45714680759589343 + ], + "peak_layer": 21, + "peak_normalized": 3.2190670201887848, + "population_cv": 0.6420341436220799, + "rest_mean": 0.05116637251167386, + "spike_contrast": 3.0462690113979045, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15586653500795364, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05303891748189926, + 0.06165467947721481, + 0.05501413345336914, + 0.05079565569758415, + 0.06543295830488205, + 0.060046885162591934, + 0.05565712973475456, + 0.04994262754917145, + 0.041366469115018845, + 0.038291461765766144, + 0.03618897870182991, + 0.03474383056163788, + 0.04922746494412422, + 0.04545453190803528, + 0.041962023824453354, + 0.03972749039530754, + 0.05088196322321892, + 0.04682547226548195, + 0.04712053760886192, + 0.05515825003385544, + 0.21736998856067657, + 0.1693350225687027, + 0.13328471779823303, + 0.10641556233167648, + 0.15292738378047943, + 0.11398839950561523, + 0.07376881688833237, + 0.058968573808670044, + 0.055413395166397095, + 0.036868199706077576, + 0.033084020018577576, + 0.030869191512465477 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.579253447265728e-06, + "first_to_last_ratio": 0.47590496457778414, + "imbalance_abs_log_ratio": 0.7425370989326324, + "last_quartile_mean": 5.419681741614113e-06, + "mean": 4.2137082871818166e-06, + "normalized": [ + 0.6771144196047112, + 0.6338426021784607, + 0.6121020910961615, + 0.6258737201152189, + 0.6050362914422536, + 0.5852204423685657, + 0.5784631886764509, + 0.5792281322851218, + 0.4697078599262646, + 0.4522117174548025, + 0.44580914592548354, + 0.4558904724294877, + 0.5773213311917524, + 0.5667240346830847, + 0.5644321177223985, + 0.5681434108598962, + 0.6503548834184826, + 0.6631947770191623, + 0.6837887360559592, + 0.7582966501774209, + 3.020973835679826, + 2.627473806530212, + 2.3058644468737186, + 2.00331262981993, + 2.819406877957528, + 2.1402761714678284, + 1.4825745701937771, + 1.1227288221975413, + 0.9855094000056028, + 0.6429466506282437, + 0.5709200007532207, + 0.5252567632614316 + ], + "peak_layer": 21, + "peak_normalized": 3.020973835679826, + "population_cv": 0.7577358346351384, + "rest_mean": 2.999999306094078e-06, + "spike_contrast": 3.589246425218318, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0767736785055605e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8531626412586775e-06, + 2.670827825568267e-06, + 2.5792196538532153e-06, + 2.6372492811788106e-06, + 2.5494464352959767e-06, + 2.465948227836634e-06, + 2.43747513195558e-06, + 2.440698381178663e-06, + 1.979211901925737e-06, + 1.9054882614000235e-06, + 1.8785096926876577e-06, + 1.920989461723366e-06, + 2.4326636776095256e-06, + 2.3880097614892293e-06, + 2.3783522919984534e-06, + 2.393990598648088e-06, + 2.7404057618696243e-06, + 2.7945093279413413e-06, + 2.8812862638005754e-06, + 3.1952408789948095e-06, + 1.2729502486763522e-05, + 1.1071408152929507e-05, + 9.716240128909703e-06, + 8.441375030088238e-06, + 1.1880158126587048e-05, + 9.018499440571759e-06, + 6.247136752790539e-06, + 4.7308517423516605e-06, + 4.152649125899188e-06, + 2.709189629968023e-06, + 2.405690338491695e-06, + 2.2132787762529915e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.826905797017389e-05, + "first_to_last_ratio": 0.9723593605402814, + "imbalance_abs_log_ratio": 0.028029830358518852, + "last_quartile_mean": 4.964117169947713e-05, + "mean": 5.395593586854375e-05, + "normalized": [ + 0.8546740011831107, + 0.9667023865336807, + 0.8780594173977533, + 0.8215201862718362, + 1.028762848378348, + 0.9492707323927138, + 0.8750383055629682, + 0.7827836490446655, + 0.6672270484578459, + 0.6202534690629704, + 0.5854228619045188, + 0.5620710233383177, + 0.781592991024926, + 0.7155185842665527, + 0.6597031604820425, + 0.621567177701999, + 0.8222052916650392, + 0.7616961441483728, + 0.7641199379358804, + 0.8753280307745216, + 3.236716477578425, + 2.479496544122304, + 1.8830004075567146, + 1.4470156791415152, + 2.05665793704697, + 1.5065644924277526, + 1.0009572516438536, + 0.8155596816984735, + 0.6595753226879756, + 0.4759799891964506, + 0.4318790517932509, + 0.413079917578251 + ], + "peak_layer": 21, + "peak_normalized": 3.236716477578425, + "population_cv": 0.6107674157639074, + "rest_mean": 4.176012171902349e-05, + "spike_contrast": 2.869084843241085, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011981333227595314, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.61147355963476e-05, + 5.2159331971779466e-05, + 4.737651761388406e-05, + 4.4325890485197306e-05, + 5.550786227104254e-05, + 5.121879075886682e-05, + 4.72135106974747e-05, + 4.2235824366798624e-05, + 3.6000859836349264e-05, + 3.3466356399003416e-05, + 3.158703839289956e-05, + 3.0327068088809028e-05, + 4.217158129904419e-05, + 3.860647484543733e-05, + 3.559490141924471e-05, + 3.353723877808079e-05, + 4.4362855987856165e-05, + 4.1098028304986656e-05, + 4.122880636714399e-05, + 4.7229143092408776e-05, + 0.00017464006668888032, + 0.00013378355652093887, + 0.00010159904923057184, + 7.807508518453687e-05, + 0.00011096890375483781, + 8.128809713525698e-05, + 5.400758527684957e-05, + 4.4004285882692784e-05, + 3.558800381142646e-05, + 2.5681945771793835e-05, + 2.330243842152413e-05, + 2.228811354143545e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.626825309765991e-05, + "first_to_last_ratio": 1.1971952602722011, + "imbalance_abs_log_ratio": 0.17998153797771219, + "last_quartile_mean": 3.0294350722215313e-05, + "mean": 3.794959516767449e-05, + "normalized": [ + 0.9342317076920968, + 1.032527492584725, + 0.9483990781630639, + 0.8885231558783968, + 1.1008949985426397, + 1.0148066560537, + 0.9027782413985963, + 0.8234017777861798, + 0.7333936750261243, + 0.6823073758552121, + 0.6464481216407498, + 0.616777331910669, + 0.8501710687237772, + 0.7702178906455867, + 0.7042577074377577, + 0.660638930723233, + 0.914060026408416, + 0.847914539167618, + 0.8394293783449699, + 0.9442746494437388, + 3.300220154812763, + 2.4489522165880904, + 1.7640184700183157, + 1.245126339879113, + 1.7643693302152461, + 1.271277790934411, + 0.9095623054467832, + 0.780451313056054, + 0.5159478762516835, + 0.40747033934370974, + 0.37633772798986104, + 0.36081233203671814 + ], + "peak_layer": 21, + "peak_normalized": 3.300220154812763, + "population_cv": 0.5858778602410362, + "rest_mean": 3.018723526582579e-05, + "spike_contrast": 2.6456990159702416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.986633863765746e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5453715099720284e-05, + 3.9184000343084335e-05, + 3.599136107368395e-05, + 3.371909406268969e-05, + 4.1778519516810775e-05, + 3.85115017706994e-05, + 3.4260068787261844e-05, + 3.124776412732899e-05, + 2.783199306577444e-05, + 2.5893288693623617e-05, + 2.453244451317005e-05, + 2.3406450054608285e-05, + 3.226364788133651e-05, + 2.9229457140900195e-05, + 2.6726294890977442e-05, + 2.5070979972952046e-05, + 3.468820796115324e-05, + 3.217801349819638e-05, + 3.185600508004427e-05, + 3.5834840673487633e-05, + 0.00012524201883934438, + 9.293674520449713e-05, + 6.694378680549562e-05, + 4.725204053102061e-05, + 6.695710180792958e-05, + 4.8244477511616424e-05, + 3.451752127148211e-05, + 2.9617811378557235e-05, + 1.95800130313728e-05, + 1.546333442092873e-05, + 1.4281864423537627e-05, + 1.3692681932298001e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012846494701079791, + "first_to_last_ratio": 0.7741545804954827, + "imbalance_abs_log_ratio": 0.25598370893064326, + "last_quartile_mean": 0.00016594224234722788, + "mean": 0.000157460202672155, + "normalized": [ + 0.7615886502877881, + 0.869859991005712, + 0.7823534284439785, + 0.735201115807512, + 0.9596786286759813, + 0.8813992955944876, + 0.8128285893567467, + 0.7239431731448671, + 0.5977864740547744, + 0.5606030792342883, + 0.5318673348078621, + 0.5130567490014816, + 0.6987134321316841, + 0.6407269063222457, + 0.6075335461477536, + 0.5704889146486948, + 0.7180428901779269, + 0.6784140170824181, + 0.6819219612414157, + 0.7908928021790416, + 3.2176296651190497, + 2.545498211708922, + 2.0288816729118655, + 1.6601468184271124, + 2.321862751456626, + 1.7738386620365922, + 1.1711907122839438, + 0.9284857615000071, + 0.7896970255450443, + 0.5236030848875767, + 0.47341239614817127, + 0.4488522586284289 + ], + "peak_layer": 21, + "peak_normalized": 3.2176296651190497, + "population_cv": 0.6625973254291921, + "rest_mean": 0.0001179550758765631, + "spike_contrast": 3.143466990402101, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037078788736835124, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011991990322712809, + 0.00013696833048015833, + 0.00012318952940404415, + 0.00011576491669984534, + 0.0001511111913714558, + 0.00013878531171940267, + 0.00012798815441783518, + 0.00011399223876651376, + 9.412757935933769e-05, + 8.82726744748652e-05, + 8.374793833354488e-05, + 8.078601968009025e-05, + 0.000110019558633212, + 0.00010088898852700368, + 9.566235530655831e-05, + 8.982930012280121e-05, + 0.0001130631790147163, + 0.00010682320862542838, + 0.00010737557022366673, + 0.00012453414092306048, + 0.0005066486191935837, + 0.00040081466431729496, + 0.00031946811941452324, + 0.0002614070544950664, + 0.0003656009794212878, + 0.00027930899523198605, + 0.00018441592692397535, + 0.00014619955618400127, + 0.00012434585369192064, + 8.24466478661634e-05, + 7.454361184500158e-05, + 7.067636761348695e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0626930519938469, + 0.09030644595623016, + 0.20229491591453552, + 0.20300470292568207, + 0.19049881398677826, + 0.12695962190628052, + 0.12132706493139267, + 0.11585728824138641, + 0.13427142798900604, + 0.10485932976007462, + 0.10419609397649765, + 0.10852954536676407, + 0.1286265105009079, + 0.09096142649650574, + 0.10080574452877045, + 0.10566592961549759, + 0.1287592649459839, + 0.09650235623121262, + 0.1065765842795372, + 0.11446042358875275, + 0.12395420670509338, + 0.07221997529268265, + 0.07677769660949707, + 0.08416415005922318, + 0.09094838052988052, + 0.059439923614263535, + 0.062182098627090454, + 0.06896103918552399, + 0.0776243805885315, + 0.06285123527050018, + 0.07231374084949493, + 0.08016443997621536 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0311521975836513e-05, + "first_to_last_ratio": 1.1418129560476864, + "imbalance_abs_log_ratio": 0.13261731152138012, + "last_quartile_mean": 9.030832870848826e-06, + "mean": 1.3945096057454975e-05, + "normalized": [ + 0.7145337044030123, + 0.932335477800909, + 0.645487324949067, + 0.558162738870377, + 0.34661223918607953, + 0.7511338800577496, + 0.9447212820009926, + 1.0225104899919193, + 0.6315142693979814, + 0.5614582240786927, + 0.4574638139025647, + 0.40615350248375953, + 0.4194348381092698, + 0.7040945119904685, + 0.6483274463519548, + 0.5709286433652233, + 0.8217341836062412, + 1.1893729541204203, + 1.2408864574521172, + 1.5700693567578727, + 2.5502861886942485, + 4.114342551839228, + 2.785136777749787, + 2.232505646034574, + 1.1806687356339707, + 1.2053474712098855, + 0.7898562388784587, + 0.5945521897391881, + 0.46830824879649285, + 0.2867747481020556, + 0.32588927285831853, + 0.32939659158711965 + ], + "peak_layer": 22, + "peak_normalized": 4.114342551839228, + "population_cv": 0.8262244130468728, + "rest_mean": 9.884005235276331e-06, + "spike_contrast": 3.629600110811422, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.587498649721965e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.964241144189145e-06, + 1.3001507795706857e-05, + 9.001382750284392e-06, + 7.783633009239566e-06, + 4.833540970139438e-06, + 1.0474634109414183e-05, + 1.3174229025025852e-05, + 1.425900700269267e-05, + 8.80652714840835e-06, + 7.82958886702545e-06, + 6.3793768276809715e-06, + 5.663849606207805e-06, + 5.849059107276844e-06, + 9.818665603233967e-06, + 9.040988516062498e-06, + 7.961654773680493e-06, + 1.1459162124083377e-05, + 1.658592009334825e-05, + 1.730428084556479e-05, + 2.189476799685508e-05, + 3.556398587534204e-05, + 5.7374902098672464e-05, + 3.883899989887141e-05, + 3.113250568276271e-05, + 1.6464538930449635e-05, + 1.68086862686323e-05, + 1.1014621122740209e-05, + 8.291087397083174e-06, + 6.530603513965616e-06, + 3.999101409135619e-06, + 4.544557214103406e-06, + 4.593467110680649e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.944479362871789e-05, + "first_to_last_ratio": 1.1135282861542686, + "imbalance_abs_log_ratio": 0.1075336103151947, + "last_quartile_mean": 3.54232524841791e-05, + "mean": 5.533300526394669e-05, + "normalized": [ + 0.5280207224829799, + 0.8999098868607942, + 0.6284748654319019, + 0.5778568458898549, + 0.34988006202074273, + 0.7764713506345448, + 0.9190909808587969, + 1.023190827540606, + 0.6226417920568781, + 0.5519145939388094, + 0.4568960180345979, + 0.4213932325876586, + 0.4074494455716704, + 0.7465319443901276, + 0.6474109861274324, + 0.5764712279753077, + 0.7979703549327781, + 1.2369697451324837, + 1.2324489817728508, + 1.5808985864286207, + 2.5860289117498976, + 4.232714993086349, + 2.835646318943954, + 2.2422528664695807, + 1.1646676435019565, + 1.2175941057769888, + 0.783401544111113, + 0.5289102464485725, + 0.46735876466607296, + 0.2966943922188728, + 0.3266155865457417, + 0.3362221758114643 + ], + "peak_layer": 22, + "peak_normalized": 4.232714993086349, + "population_cv": 0.8496980089222349, + "rest_mean": 3.881239232820614e-05, + "spike_contrast": 3.7241794809927478, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014454431511694564, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.921697341662366e-05, + 4.9794718506745994e-05, + 3.477540303720161e-05, + 3.197455589543097e-05, + 1.9359915313543752e-05, + 4.296449333196506e-05, + 5.085606608190574e-05, + 5.661622344632633e-05, + 3.4452641557436436e-05, + 3.053909313166514e-05, + 2.5281429770984687e-05, + 2.3316953956964426e-05, + 2.2545402316609398e-05, + 4.1307856008643284e-05, + 3.582319550332613e-05, + 3.189788549207151e-05, + 4.415409784996882e-05, + 6.844525341875851e-05, + 6.819510599598289e-05, + 8.747586980462074e-05, + 0.0001430927513865754, + 0.00023420884099323303, + 0.00015690483269281685, + 0.00012407058966346085, + 6.444456084864214e-05, + 6.737314106430858e-05, + 4.3347961764084175e-05, + 2.9266193450894207e-05, + 2.5860364985419437e-05, + 1.641699236643035e-05, + 1.807262196962256e-05, + 1.8604183424031362e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03498003748245537, + "first_to_last_ratio": 1.1135283344223057, + "imbalance_abs_log_ratio": 0.10753365366212776, + "last_quartile_mean": 0.03141369321383536, + "mean": 0.04906986153218895, + "normalized": [ + 0.528020735774383, + 0.8999099163028887, + 0.6284749107820657, + 0.5778568640801447, + 0.34988006993508275, + 0.776471348923419, + 0.919091002023713, + 1.0231908018037408, + 0.622641782085556, + 0.5519145889381298, + 0.45689603549337854, + 0.4213932430966296, + 0.4074494429214576, + 0.7465319861987265, + 0.6474110467766695, + 0.5764712070514831, + 0.7979703658517325, + 1.236969773835328, + 1.2324490791670337, + 1.5808986244806464, + 2.5860289675304746, + 4.232714870782151, + 2.835646138092529, + 2.242252864087746, + 1.1646675839359508, + 1.2175940341247224, + 0.7834015320190452, + 0.528910268066999, + 0.467358763362216, + 0.2966943974186558, + 0.32661557776013733, + 0.33622217729716364 + ], + "peak_layer": 22, + "peak_normalized": 4.232714870782151, + "population_cv": 0.8496979833458974, + "rest_mean": 0.034419217595347655, + "spike_contrast": 3.72417933196884, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12818333879113197, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0259099043905735, + 0.0441584549844265, + 0.03083917684853077, + 0.02835535630583763, + 0.017168566584587097, + 0.03810134157538414, + 0.04509966820478439, + 0.05020783096551895, + 0.030552946031093597, + 0.027082372456789017, + 0.022419825196266174, + 0.020677708089351654, + 0.019993487745523453, + 0.036632221192121506, + 0.03176837041974068, + 0.028287362307310104, + 0.039156295359134674, + 0.06069793552160263, + 0.06047610566020012, + 0.0775744765996933, + 0.12689608335494995, + 0.2076987326145172, + 0.13914476335048676, + 0.11002703756093979, + 0.05715007707476616, + 0.05974717065691948, + 0.038441404700279236, + 0.02595355361700058, + 0.022933229804039, + 0.014558752998709679, + 0.01602698117494583, + 0.016498375684022903 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.750052800593039e-06, + "first_to_last_ratio": 0.4308304317433478, + "imbalance_abs_log_ratio": 0.8420406961220296, + "last_quartile_mean": 6.383144267374519e-06, + "mean": 4.5530042669383874e-06, + "normalized": [ + 2.0777537806616246, + 0.8353410859824805, + 0.1910970826520623, + 0.2941663520688202, + 0.18051642612757607, + 0.3343816911052641, + 0.3564217354179324, + 0.562389007550559, + 0.34578362377189115, + 0.35956396375636174, + 0.3051237753595574, + 0.33780495264365074, + 0.3038063276744268, + 0.5221079484446093, + 0.49376666786286805, + 0.5929821493588614, + 0.3686987288124583, + 0.7392110518834059, + 0.5965749796632107, + 0.9039444986896679, + 1.6537371809131796, + 3.0803086468030596, + 2.8630898340929525, + 2.485723197420275, + 1.724168620325731, + 2.41211172564755, + 1.9299601312238395, + 1.6334426636441552, + 1.3339260437132254, + 0.9558102182238872, + 0.6013349904117006, + 0.6249509180931564 + ], + "peak_layer": 22, + "peak_normalized": 3.0803086468030596, + "population_cv": 0.8333981127885269, + "rest_mean": 3.4051366684353625e-06, + "spike_contrast": 3.1574325337710922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0751489298854722e-05, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 1 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.460021828999743e-06, + 3.80331152882718e-06, + 8.700658327143174e-07, + 1.3393406561590382e-06, + 8.218920584113221e-07, + 1.5224412663883413e-06, + 1.622789682187431e-06, + 2.56055955105694e-06, + 1.5743543144708383e-06, + 1.6370962612199946e-06, + 1.389229851156415e-06, + 1.5380273907794617e-06, + 1.383231506224547e-06, + 2.3771597170707537e-06, + 2.248121745651588e-06, + 2.699850256249192e-06, + 1.678686885497882e-06, + 3.3656310733931605e-06, + 2.71620842795528e-06, + 4.1156631596095394e-06, + 7.529472441092366e-06, + 1.4024658412381541e-05, + 1.3035660231253132e-05, + 1.1317508324282244e-05, + 7.850147085264325e-06, + 1.0982354979205411e-05, + 8.787116712483112e-06, + 7.437071417371044e-06, + 6.073370968806557e-06, + 4.35180800195667e-06, + 2.737880777203827e-06, + 2.8454041967052035e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6044932951663213e-05, + "first_to_last_ratio": 1.1246393261284713, + "imbalance_abs_log_ratio": 0.11746238525662386, + "last_quartile_mean": 2.315847609679622e-05, + "mean": 3.690820258839267e-05, + "normalized": [ + 0.45689793146127605, + 0.8738188272728166, + 0.6345310201461108, + 0.5870234156151962, + 0.3959041381169584, + 0.8145991894129532, + 0.9605505534893847, + 0.9220179377165436, + 0.6534489876582346, + 0.574567881594872, + 0.47330901138441, + 0.4295574998000546, + 0.4505301728221963, + 0.8093943395842734, + 0.6650197233282829, + 0.5723463492360998, + 0.7892367771238881, + 1.2091363576952083, + 1.2417470885367035, + 1.5367390457984618, + 2.460873737770247, + 4.227774989574034, + 3.004456821926061, + 2.2368262162922274, + 1.1669129504749511, + 1.3012136399895533, + 0.7651380597325796, + 0.47754123850698527, + 0.4891606669175266, + 0.2974921422063077, + 0.24654864385331124, + 0.27568464496229084 + ], + "peak_layer": 22, + "peak_normalized": 4.227774989574034, + "population_cv": 0.8549059646786937, + "rest_mean": 2.5840054991864168e-05, + "spike_contrast": 3.7413310319999478, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.667619960964657e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.6863281416590326e-05, + 3.2251082302536815e-05, + 2.341939944017213e-05, + 2.166597914765589e-05, + 1.4612110135203693e-05, + 3.006539191119373e-05, + 3.545219442457892e-05, + 3.4030024835374206e-05, + 2.4117627617670223e-05, + 2.120626777468715e-05, + 1.7468984879087657e-05, + 1.5854195225983858e-05, + 1.6628258890705183e-05, + 2.9873290259274654e-05, + 2.4544682673877105e-05, + 2.1124275008332916e-05, + 2.9129310860298574e-05, + 4.462704964680597e-05, + 4.583065310725942e-05, + 5.671827602782287e-05, + 9.082642645807937e-05, + 0.00015603957581333816, + 0.00011088910105172545, + 8.255723514594138e-05, + 4.3068659579148516e-05, + 4.8025456635514274e-05, + 2.823987051669974e-05, + 1.7625188775127754e-05, + 1.805404099286534e-05, + 1.0979900253005326e-05, + 9.099667295231484e-06, + 1.0175024726777337e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.700510824775847e-05, + "first_to_last_ratio": 1.1234226056354075, + "imbalance_abs_log_ratio": 0.1163799234305708, + "last_quartile_mean": 1.5136875617827172e-05, + "mean": 2.416565462226572e-05, + "normalized": [ + 0.4144047676027832, + 0.875113510868426, + 0.648392503192907, + 0.5978987553197436, + 0.4472461776170949, + 0.8581546509904864, + 0.9915956817879598, + 0.7967069167230741, + 0.6805807808463835, + 0.6196150111411964, + 0.5091007553108428, + 0.44255818216184833, + 0.49857092482915655, + 0.8652430602969526, + 0.6897944860164179, + 0.5800177420993363, + 0.7901724317041942, + 1.1871172558602836, + 1.243251110659502, + 1.5081298165112624, + 2.2807123105787017, + 4.096637658310102, + 3.1477300512572848, + 2.2202178168572866, + 1.1479124533540228, + 1.3987972507734237, + 0.7581679674914413, + 0.4356681137302007, + 0.511763265149204, + 0.3118761594955322, + 0.21315449183792956, + 0.2336979396250189 + ], + "peak_layer": 22, + "peak_normalized": 4.096637658310102, + "population_cv": 0.8395518179977153, + "rest_mean": 1.710104003938829e-05, + "spike_contrast": 3.6439054716132255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.231457336980384e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.001436248770915e-05, + 2.114769085892476e-05, + 1.5668829291826114e-05, + 1.4448614820139483e-05, + 1.0807996659423225e-05, + 2.0737868908327073e-05, + 2.396255877101794e-05, + 1.9252944184700027e-05, + 1.644668009248562e-05, + 1.497340235800948e-05, + 1.2302753020776436e-05, + 1.0694708180380985e-05, + 1.2048292774125002e-05, + 2.090916495944839e-05, + 1.6669335309416056e-05, + 1.4016508430358954e-05, + 1.9095034076599404e-05, + 2.868746560125146e-05, + 3.0043976948945783e-05, + 3.644494427135214e-05, + 5.511490599019453e-05, + 9.899793076328933e-05, + 7.606695726281032e-05, + 5.365301694837399e-05, + 2.7740055884351023e-05, + 3.3802851248765364e-05, + 1.8321625248063356e-05, + 1.0528205166338012e-05, + 1.236709431395866e-05, + 7.536691555287689e-06, + 5.1510178309399635e-06, + 5.647463694913313e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.642002219654387e-05, + "first_to_last_ratio": 1.1203608181269622, + "imbalance_abs_log_ratio": 0.11365079246685211, + "last_quartile_mean": 6.821018814662239e-05, + "mean": 0.00011010971388714097, + "normalized": [ + 0.5163109854936148, + 0.8673817094155928, + 0.5924673517780332, + 0.5654643004921971, + 0.34399113467095715, + 0.7609201839652954, + 0.9076117604661024, + 0.9981345295043799, + 0.6327009050241252, + 0.5203303165972624, + 0.4437849422865529, + 0.41423566265400885, + 0.39884159677379066, + 0.7686876572920235, + 0.6339609024838357, + 0.5703063832410542, + 0.781268072457197, + 1.1797447140196398, + 1.2760717444836593, + 1.5599326259342914, + 2.6126342040159805, + 4.419910912775975, + 2.9287458133972417, + 2.3507635425478965, + 1.1736586910474214, + 1.2201215929679614, + 0.7838637940961916, + 0.4869646019277815, + 0.46617791476107356, + 0.26920984270675297, + 0.25672408594104645, + 0.2990775247810641 + ], + "peak_layer": 22, + "peak_normalized": 4.419910912775975, + "population_cv": 0.8912016880930539, + "rest_mean": 7.550380838370916e-05, + "spike_contrast": 3.9333327677514873, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002969816036056727, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.685085488948971e-05, + 9.550715185469016e-05, + 6.523641059175134e-05, + 6.226311234058812e-05, + 3.787676541833207e-05, + 8.378470374736935e-05, + 9.993687126552686e-05, + 0.00010990430746460333, + 6.966651562834159e-05, + 5.729342228733003e-05, + 4.886503302259371e-05, + 4.561137029668316e-05, + 4.3916334107052535e-05, + 8.463997801300138e-05, + 6.980525358812883e-05, + 6.279627268668264e-05, + 8.602520392742008e-05, + 0.00012990135292056948, + 0.0001405078946845606, + 0.00017176373512484133, + 0.0002876764046959579, + 0.0004866751260124147, + 0.0003224833635613322, + 0.0002588419010862708, + 0.00012923122267238796, + 0.0001343472395092249, + 8.631101809442043e-05, + 5.3619532991433516e-05, + 5.13307168148458e-05, + 2.9642618756042793e-05, + 2.8267815650906414e-05, + 3.293134068371728e-05 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059514161199331284, + 0.08157797157764435, + 0.14292451739311218, + 0.10997853428125381, + 0.0991024449467659, + 0.1065012738108635, + 0.12155740708112717, + 0.1095810979604721, + 0.06964563578367233, + 0.08311934769153595, + 0.09363401681184769, + 0.0941167026758194, + 0.08564268052577972, + 0.09493856877088547, + 0.09063401818275452, + 0.09473779052495956, + 0.10505300015211105, + 0.09355910867452621, + 0.09618847072124481, + 0.11416846513748169, + 0.047205954790115356, + 0.053352247923612595, + 0.06512638926506042, + 0.07829347252845764, + 0.05368548259139061, + 0.05310077592730522, + 0.061235878616571426, + 0.07320034503936768, + 0.05304349213838577, + 0.0639304369688034, + 0.07404901832342148, + 0.08157160878181458 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.6955231027823174e-06, + "first_to_last_ratio": 0.14308275980953641, + "imbalance_abs_log_ratio": 1.9443320762007668, + "last_quartile_mean": 3.2816833481774665e-05, + "mean": 1.7864957968072304e-05, + "normalized": [ + 0.5367890577532618, + 0.2021050510263675, + 0.1411976282823933, + 0.2660218188543664, + 0.19890690834720978, + 0.16151559172232732, + 0.2167464101659875, + 0.3793916565295996, + 0.424775345287454, + 0.3093373886676644, + 0.2783613720705816, + 0.2825637413001494, + 0.3733172205433837, + 0.3886765425479794, + 0.44060863402002287, + 0.44323494933287805, + 0.3667341230574315, + 0.45728447370296305, + 0.42556973592368313, + 0.5658685710706646, + 2.6227066493164064, + 3.0085127149589037, + 2.58733070947387, + 2.226933740479959, + 2.8743975419651484, + 2.84666799381699, + 2.39504866979417, + 1.5908588860103314, + 1.8185955715005, + 1.4035945808786974, + 0.9127606447015644, + 0.8535860768970909 + ], + "peak_layer": 22, + "peak_normalized": 3.0085127149589037, + "population_cv": 0.9632431009960319, + "rest_mean": 1.2359982755933069e-05, + "spike_contrast": 3.850476578600324, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.759182411362417e-05, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.589713954483159e-06, + 3.6105982417211635e-06, + 2.5224896944564534e-06, + 4.7524686124233995e-06, + 3.5534635571821127e-06, + 2.8854692573077045e-06, + 3.872165507345926e-06, + 6.777815997338621e-06, + 7.588593689433765e-06, + 5.526299446501071e-06, + 4.972914211975876e-06, + 5.047989361628424e-06, + 6.669296453765128e-06, + 6.9436900957953185e-06, + 7.871454727137461e-06, + 7.918373739812523e-06, + 6.5516896938788705e-06, + 8.1693679021555e-06, + 7.6027854447602294e-06, + 1.0109218237630557e-05, + 4.685454405262135e-05, + 5.3746953199151903e-05, + 4.622255437425338e-05, + 3.97840776713565e-05, + 5.135099127073772e-05, + 5.085560405859724e-05, + 4.278744381736033e-05, + 2.8420627131708898e-05, + 3.248913344577886e-05, + 2.507515819161199e-05, + 1.6306430552504025e-05, + 1.5249279385898262e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.8878105947806034e-05, + "first_to_last_ratio": 0.1437388462831867, + "imbalance_abs_log_ratio": 1.939757193388504, + "last_quartile_mean": 0.00013133614493199275, + "mean": 7.144442986373178e-05, + "normalized": [ + 0.543944422065902, + 0.20498362677674017, + 0.14117605251679072, + 0.2675823635170904, + 0.19429426405696296, + 0.15941273479469137, + 0.21634544427316113, + 0.386139638870723, + 0.42865458135412043, + 0.3129364237141443, + 0.2788876084512782, + 0.2815826978190512, + 0.3736543391719335, + 0.3909795972451435, + 0.4389936626557141, + 0.44232523153890874, + 0.3688366584716598, + 0.45947150441297413, + 0.42734480687548726, + 0.5800824758005463, + 2.6187902433353907, + 2.987231504916195, + 2.580721746390286, + 2.2092452217225382, + 2.879951595578818, + 2.8406783220158585, + 2.397028447242086, + 1.5906421267626043, + 1.8247593500794728, + 1.4018234858691145, + 0.9198949147470885, + 0.8516049069575247 + ], + "peak_layer": 22, + "peak_normalized": 2.987231504916195, + "population_cv": 0.9601858610516129, + "rest_mean": 4.954554700547036e-05, + "spike_contrast": 3.828767846228416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00018969839729834347, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.886179911205545e-05, + 1.4644938346464187e-05, + 1.008624258247437e-05, + 1.911726940306835e-05, + 1.3881242921343073e-05, + 1.1389151950425003e-05, + 1.5456676919711754e-05, + 2.7587526346906088e-05, + 3.062498217332177e-05, + 2.2357564375852235e-05, + 1.9924966181861237e-05, + 2.0117515305173583e-05, + 2.6695521228248253e-05, + 2.7933314413530752e-05, + 3.13636519422289e-05, + 3.160167398164049e-05, + 2.6351324777351692e-05, + 3.282667967141606e-05, + 3.0531406082445756e-05, + 4.144366175751202e-05, + 0.00018709797586780041, + 0.00021342105173971504, + 0.0001843781938077882, + 0.00015783826529514045, + 0.0002057564997812733, + 0.0002029506431426853, + 0.00017125433078035712, + 0.00011364251986378804, + 0.00013036889140494168, + 0.00010015247971750796, + 6.572136771865189e-05, + 6.08424270467367e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.016741293016821146, + "first_to_last_ratio": 0.14373884545887533, + "imbalance_abs_log_ratio": 1.9397571991232891, + "last_quartile_mean": 0.11647020652890205, + "mean": 0.06335763493552804, + "normalized": [ + 0.5439444372437541, + 0.20498362168576487, + 0.14117604516877305, + 0.2675823445899417, + 0.1942942529879486, + 0.15941273039509152, + 0.2163454315588509, + 0.38613966530889043, + 0.4286545697273512, + 0.3129364189494029, + 0.278887605667477, + 0.2815826915705289, + 0.3736543321474427, + 0.3909795777927573, + 0.43899363576159356, + 0.44232520797796643, + 0.3688366746353626, + 0.45947150248290297, + 0.4273447896704886, + 0.580082454376481, + 2.6187903327724635, + 2.987231533325295, + 2.5807218070831297, + 2.2092452282911403, + 2.8799516329199686, + 2.84067822247012, + 2.397028505583422, + 1.5906421518499794, + 1.8247592808395534, + 1.4018234450909468, + 0.9198949415397243, + 0.8516049285354858 + ], + "peak_layer": 22, + "peak_normalized": 2.987231533325295, + "population_cv": 0.9601858709388058, + "rest_mean": 0.04393748607900408, + "spike_contrast": 3.8287679558695995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.16822643876075744, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.03446303308010101, + 0.01298727747052908, + 0.00894458033144474, + 0.01695338450372219, + 0.012310024350881577, + 0.010100013576447964, + 0.013707134872674942, + 0.024464895948767662, + 0.02715853974223137, + 0.01982691138982773, + 0.017669659107923508, + 0.017840413376688957, + 0.023673854768276215, + 0.024771541357040405, + 0.02781359851360321, + 0.02802467904984951, + 0.02336861938238144, + 0.029111027717590332, + 0.02707555517554283, + 0.03675265237689018, + 0.16592036187648773, + 0.18926392495632172, + 0.16350843012332916, + 0.13997255265712738, + 0.18246692419052124, + 0.1799786537885666, + 0.15187005698680878, + 0.10077932476997375, + 0.11561243236064911, + 0.08881621807813644, + 0.05828236788511276, + 0.053955674171447754 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0825735472508313e-06, + "first_to_last_ratio": 0.12187576812955331, + "imbalance_abs_log_ratio": 2.104753047082524, + "last_quartile_mean": 8.882598763193528e-06, + "mean": 4.656443639561303e-06, + "normalized": [ + 0.5462983207096154, + 0.1799290732737773, + 0.13161762512632208, + 0.2712494331106157, + 0.1613911681124062, + 0.13237661267085615, + 0.15721616286614848, + 0.2798363906909718, + 0.34662434040104295, + 0.2682122377700395, + 0.24445710198520895, + 0.24681111820039023, + 0.3364691861798734, + 0.3619231630019147, + 0.42744951901080175, + 0.43112099033231693, + 0.36249737820633765, + 0.4368387762566429, + 0.42578881412518377, + 0.38711372693052853, + 2.644134376023103, + 3.0274176414789977, + 2.680305221816438, + 2.2521785297578925, + 2.965193895729998, + 2.901136489590974, + 2.603841896063781, + 1.6862657755052308, + 1.872923926339135, + 1.3894795988927409, + 0.9615945171774488, + 0.8803069926632656 + ], + "peak_layer": 22, + "peak_normalized": 3.0274176414789977, + "population_cv": 1.0082602587575777, + "rest_mean": 3.1785867888713795e-06, + "spike_contrast": 3.9756254815914156, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2636870633286889e-05, + "top_five_layers": [ + 22, + 25, + 26, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.543807340771309e-06, + 8.3782958881784e-07, + 6.128700533736264e-07, + 1.2630576975425356e-06, + 7.515088782383827e-07, + 6.164042360978783e-07, + 7.32068201614311e-07, + 1.3030423815507675e-06, + 1.6140367051775684e-06, + 1.2489151686168043e-06, + 1.138300717684615e-06, + 1.14926206151722e-06, + 1.5667498018956394e-06, + 1.6852748103701742e-06, + 1.990394594031386e-06, + 2.0074905933142873e-06, + 1.687948611106549e-06, + 2.034115141213988e-06, + 1.9826616153295618e-06, + 1.8025732515525306e-06, + 1.2312262697378173e-05, + 1.409699962096056e-05, + 1.2480690202210099e-05, + 1.0487142390047666e-05, + 1.3807258255837951e-05, + 1.3508978554455098e-05, + 1.2124643035349436e-05, + 7.85200154496124e-06, + 8.721164704184048e-06, + 6.470033440564293e-06, + 4.477610673347954e-06, + 4.099099896848202e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.2557812397062662e-05, + "first_to_last_ratio": 0.15959566659574662, + "imbalance_abs_log_ratio": 1.8351117459907644, + "last_quartile_mean": 7.868517150200205e-05, + "mean": 4.5438396199415365e-05, + "normalized": [ + 0.5876869810338485, + 0.2186593308079107, + 0.115659623677568, + 0.24728992953724943, + 0.2138823134903866, + 0.17309343907534852, + 0.22828222046505, + 0.4264066618299725, + 0.47941738273017553, + 0.34947398487973863, + 0.31642185290041636, + 0.3326608895247323, + 0.42753132054545495, + 0.43329608717407087, + 0.4807098955780001, + 0.491094994316893, + 0.41296023678331367, + 0.5280992844682807, + 0.4827771475203192, + 0.6138123378227869, + 2.7117823469683264, + 3.061642080099734, + 2.646250140140893, + 2.167597408443836, + 2.9446942269972274, + 2.694165709502317, + 2.239316799454017, + 1.5139318638986488, + 1.6066586018929343, + 1.2634426578175135, + 0.8290147862338136, + 0.7622874643892231 + ], + "peak_layer": 22, + "peak_normalized": 3.061642080099734, + "population_cv": 0.9412918554701797, + "rest_mean": 3.107991987808082e-05, + "spike_contrast": 3.9567080229621103, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012297416833462194, + "top_five_layers": [ + 22, + 25, + 21, + 26, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.6703553885454312e-05, + 9.935529305948876e-06, + 5.255387804936618e-06, + 1.1236457794439048e-05, + 9.718469300423749e-06, + 7.86508826422505e-06, + 1.0372777978773229e-05, + 1.9375234842300415e-05, + 2.178395698138047e-05, + 1.587953738635406e-05, + 1.4377701518242247e-05, + 1.511557729827473e-05, + 1.9426337530603632e-05, + 1.968827928067185e-05, + 2.1842686692252755e-05, + 2.231456892332062e-05, + 1.876425085356459e-05, + 2.3995984520297498e-05, + 2.1936619305051863e-05, + 2.789064819808118e-05, + 0.00012321904068812728, + 0.0001391161058563739, + 0.00012024136231048033, + 9.849214984569699e-05, + 0.00013380218297243118, + 0.00012241856893524528, + 0.00010175096394959837, + 6.879063585074618e-05, + 7.300399011000991e-05, + 5.740880806115456e-05, + 3.766910231206566e-05, + 3.463711982476525e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.367274347165221e-06, + "first_to_last_ratio": 0.1920014554991945, + "imbalance_abs_log_ratio": 1.6502523262581177, + "last_quartile_mean": 4.3579223529377487e-05, + "mean": 2.787530080183842e-05, + "normalized": [ + 0.6522901960458961, + 0.23732362933486428, + 0.10383521258256681, + 0.24073339013803968, + 0.24132146388193534, + 0.19483084771733855, + 0.2475207738927628, + 0.483488789493259, + 0.550030749638162, + 0.4005903921146771, + 0.3626794403805512, + 0.398664404925254, + 0.5057551655640742, + 0.49633936232785464, + 0.5444163811542189, + 0.5645330449101175, + 0.481087325236049, + 0.6303807295780811, + 0.5534160889368265, + 0.6675682370643486, + 2.937265214456223, + 3.231070824381384, + 2.7157464748293116, + 2.0522050942602807, + 2.8791759008165254, + 2.406039643209466, + 1.9404796965168836, + 1.3618287054639213, + 1.3825321378732942, + 1.123237294980051, + 0.7355691904186271, + 0.6780441978771551 + ], + "peak_layer": 22, + "peak_normalized": 3.231070824381384, + "population_cv": 0.9137826743997257, + "rest_mean": 1.8774052764288002e-05, + "spike_contrast": 4.1025792976954865, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.702204020461067e-05, + "top_five_layers": [ + 22, + 21, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.8182785424869508e-05, + 6.615467555093346e-06, + 2.8944377845618874e-06, + 6.710515663144179e-06, + 6.726908395648934e-06, + 5.430968485597987e-06, + 6.899716026964597e-06, + 1.3477395441441331e-05, + 1.5332272596424446e-05, + 1.1166577678523026e-05, + 1.0109798495250288e-05, + 1.111289020627737e-05, + 1.409807737218216e-05, + 1.3835609024681617e-05, + 1.5175770386122167e-05, + 1.5736528439447284e-05, + 1.3410453902906738e-05, + 1.7572052456671372e-05, + 1.5426639947691e-05, + 1.86086654139217e-05, + 8.187715138774365e-05, + 9.006707114167511e-05, + 7.570224988739938e-05, + 5.720583430957049e-05, + 8.025789429666474e-05, + 6.706907879561186e-05, + 5.409145524026826e-05, + 3.796138480538502e-05, + 3.8538499211426824e-05, + 3.131057746941224e-05, + 2.0504212443483993e-05, + 1.890068597276695e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.742885974133969e-05, + "first_to_last_ratio": 0.13285814939480156, + "imbalance_abs_log_ratio": 2.018473265828943, + "last_quartile_mean": 0.00028172046586405486, + "mean": 0.00014909372896454443, + "normalized": [ + 0.5351570886371335, + 0.1998816464903299, + 0.1134003859562838, + 0.2373819769757024, + 0.18971959520637977, + 0.1492636791238371, + 0.21004413698536548, + 0.3734913450927867, + 0.42927632587153325, + 0.3024206473119055, + 0.27462175768495817, + 0.27240928064339276, + 0.3512855646324889, + 0.3655094888356237, + 0.41973324501001746, + 0.42402062842094873, + 0.360798486332806, + 0.4559662563090299, + 0.41985007515873884, + 0.5597866310716989, + 2.4844541141001115, + 2.96085466615825, + 2.5443412288142064, + 2.249909733543605, + 2.9984626521607822, + 2.895026684700318, + 2.601768175835175, + 1.7136369023458127, + 1.8313353225818492, + 1.363588302533558, + 0.8854239763588411, + 0.82717999911653 + ], + "peak_layer": 25, + "peak_normalized": 2.9984626521607822, + "population_cv": 0.9800748961570518, + "rest_mean": 0.00010360345199673989, + "spike_contrast": 3.810116525877056, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003947412245906889, + "top_five_layers": [ + 25, + 22, + 26, + 27, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.978856592671946e-05, + 2.980110002681613e-05, + 1.6907286408240907e-05, + 3.53921641362831e-05, + 2.8286001906963065e-05, + 2.2254278519540094e-05, + 3.131626363028772e-05, + 5.568521737586707e-05, + 6.400240818038583e-05, + 4.508902202360332e-05, + 4.0944381908047944e-05, + 4.061451545567252e-05, + 5.2374474762473255e-05, + 5.449517266242765e-05, + 6.257959466893226e-05, + 6.321881664916873e-05, + 5.379279173212126e-05, + 6.79817094351165e-05, + 6.259701331146061e-05, + 8.34606762509793e-05, + 0.0003704165283124894, + 0.00044144486309960485, + 0.0003793453215621412, + 0.0003354474320076406, + 0.0004470519779715687, + 0.0004316303238738328, + 0.00038790731923654675, + 0.00025549251586198807, + 0.00027304061222821474, + 0.00020330246479716152, + 0.00013201116234995425, + 0.00012332735059317201 + ] + } + } + } + } + }, + "uniform_value_backward": { + "bits_per_byte": 1.8740129107275643, + "forward": { + "activation_sha256": { + "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" + }, + "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", + "loss_nats": 1.2989667654037476, + "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" + }, + "loss_nats": 1.2989667654037476, + "loss_scale": 1.0, + "mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22886337339878082, + "group": 1, + "latest_source_mass": 0.06350618600845337, + "layer": 1, + "max_source_mass": 0.9364938139915466, + "mixer_index": 1, + "normalized_entropy": 0.3301800538435402, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9364938139915466, + "median": 0.9452826976776123, + "p05": 0.8725647330284119, + "p95": 0.9712384939193726 + }, + { + "label": "current_group_partial", + "mean": 0.06350618600845337, + "median": 0.0547172985970974, + "p05": 0.028761589899659157, + "p95": 0.12743529677391052 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.603729784488678, + "group": 1, + "latest_source_mass": 0.4133155345916748, + "layer": 2, + "max_source_mass": 0.5866844654083252, + "mixer_index": 2, + "normalized_entropy": 0.8709979661187783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5866844654083252, + "median": 0.5734395384788513, + "p05": 0.29813364148139954, + "p95": 0.8843642473220825 + }, + { + "label": "current_group_partial", + "mean": 0.4133155345916748, + "median": 0.4265604615211487, + "p05": 0.11563576757907867, + "p95": 0.7018663287162781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3967982828617096, + "group": 1, + "latest_source_mass": 0.14110636711120605, + "layer": 2, + "max_source_mass": 0.858893632888794, + "mixer_index": 3, + "normalized_entropy": 0.5724589149178446, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.858893632888794, + "median": 0.8705973029136658, + "p05": 0.7599387168884277, + "p95": 0.9270997643470764 + }, + { + "label": "current_group_partial", + "mean": 0.14110636711120605, + "median": 0.12940272688865662, + "p05": 0.07290027290582657, + "p95": 0.2400612235069275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6852496862411499, + "group": 1, + "latest_source_mass": 0.518975019454956, + "layer": 3, + "max_source_mass": 0.518975019454956, + "mixer_index": 4, + "normalized_entropy": 0.9886063241108252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48102498054504395, + "median": 0.48102718591690063, + "p05": 0.38786420226097107, + "p95": 0.580143928527832 + }, + { + "label": "current_group_partial", + "mean": 0.518975019454956, + "median": 0.5189728140830994, + "p05": 0.41985607147216797, + "p95": 0.6121357679367065 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5890734195709229, + "group": 1, + "latest_source_mass": 0.3123423159122467, + "layer": 3, + "max_source_mass": 0.6876576542854309, + "mixer_index": 5, + "normalized_entropy": 0.8498533011344741, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6876576542854309, + "median": 0.6909762620925903, + "p05": 0.48803940415382385, + "p95": 0.8900076746940613 + }, + { + "label": "current_group_partial", + "mean": 0.3123423159122467, + "median": 0.3090237081050873, + "p05": 0.10999235510826111, + "p95": 0.5119606256484985 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6873323321342468, + "group": 1, + "latest_source_mass": 0.4831485152244568, + "layer": 4, + "max_source_mass": 0.5168514251708984, + "mixer_index": 6, + "normalized_entropy": 0.9916109470127238, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5168514251708984, + "median": 0.5219584703445435, + "p05": 0.4251725673675537, + "p95": 0.5947866439819336 + }, + { + "label": "current_group_partial", + "mean": 0.4831485152244568, + "median": 0.47804147005081177, + "p05": 0.405213326215744, + "p95": 0.5748274326324463 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.487301766872406, + "group": 1, + "latest_source_mass": 0.2060384750366211, + "layer": 4, + "max_source_mass": 0.7939615249633789, + "mixer_index": 7, + "normalized_entropy": 0.7030278424832499, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7939615249633789, + "median": 0.7953404188156128, + "p05": 0.6590759754180908, + "p95": 0.9329437017440796 + }, + { + "label": "current_group_partial", + "mean": 0.2060384750366211, + "median": 0.20465964078903198, + "p05": 0.06705628335475922, + "p95": 0.3409239649772644 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674697756767273, + "group": 2, + "latest_source_mass": 0.42952626943588257, + "layer": 5, + "max_source_mass": 0.5704737305641174, + "mixer_index": 8, + "normalized_entropy": 0.9733831077870527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5704737305641174, + "median": 0.5696132779121399, + "p05": 0.4695202708244324, + "p95": 0.6750621199607849 + }, + { + "label": "completed_group_1", + "mean": 0.42952626943588257, + "median": 0.4303867816925049, + "p05": 0.3249378800392151, + "p95": 0.5304797887802124 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9691112041473389, + "group": 2, + "latest_source_mass": 0.2681158185005188, + "layer": 5, + "max_source_mass": 0.5506307482719421, + "mixer_index": 9, + "normalized_entropy": 0.8821230329784768, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5506307482719421, + "median": 0.543993353843689, + "p05": 0.41262108087539673, + "p95": 0.7161091566085815 + }, + { + "label": "completed_group_1", + "mean": 0.18125343322753906, + "median": 0.1794438362121582, + "p05": 0.11575750261545181, + "p95": 0.25031501054763794 + }, + { + "label": "current_group_partial", + "mean": 0.2681158185005188, + "median": 0.2664111256599426, + "p05": 0.1564716249704361, + "p95": 0.3817881643772125 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0745112895965576, + "group": 2, + "latest_source_mass": 0.412794291973114, + "layer": 6, + "max_source_mass": 0.412794291973114, + "mixer_index": 10, + "normalized_entropy": 0.9780623252441762, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34262996912002563, + "median": 0.3412924110889435, + "p05": 0.29556700587272644, + "p95": 0.39125555753707886 + }, + { + "label": "completed_group_1", + "mean": 0.24457570910453796, + "median": 0.24382859468460083, + "p05": 0.22120657563209534, + "p95": 0.2734297513961792 + }, + { + "label": "current_group_partial", + "mean": 0.412794291973114, + "median": 0.4151499271392822, + "p05": 0.37735897302627563, + "p95": 0.43749260902404785 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074045658111572, + "group": 2, + "latest_source_mass": 0.3775783181190491, + "layer": 6, + "max_source_mass": 0.4399106502532959, + "mixer_index": 11, + "normalized_entropy": 0.9169791528842925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4399106502532959, + "median": 0.44856905937194824, + "p05": 0.22063353657722473, + "p95": 0.6359403133392334 + }, + { + "label": "completed_group_1", + "mean": 0.18251106142997742, + "median": 0.18562838435173035, + "p05": 0.1196163073182106, + "p95": 0.23045453429222107 + }, + { + "label": "current_group_partial", + "mean": 0.3775783181190491, + "median": 0.3675365447998047, + "p05": 0.23817461729049683, + "p95": 0.5557451844215393 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.060497522354126, + "group": 2, + "latest_source_mass": 0.4025024175643921, + "layer": 7, + "max_source_mass": 0.4025024175643921, + "mixer_index": 12, + "normalized_entropy": 0.9653064445872968, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3786153197288513, + "median": 0.3907395303249359, + "p05": 0.28039106726646423, + "p95": 0.4436405599117279 + }, + { + "label": "completed_group_1", + "mean": 0.2188822478055954, + "median": 0.21226292848587036, + "p05": 0.1885872334241867, + "p95": 0.2672030031681061 + }, + { + "label": "current_group_partial", + "mean": 0.4025024175643921, + "median": 0.39815032482147217, + "p05": 0.3533596992492676, + "p95": 0.460100919008255 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9941019415855408, + "group": 2, + "latest_source_mass": 0.4801652431488037, + "layer": 7, + "max_source_mass": 0.4801652431488037, + "mixer_index": 13, + "normalized_entropy": 0.9048705824970601, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32296955585479736, + "median": 0.2803739905357361, + "p05": 0.1784922033548355, + "p95": 0.6440245509147644 + }, + { + "label": "completed_group_1", + "mean": 0.19686520099639893, + "median": 0.2053825855255127, + "p05": 0.09813719242811203, + "p95": 0.2504696547985077 + }, + { + "label": "current_group_partial", + "mean": 0.4801652431488037, + "median": 0.5057140588760376, + "p05": 0.2529003322124481, + "p95": 0.598870575428009 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0503522157669067, + "group": 2, + "latest_source_mass": 0.4001518487930298, + "layer": 8, + "max_source_mass": 0.40130966901779175, + "mixer_index": 14, + "normalized_entropy": 0.9560717885654542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40130966901779175, + "median": 0.396759569644928, + "p05": 0.3479611873626709, + "p95": 0.4733521044254303 + }, + { + "label": "completed_group_1", + "mean": 0.19853851199150085, + "median": 0.19769755005836487, + "p05": 0.17136667668819427, + "p95": 0.22788912057876587 + }, + { + "label": "current_group_partial", + "mean": 0.4001518487930298, + "median": 0.40291696786880493, + "p05": 0.3506356179714203, + "p95": 0.43941178917884827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0129402875900269, + "group": 2, + "latest_source_mass": 0.42161279916763306, + "layer": 8, + "max_source_mass": 0.42161279916763306, + "mixer_index": 15, + "normalized_entropy": 0.9220179839951123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4093267023563385, + "median": 0.4046899676322937, + "p05": 0.2895161807537079, + "p95": 0.5762131810188293 + }, + { + "label": "completed_group_1", + "mean": 0.16906049847602844, + "median": 0.16888955235481262, + "p05": 0.11233551055192947, + "p95": 0.22477403283119202 + }, + { + "label": "current_group_partial", + "mean": 0.42161279916763306, + "median": 0.4227708578109741, + "p05": 0.3131049573421478, + "p95": 0.519075334072113 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0774019956588745, + "group": 3, + "latest_source_mass": 0.3560434877872467, + "layer": 9, + "max_source_mass": 0.38510820269584656, + "mixer_index": 16, + "normalized_entropy": 0.9806935592947451, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38510820269584656, + "median": 0.3892287313938141, + "p05": 0.2934730052947998, + "p95": 0.4718902111053467 + }, + { + "label": "completed_group_1", + "mean": 0.25884830951690674, + "median": 0.25734007358551025, + "p05": 0.22217319905757904, + "p95": 0.29551801085472107 + }, + { + "label": "completed_group_2", + "mean": 0.3560434877872467, + "median": 0.35412120819091797, + "p05": 0.29444968700408936, + "p95": 0.41908568143844604 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2740377187728882, + "group": 3, + "latest_source_mass": 0.3633861243724823, + "layer": 9, + "max_source_mass": 0.3633861243724823, + "mixer_index": 17, + "normalized_entropy": 0.9190239493895668, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29550763964653015, + "median": 0.30360478162765503, + "p05": 0.16904757916927338, + "p95": 0.45146965980529785 + }, + { + "label": "completed_group_1", + "mean": 0.09541244804859161, + "median": 0.09486871212720871, + "p05": 0.061360105872154236, + "p95": 0.13037073612213135 + }, + { + "label": "completed_group_2", + "mean": 0.24569380283355713, + "median": 0.24333015084266663, + "p05": 0.1698218435049057, + "p95": 0.31589236855506897 + }, + { + "label": "current_group_partial", + "mean": 0.3633861243724823, + "median": 0.36472851037979126, + "p05": 0.2795379161834717, + "p95": 0.4399876892566681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.360302448272705, + "group": 3, + "latest_source_mass": 0.28569501638412476, + "layer": 10, + "max_source_mass": 0.3109527826309204, + "mixer_index": 18, + "normalized_entropy": 0.9812507981160736, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3109527826309204, + "median": 0.32172977924346924, + "p05": 0.23520725965499878, + "p95": 0.37052470445632935 + }, + { + "label": "completed_group_1", + "mean": 0.20984819531440735, + "median": 0.20958667993545532, + "p05": 0.1891367882490158, + "p95": 0.23184512555599213 + }, + { + "label": "completed_group_2", + "mean": 0.19350400567054749, + "median": 0.19153150916099548, + "p05": 0.16607457399368286, + "p95": 0.22877082228660583 + }, + { + "label": "current_group_partial", + "mean": 0.28569501638412476, + "median": 0.28060320019721985, + "p05": 0.23824527859687805, + "p95": 0.3439165949821472 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2722704410552979, + "group": 3, + "latest_source_mass": 0.29912811517715454, + "layer": 10, + "max_source_mass": 0.35036465525627136, + "mixer_index": 19, + "normalized_entropy": 0.9177491279900463, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35036465525627136, + "median": 0.36210548877716064, + "p05": 0.12807878851890564, + "p95": 0.5797606706619263 + }, + { + "label": "completed_group_1", + "mean": 0.12631990015506744, + "median": 0.1264108121395111, + "p05": 0.06977735459804535, + "p95": 0.1755298376083374 + }, + { + "label": "completed_group_2", + "mean": 0.22418735921382904, + "median": 0.22563719749450684, + "p05": 0.1176326721906662, + "p95": 0.32001155614852905 + }, + { + "label": "current_group_partial", + "mean": 0.29912811517715454, + "median": 0.28589946031570435, + "p05": 0.193109393119812, + "p95": 0.44119665026664734 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3558036088943481, + "group": 3, + "latest_source_mass": 0.314197838306427, + "layer": 11, + "max_source_mass": 0.314197838306427, + "mixer_index": 20, + "normalized_entropy": 0.978005571485618, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298976868391037, + "median": 0.2961939573287964, + "p05": 0.24950163066387177, + "p95": 0.35835936665534973 + }, + { + "label": "completed_group_1", + "mean": 0.1941717267036438, + "median": 0.1918032318353653, + "p05": 0.16809208691120148, + "p95": 0.22895419597625732 + }, + { + "label": "completed_group_2", + "mean": 0.1926535815000534, + "median": 0.19306811690330505, + "p05": 0.16983944177627563, + "p95": 0.2155935913324356 + }, + { + "label": "current_group_partial", + "mean": 0.314197838306427, + "median": 0.31541913747787476, + "p05": 0.2673022747039795, + "p95": 0.35941022634506226 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2696614265441895, + "group": 3, + "latest_source_mass": 0.2862822115421295, + "layer": 11, + "max_source_mass": 0.3667730689048767, + "mixer_index": 21, + "normalized_entropy": 0.9158671218416545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3667730689048767, + "median": 0.4067659378051758, + "p05": 0.10593366622924805, + "p95": 0.5672274827957153 + }, + { + "label": "completed_group_1", + "mean": 0.15310388803482056, + "median": 0.15050218999385834, + "p05": 0.08651046454906464, + "p95": 0.22459912300109863 + }, + { + "label": "completed_group_2", + "mean": 0.19384080171585083, + "median": 0.18732509016990662, + "p05": 0.1267009675502777, + "p95": 0.281107634305954 + }, + { + "label": "current_group_partial", + "mean": 0.2862822115421295, + "median": 0.26271265745162964, + "p05": 0.17024393379688263, + "p95": 0.475044310092926 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3606936931610107, + "group": 3, + "latest_source_mass": 0.3028184771537781, + "layer": 12, + "max_source_mass": 0.3028184771537781, + "mixer_index": 22, + "normalized_entropy": 0.9815330216461395, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298227459192276, + "median": 0.2954302132129669, + "p05": 0.25287362933158875, + "p95": 0.35405808687210083 + }, + { + "label": "completed_group_1", + "mean": 0.18611131608486176, + "median": 0.1850002110004425, + "p05": 0.15901757776737213, + "p95": 0.21725836396217346 + }, + { + "label": "completed_group_2", + "mean": 0.21284276247024536, + "median": 0.21402311325073242, + "p05": 0.18819917738437653, + "p95": 0.23418807983398438 + }, + { + "label": "current_group_partial", + "mean": 0.3028184771537781, + "median": 0.3041122555732727, + "p05": 0.25661808252334595, + "p95": 0.34458184242248535 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.269316554069519, + "group": 3, + "latest_source_mass": 0.2653314769268036, + "layer": 12, + "max_source_mass": 0.4081852436065674, + "mixer_index": 23, + "normalized_entropy": 0.9156183489371815, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4081852436065674, + "median": 0.4254342317581177, + "p05": 0.20765496790409088, + "p95": 0.587238073348999 + }, + { + "label": "completed_group_1", + "mean": 0.14420127868652344, + "median": 0.1438048928976059, + "p05": 0.1064920499920845, + "p95": 0.1871720254421234 + }, + { + "label": "completed_group_2", + "mean": 0.18228203058242798, + "median": 0.17507661879062653, + "p05": 0.12183455377817154, + "p95": 0.26505547761917114 + }, + { + "label": "current_group_partial", + "mean": 0.2653314769268036, + "median": 0.24924281239509583, + "p05": 0.15609969198703766, + "p95": 0.4061311185359955 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3636136054992676, + "group": 4, + "latest_source_mass": 0.3136237859725952, + "layer": 13, + "max_source_mass": 0.3136237859725952, + "mixer_index": 24, + "normalized_entropy": 0.9836392931712563, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27427661418914795, + "median": 0.27403637766838074, + "p05": 0.22928935289382935, + "p95": 0.319773405790329 + }, + { + "label": "completed_group_1", + "mean": 0.21341991424560547, + "median": 0.20697325468063354, + "p05": 0.18552586436271667, + "p95": 0.26688331365585327 + }, + { + "label": "completed_group_2", + "mean": 0.19867971539497375, + "median": 0.197545126080513, + "p05": 0.1794913411140442, + "p95": 0.222100168466568 + }, + { + "label": "completed_group_3", + "mean": 0.3136237859725952, + "median": 0.31875866651535034, + "p05": 0.24156072735786438, + "p95": 0.3695686459541321 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.570773959159851, + "group": 4, + "latest_source_mass": 0.2294243425130844, + "layer": 13, + "max_source_mass": 0.2658740282058716, + "mixer_index": 25, + "normalized_entropy": 0.9759767351225285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2658740282058716, + "median": 0.2747163474559784, + "p05": 0.17927880585193634, + "p95": 0.3512483835220337 + }, + { + "label": "completed_group_1", + "mean": 0.13419361412525177, + "median": 0.13374894857406616, + "p05": 0.11072897166013718, + "p95": 0.1586270034313202 + }, + { + "label": "completed_group_2", + "mean": 0.1907704919576645, + "median": 0.19074805080890656, + "p05": 0.14699944853782654, + "p95": 0.23316305875778198 + }, + { + "label": "completed_group_3", + "mean": 0.17973749339580536, + "median": 0.17616114020347595, + "p05": 0.1363743096590042, + "p95": 0.23202598094940186 + }, + { + "label": "current_group_partial", + "mean": 0.2294243425130844, + "median": 0.22995705902576447, + "p05": 0.17975632846355438, + "p95": 0.2789157032966614 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5560038089752197, + "group": 4, + "latest_source_mass": 0.32468360662460327, + "layer": 14, + "max_source_mass": 0.32468360662460327, + "mixer_index": 26, + "normalized_entropy": 0.9667995248241249, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20477107167243958, + "median": 0.206403911113739, + "p05": 0.16619175672531128, + "p95": 0.2431035190820694 + }, + { + "label": "completed_group_1", + "mean": 0.15721669793128967, + "median": 0.15658551454544067, + "p05": 0.13878680765628815, + "p95": 0.17778633534908295 + }, + { + "label": "completed_group_2", + "mean": 0.13811248540878296, + "median": 0.13743364810943604, + "p05": 0.12352626770734787, + "p95": 0.15553249418735504 + }, + { + "label": "completed_group_3", + "mean": 0.1752161681652069, + "median": 0.17309701442718506, + "p05": 0.15223677456378937, + "p95": 0.20474471151828766 + }, + { + "label": "current_group_partial", + "mean": 0.32468360662460327, + "median": 0.32497870922088623, + "p05": 0.2946181893348694, + "p95": 0.35203880071640015 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5659973621368408, + "group": 4, + "latest_source_mass": 0.26703086495399475, + "layer": 14, + "max_source_mass": 0.26703086495399475, + "mixer_index": 27, + "normalized_entropy": 0.9730088685238187, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20531430840492249, + "median": 0.20229148864746094, + "p05": 0.10698921978473663, + "p95": 0.3393132984638214 + }, + { + "label": "completed_group_1", + "mean": 0.15986156463623047, + "median": 0.15952220559120178, + "p05": 0.11969029903411865, + "p95": 0.20180393755435944 + }, + { + "label": "completed_group_2", + "mean": 0.21557100117206573, + "median": 0.21654027700424194, + "p05": 0.1673085242509842, + "p95": 0.2610440254211426 + }, + { + "label": "completed_group_3", + "mean": 0.15222224593162537, + "median": 0.1510905921459198, + "p05": 0.11800260841846466, + "p95": 0.18893754482269287 + }, + { + "label": "current_group_partial", + "mean": 0.26703086495399475, + "median": 0.258539617061615, + "p05": 0.19330690801143646, + "p95": 0.36521556973457336 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5552496910095215, + "group": 4, + "latest_source_mass": 0.31774693727493286, + "layer": 15, + "max_source_mass": 0.31774693727493286, + "mixer_index": 28, + "normalized_entropy": 0.9663309649872576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2162160575389862, + "median": 0.2145705670118332, + "p05": 0.17727568745613098, + "p95": 0.26878347992897034 + }, + { + "label": "completed_group_1", + "mean": 0.17982497811317444, + "median": 0.18024154007434845, + "p05": 0.15453973412513733, + "p95": 0.2030555009841919 + }, + { + "label": "completed_group_2", + "mean": 0.1307673156261444, + "median": 0.13107559084892273, + "p05": 0.11463287472724915, + "p95": 0.14630235731601715 + }, + { + "label": "completed_group_3", + "mean": 0.1554446816444397, + "median": 0.15516987442970276, + "p05": 0.13627474009990692, + "p95": 0.1754152774810791 + }, + { + "label": "current_group_partial", + "mean": 0.31774693727493286, + "median": 0.319039523601532, + "p05": 0.27606821060180664, + "p95": 0.35650834441185 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5068919658660889, + "group": 4, + "latest_source_mass": 0.3298327624797821, + "layer": 15, + "max_source_mass": 0.3298327624797821, + "mixer_index": 29, + "normalized_entropy": 0.9362846209998111, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20646697282791138, + "median": 0.18775638937950134, + "p05": 0.0806753933429718, + "p95": 0.4274584949016571 + }, + { + "label": "completed_group_1", + "mean": 0.1197461187839508, + "median": 0.11989086866378784, + "p05": 0.0809398666024208, + "p95": 0.15743419528007507 + }, + { + "label": "completed_group_2", + "mean": 0.19215428829193115, + "median": 0.19620640575885773, + "p05": 0.1148109883069992, + "p95": 0.2491212636232376 + }, + { + "label": "completed_group_3", + "mean": 0.15179988741874695, + "median": 0.15212899446487427, + "p05": 0.10697861760854721, + "p95": 0.19542087614536285 + }, + { + "label": "current_group_partial", + "mean": 0.3298327624797821, + "median": 0.32837340235710144, + "p05": 0.22746729850769043, + "p95": 0.4339301884174347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5435223579406738, + "group": 4, + "latest_source_mass": 0.3224271535873413, + "layer": 16, + "max_source_mass": 0.3224271535873413, + "mixer_index": 30, + "normalized_entropy": 0.9590443632623663, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22410975396633148, + "median": 0.22760415077209473, + "p05": 0.14593973755836487, + "p95": 0.30093055963516235 + }, + { + "label": "completed_group_1", + "mean": 0.16986176371574402, + "median": 0.16963240504264832, + "p05": 0.13569067418575287, + "p95": 0.20651353895664215 + }, + { + "label": "completed_group_2", + "mean": 0.13080403208732605, + "median": 0.13075244426727295, + "p05": 0.10945598781108856, + "p95": 0.15187400579452515 + }, + { + "label": "completed_group_3", + "mean": 0.15279728174209595, + "median": 0.15036612749099731, + "p05": 0.12810495495796204, + "p95": 0.18468116223812103 + }, + { + "label": "current_group_partial", + "mean": 0.3224271535873413, + "median": 0.3164973258972168, + "p05": 0.2600978910923004, + "p95": 0.40530598163604736 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4949920177459717, + "group": 4, + "latest_source_mass": 0.31847020983695984, + "layer": 16, + "max_source_mass": 0.31847020983695984, + "mixer_index": 31, + "normalized_entropy": 0.9288907675133354, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27174079418182373, + "median": 0.2585451304912567, + "p05": 0.13639721274375916, + "p95": 0.4698348045349121 + }, + { + "label": "completed_group_1", + "mean": 0.12178683280944824, + "median": 0.11714950203895569, + "p05": 0.08594125509262085, + "p95": 0.1723237782716751 + }, + { + "label": "completed_group_2", + "mean": 0.1529826819896698, + "median": 0.15223106741905212, + "p05": 0.10371473431587219, + "p95": 0.203575000166893 + }, + { + "label": "completed_group_3", + "mean": 0.1350194811820984, + "median": 0.13363957405090332, + "p05": 0.08964549005031586, + "p95": 0.18468207120895386 + }, + { + "label": "current_group_partial", + "mean": 0.31847020983695984, + "median": 0.3185216188430786, + "p05": 0.21444711089134216, + "p95": 0.4301546812057495 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.544264793395996, + "group": 5, + "latest_source_mass": 0.2861037850379944, + "layer": 17, + "max_source_mass": 0.2861037850379944, + "mixer_index": 32, + "normalized_entropy": 0.9595056643474137, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24016115069389343, + "median": 0.23868846893310547, + "p05": 0.18435582518577576, + "p95": 0.2983453571796417 + }, + { + "label": "completed_group_1", + "mean": 0.20730282366275787, + "median": 0.1998077780008316, + "p05": 0.16255050897598267, + "p95": 0.28107208013534546 + }, + { + "label": "completed_group_2", + "mean": 0.09692573547363281, + "median": 0.0967404842376709, + "p05": 0.08098706603050232, + "p95": 0.11221462488174438 + }, + { + "label": "completed_group_3", + "mean": 0.1695065200328827, + "median": 0.1696585714817047, + "p05": 0.136396586894989, + "p95": 0.20130634307861328 + }, + { + "label": "completed_group_4", + "mean": 0.2861037850379944, + "median": 0.28883200883865356, + "p05": 0.22401469945907593, + "p95": 0.3409370183944702 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.686682939529419, + "group": 5, + "latest_source_mass": 0.12281867861747742, + "layer": 17, + "max_source_mass": 0.2885604202747345, + "mixer_index": 33, + "normalized_entropy": 0.9413556721740635, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16755425930023193, + "median": 0.1327776312828064, + "p05": 0.0790192037820816, + "p95": 0.3667412996292114 + }, + { + "label": "completed_group_1", + "mean": 0.10962115228176117, + "median": 0.10525335371494293, + "p05": 0.07840172946453094, + "p95": 0.15405339002609253 + }, + { + "label": "completed_group_2", + "mean": 0.1783454418182373, + "median": 0.17896884679794312, + "p05": 0.10802385956048965, + "p95": 0.24048911035060883 + }, + { + "label": "completed_group_3", + "mean": 0.13310004770755768, + "median": 0.13033035397529602, + "p05": 0.0904831737279892, + "p95": 0.17934636771678925 + }, + { + "label": "completed_group_4", + "mean": 0.2885604202747345, + "median": 0.2831825315952301, + "p05": 0.16967125236988068, + "p95": 0.4189499616622925 + }, + { + "label": "current_group_partial", + "mean": 0.12281867861747742, + "median": 0.12062528729438782, + "p05": 0.07461707293987274, + "p95": 0.1782219111919403 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7070834636688232, + "group": 5, + "latest_source_mass": 0.2988377809524536, + "layer": 18, + "max_source_mass": 0.2988377809524536, + "mixer_index": 34, + "normalized_entropy": 0.9527414214834803, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.181177020072937, + "median": 0.18419909477233887, + "p05": 0.13624237477779388, + "p95": 0.22192169725894928 + }, + { + "label": "completed_group_1", + "mean": 0.14557164907455444, + "median": 0.144476979970932, + "p05": 0.1293315887451172, + "p95": 0.16486899554729462 + }, + { + "label": "completed_group_2", + "mean": 0.08318156749010086, + "median": 0.0820261612534523, + "p05": 0.07088497281074524, + "p95": 0.0990293100476265 + }, + { + "label": "completed_group_3", + "mean": 0.11424209177494049, + "median": 0.11267061531543732, + "p05": 0.0979941189289093, + "p95": 0.1353992223739624 + }, + { + "label": "completed_group_4", + "mean": 0.17698991298675537, + "median": 0.17597860097885132, + "p05": 0.14837387204170227, + "p95": 0.21050144731998444 + }, + { + "label": "current_group_partial", + "mean": 0.2988377809524536, + "median": 0.2989804744720459, + "p05": 0.2721370458602905, + "p95": 0.32572072744369507 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7419235706329346, + "group": 5, + "latest_source_mass": 0.26016175746917725, + "layer": 18, + "max_source_mass": 0.26016175746917725, + "mixer_index": 35, + "normalized_entropy": 0.972186055410333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1532188355922699, + "median": 0.1451343297958374, + "p05": 0.11514341086149216, + "p95": 0.22103679180145264 + }, + { + "label": "completed_group_1", + "mean": 0.11420352011919022, + "median": 0.11261284351348877, + "p05": 0.0928458571434021, + "p95": 0.14123988151550293 + }, + { + "label": "completed_group_2", + "mean": 0.16597424447536469, + "median": 0.1655328869819641, + "p05": 0.13871800899505615, + "p95": 0.19611573219299316 + }, + { + "label": "completed_group_3", + "mean": 0.1250365972518921, + "median": 0.12369821965694427, + "p05": 0.10375828295946121, + "p95": 0.14912742376327515 + }, + { + "label": "completed_group_4", + "mean": 0.18140508234500885, + "median": 0.1796584278345108, + "p05": 0.14511024951934814, + "p95": 0.22286373376846313 + }, + { + "label": "current_group_partial", + "mean": 0.26016175746917725, + "median": 0.26266711950302124, + "p05": 0.1889389455318451, + "p95": 0.3234381675720215 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7127439975738525, + "group": 5, + "latest_source_mass": 0.2907913327217102, + "layer": 19, + "max_source_mass": 0.2907913327217102, + "mixer_index": 36, + "normalized_entropy": 0.9559006256078307, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18731768429279327, + "median": 0.19042012095451355, + "p05": 0.13879702985286713, + "p95": 0.23792000114917755 + }, + { + "label": "completed_group_1", + "mean": 0.16715484857559204, + "median": 0.1667211651802063, + "p05": 0.14574480056762695, + "p95": 0.19026139378547668 + }, + { + "label": "completed_group_2", + "mean": 0.08496320247650146, + "median": 0.08452782034873962, + "p05": 0.07410907745361328, + "p95": 0.097386933863163 + }, + { + "label": "completed_group_3", + "mean": 0.11362075805664062, + "median": 0.11248823255300522, + "p05": 0.0977015346288681, + "p95": 0.13219426572322845 + }, + { + "label": "completed_group_4", + "mean": 0.15615218877792358, + "median": 0.15540224313735962, + "p05": 0.1289147436618805, + "p95": 0.18598109483718872 + }, + { + "label": "current_group_partial", + "mean": 0.2907913327217102, + "median": 0.2900415062904358, + "p05": 0.25822532176971436, + "p95": 0.32475030422210693 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076834201812744, + "group": 5, + "latest_source_mass": 0.3738824725151062, + "layer": 19, + "max_source_mass": 0.3738824725151062, + "mixer_index": 37, + "normalized_entropy": 0.8972652009334232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15032559633255005, + "median": 0.12544110417366028, + "p05": 0.06976518034934998, + "p95": 0.35840943455696106 + }, + { + "label": "completed_group_1", + "mean": 0.08624927699565887, + "median": 0.08602266013622284, + "p05": 0.061998721212148666, + "p95": 0.11138717085123062 + }, + { + "label": "completed_group_2", + "mean": 0.15736976265907288, + "median": 0.15802210569381714, + "p05": 0.10121746361255646, + "p95": 0.20687000453472137 + }, + { + "label": "completed_group_3", + "mean": 0.08584291487932205, + "median": 0.0827617198228836, + "p05": 0.060561805963516235, + "p95": 0.11351694166660309 + }, + { + "label": "completed_group_4", + "mean": 0.14632999897003174, + "median": 0.1426895558834076, + "p05": 0.10106660425662994, + "p95": 0.20179018378257751 + }, + { + "label": "current_group_partial", + "mean": 0.3738824725151062, + "median": 0.3799295425415039, + "p05": 0.23631010949611664, + "p95": 0.4884527325630188 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.709290862083435, + "group": 5, + "latest_source_mass": 0.2901708781719208, + "layer": 20, + "max_source_mass": 0.2901708781719208, + "mixer_index": 38, + "normalized_entropy": 0.9539733939957076, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18991519510746002, + "median": 0.1924753338098526, + "p05": 0.14201581478118896, + "p95": 0.23709386587142944 + }, + { + "label": "completed_group_1", + "mean": 0.17091524600982666, + "median": 0.17006167769432068, + "p05": 0.15297070145606995, + "p95": 0.19153237342834473 + }, + { + "label": "completed_group_2", + "mean": 0.07822999358177185, + "median": 0.07778562605381012, + "p05": 0.06707640737295151, + "p95": 0.09089364111423492 + }, + { + "label": "completed_group_3", + "mean": 0.116765096783638, + "median": 0.11516137421131134, + "p05": 0.10162825882434845, + "p95": 0.13653048872947693 + }, + { + "label": "completed_group_4", + "mean": 0.1540035903453827, + "median": 0.15287795662879944, + "p05": 0.12593276798725128, + "p95": 0.18590238690376282 + }, + { + "label": "current_group_partial", + "mean": 0.2901708781719208, + "median": 0.29104921221733093, + "p05": 0.25643545389175415, + "p95": 0.3205650746822357 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6690887212753296, + "group": 5, + "latest_source_mass": 0.3342471718788147, + "layer": 20, + "max_source_mass": 0.3342471718788147, + "mixer_index": 39, + "normalized_entropy": 0.9315361520005944, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19029226899147034, + "median": 0.19644704461097717, + "p05": 0.09456793963909149, + "p95": 0.30468082427978516 + }, + { + "label": "completed_group_1", + "mean": 0.1100241169333458, + "median": 0.10808776319026947, + "p05": 0.09035106748342514, + "p95": 0.1359245330095291 + }, + { + "label": "completed_group_2", + "mean": 0.11471512168645859, + "median": 0.11494918912649155, + "p05": 0.08799737691879272, + "p95": 0.14024731516838074 + }, + { + "label": "completed_group_3", + "mean": 0.09999909996986389, + "median": 0.09682698547840118, + "p05": 0.07921088486909866, + "p95": 0.1264391839504242 + }, + { + "label": "completed_group_4", + "mean": 0.1507222205400467, + "median": 0.1476232260465622, + "p05": 0.11397356539964676, + "p95": 0.19991227984428406 + }, + { + "label": "current_group_partial", + "mean": 0.3342471718788147, + "median": 0.33802270889282227, + "p05": 0.2548266053199768, + "p95": 0.40130624175071716 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7458693981170654, + "group": 6, + "latest_source_mass": 0.2361394464969635, + "layer": 21, + "max_source_mass": 0.2361394464969635, + "mixer_index": 40, + "normalized_entropy": 0.9743882636597644, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20528709888458252, + "median": 0.2140858769416809, + "p05": 0.13168928027153015, + "p95": 0.27098023891448975 + }, + { + "label": "completed_group_1", + "mean": 0.1722964197397232, + "median": 0.17238952219486237, + "p05": 0.15053534507751465, + "p95": 0.19365043938159943 + }, + { + "label": "completed_group_2", + "mean": 0.10435444861650467, + "median": 0.10361101478338242, + "p05": 0.08888213336467743, + "p95": 0.12212536484003067 + }, + { + "label": "completed_group_3", + "mean": 0.12075909972190857, + "median": 0.1180175170302391, + "p05": 0.10358668118715286, + "p95": 0.14458929002285004 + }, + { + "label": "completed_group_4", + "mean": 0.16116347908973694, + "median": 0.15841388702392578, + "p05": 0.1337081342935562, + "p95": 0.19791515171527863 + }, + { + "label": "completed_group_5", + "mean": 0.2361394464969635, + "median": 0.23709020018577576, + "p05": 0.20446214079856873, + "p95": 0.26509594917297363 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4612125158309937, + "group": 6, + "latest_source_mass": 0.5543455481529236, + "layer": 21, + "max_source_mass": 0.5543455481529236, + "mixer_index": 41, + "normalized_entropy": 0.7509146897354808, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08934466540813446, + "median": 0.07911523431539536, + "p05": 0.04488275945186615, + "p95": 0.1737232506275177 + }, + { + "label": "completed_group_1", + "mean": 0.054379627108573914, + "median": 0.05344422906637192, + "p05": 0.04221750423312187, + "p95": 0.06924191862344742 + }, + { + "label": "completed_group_2", + "mean": 0.07567165046930313, + "median": 0.07438549399375916, + "p05": 0.05811355262994766, + "p95": 0.09632256627082825 + }, + { + "label": "completed_group_3", + "mean": 0.05758609622716904, + "median": 0.05612001568078995, + "p05": 0.044792283326387405, + "p95": 0.07347827404737473 + }, + { + "label": "completed_group_4", + "mean": 0.07884950935840607, + "median": 0.07757881283760071, + "p05": 0.05758974328637123, + "p95": 0.10452556610107422 + }, + { + "label": "completed_group_5", + "mean": 0.08982288837432861, + "median": 0.08912864327430725, + "p05": 0.0663854256272316, + "p95": 0.11604885756969452 + }, + { + "label": "current_group_partial", + "mean": 0.5543455481529236, + "median": 0.5596089363098145, + "p05": 0.45455610752105713, + "p95": 0.6367484331130981 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7241814136505127, + "group": 6, + "latest_source_mass": 0.39484071731567383, + "layer": 22, + "max_source_mass": 0.39484071731567383, + "mixer_index": 42, + "normalized_entropy": 0.886053970419732, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13360010087490082, + "median": 0.14072942733764648, + "p05": 0.07699814438819885, + "p95": 0.17506597936153412 + }, + { + "label": "completed_group_1", + "mean": 0.10609637200832367, + "median": 0.10586297512054443, + "p05": 0.08648421615362167, + "p95": 0.12602579593658447 + }, + { + "label": "completed_group_2", + "mean": 0.06363911926746368, + "median": 0.06324908882379532, + "p05": 0.054980404675006866, + "p95": 0.07316673547029495 + }, + { + "label": "completed_group_3", + "mean": 0.07426095008850098, + "median": 0.07362888753414154, + "p05": 0.06318790465593338, + "p95": 0.08665015548467636 + }, + { + "label": "completed_group_4", + "mean": 0.09383076429367065, + "median": 0.09294568747282028, + "p05": 0.07638302445411682, + "p95": 0.11445100605487823 + }, + { + "label": "completed_group_5", + "mean": 0.13373197615146637, + "median": 0.13327741622924805, + "p05": 0.11399447172880173, + "p95": 0.15641780197620392 + }, + { + "label": "current_group_partial", + "mean": 0.39484071731567383, + "median": 0.3879151940345764, + "p05": 0.3302172124385834, + "p95": 0.4852263331413269 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3979300260543823, + "group": 6, + "latest_source_mass": 0.5830056071281433, + "layer": 22, + "max_source_mass": 0.5830056071281433, + "mixer_index": 43, + "normalized_entropy": 0.7183939231382495, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08959861099720001, + "median": 0.07544538378715515, + "p05": 0.04165133833885193, + "p95": 0.20099139213562012 + }, + { + "label": "completed_group_1", + "mean": 0.05414731800556183, + "median": 0.05336771905422211, + "p05": 0.04069708660244942, + "p95": 0.07046990841627121 + }, + { + "label": "completed_group_2", + "mean": 0.07668881118297577, + "median": 0.07607124000787735, + "p05": 0.0555989071726799, + "p95": 0.09980960935354233 + }, + { + "label": "completed_group_3", + "mean": 0.05603286623954773, + "median": 0.054573021829128265, + "p05": 0.04162490367889404, + "p95": 0.0730118378996849 + }, + { + "label": "completed_group_4", + "mean": 0.06979228556156158, + "median": 0.06884932518005371, + "p05": 0.04576495662331581, + "p95": 0.09806846082210541 + }, + { + "label": "completed_group_5", + "mean": 0.07073447108268738, + "median": 0.07023533433675766, + "p05": 0.04956750199198723, + "p95": 0.09379896521568298 + }, + { + "label": "current_group_partial", + "mean": 0.5830056071281433, + "median": 0.5862616896629333, + "p05": 0.4658474922180176, + "p95": 0.6849814057350159 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7459560632705688, + "group": 6, + "latest_source_mass": 0.37691548466682434, + "layer": 23, + "max_source_mass": 0.37691548466682434, + "mixer_index": 44, + "normalized_entropy": 0.897243926765161, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14318783581256866, + "median": 0.1493408977985382, + "p05": 0.08292607218027115, + "p95": 0.19464610517024994 + }, + { + "label": "completed_group_1", + "mean": 0.1094653457403183, + "median": 0.10858799517154694, + "p05": 0.09432064741849899, + "p95": 0.1279931515455246 + }, + { + "label": "completed_group_2", + "mean": 0.06349228322505951, + "median": 0.06325754523277283, + "p05": 0.05439673364162445, + "p95": 0.0737837627530098 + }, + { + "label": "completed_group_3", + "mean": 0.07859592139720917, + "median": 0.0775163471698761, + "p05": 0.06603655964136124, + "p95": 0.09357087314128876 + }, + { + "label": "completed_group_4", + "mean": 0.09567482024431229, + "median": 0.09468592703342438, + "p05": 0.07671141624450684, + "p95": 0.11774884164333344 + }, + { + "label": "completed_group_5", + "mean": 0.13266831636428833, + "median": 0.13199329376220703, + "p05": 0.11244359612464905, + "p95": 0.15504513680934906 + }, + { + "label": "current_group_partial", + "mean": 0.37691548466682434, + "median": 0.37180638313293457, + "p05": 0.31207239627838135, + "p95": 0.4595233201980591 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4055393934249878, + "group": 6, + "latest_source_mass": 0.5772827863693237, + "layer": 23, + "max_source_mass": 0.5772827863693237, + "mixer_index": 45, + "normalized_entropy": 0.7223043644164862, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10154955089092255, + "median": 0.08766824007034302, + "p05": 0.04901552200317383, + "p95": 0.20287738740444183 + }, + { + "label": "completed_group_1", + "mean": 0.05619276314973831, + "median": 0.055477917194366455, + "p05": 0.041608311235904694, + "p95": 0.07371112704277039 + }, + { + "label": "completed_group_2", + "mean": 0.07791627198457718, + "median": 0.07736755907535553, + "p05": 0.055920060724020004, + "p95": 0.10158214718103409 + }, + { + "label": "completed_group_3", + "mean": 0.053915880620479584, + "median": 0.05215863138437271, + "p05": 0.03798118978738785, + "p95": 0.07156119495630264 + }, + { + "label": "completed_group_4", + "mean": 0.06398355215787888, + "median": 0.06229078769683838, + "p05": 0.040616974234580994, + "p95": 0.09270931780338287 + }, + { + "label": "completed_group_5", + "mean": 0.06915920227766037, + "median": 0.0684593990445137, + "p05": 0.046986356377601624, + "p95": 0.09408729523420334 + }, + { + "label": "current_group_partial", + "mean": 0.5772827863693237, + "median": 0.5824689269065857, + "p05": 0.4518115520477295, + "p95": 0.6926202178001404 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7853620052337646, + "group": 6, + "latest_source_mass": 0.3407441973686218, + "layer": 24, + "max_source_mass": 0.3407441973686218, + "mixer_index": 46, + "normalized_entropy": 0.9174945750195659, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15284113585948944, + "median": 0.1593906283378601, + "p05": 0.08678612112998962, + "p95": 0.211342453956604 + }, + { + "label": "completed_group_1", + "mean": 0.12206703424453735, + "median": 0.12153112143278122, + "p05": 0.1060115247964859, + "p95": 0.1399766057729721 + }, + { + "label": "completed_group_2", + "mean": 0.06508444249629974, + "median": 0.06482555717229843, + "p05": 0.055350616574287415, + "p95": 0.07557765394449234 + }, + { + "label": "completed_group_3", + "mean": 0.08347313106060028, + "median": 0.08190013468265533, + "p05": 0.06966532766819, + "p95": 0.10050469636917114 + }, + { + "label": "completed_group_4", + "mean": 0.09811970591545105, + "median": 0.09664031863212585, + "p05": 0.07688035070896149, + "p95": 0.12369741499423981 + }, + { + "label": "completed_group_5", + "mean": 0.13767032325267792, + "median": 0.1373325139284134, + "p05": 0.11642646044492722, + "p95": 0.1606694608926773 + }, + { + "label": "current_group_partial", + "mean": 0.3407441973686218, + "median": 0.33805108070373535, + "p05": 0.271301805973053, + "p95": 0.41781002283096313 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.509842872619629, + "group": 6, + "latest_source_mass": 0.5227814316749573, + "layer": 24, + "max_source_mass": 0.5227814316749573, + "mixer_index": 47, + "normalized_entropy": 0.7759057494780099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11788781732320786, + "median": 0.10097194463014603, + "p05": 0.06265684217214584, + "p95": 0.2408325970172882 + }, + { + "label": "completed_group_1", + "mean": 0.06822571158409119, + "median": 0.0673636943101883, + "p05": 0.05209728330373764, + "p95": 0.08648915588855743 + }, + { + "label": "completed_group_2", + "mean": 0.08556114137172699, + "median": 0.08463940769433975, + "p05": 0.06234566867351532, + "p95": 0.1125786080956459 + }, + { + "label": "completed_group_3", + "mean": 0.06067090481519699, + "median": 0.058974385261535645, + "p05": 0.043267179280519485, + "p95": 0.07997901737689972 + }, + { + "label": "completed_group_4", + "mean": 0.07228855788707733, + "median": 0.06972275674343109, + "p05": 0.04689513146877289, + "p95": 0.106512151658535 + }, + { + "label": "completed_group_5", + "mean": 0.07258446514606476, + "median": 0.07082539796829224, + "p05": 0.050949547439813614, + "p95": 0.09970614314079285 + }, + { + "label": "current_group_partial", + "mean": 0.5227814316749573, + "median": 0.5290273427963257, + "p05": 0.38773614168167114, + "p95": 0.6392589807510376 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.839367389678955, + "group": 7, + "latest_source_mass": 0.29583215713500977, + "layer": 25, + "max_source_mass": 0.29583215713500977, + "mixer_index": 48, + "normalized_entropy": 0.9452478525649903, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15906387567520142, + "median": 0.15846745669841766, + "p05": 0.09519976377487183, + "p95": 0.24517810344696045 + }, + { + "label": "completed_group_1", + "mean": 0.13139380514621735, + "median": 0.1308426856994629, + "p05": 0.11478274315595627, + "p95": 0.15000829100608826 + }, + { + "label": "completed_group_2", + "mean": 0.07717745751142502, + "median": 0.07715500891208649, + "p05": 0.06616044044494629, + "p95": 0.08791667222976685 + }, + { + "label": "completed_group_3", + "mean": 0.09607303142547607, + "median": 0.09436552226543427, + "p05": 0.08152313530445099, + "p95": 0.11384639143943787 + }, + { + "label": "completed_group_4", + "mean": 0.1107850894331932, + "median": 0.1089654266834259, + "p05": 0.08814431726932526, + "p95": 0.139452263712883 + }, + { + "label": "completed_group_5", + "mean": 0.12967455387115479, + "median": 0.1299237608909607, + "p05": 0.10832332074642181, + "p95": 0.15138982236385345 + }, + { + "label": "completed_group_6", + "mean": 0.29583215713500977, + "median": 0.29647016525268555, + "p05": 0.2276897132396698, + "p95": 0.3604946434497833 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5841834545135498, + "group": 7, + "latest_source_mass": 0.40767741203308105, + "layer": 25, + "max_source_mass": 0.40767741203308105, + "mixer_index": 49, + "normalized_entropy": 0.7618312045616817, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07896879315376282, + "median": 0.06809917092323303, + "p05": 0.025932887569069862, + "p95": 0.19368338584899902 + }, + { + "label": "completed_group_1", + "mean": 0.048752427101135254, + "median": 0.04767637699842453, + "p05": 0.034443892538547516, + "p95": 0.0667153000831604 + }, + { + "label": "completed_group_2", + "mean": 0.055707164108753204, + "median": 0.0549345538020134, + "p05": 0.038696639239788055, + "p95": 0.0750277042388916 + }, + { + "label": "completed_group_3", + "mean": 0.04139826446771622, + "median": 0.03878612071275711, + "p05": 0.026673054322600365, + "p95": 0.05746251344680786 + }, + { + "label": "completed_group_4", + "mean": 0.04449018836021423, + "median": 0.04196460545063019, + "p05": 0.026162121444940567, + "p95": 0.07164917886257172 + }, + { + "label": "completed_group_5", + "mean": 0.04285799711942673, + "median": 0.04109475016593933, + "p05": 0.029393533244729042, + "p95": 0.06277710199356079 + }, + { + "label": "completed_group_6", + "mean": 0.2801477909088135, + "median": 0.2763082981109619, + "p05": 0.1455708146095276, + "p95": 0.4254714846611023 + }, + { + "label": "current_group_partial", + "mean": 0.40767741203308105, + "median": 0.4120897948741913, + "p05": 0.21399420499801636, + "p95": 0.5853031277656555 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8125314712524414, + "group": 7, + "latest_source_mass": 0.3700707256793976, + "layer": 26, + "max_source_mass": 0.3700707256793976, + "mixer_index": 50, + "normalized_entropy": 0.8716433883436914, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10889513790607452, + "median": 0.11333495378494263, + "p05": 0.04850132763385773, + "p95": 0.1620434820652008 + }, + { + "label": "completed_group_1", + "mean": 0.08656036853790283, + "median": 0.08640051633119583, + "p05": 0.0650845319032669, + "p95": 0.1084962785243988 + }, + { + "label": "completed_group_2", + "mean": 0.048123255372047424, + "median": 0.04800213500857353, + "p05": 0.03576180338859558, + "p95": 0.06107594817876816 + }, + { + "label": "completed_group_3", + "mean": 0.06561170518398285, + "median": 0.06379580497741699, + "p05": 0.050968848168849945, + "p95": 0.08332888036966324 + }, + { + "label": "completed_group_4", + "mean": 0.06917259097099304, + "median": 0.06724178791046143, + "p05": 0.050207968801259995, + "p95": 0.09373342245817184 + }, + { + "label": "completed_group_5", + "mean": 0.08286689221858978, + "median": 0.08183546364307404, + "p05": 0.06535022705793381, + "p95": 0.10340933501720428 + }, + { + "label": "completed_group_6", + "mean": 0.16869932413101196, + "median": 0.16509854793548584, + "p05": 0.11059704422950745, + "p95": 0.23654375970363617 + }, + { + "label": "current_group_partial", + "mean": 0.3700707256793976, + "median": 0.3694133758544922, + "p05": 0.23441238701343536, + "p95": 0.5025027990341187 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.50089693069458, + "group": 7, + "latest_source_mass": 0.49096518754959106, + "layer": 26, + "max_source_mass": 0.49096518754959106, + "mixer_index": 51, + "normalized_entropy": 0.7217788529328457, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07974998652935028, + "median": 0.07402504235506058, + "p05": 0.04579866677522659, + "p95": 0.1359037309885025 + }, + { + "label": "completed_group_1", + "mean": 0.04166398197412491, + "median": 0.04038450866937637, + "p05": 0.027992289513349533, + "p95": 0.05969971418380737 + }, + { + "label": "completed_group_2", + "mean": 0.04607272148132324, + "median": 0.044438350945711136, + "p05": 0.030038591474294662, + "p95": 0.06808309257030487 + }, + { + "label": "completed_group_3", + "mean": 0.03586764633655548, + "median": 0.03317485377192497, + "p05": 0.02203454077243805, + "p95": 0.05572832003235817 + }, + { + "label": "completed_group_4", + "mean": 0.04223340004682541, + "median": 0.03930673003196716, + "p05": 0.02465122565627098, + "p95": 0.06968094408512115 + }, + { + "label": "completed_group_5", + "mean": 0.04095568507909775, + "median": 0.03919431194663048, + "p05": 0.026239857077598572, + "p95": 0.06058088690042496 + }, + { + "label": "completed_group_6", + "mean": 0.22249138355255127, + "median": 0.2125401496887207, + "p05": 0.10647299885749817, + "p95": 0.3711881935596466 + }, + { + "label": "current_group_partial", + "mean": 0.49096518754959106, + "median": 0.4988659620285034, + "p05": 0.2878011167049408, + "p95": 0.6676807403564453 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8059929609298706, + "group": 7, + "latest_source_mass": 0.37139976024627686, + "layer": 27, + "max_source_mass": 0.37139976024627686, + "mixer_index": 52, + "normalized_entropy": 0.8684990295379666, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10711923241615295, + "median": 0.10972198843955994, + "p05": 0.04896187782287598, + "p95": 0.16201043128967285 + }, + { + "label": "completed_group_1", + "mean": 0.07651707530021667, + "median": 0.07666032761335373, + "p05": 0.05141128972172737, + "p95": 0.10152198374271393 + }, + { + "label": "completed_group_2", + "mean": 0.058284737169742584, + "median": 0.05812416970729828, + "p05": 0.039273619651794434, + "p95": 0.07745087146759033 + }, + { + "label": "completed_group_3", + "mean": 0.06609587371349335, + "median": 0.06383530795574188, + "p05": 0.04481930285692215, + "p95": 0.09336265176534653 + }, + { + "label": "completed_group_4", + "mean": 0.07139429450035095, + "median": 0.0694514587521553, + "p05": 0.0485839769244194, + "p95": 0.10034126043319702 + }, + { + "label": "completed_group_5", + "mean": 0.07493151724338531, + "median": 0.07319007813930511, + "p05": 0.051876723766326904, + "p95": 0.10402432829141617 + }, + { + "label": "completed_group_6", + "mean": 0.17425751686096191, + "median": 0.16937679052352905, + "p05": 0.10449051856994629, + "p95": 0.2624811828136444 + }, + { + "label": "current_group_partial", + "mean": 0.37139976024627686, + "median": 0.3732905089855194, + "p05": 0.23830299079418182, + "p95": 0.5014542937278748 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5622732639312744, + "group": 7, + "latest_source_mass": 0.4768889844417572, + "layer": 27, + "max_source_mass": 0.4768889844417572, + "mixer_index": 53, + "normalized_entropy": 0.7512946301290215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08775827288627625, + "median": 0.08504067361354828, + "p05": 0.0428270623087883, + "p95": 0.144285649061203 + }, + { + "label": "completed_group_1", + "mean": 0.047765426337718964, + "median": 0.04646281152963638, + "p05": 0.031814731657505035, + "p95": 0.06807165592908859 + }, + { + "label": "completed_group_2", + "mean": 0.053928617388010025, + "median": 0.052262187004089355, + "p05": 0.035437777638435364, + "p95": 0.0784272700548172 + }, + { + "label": "completed_group_3", + "mean": 0.04293380305171013, + "median": 0.03996016085147858, + "p05": 0.02556057646870613, + "p95": 0.06623870134353638 + }, + { + "label": "completed_group_4", + "mean": 0.04174989461898804, + "median": 0.03903091698884964, + "p05": 0.025013400241732597, + "p95": 0.06869364529848099 + }, + { + "label": "completed_group_5", + "mean": 0.04734005779027939, + "median": 0.04609068110585213, + "p05": 0.029884062707424164, + "p95": 0.06911227852106094 + }, + { + "label": "completed_group_6", + "mean": 0.20163494348526, + "median": 0.19222964346408844, + "p05": 0.09963390231132507, + "p95": 0.3307305574417114 + }, + { + "label": "current_group_partial", + "mean": 0.4768889844417572, + "median": 0.48383551836013794, + "p05": 0.276212215423584, + "p95": 0.6487575173377991 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8757455348968506, + "group": 7, + "latest_source_mass": 0.3109493851661682, + "layer": 28, + "max_source_mass": 0.3109493851661682, + "mixer_index": 54, + "normalized_entropy": 0.9020429270551009, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11563669890165329, + "median": 0.12087491154670715, + "p05": 0.05733206868171692, + "p95": 0.1688953936100006 + }, + { + "label": "completed_group_1", + "mean": 0.08108697831630707, + "median": 0.08185897022485733, + "p05": 0.05175650864839554, + "p95": 0.10753051191568375 + }, + { + "label": "completed_group_2", + "mean": 0.07280858606100082, + "median": 0.07357072830200195, + "p05": 0.04870978742837906, + "p95": 0.0948689877986908 + }, + { + "label": "completed_group_3", + "mean": 0.07553227245807648, + "median": 0.07362619787454605, + "p05": 0.056317005306482315, + "p95": 0.09756095707416534 + }, + { + "label": "completed_group_4", + "mean": 0.07606014609336853, + "median": 0.07456053793430328, + "p05": 0.05528895929455757, + "p95": 0.1022825762629509 + }, + { + "label": "completed_group_5", + "mean": 0.06935927271842957, + "median": 0.06843917071819305, + "p05": 0.04978285729885101, + "p95": 0.09167985618114471 + }, + { + "label": "completed_group_6", + "mean": 0.19856664538383484, + "median": 0.19508783519268036, + "p05": 0.1281973123550415, + "p95": 0.27796900272369385 + }, + { + "label": "current_group_partial", + "mean": 0.3109493851661682, + "median": 0.3105033040046692, + "p05": 0.18172834813594818, + "p95": 0.43916669487953186 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6924852132797241, + "group": 7, + "latest_source_mass": 0.3315581679344177, + "layer": 28, + "max_source_mass": 0.3315581679344177, + "mixer_index": 55, + "normalized_entropy": 0.8139133413255193, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08638577908277512, + "median": 0.058608733117580414, + "p05": 0.026544082909822464, + "p95": 0.2740764319896698 + }, + { + "label": "completed_group_1", + "mean": 0.059912845492362976, + "median": 0.05901291221380234, + "p05": 0.03629481792449951, + "p95": 0.08707985281944275 + }, + { + "label": "completed_group_2", + "mean": 0.06609142571687698, + "median": 0.0647939145565033, + "p05": 0.038691695779561996, + "p95": 0.09703366458415985 + }, + { + "label": "completed_group_3", + "mean": 0.056745611131191254, + "median": 0.05418796092271805, + "p05": 0.032174739986658096, + "p95": 0.08370660990476608 + }, + { + "label": "completed_group_4", + "mean": 0.06277773529291153, + "median": 0.0600711889564991, + "p05": 0.03501531109213829, + "p95": 0.10094858705997467 + }, + { + "label": "completed_group_5", + "mean": 0.05352145433425903, + "median": 0.05179818347096443, + "p05": 0.032271772623062134, + "p95": 0.08084158599376678 + }, + { + "label": "completed_group_6", + "mean": 0.2830069661140442, + "median": 0.2759752869606018, + "p05": 0.12981083989143372, + "p95": 0.46204638481140137 + }, + { + "label": "current_group_partial", + "mean": 0.3315581679344177, + "median": 0.3204721510410309, + "p05": 0.13446375727653503, + "p95": 0.5667697787284851 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8980522155761719, + "group": 8, + "latest_source_mass": 0.20123262703418732, + "layer": 29, + "max_source_mass": 0.26602888107299805, + "mixer_index": 56, + "normalized_entropy": 0.9127701729200177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11603894829750061, + "median": 0.09991352260112762, + "p05": 0.030035395175218582, + "p95": 0.2941743731498718 + }, + { + "label": "completed_group_1", + "mean": 0.08804173022508621, + "median": 0.08880250155925751, + "p05": 0.05664174258708954, + "p95": 0.11779069900512695 + }, + { + "label": "completed_group_2", + "mean": 0.07975024729967117, + "median": 0.07984748482704163, + "p05": 0.05428915470838547, + "p95": 0.10499341040849686 + }, + { + "label": "completed_group_3", + "mean": 0.09027000516653061, + "median": 0.08898856490850449, + "p05": 0.06189651042222977, + "p95": 0.11996553838253021 + }, + { + "label": "completed_group_4", + "mean": 0.08430293202400208, + "median": 0.0819794088602066, + "p05": 0.05464346706867218, + "p95": 0.12159879505634308 + }, + { + "label": "completed_group_5", + "mean": 0.07433462142944336, + "median": 0.07287181913852692, + "p05": 0.052520088851451874, + "p95": 0.1010315865278244 + }, + { + "label": "completed_group_6", + "mean": 0.26602888107299805, + "median": 0.2639757990837097, + "p05": 0.15474572777748108, + "p95": 0.3846370279788971 + }, + { + "label": "completed_group_7", + "mean": 0.20123262703418732, + "median": 0.18140269815921783, + "p05": 0.08419936150312424, + "p95": 0.3782624900341034 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7512003183364868, + "group": 8, + "latest_source_mass": 0.33482813835144043, + "layer": 29, + "max_source_mass": 0.33482813835144043, + "mixer_index": 57, + "normalized_entropy": 0.7970056117156376, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07744461297988892, + "median": 0.06970229744911194, + "p05": 0.014401955530047417, + "p95": 0.17144526541233063 + }, + { + "label": "completed_group_1", + "mean": 0.046086348593235016, + "median": 0.04394831880927086, + "p05": 0.028066720813512802, + "p95": 0.0718703418970108 + }, + { + "label": "completed_group_2", + "mean": 0.05025433003902435, + "median": 0.04861236736178398, + "p05": 0.029644077643752098, + "p95": 0.07761462777853012 + }, + { + "label": "completed_group_3", + "mean": 0.03748488053679466, + "median": 0.03434944897890091, + "p05": 0.020774798467755318, + "p95": 0.05938909202814102 + }, + { + "label": "completed_group_4", + "mean": 0.04005061462521553, + "median": 0.037498839199543, + "p05": 0.019541850313544273, + "p95": 0.06922487914562225 + }, + { + "label": "completed_group_5", + "mean": 0.041623204946517944, + "median": 0.039552345871925354, + "p05": 0.022081367671489716, + "p95": 0.06821337342262268 + }, + { + "label": "completed_group_6", + "mean": 0.198423832654953, + "median": 0.18318486213684082, + "p05": 0.07548617571592331, + "p95": 0.3721975088119507 + }, + { + "label": "completed_group_7", + "mean": 0.17380405962467194, + "median": 0.1514919549226761, + "p05": 0.05777408182621002, + "p95": 0.36326974630355835 + }, + { + "label": "current_group_partial", + "mean": 0.33482813835144043, + "median": 0.33602219820022583, + "p05": 0.0795319601893425, + "p95": 0.5979493856430054 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0160694122314453, + "group": 8, + "latest_source_mass": 0.21711516380310059, + "layer": 30, + "max_source_mass": 0.21711516380310059, + "mixer_index": 58, + "normalized_entropy": 0.9175527313077866, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09385271370410919, + "median": 0.09211117774248123, + "p05": 0.032377734780311584, + "p95": 0.16496515274047852 + }, + { + "label": "completed_group_1", + "mean": 0.06208505108952522, + "median": 0.06140740588307381, + "p05": 0.0407925546169281, + "p95": 0.0863042101264 + }, + { + "label": "completed_group_2", + "mean": 0.06560768187046051, + "median": 0.06516212224960327, + "p05": 0.041897669434547424, + "p95": 0.09085705131292343 + }, + { + "label": "completed_group_3", + "mean": 0.07029052078723907, + "median": 0.06780634075403214, + "p05": 0.04846132919192314, + "p95": 0.09917552024126053 + }, + { + "label": "completed_group_4", + "mean": 0.06935097277164459, + "median": 0.06708167493343353, + "p05": 0.04668639227747917, + "p95": 0.09815650433301926 + }, + { + "label": "completed_group_5", + "mean": 0.056832700967788696, + "median": 0.05595892667770386, + "p05": 0.038822539150714874, + "p95": 0.0771557092666626 + }, + { + "label": "completed_group_6", + "mean": 0.19425815343856812, + "median": 0.19145764410495758, + "p05": 0.12249615788459778, + "p95": 0.27496030926704407 + }, + { + "label": "completed_group_7", + "mean": 0.17060703039169312, + "median": 0.16317546367645264, + "p05": 0.09239593893289566, + "p95": 0.2726203501224518 + }, + { + "label": "current_group_partial", + "mean": 0.21711516380310059, + "median": 0.21198657155036926, + "p05": 0.11019522696733475, + "p95": 0.3391069769859314 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7837426662445068, + "group": 8, + "latest_source_mass": 0.29026859998703003, + "layer": 30, + "max_source_mass": 0.29026859998703003, + "mixer_index": 59, + "normalized_entropy": 0.8118162725118464, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08836327493190765, + "median": 0.07904411852359772, + "p05": 0.02631576545536518, + "p95": 0.17534467577934265 + }, + { + "label": "completed_group_1", + "mean": 0.04953558370471001, + "median": 0.04658038914203644, + "p05": 0.025181813165545464, + "p95": 0.08318150043487549 + }, + { + "label": "completed_group_2", + "mean": 0.04813014343380928, + "median": 0.045740917325019836, + "p05": 0.026714935898780823, + "p95": 0.07843323051929474 + }, + { + "label": "completed_group_3", + "mean": 0.0373263955116272, + "median": 0.03401809185743332, + "p05": 0.01990434154868126, + "p95": 0.06290259957313538 + }, + { + "label": "completed_group_4", + "mean": 0.037015400826931, + "median": 0.03405017405748367, + "p05": 0.018646355718374252, + "p95": 0.06544196605682373 + }, + { + "label": "completed_group_5", + "mean": 0.03390160948038101, + "median": 0.03198609501123428, + "p05": 0.018189728260040283, + "p95": 0.05570102855563164 + }, + { + "label": "completed_group_6", + "mean": 0.1840541511774063, + "median": 0.17327937483787537, + "p05": 0.07966600358486176, + "p95": 0.32982927560806274 + }, + { + "label": "completed_group_7", + "mean": 0.23140482604503632, + "median": 0.22406929731369019, + "p05": 0.08831030130386353, + "p95": 0.40350067615509033 + }, + { + "label": "current_group_partial", + "mean": 0.29026859998703003, + "median": 0.2745037376880646, + "p05": 0.10280418395996094, + "p95": 0.5399085879325867 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9973840713500977, + "group": 8, + "latest_source_mass": 0.19101998209953308, + "layer": 31, + "max_source_mass": 0.20831164717674255, + "mixer_index": 60, + "normalized_entropy": 0.9090486661912383, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10261696577072144, + "median": 0.10389091819524765, + "p05": 0.01371697522699833, + "p95": 0.21163305640220642 + }, + { + "label": "completed_group_1", + "mean": 0.06039813905954361, + "median": 0.06085003539919853, + "p05": 0.026107093319296837, + "p95": 0.09348798543214798 + }, + { + "label": "completed_group_2", + "mean": 0.06671924144029617, + "median": 0.06656201928853989, + "p05": 0.034684643149375916, + "p95": 0.09819839894771576 + }, + { + "label": "completed_group_3", + "mean": 0.08377155661582947, + "median": 0.0804532840847969, + "p05": 0.05448959022760391, + "p95": 0.12039553374052048 + }, + { + "label": "completed_group_4", + "mean": 0.07002593576908112, + "median": 0.06737451255321503, + "p05": 0.044841229915618896, + "p95": 0.10655929893255234 + }, + { + "label": "completed_group_5", + "mean": 0.05768897384405136, + "median": 0.056077782064676285, + "p05": 0.039495617151260376, + "p95": 0.08242568373680115 + }, + { + "label": "completed_group_6", + "mean": 0.20831164717674255, + "median": 0.2021285891532898, + "p05": 0.11652229726314545, + "p95": 0.31706833839416504 + }, + { + "label": "completed_group_7", + "mean": 0.1594475358724594, + "median": 0.1546279788017273, + "p05": 0.06593266129493713, + "p95": 0.2735573351383209 + }, + { + "label": "current_group_partial", + "mean": 0.19101998209953308, + "median": 0.1713225245475769, + "p05": 0.06408411264419556, + "p95": 0.3850628137588501 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.969987154006958, + "group": 8, + "latest_source_mass": 0.16864418983459473, + "layer": 31, + "max_source_mass": 0.24503237009048462, + "mixer_index": 61, + "normalized_entropy": 0.8965797917640489, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08120635151863098, + "median": 0.061164144426584244, + "p05": 0.0259944386780262, + "p95": 0.23622018098831177 + }, + { + "label": "completed_group_1", + "mean": 0.07894632965326309, + "median": 0.07658770680427551, + "p05": 0.0496862456202507, + "p95": 0.11596276611089706 + }, + { + "label": "completed_group_2", + "mean": 0.08615730702877045, + "median": 0.08467075228691101, + "p05": 0.050409119576215744, + "p95": 0.12811316549777985 + }, + { + "label": "completed_group_3", + "mean": 0.07609779387712479, + "median": 0.0722871869802475, + "p05": 0.04099677503108978, + "p95": 0.11950768530368805 + }, + { + "label": "completed_group_4", + "mean": 0.058728985488414764, + "median": 0.056080836802721024, + "p05": 0.03323862701654434, + "p95": 0.09344407916069031 + }, + { + "label": "completed_group_5", + "mean": 0.05076971277594566, + "median": 0.04946017265319824, + "p05": 0.03047419898211956, + "p95": 0.07563400268554688 + }, + { + "label": "completed_group_6", + "mean": 0.24503237009048462, + "median": 0.24110376834869385, + "p05": 0.10613486170768738, + "p95": 0.3991503119468689 + }, + { + "label": "completed_group_7", + "mean": 0.1544169783592224, + "median": 0.1400495171546936, + "p05": 0.06199025735259056, + "p95": 0.2995939254760742 + }, + { + "label": "current_group_partial", + "mean": 0.16864418983459473, + "median": 0.15222924947738647, + "p05": 0.0543956533074379, + "p95": 0.3329208791255951 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.071648359298706, + "group": 8, + "latest_source_mass": 0.11520501971244812, + "layer": 32, + "max_source_mass": 0.20955611765384674, + "mixer_index": 62, + "normalized_entropy": 0.9428478002054053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10569609701633453, + "median": 0.08689188957214355, + "p05": 0.03173588588833809, + "p95": 0.24571751058101654 + }, + { + "label": "completed_group_1", + "mean": 0.09922641515731812, + "median": 0.09795533865690231, + "p05": 0.06407676637172699, + "p95": 0.14120742678642273 + }, + { + "label": "completed_group_2", + "mean": 0.08180510997772217, + "median": 0.080217644572258, + "p05": 0.054607540369033813, + "p95": 0.11442367732524872 + }, + { + "label": "completed_group_3", + "mean": 0.10646799206733704, + "median": 0.10510502755641937, + "p05": 0.07415887713432312, + "p95": 0.14216329157352448 + }, + { + "label": "completed_group_4", + "mean": 0.0826798751950264, + "median": 0.08022409677505493, + "p05": 0.05491747707128525, + "p95": 0.11860857903957367 + }, + { + "label": "completed_group_5", + "mean": 0.06615597754716873, + "median": 0.06485182791948318, + "p05": 0.04477996751666069, + "p95": 0.09238749742507935 + }, + { + "label": "completed_group_6", + "mean": 0.20955611765384674, + "median": 0.206287682056427, + "p05": 0.129981130361557, + "p95": 0.2996031939983368 + }, + { + "label": "completed_group_7", + "mean": 0.13320739567279816, + "median": 0.12465103715658188, + "p05": 0.06303662061691284, + "p95": 0.23065781593322754 + }, + { + "label": "current_group_partial", + "mean": 0.11520501971244812, + "median": 0.10386130958795547, + "p05": 0.03912060335278511, + "p95": 0.22735844552516937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781701564788818, + "group": 8, + "latest_source_mass": 0.16112206876277924, + "layer": 32, + "max_source_mass": 0.22251944243907928, + "mixer_index": 63, + "normalized_entropy": 0.9003040366848136, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09446848928928375, + "median": 0.074150450527668, + "p05": 0.042592622339725494, + "p95": 0.19443637132644653 + }, + { + "label": "completed_group_1", + "mean": 0.07056106626987457, + "median": 0.06868506222963333, + "p05": 0.04818308353424072, + "p95": 0.0982733890414238 + }, + { + "label": "completed_group_2", + "mean": 0.08033448457717896, + "median": 0.08047192543745041, + "p05": 0.048296794295310974, + "p95": 0.11253741383552551 + }, + { + "label": "completed_group_3", + "mean": 0.0658978745341301, + "median": 0.06256154924631119, + "p05": 0.04223886504769325, + "p95": 0.0967884361743927 + }, + { + "label": "completed_group_4", + "mean": 0.058234501630067825, + "median": 0.0563921295106411, + "p05": 0.03529864549636841, + "p95": 0.08846980333328247 + }, + { + "label": "completed_group_5", + "mean": 0.04316519945859909, + "median": 0.042268507182598114, + "p05": 0.029443470761179924, + "p95": 0.05996086448431015 + }, + { + "label": "completed_group_6", + "mean": 0.2036968618631363, + "median": 0.19645483791828156, + "p05": 0.09398645162582397, + "p95": 0.3395216464996338 + }, + { + "label": "completed_group_7", + "mean": 0.22251944243907928, + "median": 0.2130982130765915, + "p05": 0.11867167800664902, + "p95": 0.35465875267982483 + }, + { + "label": "current_group_partial", + "mean": 0.16112206876277924, + "median": 0.14023703336715698, + "p05": 0.06008770316839218, + "p95": 0.3247341811656952 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.6036605834960938, + "group": null, + "latest_source_mass": 0.31639498472213745, + "layer": null, + "max_source_mass": 0.3412485122680664, + "mixer_index": null, + "normalized_entropy": 0.7298573846467136, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09759747982025146, + "median": 0.09915056824684143, + "p05": 0.025461740791797638, + "p95": 0.16398483514785767 + }, + { + "label": "completed_group_1", + "mean": 0.04215940088033676, + "median": 0.04050712287425995, + "p05": 0.025720715522766113, + "p95": 0.06398092210292816 + }, + { + "label": "completed_group_2", + "mean": 0.041145481169223785, + "median": 0.03881259262561798, + "p05": 0.023510465398430824, + "p95": 0.0662468671798706 + }, + { + "label": "completed_group_3", + "mean": 0.029554028064012527, + "median": 0.027420252561569214, + "p05": 0.01627083122730255, + "p95": 0.04924973472952843 + }, + { + "label": "completed_group_4", + "mean": 0.0289996936917305, + "median": 0.027226522564888, + "p05": 0.01560397818684578, + "p95": 0.048351090401411057 + }, + { + "label": "completed_group_5", + "mean": 0.03647579252719879, + "median": 0.0347352996468544, + "p05": 0.020744025707244873, + "p95": 0.058257561177015305 + }, + { + "label": "completed_group_6", + "mean": 0.06642461568117142, + "median": 0.057809896767139435, + "p05": 0.022837381809949875, + "p95": 0.14078694581985474 + }, + { + "label": "completed_group_7", + "mean": 0.3412485122680664, + "median": 0.3336990475654602, + "p05": 0.0819513276219368, + "p95": 0.6423859596252441 + }, + { + "label": "completed_group_8", + "mean": 0.31639498472213745, + "median": 0.30713334679603577, + "p05": 0.05841828137636185, + "p95": 0.6156357526779175 + } + ], + "sources": 9 + } + ], + "mode": "uniform_value_backward", + "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "optimizer_state_unchanged": true, + "positions": { + "attention_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.24537226557731628, + 0.07698935270309448, + 0.0523640513420105, + 0.09380833059549332, + 0.03647654876112938, + 0.039120711386203766, + 0.06371714919805527, + 0.05156813934445381, + 0.03728010877966881, + 0.03983496502041817, + 0.040284499526023865, + 0.046871740370988846, + 0.04314620420336723, + 0.04033593460917473, + 0.04471700266003609, + 0.08397205173969269, + 0.0528092160820961, + 0.04443390294909477, + 0.07560808956623077, + 0.03708760067820549, + 0.03599191829562187, + 0.033304885029792786, + 0.04061548039317131, + 0.025487789884209633, + 0.021565141156315804, + 0.02342291921377182, + 0.017244853079319, + 0.024166326969861984, + 0.016819356009364128, + 0.021590016782283783, + 0.026201345026493073 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.459922923975682e-06, + "first_to_last_ratio": 2.328407895763024, + "imbalance_abs_log_ratio": 0.8451847274778195, + "last_quartile_mean": 3.6333509001451603e-06, + "mean": 5.48445950698806e-06, + "normalized": [ + 1.7981492239015215, + 1.9357513407040896, + 1.6672073870519302, + 1.5017840953582675, + 1.4824821685817133, + 1.4014170610824435, + 1.3280238040790173, + 1.2253951633993214, + 1.156695262052324, + 1.0641431906294738, + 1.0106344255365511, + 0.9662588157958233, + 0.9475559543612537, + 0.9090700068173221, + 0.8837794995113782, + 0.8574858790556309, + 0.8607721566700557, + 0.837358195636748, + 0.8360590737297285, + 0.824566471544106, + 0.8217232949937042, + 0.8060327579708028, + 0.7938760845984737, + 0.7839296928623085, + 0.7744755711517991, + 0.7571739836659549, + 0.7307992717355416, + 0.68650330244616, + 0.6478013597166409, + 0.6078235943688355, + 0.5594006660114121, + 0.5358712449796666 + ], + "peak_layer": 2, + "peak_normalized": 1.9357513407040896, + "population_cv": 0.35649081874965816, + "rest_mean": 5.691642602161866e-06, + "spike_contrast": 0.7670317864637676, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.365670793049503e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.8618766060099e-06, + 1.0616549843689427e-05, + 9.14373140403768e-06, + 8.236474059231114e-06, + 8.130613423418254e-06, + 7.686015123908874e-06, + 7.283492777787615e-06, + 6.720630153722595e-06, + 6.343848326650914e-06, + 5.836250238644425e-06, + 5.542783583223354e-06, + 5.299407348502427e-06, + 5.196832262299722e-06, + 4.985757641406963e-06, + 4.847052878176328e-06, + 4.7028465814946685e-06, + 4.720870037999703e-06, + 4.59245711681433e-06, + 4.585332135320641e-06, + 4.522301424003672e-06, + 4.506708137341775e-06, + 4.420654022396775e-06, + 4.353981239546556e-06, + 4.2994306568289176e-06, + 4.247579909133492e-06, + 4.152690053160768e-06, + 4.008039013569942e-06, + 3.7650995636795415e-06, + 3.552840325937723e-06, + 3.333583890707814e-06, + 3.068010300921742e-06, + 2.93896414405026e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.357288642291678e-05, + "first_to_last_ratio": 2.3058080996835404, + "imbalance_abs_log_ratio": 0.8354312005538636, + "last_quartile_mean": 1.456013899314712e-05, + "mean": 2.1838724165945678e-05, + "normalized": [ + 1.8092622254038118, + 1.9431844817635626, + 1.6767107875669782, + 1.4950940749576347, + 1.4674045069810522, + 1.3807616632021802, + 1.3135886403618173, + 1.2124726583027057, + 1.1458806799401573, + 1.058561922179424, + 1.0064538267556071, + 0.9652676350426349, + 0.9479497443118112, + 0.9127224235896364, + 0.8880734273823512, + 0.8621668896719061, + 0.8678176638273486, + 0.8415578850574983, + 0.8383782990632195, + 0.8250966517885899, + 0.8217974585881554, + 0.8072470254003167, + 0.7943189523527976, + 0.7845346497712516, + 0.7759095209279144, + 0.7590478200773931, + 0.7335564093275885, + 0.6907997066984136, + 0.653253330518448, + 0.6137700422385202, + 0.5655418513252468, + 0.5418171456240263 + ], + "peak_layer": 2, + "peak_normalized": 1.9431844817635626, + "population_cv": 0.3545327058404385, + "rest_mean": 2.266066288332998e-05, + "spike_contrast": 0.7678616985591674, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.740025509207044e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.951197868445888e-05, + 4.2436669900780544e-05, + 3.661722439574078e-05, + 3.2650947105139494e-05, + 3.204624226782471e-05, + 3.01540731015848e-05, + 2.8687099984381348e-05, + 2.6478855943423696e-05, + 2.502457209629938e-05, + 2.3117641831049696e-05, + 2.197966750827618e-05, + 2.1080213628010824e-05, + 2.0702012989204377e-05, + 1.99326932488475e-05, + 1.9394390619709156e-05, + 1.8828624888556078e-05, + 1.895203058666084e-05, + 1.8378550521447323e-05, + 1.8309112419956364e-05, + 1.8019058188656345e-05, + 1.7947008018381894e-05, + 1.762924512149766e-05, + 1.7346912500215694e-05, + 1.7133235814981163e-05, + 1.694487400527578e-05, + 1.6576635971432552e-05, + 1.6019936083466746e-05, + 1.5086184248502832e-05, + 1.4266219295677729e-05, + 1.3403954653767869e-05, + 1.2350712495390326e-05, + 1.1832595191663131e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.029772771056741476, + "first_to_last_ratio": 2.3058080657449813, + "imbalance_abs_log_ratio": 0.8354311858351369, + "last_quartile_mean": 0.012912076897919178, + "mean": 0.01936679895152338, + "normalized": [ + 1.8092622688469415, + 1.9431844502992837, + 1.6767106129751548, + 1.4950940451422423, + 1.4674045233760984, + 1.3807616273635772, + 1.3135886581915137, + 1.2124727235166237, + 1.1458806648362485, + 1.0585619314023544, + 1.0064538755889925, + 0.965267610260004, + 0.9479497452013365, + 0.91272242891937, + 0.8880734542201515, + 0.8621668660127682, + 0.8678176632029413, + 0.8415579121933264, + 0.838378389023671, + 0.8250966015781978, + 0.8217974339297908, + 0.8072470687715084, + 0.7943189250144355, + 0.7845346707619233, + 0.7759095445302433, + 0.7590478000025082, + 0.7335564570017248, + 0.69079971362327, + 0.6532533237931528, + 0.6137700206530842, + 0.56554185062535, + 0.5418171391422113 + ], + "peak_layer": 2, + "peak_normalized": 1.9431844502992837, + "population_cv": 0.35453269447871943, + "rest_mean": 0.020095702398706367, + "spike_contrast": 0.7678617064775293, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01543072033673525, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.035039618611335754, + 0.0376332625746727, + 0.03247251734137535, + 0.028955185785889626, + 0.028418928384780884, + 0.02674093283712864, + 0.02544000744819641, + 0.023481715470552444, + 0.02219204045832157, + 0.02050095610320568, + 0.019491789862513542, + 0.018694143742322922, + 0.018358752131462097, + 0.01767651177942753, + 0.017199140042066574, + 0.016697412356734276, + 0.01680685020983219, + 0.016298282891511917, + 0.01623670570552349, + 0.015979479998350143, + 0.015915585681796074, + 0.01563379168510437, + 0.015383414924144745, + 0.015193925239145756, + 0.015026884153485298, + 0.014700326137244701, + 0.014206640422344208, + 0.013378579169511795, + 0.012651425786316395, + 0.011886760592460632, + 0.01095273531973362, + 0.010493263602256775 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2918531382410947e-06, + "first_to_last_ratio": 1.348602552566167, + "imbalance_abs_log_ratio": 0.2990689100691611, + "last_quartile_mean": 9.57919837674126e-07, + "mean": 1.0475390865849477e-06, + "normalized": [ + 2.4104888609370185, + 1.341971980631633, + 1.2193546382838214, + 1.0858339683324785, + 0.9788660798673142, + 0.9472203180914999, + 0.9376150883665126, + 0.9444624163032238, + 0.9288836138476254, + 0.8900745462090585, + 0.8740769359854389, + 0.8711050178927995, + 0.8545471342557541, + 0.8634403690695975, + 0.8786271700589888, + 0.8875789012156576, + 0.8758221673193468, + 0.9032444171738859, + 0.9303783636244995, + 0.9552629683285443, + 0.9757615411235937, + 1.0157402642998257, + 1.045251506463968, + 1.0688091449644375, + 1.0624071055149489, + 1.0529656438443655, + 1.0284038000374691, + 0.9523781978486857, + 0.8838469105126692, + 0.8403571044364565, + 0.7713914356307573, + 0.7238323895281237 + ], + "peak_layer": 1, + "peak_normalized": 2.4104888609370185, + "population_cv": 0.279812062407651, + "rest_mean": 1.0410222465132527e-06, + "spike_contrast": 1.040064250882766, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0827300229721004e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 24 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.5250812996091554e-06, + 1.4057681028134539e-06, + 1.2773216440109536e-06, + 1.1374535233699135e-06, + 1.0254004791931948e-06, + 9.922503068082733e-07, + 9.821884532357217e-07, + 9.893612968880916e-07, + 9.730418923936668e-07, + 9.323878771283489e-07, + 9.156297551271564e-07, + 9.125165547629877e-07, + 8.951715244620573e-07, + 9.044875355357362e-07, + 9.203963031723106e-07, + 9.297735914515215e-07, + 9.174579531645577e-07, + 9.461838317292859e-07, + 9.746077012096066e-07, + 1.0006752972913091e-06, + 1.0221483535133302e-06, + 1.0640276286721928e-06, + 1.0949418083328055e-06, + 1.1196193554496858e-06, + 1.1129129688924877e-06, + 1.103022668758058e-06, + 1.0772931773317396e-06, + 9.976533874578308e-07, + 9.258641853193694e-07, + 8.803069135865371e-07, + 8.08062679880095e-07, + 7.582427201668906e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.57942979260406e-05, + "first_to_last_ratio": 2.5994532681374096, + "imbalance_abs_log_ratio": 0.9553011414295731, + "last_quartile_mean": 9.922970434672607e-06, + "mean": 1.6124295910913133e-05, + "normalized": [ + 1.8750767487966784, + 2.0012252521436076, + 1.745806554118046, + 1.5722176381119681, + 1.5471673963385955, + 1.4484453745095827, + 1.357829461828116, + 1.2499614439371438, + 1.1713452937825866, + 1.079951327288068, + 1.0230197185281116, + 0.9769120487640461, + 0.96163457802513, + 0.918624119720732, + 0.8885326600597652, + 0.8580725357821459, + 0.8699071437806652, + 0.8325299891167686, + 0.8310410601256768, + 0.8095951692808093, + 0.8030390759939747, + 0.7755468823771603, + 0.7515517005264444, + 0.7277277649695101, + 0.7158336494479539, + 0.6950266607011315, + 0.6717468100420292, + 0.6353600749013578, + 0.6043990157307626, + 0.5679033620811016, + 0.527236654105826, + 0.5057328350845046 + ], + "peak_layer": 2, + "peak_normalized": 2.0012252521436076, + "population_cv": 0.3945192515369112, + "rest_mean": 1.6856638096753698e-05, + "spike_contrast": 0.7219498951999067, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2169648107374087e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.0234292353270575e-05, + 3.226834814995527e-05, + 2.8149901481810957e-05, + 2.5350902433274314e-05, + 2.4946984922280535e-05, + 2.3355161829385906e-05, + 2.189404403907247e-05, + 2.0154748199274763e-05, + 1.8887118130805902e-05, + 1.741345477057621e-05, + 1.6495472664246336e-05, + 1.575201895320788e-05, + 1.550568049424328e-05, + 1.4812167137279175e-05, + 1.4326963537314441e-05, + 1.3835815479978919e-05, + 1.4026640201336704e-05, + 1.3423959899228066e-05, + 1.3399951967585366e-05, + 1.305415207752958e-05, + 1.2948439689353108e-05, + 1.2505147424235474e-05, + 1.2118242011638358e-05, + 1.1734097824955825e-05, + 1.154231358668767e-05, + 1.1206815543118864e-05, + 1.0831444342329632e-05, + 1.0244733857689425e-05, + 9.745508577907458e-06, + 9.157041858998127e-06, + 8.501319825882092e-06, + 8.154585884767585e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.933050248226209e-05, + "first_to_last_ratio": 2.5350541410379415, + "imbalance_abs_log_ratio": 0.930214994226497, + "last_quartile_mean": 7.6252819098954205e-06, + "mean": 1.202020879986776e-05, + "normalized": [ + 1.8583274986098608, + 2.0026775386155595, + 1.77731729208076, + 1.615557162502337, + 1.5930029936690748, + 1.4693486681854961, + 1.3292468480215616, + 1.2198575705059131, + 1.1444407348951777, + 1.053479376420273, + 1.0059382961834429, + 0.9516573870963836, + 0.9421936613647472, + 0.8964225087904136, + 0.8652818603489327, + 0.841617098226079, + 0.8649772378810286, + 0.8234169272301419, + 0.8325562065776818, + 0.8056805753420667, + 0.8017546832979361, + 0.7661992033221119, + 0.7431607861543773, + 0.720913207047878, + 0.7139696155781592, + 0.6926020821557612, + 0.6726446950161074, + 0.6435870710508805, + 0.6274403778164145, + 0.5970363543218487, + 0.569837411659171, + 0.5578570700324221 + ], + "peak_layer": 2, + "peak_normalized": 2.0026775386155595, + "population_cv": 0.3945939876253568, + "rest_mean": 1.2578481834713818e-05, + "spike_contrast": 0.7159476421745723, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.005534411699046e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.2337484551826492e-05, + 2.4072602172964253e-05, + 2.136372495442629e-05, + 1.941933442139998e-05, + 1.91482286027167e-05, + 1.7661877791397274e-05, + 1.5977824659785256e-05, + 1.4662942703580484e-05, + 1.3756416592514142e-05, + 1.2663042070926167e-05, + 1.20915883599082e-05, + 1.143912049883511e-05, + 1.1325364539516158e-05, + 1.0775185728562064e-05, + 1.0400868632132187e-05, + 1.0116413250216283e-05, + 1.039720700646285e-05, + 9.897643394651823e-06, + 1.0007499440689571e-05, + 9.68444874160923e-06, + 9.63725869951304e-06, + 9.209874406224117e-06, + 8.932947821449488e-06, + 8.66552727529779e-06, + 8.582063856010791e-06, + 8.325221642735414e-06, + 8.08532968221698e-06, + 7.736050974926911e-06, + 7.5419643508212175e-06, + 7.176501640060451e-06, + 6.849564670119435e-06, + 6.705558462272165e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.031241830190993e-05, + "first_to_last_ratio": 2.261102148101721, + "imbalance_abs_log_ratio": 0.8158523705600311, + "last_quartile_mean": 3.109652448074485e-05, + "mean": 4.619880877498872e-05, + "normalized": [ + 1.7997064857450336, + 1.904359030499371, + 1.6374224610397496, + 1.477941957137803, + 1.4602990373876177, + 1.3729930118356377, + 1.3118711870701552, + 1.2110312951649602, + 1.137824927128826, + 1.0604309170276205, + 1.0061171703075658, + 0.9633053240172016, + 0.949330006688662, + 0.9216248238136175, + 0.906779203920096, + 0.8687912686710114, + 0.8800028320072268, + 0.8475838252231725, + 0.8463171144866274, + 0.8286039531613457, + 0.8201877997739442, + 0.8102510580287974, + 0.7986912791711968, + 0.7937156242787844, + 0.7857004463402506, + 0.7744091131435279, + 0.7453649305566711, + 0.6981461341444608, + 0.6620680295240863, + 0.614979715499695, + 0.5681574453782887, + 0.5359925918269958 + ], + "peak_layer": 2, + "peak_normalized": 1.904359030499371, + "population_cv": 0.3447198298829091, + "rest_mean": 4.7895252632938706e-05, + "spike_contrast": 0.7733128004546086, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.70380119420588e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.314429578604177e-05, + 8.797911868896335e-05, + 7.564696716144681e-05, + 6.827915785834193e-05, + 6.746407598257065e-05, + 6.343064160319045e-05, + 6.060688610887155e-05, + 5.594820322585292e-05, + 5.256615622784011e-05, + 4.899064515484497e-05, + 4.648141475627199e-05, + 4.4503558456199244e-05, + 4.3857915443368256e-05, + 4.2577968997647986e-05, + 4.189211904304102e-05, + 4.013712168671191e-05, + 4.06550825573504e-05, + 3.915736306225881e-05, + 3.909884253516793e-05, + 3.828051558230072e-05, + 3.789169932133518e-05, + 3.74326336896047e-05, + 3.689858567668125e-05, + 3.666871634777635e-05, + 3.629842467489652e-05, + 3.577677853172645e-05, + 3.443497189437039e-05, + 3.225351974833757e-05, + 3.058675429201685e-05, + 2.8411330276867375e-05, + 2.624819717311766e-05, + 2.476221925462596e-05 + ] + } + } + }, + "mlp_branch_output": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", + "count": 32, + "dtypes": [ + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16", + "torch.bfloat16" + ], + "output_rms_by_layer": [ + 0.13583384454250336, + 0.0415157787501812, + 0.018140379339456558, + 0.039518751204013824, + 0.022877270355820656, + 0.015532293356955051, + 0.02314678393304348, + 0.05432315170764923, + 0.03743875026702881, + 0.04006272554397583, + 0.045583780854940414, + 0.05995568633079529, + 0.038651175796985626, + 0.04559725895524025, + 0.05632574483752251, + 0.07952121645212173, + 0.042337141931056976, + 0.05044656619429588, + 0.04418695345520973, + 0.06310147047042847, + 0.029010163620114326, + 0.04586305841803551, + 0.06033914536237717, + 0.07656749337911606, + 0.045884523540735245, + 0.051604680716991425, + 0.06502768397331238, + 0.06792232394218445, + 0.060306549072265625, + 0.08025261014699936, + 0.09663844108581543, + 0.12230555713176727 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.247537380157155e-06, + "first_to_last_ratio": 2.3850511827649155, + "imbalance_abs_log_ratio": 0.8692205843888265, + "last_quartile_mean": 3.458012741930361e-06, + "mean": 5.339330805043119e-06, + "normalized": [ + 1.6764909218312523, + 1.9617954794601216, + 1.6992151272822646, + 1.4983517248278486, + 1.5139649706461762, + 1.4256520166778042, + 1.3385091192829222, + 1.2434304822395104, + 1.131080096564001, + 1.0546107928438107, + 1.0037428856625374, + 0.962940037127743, + 0.9549245789646706, + 0.9150321056456306, + 0.8876946170122294, + 0.8650767880832936, + 0.8770248559584294, + 0.8519848957954361, + 0.8533470944296186, + 0.848236507394223, + 0.8353817265855179, + 0.8173190953401269, + 0.8073880080861434, + 0.7956133450140085, + 0.7777311024585761, + 0.7505722139587175, + 0.705927759165694, + 0.665757931848141, + 0.624138138877991, + 0.5752620873586926, + 0.5511417007015933, + 0.530661792875274 + ], + "peak_layer": 2, + "peak_normalized": 1.9617954794601216, + "population_cv": 0.3568485733648799, + "rest_mean": 5.530472267141535e-06, + "spike_contrast": 0.7788063481128092, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.307166909711668e-06, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.95133962330874e-06, + 1.0474675036675762e-05, + 9.07267167349346e-06, + 8.000195521162823e-06, + 8.08355980552733e-06, + 7.612027729919646e-06, + 7.146742973418441e-06, + 6.639086677751038e-06, + 6.039210802555317e-06, + 5.630915893561905e-06, + 5.359315309760859e-06, + 5.141455403645523e-06, + 5.098658220958896e-06, + 4.885659109277185e-06, + 4.7396952140843496e-06, + 4.6189311433408875e-06, + 4.682725830207346e-06, + 4.549029199552024e-06, + 4.556302428682102e-06, + 4.52901531389216e-06, + 4.460379386728164e-06, + 4.363937023299513e-06, + 4.310911663196748e-06, + 4.2480428419366945e-06, + 4.1525636333972216e-06, + 4.007553343399195e-06, + 3.7691818306484492e-06, + 3.5547018342185766e-06, + 3.3324799915135372e-06, + 3.071514584007673e-06, + 2.942727860499872e-06, + 2.833378857758362e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.269888588874892e-05, + "first_to_last_ratio": 2.35686858292009, + "imbalance_abs_log_ratio": 0.8573338663198418, + "last_quartile_mean": 1.3873868965674774e-05, + "mean": 2.1262430664137355e-05, + "normalized": [ + 1.6815888767055769, + 1.9686225336479115, + 1.7075392779747713, + 1.4904980805963362, + 1.4972655559630919, + 1.4034192556792997, + 1.3238794766841318, + 1.2301588485530683, + 1.121552358208752, + 1.0499507133536148, + 1.0011277236394078, + 0.963324273966619, + 0.9560838812000779, + 0.9203885412533225, + 0.8930508699850467, + 0.8703824880945366, + 0.8841528709344532, + 0.8558102501916335, + 0.8550558750774316, + 0.8483637687825304, + 0.8350347355926667, + 0.8178886567183555, + 0.8078366810386547, + 0.7969743808996977, + 0.7797462600958652, + 0.7532932557652127, + 0.7097594040214169, + 0.6713244282550651, + 0.6299481338236045, + 0.5812721180769863, + 0.5573951790036309, + 0.5373112462172303 + ], + "peak_layer": 2, + "peak_normalized": 1.9686225336479115, + "population_cv": 0.35389080715057486, + "rest_mean": 2.202041212984585e-05, + "spike_contrast": 0.7797006998811187, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7169330749311485e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5754666896536946e-05, + 4.1857700125547126e-05, + 3.630643550422974e-05, + 3.169161209370941e-05, + 3.183550506946631e-05, + 2.9840104616596363e-05, + 2.8148895580670796e-05, + 2.615616722323466e-05, + 2.3846929252613336e-05, + 2.232450424344279e-05, + 2.1286408809828572e-05, + 2.0482615582295693e-05, + 2.0328667233115993e-05, + 1.9569697542465292e-05, + 1.89884322026046e-05, + 1.850644730438944e-05, + 1.8799239114741795e-05, + 1.819660610635765e-05, + 1.818056625779718e-05, + 1.8038275811704807e-05, + 1.7754868167685345e-05, + 1.7390300854458474e-05, + 1.717657141853124e-05, + 1.6945612514973618e-05, + 1.6579300790908746e-05, + 1.601684562047012e-05, + 1.5091210116224829e-05, + 1.4273989108914975e-05, + 1.339422851742711e-05, + 1.2359258107608184e-05, + 1.1851576346089132e-05, + 1.1424543117755093e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.02899769856594503, + "first_to_last_ratio": 2.35686858681256, + "imbalance_abs_log_ratio": 0.8573338679713849, + "last_quartile_mean": 0.012303485535085201, + "mean": 0.01885573595063761, + "normalized": [ + 1.6815888871597915, + 1.9686224147635019, + 1.7075392481901774, + 1.4904981407970117, + 1.4972654379699921, + 1.4034192484903574, + 1.3238795515861768, + 1.230158916579243, + 1.1215523220126975, + 1.0499507103306343, + 1.001127702787151, + 0.9633242487753337, + 0.9560838749935686, + 0.9203885704718483, + 0.8930508868493195, + 0.8703824268783725, + 0.8841529174326813, + 0.8558103041085099, + 0.8550558906482283, + 0.8483637681044575, + 0.8350347432767428, + 0.8178887044245051, + 0.8078366908478906, + 0.796974401455123, + 0.7797462730931964, + 0.7532932002519721, + 0.7097594341790385, + 0.671324454011456, + 0.6299481306480355, + 0.5812721137067387, + 0.5573951795880433, + 0.5373112055882039 + ], + "peak_layer": 2, + "peak_normalized": 1.9686224147635019, + "population_cv": 0.35389079505518783, + "rest_mean": 0.019527921418624895, + "spike_contrast": 0.7797007217053022, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01522593442350626, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03170759603381157, + 0.037119824439287186, + 0.03219690918922424, + 0.02810443937778473, + 0.028232041746377945, + 0.026462502777576447, + 0.02496272325515747, + 0.023195551708340645, + 0.021147694438695908, + 0.019797593355178833, + 0.018876999616622925, + 0.01816418766975403, + 0.018027665093541145, + 0.017354603856801987, + 0.016839131712913513, + 0.01641170121729374, + 0.016671353951096535, + 0.016136933118104935, + 0.016122708097100258, + 0.015996523201465607, + 0.015745194628834724, + 0.015421893447637558, + 0.015232355333864689, + 0.015027538873255253, + 0.014702689833939075, + 0.014203897677361965, + 0.013383036479353905, + 0.012658316642045975, + 0.011878135614097118, + 0.010960313491523266, + 0.010510096326470375, + 0.01013139821588993 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.2314776824950968e-06, + "first_to_last_ratio": 1.358248310832468, + "imbalance_abs_log_ratio": 0.30619586281350625, + "last_quartile_mean": 9.066660879852861e-07, + "mean": 1.0095481766114744e-06, + "normalized": [ + 2.2510647132267514, + 1.3455279053230764, + 1.2283491143156429, + 1.0526455105583856, + 1.0056690176197902, + 0.9692221599384264, + 0.9572558258596283, + 0.9489099547933215, + 0.9218216280115407, + 0.8905546788106925, + 0.8763660110671972, + 0.8694242377636043, + 0.8685998082234971, + 0.8764693322122661, + 0.8891057616302812, + 0.8990022944596364, + 0.9003146700723137, + 0.9275413402074646, + 0.9581989480285121, + 0.9876176030015524, + 1.000156116631407, + 1.0336678655116023, + 1.068099651182219, + 1.0896881962272793, + 1.081885225695443, + 1.0551976273002857, + 0.9988041017347361, + 0.9238330401452145, + 0.8601167762256441, + 0.7958830056879117, + 0.7583180274077601, + 0.7106898511269164 + ], + "peak_layer": 1, + "peak_normalized": 2.2510647132267514, + "population_cv": 0.2561818803841662, + "rest_mean": 9.993219375953938e-07, + "spike_contrast": 1.0654923375948289, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0647698672983098e-06, + "top_five_layers": [ + 1, + 2, + 3, + 24, + 25 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.2725582766724983e-06, + 1.3583752433987684e-06, + 1.2400776085996768e-06, + 1.0626963558024727e-06, + 1.015271323012712e-06, + 9.784764642972732e-07, + 9.663958735472988e-07, + 9.579703146300744e-07, + 9.306233437200717e-07, + 8.990578521661519e-07, + 8.847337085171603e-07, + 8.777256539360678e-07, + 8.768933525971079e-07, + 8.848380161907698e-07, + 8.975951004686067e-07, + 9.075861271412577e-07, + 9.089110335480655e-07, + 9.363976687382092e-07, + 9.673480008132174e-07, + 9.970475502996123e-07, + 1.0097057838720502e-06, + 1.0435375088491128e-06, + 1.078298055290361e-06, + 1.1000927315762965e-06, + 1.092215256903728e-06, + 1.0652728406057577e-06, + 1.0083408596983645e-06, + 9.326539611720364e-07, + 8.683293231115385e-07, + 8.03482237188291e-07, + 7.655585818611144e-07, + 7.174756433414586e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.5231804784198175e-05, + "first_to_last_ratio": 2.6518592370690897, + "imbalance_abs_log_ratio": 0.975260992886681, + "last_quartile_mean": 9.514760222373297e-06, + "mean": 1.575830160049918e-05, + "normalized": [ + 1.7680072391258728, + 2.020274701367778, + 1.7747981337484644, + 1.5702283057720683, + 1.5745659549387958, + 1.468575316323847, + 1.366589194311488, + 1.2663643116143115, + 1.1465258163389551, + 1.0704343447928568, + 1.0164163108220898, + 0.9740484512911409, + 0.9677541966052103, + 0.9245201462352831, + 0.8913652028193111, + 0.8644158057124367, + 0.8839251267896815, + 0.845522595189969, + 0.8460497665068152, + 0.8329662909412521, + 0.8145821291394524, + 0.7836795820242509, + 0.7613757450733065, + 0.7366672589275867, + 0.716422356677449, + 0.689594392508022, + 0.652884419046894, + 0.6205919149770609, + 0.5818976326649927, + 0.5425153265661802, + 0.5218602418605108, + 0.5045817892866661 + ], + "peak_layer": 2, + "peak_normalized": 2.020274701367778, + "population_cv": 0.39415073781885135, + "rest_mean": 1.645124252212958e-05, + "spike_contrast": 0.7304263253994938, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2016420623695012e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.786079130601138e-05, + 3.1836098060011864e-05, + 2.7967804271611385e-05, + 2.47441312239971e-05, + 2.4812485207803547e-05, + 2.314225275767967e-05, + 2.1535124687943608e-05, + 1.9955750758526847e-05, + 1.8067299606627785e-05, + 1.6868227248778567e-05, + 1.6016994777601212e-05, + 1.5349349268944934e-05, + 1.5250162505253684e-05, + 1.4568867300113197e-05, + 1.4046401702216826e-05, + 1.362172497465508e-05, + 1.3929158740211278e-05, + 1.3324000065040309e-05, + 1.3332307389646303e-05, + 1.3126134035701398e-05, + 1.2836430869356263e-05, + 1.2349459211691283e-05, + 1.1997988622169942e-05, + 1.1608624845393933e-05, + 1.128959956986364e-05, + 1.0866836419154424e-05, + 1.0288349585607648e-05, + 9.77947456703987e-06, + 9.169718396151438e-06, + 8.549120138923172e-06, + 8.223631084547378e-06, + 7.95135201769881e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.902213432458666e-05, + "first_to_last_ratio": 2.570514437449803, + "imbalance_abs_log_ratio": 0.9441060490896753, + "last_quartile_mean": 7.4001274015245144e-06, + "mean": 1.1811577877551827e-05, + "normalized": [ + 1.7836661639182594, + 2.007884486098661, + 1.801766001704916, + 1.6166376372836335, + 1.6162792780207194, + 1.484195322536222, + 1.334217273990145, + 1.2390748916935237, + 1.1152682343795186, + 1.047183670438877, + 0.9989511624599826, + 0.9468241347561922, + 0.942586809823197, + 0.9015408911709663, + 0.8679632288380377, + 0.8434512704540663, + 0.8735354304753469, + 0.8326703084602718, + 0.8456967523674954, + 0.8281893546686905, + 0.8104136880244638, + 0.7723543788975116, + 0.750047939286133, + 0.727483933767052, + 0.7080168021618385, + 0.6848377954753971, + 0.6576536196741141, + 0.6404010158732426, + 0.6084114841371344, + 0.5799389397954393, + 0.5702129585195616, + 0.5626451408493897 + ], + "peak_layer": 2, + "peak_normalized": 2.007884486098661, + "population_cv": 0.39402258867818274, + "rest_mean": 1.2350397237578701e-05, + "spike_contrast": 0.720782753960385, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.901953333406708e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1067911802674644e-05, + 2.3716283976682462e-05, + 2.1281699446262792e-05, + 1.909504135255702e-05, + 1.9090808564214967e-05, + 1.753068863763474e-05, + 1.57592112373095e-05, + 1.4635429579357151e-05, + 1.3173077604733407e-05, + 1.2368891475489363e-05, + 1.1799189451267011e-05, + 1.1183487004018389e-05, + 1.1133437510579824e-05, + 1.0648620445863344e-05, + 1.025201527227182e-05, + 9.962490366888233e-06, + 1.0317831765860319e-05, + 9.835150194703601e-06, + 9.989013051381335e-06, + 9.782223060028628e-06, + 9.572264389134943e-06, + 9.12272389541613e-06, + 8.859249646775424e-06, + 8.592733138357289e-06, + 8.36279559734976e-06, + 8.089014954748563e-06, + 7.767926945234649e-06, + 7.564146471850108e-06, + 7.186299626482651e-06, + 6.849993951618671e-06, + 6.735114766343031e-06, + 6.645726898568682e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.828012237747316e-05, + "first_to_last_ratio": 2.312494882445189, + "imbalance_abs_log_ratio": 0.8383269774053195, + "last_quartile_mean": 2.9526604748753016e-05, + "mean": 4.489596511803029e-05, + "normalized": [ + 1.6577187409379064, + 1.9152258678535772, + 1.6744752052558183, + 1.4840054420576252, + 1.4912917773323804, + 1.3961062438002583, + 1.3297764454700043, + 1.2182169964281564, + 1.1079961846053805, + 1.047881406618081, + 0.999963947398217, + 0.9691026873070004, + 0.9556277478959141, + 0.9353258396988691, + 0.9030474230504885, + 0.8874013307551517, + 0.8935396144488046, + 0.8678070689766392, + 0.8641197392760416, + 0.8577613745480667, + 0.8343135878548937, + 0.8204103169462279, + 0.8193971014627366, + 0.8081501178832716, + 0.7960663239792694, + 0.7673062057216204, + 0.7200712732146042, + 0.6825993169002439, + 0.6333797250658595, + 0.5852510577399719, + 0.5516412156589899, + 0.5250226738579302 + ], + "peak_layer": 2, + "peak_normalized": 1.9152258678535772, + "population_cv": 0.3428609763636811, + "rest_mean": 4.642851807282703e-05, + "spike_contrast": 0.7887432268392901, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6620179162127894e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.442488276865333e-05, + 8.598591375630349e-05, + 7.517718040617183e-05, + 6.662585656158626e-05, + 6.695298361591995e-05, + 6.267953722272068e-05, + 5.970159691059962e-05, + 5.469302777783014e-05, + 4.9744558054953814e-05, + 4.704564707935788e-05, + 4.489434650167823e-05, + 4.350880044512451e-05, + 4.2903830035356805e-05, + 4.199235627311282e-05, + 4.054318560520187e-05, + 3.984073919127695e-05, + 4.0116323361871764e-05, + 3.89610358979553e-05, + 3.879548967233859e-05, + 3.851002475130372e-05, + 3.745731373783201e-05, + 3.683311297209002e-05, + 3.6787623685086146e-05, + 3.628267950261943e-05, + 3.574016591301188e-05, + 3.4448952646926045e-05, + 3.232829476473853e-05, + 3.064595512114465e-05, + 2.8436194043024443e-05, + 2.627541107358411e-05, + 2.4766464775893837e-05, + 2.357139965170063e-05 + ] + } + } + }, + "post_attention_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0933690071105957, + 0.35791492462158203, + 0.3940892517566681, + 0.4057151973247528, + 0.09380833059549332, + 0.11286329478025436, + 0.13231655955314636, + 0.1621972620487213, + 0.05156813934445381, + 0.08533531427383423, + 0.12003090232610703, + 0.16063299775123596, + 0.046871740370988846, + 0.079384446144104, + 0.11459959298372269, + 0.15625107288360596, + 0.08397205173969269, + 0.11654024571180344, + 0.15294702351093292, + 0.21572710573673248, + 0.03708760067820549, + 0.06553087383508682, + 0.09390194714069366, + 0.12852425873279572, + 0.025487789884209633, + 0.06059965491294861, + 0.08860157430171967, + 0.12100546807050705, + 0.024166326969861984, + 0.06877554953098297, + 0.12682415544986725, + 0.2025238573551178 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.4598292460214e-06, + "first_to_last_ratio": 2.328358854359362, + "imbalance_abs_log_ratio": 0.8451636650513356, + "last_quartile_mean": 3.633387194668103e-06, + "mean": 5.48445886039417e-06, + "normalized": [ + 1.7981292044822659, + 1.9357505739332324, + 1.6671812164387254, + 1.501771337574118, + 1.4824621119466825, + 1.4013991506965175, + 1.3280020709214548, + 1.225379388067507, + 1.156718780628782, + 1.0641415748591472, + 1.0106456553801202, + 0.9662668896138349, + 0.9475625334929744, + 0.9090604128642483, + 0.8837694050828951, + 0.8574965104342549, + 0.860785690483099, + 0.8373501686260958, + 0.8360756725074443, + 0.8245861368451315, + 0.8217235577025693, + 0.8060327700829376, + 0.7938754319523651, + 0.7839311948498927, + 0.7744962255348613, + 0.7571785503773947, + 0.7308065715532428, + 0.6865019323584283, + 0.6478069914366877, + 0.6078375958541098, + 0.5594073652125656, + 0.5358673282064139 + ], + "peak_layer": 2, + "peak_normalized": 1.9357505739332324, + "population_cv": 0.3564831577662627, + "rest_mean": 5.691637507307287e-06, + "spike_contrast": 0.7670365797994658, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.365694167063339e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 9.861765647656284e-06, + 1.0616544386721216e-05, + 9.143586794380099e-06, + 8.236403118644375e-06, + 8.130502465064637e-06, + 7.68591598898638e-06, + 7.283372724486981e-06, + 6.720542842231225e-06, + 6.343976565403864e-06, + 5.836240688950056e-06, + 5.542844519368373e-06, + 5.2994510042481124e-06, + 5.196867732593091e-06, + 4.985704435966909e-06, + 4.846996944252169e-06, + 4.702904334408231e-06, + 4.720943707070546e-06, + 4.592412551573943e-06, + 4.5854226300434675e-06, + 4.522408744378481e-06, + 4.506709046836477e-06, + 4.420653567649424e-06, + 4.353977146820398e-06, + 4.299438387533883e-06, + 4.247692686476512e-06, + 4.152714609517716e-06, + 4.008078576589469e-06, + 3.765091605600901e-06, + 3.552870794010232e-06, + 3.333660288262763e-06, + 3.068046680709813e-06, + 2.9389423161774175e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.3572764550626744e-05, + "first_to_last_ratio": 2.3057895392595262, + "imbalance_abs_log_ratio": 0.8354231510987864, + "last_quartile_mean": 1.456020333989727e-05, + "mean": 2.183873047556517e-05, + "normalized": [ + 1.80924687671537, + 1.9431940819531333, + 1.676696643261768, + 1.4950864798932781, + 1.467401917430866, + 1.3807559335924544, + 1.31358592866812, + 1.212462979303528, + 1.1458997558869217, + 1.0585578682053984, + 1.0064671958441547, + 0.9652690219967264, + 0.9479597986271983, + 0.9127134975291503, + 0.888075836142126, + 0.862166224116184, + 0.867826991667279, + 0.841547063844033, + 0.83837247628253, + 0.8251061585552277, + 0.8217963049445071, + 0.8072479582581175, + 0.7943165572696724, + 0.7845285926712973, + 0.7759202912841717, + 0.7590490167365564, + 0.7335626108658329, + 0.6907982993808188, + 0.6532520589865599, + 0.6137777359935852, + 0.565539938799635, + 0.5418179052937989 + ], + "peak_layer": 2, + "peak_normalized": 1.9431940819531333, + "population_cv": 0.3545305270645445, + "rest_mean": 2.2660667733968392e-05, + "spike_contrast": 0.76786216030628, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.7400269280187784e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.951165490434505e-05, + 4.2436891817487776e-05, + 3.6616926081478596e-05, + 3.265079067205079e-05, + 3.204619497410022e-05, + 3.0153956686262973e-05, + 2.868704905267805e-05, + 2.64786522166105e-05, + 2.5024995920830406e-05, + 2.3117559976526536e-05, + 2.197996582253836e-05, + 2.1080250007798895e-05, + 2.0702238543890417e-05, + 1.993250407394953e-05, + 1.939444882737007e-05, + 1.882861579360906e-05, + 1.8952239770442247e-05, + 1.8378319509793073e-05, + 1.8308990547666326e-05, + 1.801927101041656e-05, + 1.7946988009498455e-05, + 1.762927058734931e-05, + 1.7346865206491202e-05, + 1.7133108485722914e-05, + 1.6945114111877047e-05, + 1.6576666894252412e-05, + 1.602007614565082e-05, + 1.508615787315648e-05, + 1.4266195648815483e-05, + 1.3404126548266504e-05, + 1.2350674296612851e-05, + 1.183261520054657e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.029772662557661533, + "first_to_last_ratio": 2.3057895177959966, + "imbalance_abs_log_ratio": 0.8354231417902483, + "last_quartile_mean": 0.01291213370859623, + "mean": 0.019366804277524352, + "normalized": [ + 1.809246767643357, + 1.9431941106945074, + 1.6766966870594806, + 1.495086420690814, + 1.4674019077542524, + 1.3807559578984756, + 1.3135858925156472, + 1.2124629647121858, + 1.1458997775072552, + 1.0585579855568523, + 1.0064672559708643, + 0.9652689798174655, + 0.9479597754699817, + 0.9127135219659304, + 0.888075806778596, + 0.8621662442022726, + 0.8678270422679951, + 0.841547005089253, + 0.8383724840087661, + 0.8251060885690371, + 0.8217963423351232, + 0.8072480008996415, + 0.7943165425842438, + 0.7845285881999872, + 0.7759202953515476, + 0.7590489858287814, + 0.7335626029644629, + 0.6907983214336281, + 0.653252038106227, + 0.6137777383932114, + 0.5655399639078199, + 0.5418179038223359 + ], + "peak_layer": 2, + "peak_normalized": 1.9431941106945074, + "population_cv": 0.35453052191257556, + "rest_mean": 0.020095706399944093, + "spike_contrast": 0.7678621746036595, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.015430732816457748, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.035039328038692474, + 0.03763346001505852, + 0.03247225657105446, + 0.028955046087503433, + 0.02841888554394245, + 0.02674083039164543, + 0.02543996088206768, + 0.02348153293132782, + 0.022192416712641716, + 0.02050088532269001, + 0.019492054358124733, + 0.01869417540729046, + 0.018358951434493065, + 0.017676344141364098, + 0.017199190333485603, + 0.01669740490615368, + 0.016807036474347115, + 0.016298076137900352, + 0.016236595809459686, + 0.015979668125510216, + 0.01591556891798973, + 0.01563381403684616, + 0.015383373014628887, + 0.015193811617791653, + 0.01502709649503231, + 0.014700353145599365, + 0.014206763356924057, + 0.01337855588644743, + 0.012651404365897179, + 0.01188691332936287, + 0.010952701792120934, + 0.010493281297385693 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.291835857841761e-06, + "first_to_last_ratio": 1.3485833026747331, + "imbalance_abs_log_ratio": 0.299054636012817, + "last_quartile_mean": 9.579206974308363e-07, + "mean": 1.0475393885656104e-06, + "normalized": [ + 2.4104197937214296, + 1.3419993768147092, + 1.2193873876623726, + 1.0857681046985204, + 0.9788139215354608, + 0.9471793472155275, + 0.9376610507921448, + 0.9444495548464527, + 0.9288857336777386, + 0.8900930106166185, + 0.8741329012581984, + 0.8711139916120108, + 0.8545363607420399, + 0.8634605233315061, + 0.8786425447326217, + 0.887541854523117, + 0.8758151318715187, + 0.9032250016851439, + 0.9304164056283383, + 0.9552919953757107, + 0.9757865467437721, + 1.0156631340115454, + 1.0452801819870623, + 1.0688491005570937, + 1.0624669234868362, + 1.0529715263673087, + 1.0284586355464043, + 0.9523438456637114, + 0.8838175160851754, + 0.8403596296329269, + 0.7713843760240694, + 0.7237845915529142 + ], + "peak_layer": 1, + "peak_normalized": 2.4104197937214296, + "population_cv": 0.27980440532718537, + "rest_mean": 1.0410195853991254e-06, + "spike_contrast": 1.0400825698677958, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0827463256646297e-06, + "top_five_layers": [ + 1, + 2, + 3, + 4, + 24 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 2.5250096769013908e-06, + 1.4057972066439106e-06, + 1.2773563184964587e-06, + 1.1373848565199296e-06, + 1.0253461368847638e-06, + 9.922076742441277e-07, + 9.82236883828591e-07, + 9.893481092149159e-07, + 9.730443935040967e-07, + 9.324074881078559e-07, + 9.156886449090962e-07, + 9.12526218144194e-07, + 8.951604968387983e-07, + 9.045089086612279e-07, + 9.204126740769425e-07, + 9.297350516135339e-07, + 9.174508477372001e-07, + 9.46163766002428e-07, + 9.746478326633223e-07, + 1.000705992737494e-06, + 1.0221748425465194e-06, + 1.063947138391086e-06, + 1.0949721627184772e-06, + 1.1196615332664805e-06, + 1.1129759514005855e-06, + 1.1030291489078081e-06, + 1.0773509302453022e-06, + 9.976176897907862e-07, + 9.258336604034412e-07, + 8.80309812600899e-07, + 8.080555176093185e-07, + 7.581928684885497e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.5794245857468923e-05, + "first_to_last_ratio": 2.5994382822780953, + "imbalance_abs_log_ratio": 0.9552953764086378, + "last_quartile_mean": 9.923007610268542e-06, + "mean": 1.6124308444886992e-05, + "normalized": [ + 1.875069425095799, + 2.001233172595225, + 1.7457889523480976, + 1.5722148366283213, + 1.547165404000032, + 1.4484413155137394, + 1.3578243451672811, + 1.2499566367465564, + 1.1713539721385289, + 1.0799457497690719, + 1.023025015060715, + 0.976913996826235, + 0.9616437578274982, + 0.9186228415910193, + 0.888533041071715, + 0.8580748018430988, + 0.8699143078938142, + 0.8325216144516643, + 0.8310392860253453, + 0.8096019290349791, + 0.8030372108501562, + 0.7755478024583314, + 0.7515516803716358, + 0.7277252251002508, + 0.7158404256809536, + 0.6950255563813353, + 0.6717498413974154, + 0.6353591861780038, + 0.6043991099627453, + 0.567907771476934, + 0.5272362442668266, + 0.5057355442466751 + ], + "peak_layer": 2, + "peak_normalized": 2.001233172595225, + "population_cv": 0.3945173779757133, + "rest_mean": 1.685664924648504e-05, + "spike_contrast": 0.7219506046728209, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2169668116257526e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.023419776582159e-05, + 3.226850094506517e-05, + 2.8149639547336847e-05, + 2.5350876967422664e-05, + 2.494697218935471e-05, + 2.3355114535661414e-05, + 2.189397855545394e-05, + 2.0154686353635043e-05, + 1.8887272744905204e-05, + 1.741337837302126e-05, + 1.6495570889674127e-05, + 1.5752062608953565e-05, + 1.550584056531079e-05, + 1.4812158042332157e-05, + 1.4326980817713775e-05, + 1.3835862773703411e-05, + 1.402676662110025e-05, + 1.3423835298453923e-05, + 1.339993377769133e-05, + 1.3054271221335512e-05, + 1.294841968046967e-05, + 1.2505171980592422e-05, + 1.2118251106585376e-05, + 1.1734065992641263e-05, + 1.15424318209989e-05, + 1.1206806448171847e-05, + 1.0831501640495844e-05, + 1.0244727491226513e-05, + 9.745517672854476e-06, + 9.15712007554248e-06, + 8.501319825882092e-06, + 8.154635906976182e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9329483848196105e-05, + "first_to_last_ratio": 2.5350525368598045, + "imbalance_abs_log_ratio": 0.9302143614279234, + "last_quartile_mean": 7.624884915458097e-06, + "mean": 1.201984051135696e-05, + "normalized": [ + 1.8583357088751307, + 2.0027277021690506, + 1.7771668453163478, + 1.615570343516219, + 1.592938001514604, + 1.4694493793664982, + 1.32898385240296, + 1.2198799651106356, + 1.144267264787803, + 1.0533998962206053, + 1.0058282278662807, + 0.9519161926020662, + 0.9421831838426558, + 0.8966453451440455, + 0.8650037408265971, + 0.8416374374551236, + 0.8651561323934935, + 0.8233100437191515, + 0.8327440956360556, + 0.8058278405503257, + 0.8017291581294494, + 0.7663098470747268, + 0.7431513228704318, + 0.7209725235957172, + 0.7140172181236788, + 0.6925010270670932, + 0.6728195124367922, + 0.6435648715599169, + 0.6274519981756614, + 0.5969246531502274, + 0.5697514359189264, + 0.5578352325817294 + ], + "peak_layer": 2, + "peak_normalized": 2.0027277021690506, + "population_cv": 0.3945781723821696, + "rest_mean": 1.2578015162876841e-05, + "spike_contrast": 0.7159871630405812, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.005697393149603e-06, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.233689883723855e-05, + 2.407246756774839e-05, + 2.1361262042773888e-05, + 1.941889786394313e-05, + 1.9146860722685233e-05, + 1.766254717949778e-05, + 1.597417394805234e-05, + 1.4662762623629533e-05, + 1.3753910025116056e-05, + 1.2661698747251648e-05, + 1.20898948807735e-05, + 1.144188081525499e-05, + 1.1324891602271236e-05, + 1.0777534043882042e-05, + 1.039720700646285e-05, + 1.0116347766597755e-05, + 1.039903872879222e-05, + 9.896055416902527e-06, + 1.0009451216319576e-05, + 9.685922123026103e-06, + 9.636656614020467e-06, + 9.210922144120559e-06, + 8.932560376706533e-06, + 8.665974746691063e-06, + 8.582373084209394e-06, + 8.32375189929735e-06, + 8.087183232419193e-06, + 7.735547114862129e-06, + 7.541872946603689e-06, + 7.174939128162805e-06, + 6.848321390862111e-06, + 6.705090527248103e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.030938422758481e-05, + "first_to_last_ratio": 2.2606879822899777, + "imbalance_abs_log_ratio": 0.8156691839012263, + "last_quartile_mean": 3.110087936875061e-05, + "mean": 4.619929336513451e-05, + "normalized": [ + 1.8000040071532064, + 1.904408036353138, + 1.637904058859854, + 1.4777606171460849, + 1.4597770726472121, + 1.372671031064488, + 1.3115103960008867, + 1.2109361461416157, + 1.137534863820107, + 1.0602352149716379, + 1.0062919047911225, + 0.9633358523801769, + 0.9491225557491253, + 0.9216727196154594, + 0.9066025950003278, + 0.8688487743640587, + 0.8802774784785187, + 0.8475087887298653, + 0.846208545767534, + 0.8286626678374982, + 0.8203744064059802, + 0.810217596933452, + 0.7988918917164441, + 0.7937267489942904, + 0.7857342550389422, + 0.7744580018919649, + 0.7454551502724039, + 0.6984329250267984, + 0.662100300186566, + 0.6150609478307189, + 0.5682149152709306, + 0.5360595335595907 + ], + "peak_layer": 2, + "peak_normalized": 1.904408036353138, + "population_cv": 0.3446893198253748, + "rest_mean": 4.789507282246767e-05, + "spike_contrast": 0.7734007302345921, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.704208429553546e-05, + "top_five_layers": [ + 2, + 1, + 3, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.315891318488866e-05, + 8.798230555839837e-05, + 7.567001011921093e-05, + 6.82714962749742e-05, + 6.744066922692582e-05, + 6.341643165796995e-05, + 6.0590853536268696e-05, + 5.5944394262041897e-05, + 5.255330688669346e-05, + 4.8982117732521147e-05, + 4.6489974920405075e-05, + 4.45054356532637e-05, + 4.384879139252007e-05, + 4.258062836015597e-05, + 4.188439925201237e-05, + 4.014019941678271e-05, + 4.066819747094996e-05, + 3.915430716006085e-05, + 3.9094236853998154e-05, + 3.828362969215959e-05, + 3.790071787079796e-05, + 3.743148045032285e-05, + 3.690824087243527e-05, + 3.6669614928541705e-05, + 3.630036735557951e-05, + 3.5779412428382784e-05, + 3.443950117798522e-05, + 3.2267107599182054e-05, + 3.058856600546278e-05, + 2.841538116626907e-05, + 2.6251127565046772e-05, + 2.4765571652096696e-05 + ] + } + } + }, + "post_mlp_state": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.16915453970432281, + 0.374446302652359, + 0.39966297149658203, + 0.4194087088108063, + 0.09892481565475464, + 0.11664740741252899, + 0.13951337337493896, + 0.18593810498714447, + 0.06607726216316223, + 0.10425819456577301, + 0.14551182091236115, + 0.19865906238555908, + 0.06484340131282806, + 0.10430748760700226, + 0.14720691740512848, + 0.20439592003822327, + 0.10052042454481125, + 0.14085541665554047, + 0.1745898574590683, + 0.2420409768819809, + 0.049456026405096054, + 0.08509506285190582, + 0.11783231794834137, + 0.15649692714214325, + 0.05533855780959129, + 0.08507220447063446, + 0.11942370235919952, + 0.1559232622385025, + 0.06628894805908203, + 0.12384890764951706, + 0.19928614795207977, + 0.30097496509552 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 8.247530786320567e-06, + "first_to_last_ratio": 2.3850533729557593, + "imbalance_abs_log_ratio": 0.86922150268771, + "last_quartile_mean": 3.45800680179309e-06, + "mean": 5.339328467357518e-06, + "normalized": [ + 1.676519761737885, + 1.9618007671893674, + 1.6991821441606432, + 1.498358513038152, + 1.5139533691039981, + 1.4256686527077282, + 1.3385028917629673, + 1.243419273267211, + 1.1310636430692016, + 1.054615257538969, + 1.0037504793539562, + 0.9629245320498924, + 0.9549124771530781, + 0.9150161537455905, + 0.8877036077745802, + 0.8650752931079952, + 0.8770263471429458, + 0.8519858650005171, + 0.8533548777823373, + 0.8482425851215958, + 0.8353975931646603, + 0.8173266925801457, + 0.8073981560598078, + 0.7956149708934007, + 0.7777226705206242, + 0.7505720315608271, + 0.7059216805334544, + 0.665764185190204, + 0.6241330890536816, + 0.5752674068009126, + 0.5511430066219813, + 0.5306620252116886 + ], + "peak_layer": 2, + "peak_normalized": 1.9618007671893674, + "population_cv": 0.3568479486512281, + "rest_mean": 5.530464544857817e-06, + "spike_contrast": 0.7788122704557785, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.3071936488559e-06, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.951489689934533e-06, + 1.0474698683538008e-05, + 9.072491593542509e-06, + 8.000228262972087e-06, + 8.083494321908802e-06, + 7.612113222421613e-06, + 7.14670659363037e-06, + 6.639023922616616e-06, + 6.039120307832491e-06, + 5.630937266687397e-06, + 5.359353508538334e-06, + 5.141370365890907e-06, + 5.098591373098316e-06, + 4.885571797785815e-06, + 4.739741143566789e-06, + 4.618921138899168e-06, + 4.6827317419229075e-06, + 4.54903238278348e-06, + 4.556341991701629e-06, + 4.529045781964669e-06, + 4.460462150746025e-06, + 4.363975676824339e-06, + 4.3109639591421e-06, + 4.248049663146958e-06, + 4.15251679442008e-06, + 4.00755061491509e-06, + 3.7691477245971328e-06, + 3.5547336665331386e-06, + 3.332451569804107e-06, + 3.0715416414750507e-06, + 2.942733544841758e-06, + 2.833378857758362e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.269887292844942e-05, + "first_to_last_ratio": 2.3568639793204684, + "imbalance_abs_log_ratio": 0.8573319130483905, + "last_quartile_mean": 1.3873890566173941e-05, + "mean": 2.12624314031018e-05, + "normalized": [ + 1.6816001107908207, + 1.968609461712434, + 1.7075260440141944, + 1.4905139409933708, + 1.4972595154646264, + 1.4034185225085787, + 1.3238781474315933, + 1.2301608589864579, + 1.121553773570522, + 1.0499501635664128, + 1.001131281922829, + 0.9633193641679035, + 0.9560899219831485, + 0.9203740514079343, + 0.8930538331784411, + 0.8703809179546808, + 0.8841528402061924, + 0.8558056863273277, + 0.8550540488218812, + 0.8483642525948266, + 0.8350365031100208, + 0.8178976965352276, + 0.8078428125234317, + 0.7969682791900239, + 0.7797478584358322, + 0.7532962238158083, + 0.7097589943815636, + 0.6713244476982704, + 0.629948240254275, + 0.5812766747709892, + 0.5573943041371163, + 0.5373112275432649 + ], + "peak_layer": 2, + "peak_normalized": 1.968609461712434, + "population_cv": 0.35388972104860866, + "rest_mean": 2.2020403102268808e-05, + "spike_contrast": 0.7797034481094922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.716938422759995e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5754907003138214e-05, + 4.185742363915779e-05, + 3.630615537986159e-05, + 3.169195042573847e-05, + 3.183537774020806e-05, + 2.9840090064681135e-05, + 2.8148868295829743e-05, + 2.6156210878980346e-05, + 2.3846960175433196e-05, + 2.2324493329506367e-05, + 2.128648520738352e-05, + 2.048251189989969e-05, + 2.0328796381363645e-05, + 1.9569390133256093e-05, + 1.8988495867233723e-05, + 1.8506414562580176e-05, + 1.8799239114741795e-05, + 1.819650969991926e-05, + 1.8180528059019707e-05, + 1.8038286725641228e-05, + 1.775490636646282e-05, + 1.739049366733525e-05, + 1.7176702385768294e-05, + 1.6945483366725966e-05, + 1.6579335351707414e-05, + 1.6016909285099246e-05, + 1.5091201930772513e-05, + 1.4273990018409677e-05, + 1.3394231245911215e-05, + 1.2359355423541274e-05, + 1.1851558156195097e-05, + 1.1424543117755093e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.02899768785573542, + "first_to_last_ratio": 2.3568640590123793, + "imbalance_abs_log_ratio": 0.8573319468610798, + "last_quartile_mean": 0.012303504627197981, + "mean": 0.018855736823752522, + "normalized": [ + 1.6816000706700753, + 1.9686094816861437, + 1.707526129634271, + 1.4905139760038595, + 1.4972595403829805, + 1.403418590801014, + 1.323878107307901, + 1.2301608352967786, + 1.12155375183925, + 1.0499501677926542, + 1.0011313114380156, + 0.963319363752492, + 0.9560898565462423, + 0.9203740066049206, + 0.8930539078006676, + 0.8703809048152793, + 0.8841528764919796, + 0.8558057204161084, + 0.8550539741588212, + 0.848364321524986, + 0.8350365815065012, + 0.8178976558963745, + 0.8078427780491525, + 0.7969682399430816, + 0.7797478175310808, + 0.7532962276748001, + 0.7097590061776181, + 0.6713244229257764, + 0.6299482002622888, + 0.5812766802470385, + 0.5573943141138397, + 0.5373111807080079 + ], + "peak_layer": 2, + "peak_normalized": 1.9686094816861437, + "population_cv": 0.3538897326937565, + "rest_mean": 0.019527913761083728, + "spike_contrast": 0.7797034311216169, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.01522598136216402, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.03170780837535858, + 0.037119582295417786, + 0.032196663320064545, + 0.028104739263653755, + 0.02823193185031414, + 0.02646249160170555, + 0.02496269717812538, + 0.02319558896124363, + 0.021147722378373146, + 0.019797584041953087, + 0.018877068534493446, + 0.018164096400141716, + 0.01802777871489525, + 0.01735433004796505, + 0.01683918945491314, + 0.0164116732776165, + 0.016671353951096535, + 0.01613684743642807, + 0.016122672706842422, + 0.015996534377336502, + 0.01574523001909256, + 0.015422062948346138, + 0.015232470817863941, + 0.015027423389256, + 0.014702719636261463, + 0.014203955419361591, + 0.013383029028773308, + 0.012658316642045975, + 0.011878137476742268, + 0.010960400104522705, + 0.010510080493986607, + 0.01013139821588993 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.231474328733384e-06, + "first_to_last_ratio": 1.3582476774197694, + "imbalance_abs_log_ratio": 0.3061953964681102, + "last_quartile_mean": 9.066640416222072e-07, + "mean": 1.0095540616816834e-06, + "normalized": [ + 2.2510689330098104, + 1.3453781719433024, + 1.2282097485564385, + 1.0527316026565288, + 1.0056637182516928, + 0.969318760716823, + 0.9573356047524155, + 0.9488541987585868, + 0.9217951398143368, + 0.8906270200688119, + 0.8763270628110001, + 0.8694286288822738, + 0.8685312885606712, + 0.8765222738855435, + 0.8891328417344724, + 0.8990263889886259, + 0.900299568341834, + 0.9275418452948997, + 0.9582330013794911, + 0.9876130845276129, + 1.0001282145999768, + 1.0337196092881593, + 1.0681495050787642, + 1.0896942312291085, + 1.0818926575150707, + 1.0552217685009913, + 0.9988175358018173, + 0.923772869549939, + 0.860068575983845, + 0.7959012825140968, + 0.7583091587544984, + 0.710685708248561 + ], + "peak_layer": 1, + "peak_normalized": 2.2510689330098104, + "population_cv": 0.25617417026860584, + "rest_mean": 9.993245039704899e-07, + "spike_contrast": 1.065513423409031, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0647936733221285e-06, + "top_five_layers": [ + 1, + 2, + 3, + 24, + 25 + ], + "top_five_spike_overlap": 2 + }, + "values": [ + 2.2725757844455075e-06, + 1.3582319979832391e-06, + 1.2399441402521916e-06, + 1.0627894653225667e-06, + 1.0152718914469006e-06, + 9.785796919459244e-07, + 9.664820481702918e-07, + 9.579196103004506e-07, + 9.306020274379989e-07, + 8.991361255539232e-07, + 8.846995456224249e-07, + 8.777352036304364e-07, + 8.768292900640517e-07, + 8.848966217556153e-07, + 8.976276717476139e-07, + 9.076157425624842e-07, + 9.089010859497648e-07, + 9.364036372971896e-07, + 9.673880185800954e-07, + 9.970488008548273e-07, + 1.0096835012518568e-06, + 1.043595830196864e-06, + 1.0783546713355463e-06, + 1.1001052371284459e-06, + 1.09222912669793e-06, + 1.0653034223651048e-06, + 1.008360300147615e-06, + 9.325986525254848e-07, + 8.682857242092723e-07, + 8.035053724597674e-07, + 7.655540912310244e-07, + 7.174756433414586e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.523178955016192e-05, + "first_to_last_ratio": 2.651853802016533, + "imbalance_abs_log_ratio": 0.9752589433592965, + "last_quartile_mean": 9.514773978480662e-06, + "mean": 1.5758304414248414e-05, + "normalized": [ + 1.768014772712438, + 2.0202664913584756, + 1.7747876589600569, + 1.5702354129512541, + 1.5745618645822212, + 1.4685789787379961, + 1.3665832942020837, + 1.266364662649249, + 1.14652665049396, + 1.0704311524660057, + 1.0164222471522504, + 0.9740486236599476, + 0.9677569672847843, + 0.9245129976091487, + 0.891368391145493, + 0.8644143239142155, + 0.883928662735923, + 0.8455178269951987, + 0.8460475954048016, + 0.8329649879043416, + 0.8145835420025702, + 0.783683597592109, + 0.7613793029015486, + 0.7366657999397114, + 0.716423036769208, + 0.6895960585495816, + 0.6528852259142993, + 0.6205906498609965, + 0.5818984522075571, + 0.54251886575813, + 0.5218602063941854, + 0.5045816991902581 + ], + "peak_layer": 2, + "peak_normalized": 2.0202664913584756, + "population_cv": 0.3941497713845743, + "rest_mean": 1.645124066945519e-05, + "spike_contrast": 0.7304281104125244, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2016448636131827e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.786091499729082e-05, + 3.183597436873242e-05, + 2.7967644200543873e-05, + 2.4744247639318928e-05, + 2.481242518115323e-05, + 2.314231460331939e-05, + 2.1535035557462834e-05, + 1.9955759853473864e-05, + 1.8067315977532417e-05, + 1.6868179955054075e-05, + 1.60170911840396e-05, + 1.5349354725913145e-05, + 1.5250208889483474e-05, + 1.4568757251254283e-05, + 1.404645445290953e-05, + 1.362170405627694e-05, + 1.3929216947872192e-05, + 1.3323927305464167e-05, + 1.3332275557331741e-05, + 1.3126115845807362e-05, + 1.2836455425713211e-05, + 1.234952469530981e-05, + 1.1998046829830855e-05, + 1.1608603927015793e-05, + 1.1289612302789465e-05, + 1.086686461349018e-05, + 1.0288364137522876e-05, + 9.779456377145834e-06, + 9.169732948066667e-06, + 8.549177437089384e-06, + 8.22363199404208e-06, + 7.95135201769881e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.9022438777938078e-05, + "first_to_last_ratio": 2.5706811468129205, + "imbalance_abs_log_ratio": 0.9441709014612467, + "last_quartile_mean": 7.3997659342239785e-06, + "mean": 1.1811503938474743e-05, + "normalized": [ + 1.7838575112897632, + 2.0081895041574866, + 1.8013630165639003, + 1.6167453942621992, + 1.616253821452643, + 1.484346910873369, + 1.334088872746376, + 1.2391628829890091, + 1.1149528907165038, + 1.0471548053789776, + 0.9987803140821989, + 0.9467453609714046, + 0.9424805202441363, + 0.9017388056265639, + 0.868090708413055, + 0.8437486142496254, + 0.8736503938091038, + 0.8327134052809684, + 0.8458575878838563, + 0.8280086592496597, + 0.8103372943527495, + 0.7723135523278634, + 0.7500419314219764, + 0.7274729336065492, + 0.7078485216009954, + 0.6848456245402912, + 0.657709866043606, + 0.6402184904077136, + 0.6083853394516672, + 0.5800023997525418, + 0.5702454032899359, + 0.5626486629633098 + ], + "peak_layer": 2, + "peak_normalized": 2.0081895041574866, + "population_cv": 0.39403850931287454, + "rest_mean": 1.2350452262008158e-05, + "spike_contrast": 0.7207171691012223, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.901182991394307e-06, + "top_five_layers": [ + 2, + 3, + 1, + 4, + 5 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.1070040020276792e-05, + 2.3719738237559795e-05, + 2.1276806364767253e-05, + 1.9096194591838866e-05, + 1.9090388377662748e-05, + 1.7532369383843616e-05, + 1.5757595974719152e-05, + 1.4636377272836398e-05, + 1.3169270459911786e-05, + 1.2368473107926548e-05, + 1.1797097613452934e-05, + 1.1182486559846438e-05, + 1.1132112376799341e-05, + 1.065089145413367e-05, + 1.025345682137413e-05, + 9.965940080292057e-06, + 1.031912506732624e-05, + 9.835597666096874e-06, + 9.990850230678916e-06, + 9.780027539818548e-06, + 9.571302143740468e-06, + 9.122184565057978e-06, + 8.859123227011878e-06, + 8.592549420427531e-06, + 8.360755600733683e-06, + 8.089056791504845e-06, + 7.76854267314775e-06, + 7.5619432209350634e-06, + 7.185945833043661e-06, + 6.850700629001949e-06, + 6.735455826856196e-06, + 6.645726898568682e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 6.828027426308836e-05, + "first_to_last_ratio": 2.3127230006738477, + "imbalance_abs_log_ratio": 0.838425618478933, + "last_quartile_mean": 2.9523758030336467e-05, + "mean": 4.4895063126659807e-05, + "normalized": [ + 1.6579137880786858, + 1.9150708402023993, + 1.6745616807855777, + 1.4835198878876907, + 1.4915155698146907, + 1.3960120954471311, + 1.3299713054901032, + 1.2185230612071285, + 1.1078646449578662, + 1.0477053876101448, + 0.999765654013293, + 0.9692189920089408, + 0.955488933285625, + 0.9355193384387992, + 0.9031699366923724, + 0.8872109049801036, + 0.8933262176117003, + 0.8675434009211797, + 0.864331741509242, + 0.8580824004003949, + 0.8343570910048359, + 0.8204903297173597, + 0.8195590181762232, + 0.8083415477202021, + 0.7958344380770943, + 0.7671025897223802, + 0.7201771453938138, + 0.6826439856557369, + 0.6331928257241357, + 0.58535097991478, + 0.5516010454074354, + 0.5250332221429341 + ], + "peak_layer": 2, + "peak_normalized": 1.9150708402023993, + "population_cv": 0.34285892534032564, + "rest_mean": 4.642717794922646e-05, + "spike_contrast": 0.788797525553888, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.662164308479987e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.443214417435229e-05, + 8.597722626291215e-05, + 7.517955236835405e-05, + 6.660271901637316e-05, + 6.696168566122651e-05, + 6.267405115067959e-05, + 5.9709145716624334e-05, + 5.470566975418478e-05, + 4.9737653171177953e-05, + 4.703679951489903e-05, + 4.4884542148793116e-05, + 4.351314782979898e-05, + 4.2896735976682976e-05, + 4.200019975542091e-05, + 4.0547871321905404e-05, + 3.983138958574273e-05, + 4.010593693237752e-05, + 3.89484157494735e-05, + 3.8804228097433224e-05, + 3.8523663533851504e-05, + 3.745851427083835e-05, + 3.683596514747478e-05, + 3.6794153857044876e-05, + 3.629054481280036e-05, + 3.5729037335840985e-05, + 3.4439119190210477e-05, + 3.233239840483293e-05, + 3.064734482904896e-05, + 2.8427231882233173e-05, + 2.6279369194526225e-05, + 2.4764163754298352e-05, + 2.357139965170063e-05 + ] + } + } + }, + "pre_attention_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.0626930519938469, + 0.09030644595623016, + 0.20229491591453552, + 0.20300470292568207, + 0.19049881398677826, + 0.12695962190628052, + 0.12132706493139267, + 0.11585728824138641, + 0.13427142798900604, + 0.10485932976007462, + 0.10419609397649765, + 0.10852954536676407, + 0.1286265105009079, + 0.09096142649650574, + 0.10080574452877045, + 0.10566592961549759, + 0.1287592649459839, + 0.09650235623121262, + 0.1065765842795372, + 0.11446042358875275, + 0.12395420670509338, + 0.07221997529268265, + 0.07677769660949707, + 0.08416415005922318, + 0.09094838052988052, + 0.059439923614263535, + 0.062182098627090454, + 0.06896103918552399, + 0.0776243805885315, + 0.06285123527050018, + 0.07231374084949493, + 0.08016443997621536 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 6.4383448545868305e-06, + "first_to_last_ratio": 5.042948145238695, + "imbalance_abs_log_ratio": 1.6179908605191557, + "last_quartile_mean": 1.276702569441568e-06, + "mean": 3.687052599232743e-06, + "normalized": [ + 3.304965820363612, + 2.227169292436081, + 1.6521910965438824, + 1.3947865539668498, + 0.5800793227551304, + 1.2714767915022098, + 1.750193241091026, + 1.7887674026051423, + 1.1778703519658833, + 1.1179830646270124, + 0.8417156564271726, + 0.7655624105040819, + 0.5268426976300161, + 1.047779560839775, + 0.8380697743291855, + 0.7915788420213868, + 0.7099070375965003, + 1.3022194692601015, + 1.0739447359104763, + 1.3540969285822078, + 0.6285621976492703, + 1.167508624430284, + 0.9740363058232884, + 0.9425613189042933, + 0.35040916273624656, + 0.44668469186889276, + 0.3295723924580137, + 0.4329866871315858, + 0.2917160480921282, + 0.22501238083060152, + 0.3167851028104899, + 0.37696503630717276 + ], + "peak_layer": 1, + "peak_normalized": 3.304965820363612, + "population_cv": 0.641995540013263, + "rest_mean": 3.814996382010981e-06, + "spike_contrast": 0.7853627820875961, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.9961561722302575e-06, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.218558281834703e-05, + 8.211690328607801e-06, + 6.091715476941317e-06, + 5.142651389178354e-06, + 2.1387829747254727e-06, + 4.688001808972331e-06, + 6.4530545387242455e-06, + 6.5952795011980925e-06, + 4.342869942774996e-06, + 4.122062364331214e-06, + 3.1034498988447012e-06, + 2.8226688755239593e-06, + 1.942496737683541e-06, + 3.863218353217235e-06, + 3.090007339778822e-06, + 2.918592826972599e-06, + 2.6174645881837932e-06, + 4.80135167890694e-06, + 3.959690729971044e-06, + 4.992626600142103e-06, + 2.3175418846221874e-06, + 4.3046657083323225e-06, + 3.5913230931328144e-06, + 3.475273160802317e-06, + 1.2919770142616471e-06, + 1.646949954192678e-06, + 1.215150746247673e-06, + 1.5964446902216878e-06, + 1.0755724133559852e-06, + 8.296324836010172e-07, + 1.1680033367156284e-06, + 1.3898899169362267e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.5929461344276206e-05, + "first_to_last_ratio": 5.201390647867945, + "imbalance_abs_log_ratio": 1.6489260221160402, + "last_quartile_mean": 4.985101696775018e-06, + "mean": 1.4714017034123117e-05, + "normalized": [ + 3.407292787305506, + 2.2558164668132807, + 1.6137698282608655, + 1.433949322998548, + 0.5790123126574623, + 1.305711460245382, + 1.7258967842855322, + 1.7763796775082434, + 1.1315333512208903, + 1.0856488588758444, + 0.8714057435432361, + 0.7593394863607054, + 0.5216263168993628, + 1.0262173986737884, + 0.8250528440437026, + 0.7676154211606976, + 0.723733557926003, + 1.3830217582174593, + 1.0774149558606307, + 1.3018941088272185, + 0.646511650324857, + 1.1377981887421273, + 0.9823485776546648, + 0.9506130966721329, + 0.33641925298333475, + 0.4509311419254927, + 0.32185944273605893, + 0.4000452689662174, + 0.290655292962286, + 0.22712751082346105, + 0.30970536495859485, + 0.3736527695664126 + ], + "peak_layer": 1, + "peak_normalized": 3.407292787305506, + "population_cv": 0.6554716761859383, + "rest_mean": 1.522972111497775e-05, + "spike_contrast": 0.7832851900207317, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.1929214997508097e-05, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.0134964112658054e-05, + 3.3192121918546036e-05, + 2.374503674218431e-05, + 2.1099154764669947e-05, + 8.519597031408921e-06, + 1.921226066770032e-05, + 2.539487468311563e-05, + 2.6137680833926424e-05, + 1.6649401004542597e-05, + 1.59742558025755e-05, + 1.2821878954127897e-05, + 1.1172934136993717e-05, + 7.675218512304127e-06, + 1.5099780284799635e-05, + 1.2139841601310764e-05, + 1.1294706382614095e-05, + 1.0649027899489738e-05, + 2.03498057089746e-05, + 1.5853102013352327e-05, + 1.9156092093908228e-05, + 9.512783435638994e-06, + 1.674158193054609e-05, + 1.4454293705057353e-05, + 1.3987337297294289e-05, + 4.9500786190037616e-06, + 6.635008503508288e-06, + 4.735845323011745e-06, + 5.886272901989287e-06, + 4.276706931705121e-06, + 3.3419580631743884e-06, + 4.557010015560081e-06, + 5.497933216247475e-06 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.022994505590759218, + "first_to_last_ratio": 5.20139065184949, + "imbalance_abs_log_ratio": 1.6489260228815172, + "last_quartile_mean": 0.004420838027726859, + "mean": 0.01304853729379829, + "normalized": [ + 3.4072926237702874, + 2.255816491845768, + 1.613769818466143, + 1.4339493120942854, + 0.5790123351120128, + 1.3057114456299215, + 1.725896924613148, + 1.7763798383286113, + 1.13153331687909, + 1.0856488688373112, + 0.8714057478512427, + 0.7593395234826519, + 0.5216262984837251, + 1.0262174043007157, + 0.8250528027479058, + 0.7676154484189387, + 0.7237335326830917, + 1.3830217401378855, + 1.0774149116451128, + 1.3018940938870247, + 0.6465115988335997, + 1.1377982160374271, + 0.9823485499546105, + 0.9506130843152019, + 0.3364192535702373, + 0.45093116963769325, + 0.3218594440675733, + 0.4000452442436667, + 0.2906553022018551, + 0.22712752175948964, + 0.30970535698369017, + 0.3736527791800823 + ], + "peak_layer": 1, + "peak_normalized": 3.4072926237702874, + "population_cv": 0.6554716682951498, + "rest_mean": 0.013505868855173941, + "spike_contrast": 0.7832851759342461, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.010578946862369775, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.04446018487215042, + 0.029435105621814728, + 0.021057335659861565, + 0.018710941076278687, + 0.007555264048278332, + 0.01703762449324131, + 0.022520430386066437, + 0.023179158568382263, + 0.014764854684472084, + 0.014166129752993584, + 0.01137057039886713, + 0.009908270090818405, + 0.0068064602091908455, + 0.013390636071562767, + 0.010765732266008854, + 0.010016258805990219, + 0.009443663991987705, + 0.018046410754323006, + 0.014058688655495644, + 0.016987813636660576, + 0.008436030708253384, + 0.014846602454781532, + 0.012818211689591408, + 0.01240411028265953, + 0.0043897791765630245, + 0.005883992183953524, + 0.004199794959276915, + 0.005220005288720131, + 0.0037926265504211187, + 0.0029636819381266832, + 0.004041201900690794, + 0.004875622224062681 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.464145257619066e-06, + "first_to_last_ratio": 2.649593113901558, + "imbalance_abs_log_ratio": 0.9744060862854225, + "last_quartile_mean": 9.300089303110326e-07, + "mean": 1.2621179932281734e-06, + "normalized": [ + 9.98073298735891, + 2.6081595638248576, + 0.448046131778418, + 0.5712169168045845, + 0.31851900459136234, + 0.5125315286968626, + 0.5097468218730781, + 0.6701588608628778, + 0.5743479667542774, + 0.6374764102214231, + 0.546790403537672, + 0.5390341640801587, + 0.35740547847929505, + 0.6625310250792219, + 0.6002964531361203, + 0.6405992632336756, + 0.4466467073820306, + 0.8660901213820976, + 0.747225340722879, + 0.8852501473817866, + 0.3909436983971323, + 0.8018997241352653, + 0.8266134004412924, + 0.9628283052981523, + 0.5300152809900267, + 0.7412196878662661, + 0.7889073080638102, + 1.074944484309473, + 0.6781360573263348, + 0.7166665274120905, + 0.6792494446365528, + 0.6857707839420146 + ], + "peak_layer": 1, + "peak_normalized": 9.98073298735891, + "population_cv": 1.6572947531819389, + "rest_mean": 1.3316606755240543e-06, + "spike_contrast": 0.6657758429950736, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.865875088304165e-07, + "top_five_layers": [ + 1, + 2, + 28, + 24, + 20 + ], + "top_five_spike_overlap": 1 + }, + "values": [ + 1.259686268895166e-05, + 3.2918051147134975e-06, + 5.654870847138227e-07, + 7.209431487353868e-07, + 4.020085668798856e-07, + 6.468752644650522e-07, + 6.433606358768884e-07, + 8.45819556616334e-07, + 7.248949032145902e-07, + 8.045704475989623e-07, + 6.901140068293898e-07, + 6.803247174502758e-07, + 4.51087885267043e-07, + 8.361923278243921e-07, + 7.576449547741504e-07, + 8.08511856575933e-07, + 5.637208460029797e-07, + 1.0931079259535181e-06, + 9.430865475223982e-07, + 1.1172901395184454e-06, + 4.934170760861889e-07, + 1.012092070595827e-06, + 1.0432836461404804e-06, + 1.2152029285061872e-06, + 6.68941822823399e-07, + 9.355067049909849e-07, + 9.956941084965365e-07, + 1.356706775368366e-06, + 8.558877198083792e-07, + 9.045177193911513e-07, + 8.572929459660372e-07, + 8.655236456434068e-07 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.6010561125767708e-05, + "first_to_last_ratio": 4.917014092262474, + "imbalance_abs_log_ratio": 1.5927014544542644, + "last_quartile_mean": 3.2561552245624625e-06, + "mean": 9.725274459526645e-06, + "normalized": [ + 2.0495551017102773, + 2.324610334145404, + 1.6789860752319785, + 1.469067342376247, + 0.683331640000088, + 1.4508358782825064, + 1.8574881471610558, + 1.6563953320037643, + 1.2394147680445438, + 1.1757596976660154, + 0.9385611021097731, + 0.8217156066495895, + 0.5863388014820525, + 1.1122247987743321, + 0.8990502135337656, + 0.7981592831866082, + 0.7526244364476551, + 1.397025330921264, + 1.1617275022678442, + 1.3251897151639205, + 0.6397940246687674, + 1.2096957514764108, + 1.0918700868694384, + 1.0020693481814167, + 0.3582271576029711, + 0.48201484763579705, + 0.332690365953382, + 0.356368076645877, + 0.30908080698932155, + 0.24694487721404904, + 0.26695462213676374, + 0.32622892746712 + ], + "peak_layer": 2, + "peak_normalized": 2.324610334145404, + "population_cv": 0.5441079654169187, + "rest_mean": 9.97681458843376e-06, + "spike_contrast": 0.83864019815785, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 8.366957763428217e-06, + "top_five_layers": [ + 2, + 1, + 7, + 3, + 8 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.9932485884055495e-05, + 2.2607473511015996e-05, + 1.6328600395354442e-05, + 1.42870831041364e-05, + 6.645587745879311e-06, + 1.4109777112025768e-05, + 1.8064582036458887e-05, + 1.6108899217215367e-05, + 1.2053648788423743e-05, + 1.143458575825207e-05, + 9.127764315053355e-06, + 7.991409802343696e-06, + 5.7023057706828695e-06, + 1.0816691428772174e-05, + 8.743510079511907e-06, + 7.762318091408815e-06, + 7.319479209400015e-06, + 1.3586454770120326e-05, + 1.1298118806735147e-05, + 1.2887833690911066e-05, + 6.222172487468924e-06, + 1.176462319563143e-05, + 1.0618736268952489e-05, + 9.745399438543245e-06, + 3.4838574265450006e-06, + 4.6877266868250445e-06, + 3.2355051189369988e-06, + 3.465777353994781e-06, + 3.0058956781431334e-06, + 2.4016067072807346e-06, + 2.5962069685192546e-06, + 3.1726658562547527e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 9.92899242646672e-06, + "first_to_last_ratio": 4.574804953054558, + "imbalance_abs_log_ratio": 1.520564064606545, + "last_quartile_mean": 2.1703640982195793e-06, + "mean": 6.422408389994416e-06, + "normalized": [ + 1.2694651099378356, + 2.2660081023317744, + 1.6776221325435599, + 1.4243775268645795, + 0.778686696628982, + 1.544586331363982, + 1.9675189416357637, + 1.4396717003002808, + 1.3121814531183145, + 1.2749800755885756, + 1.028924014649809, + 0.867611835816526, + 0.6467747339714943, + 1.1792708375805625, + 0.9308967888836265, + 0.8222106571344964, + 0.7916222394567357, + 1.4570716548377307, + 1.2330890513227566, + 1.308271526002021, + 0.6046831152920485, + 1.2497078701132496, + 1.207744340136328, + 1.0135338985983424, + 0.3691120245165777, + 0.5161808316774584, + 0.34931346918180795, + 0.3250152051676304, + 0.32914583570336414, + 0.26959120942422543, + 0.2525212479205583, + 0.29260954229900277 + ], + "peak_layer": 2, + "peak_normalized": 2.2660081023317744, + "population_cv": 0.5120965787375209, + "rest_mean": 6.5544765961762836e-06, + "spike_contrast": 0.8710443912398678, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.7092400766123316e-06, + "top_five_layers": [ + 2, + 7, + 3, + 6, + 18 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 8.153023372869939e-06, + 1.4553229448210914e-05, + 1.0774374459288083e-05, + 9.147934179054573e-06, + 5.001043973607011e-06, + 9.919964213622734e-06, + 1.2636210158234462e-05, + 9.24615960684605e-06, + 8.427365173702128e-06, + 8.188442734535784e-06, + 6.608170224353671e-06, + 5.572157533606514e-06, + 4.153851477894932e-06, + 7.5737589213531464e-06, + 5.978599347145064e-06, + 5.280572622723412e-06, + 5.084121312393108e-06, + 9.357909220852889e-06, + 7.919401468825527e-06, + 8.402254024986178e-06, + 3.883521912939614e-06, + 8.026134310057387e-06, + 7.756627383059822e-06, + 6.509328613901744e-06, + 2.3705881631030934e-06, + 3.3151241041196045e-06, + 2.2434337552112993e-06, + 2.087380380544346e-06, + 2.1139089767530095e-06, + 1.731424845274887e-06, + 1.6217945812968537e-06, + 1.8792579794535413e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 4.949699314238387e-05, + "first_to_last_ratio": 5.262607641337885, + "imbalance_abs_log_ratio": 1.6606266532103107, + "last_quartile_mean": 9.405412015439651e-06, + "mean": 2.8804784136582384e-05, + "normalized": [ + 2.499900864670067, + 2.416668640808701, + 1.6691856383541526, + 1.5215442251975408, + 0.6043399354947968, + 1.3939313584258168, + 1.817946933469592, + 1.823363590450555, + 1.1883363380697656, + 1.0810267152424702, + 0.8648382834598256, + 0.8031803638827597, + 0.533176912017834, + 1.0261239559732709, + 0.8554824003261805, + 0.7869370867194827, + 0.7081002132573654, + 1.3561635382832782, + 1.1101608226630673, + 1.398379059233539, + 0.6739882703135641, + 1.211767597856563, + 0.9998718364198766, + 1.043404895274106, + 0.34963454859286996, + 0.4704717001014101, + 0.31939467298460794, + 0.36230101093546613, + 0.28800141494502707, + 0.22353921217823322, + 0.2657184225315432, + 0.3331195418666723 + ], + "peak_layer": 1, + "peak_normalized": 2.499900864670067, + "population_cv": 0.5921886777524846, + "rest_mean": 2.957433365803876e-05, + "spike_contrast": 0.8334665120685795, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4649216720717958e-05, + "top_five_layers": [ + 1, + 2, + 8, + 7, + 3 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 7.200910476967692e-05, + 6.961161852814257e-05, + 4.8080531996674836e-05, + 4.382775296107866e-05, + 1.7407881387043744e-05, + 4.01518918806687e-05, + 5.2365568990353495e-05, + 5.2521594625432044e-05, + 3.4229771699756384e-05, + 3.113874117843807e-05, + 2.4911480068112724e-05, + 2.3135437004384585e-05, + 1.5358045857283287e-05, + 2.9557279049186036e-05, + 2.4641985874040984e-05, + 2.2667552912025712e-05, + 2.0396673789946362e-05, + 3.906399797415361e-05, + 3.197794285370037e-05, + 4.028000694233924e-05, + 1.941408663697075e-05, + 3.490470407996327e-05, + 2.880109241232276e-05, + 3.0055052775423974e-05, + 1.0071147698909044e-05, + 1.3551835763792042e-05, + 9.200094609695952e-06, + 1.0436002412461676e-05, + 8.295818588521797e-06, + 6.438998752855696e-06, + 7.65396180213429e-06, + 9.595436495146714e-06 + ] + } + } + }, + "pre_mlp_input": { + "all_gradients_finite": true, + "all_gradients_present": true, + "capture": { + "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", + "count": 32, + "dtypes": [ + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32", + "torch.float32" + ], + "output_rms_by_layer": [ + 0.059514161199331284, + 0.08157797157764435, + 0.14292451739311218, + 0.10997853428125381, + 0.0991024449467659, + 0.1065012738108635, + 0.12155740708112717, + 0.1095810979604721, + 0.06964563578367233, + 0.08311934769153595, + 0.09363401681184769, + 0.0941167026758194, + 0.08564268052577972, + 0.09493856877088547, + 0.09063401818275452, + 0.09473779052495956, + 0.10505300015211105, + 0.09355910867452621, + 0.09618847072124481, + 0.11416846513748169, + 0.047205954790115356, + 0.053352247923612595, + 0.06512638926506042, + 0.07829347252845764, + 0.05368548259139061, + 0.05310077592730522, + 0.061235878616571426, + 0.07320034503936768, + 0.05304349213838577, + 0.0639304369688034, + 0.07404901832342148, + 0.08157160878181458 + ], + "shape": [ + 16, + 256, + 192 + ], + "storage_unique": true + }, + "reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.6960831434053034e-06, + "first_to_last_ratio": 0.38455700861487224, + "imbalance_abs_log_ratio": 0.9556632341481542, + "last_quartile_mean": 7.0108802674440085e-06, + "mean": 4.081287496404684e-06, + "normalized": [ + 1.6016557623725205, + 0.5624254649150265, + 0.3792516557450063, + 0.6472346627621453, + 0.3825774509297101, + 0.309883335456179, + 0.41924156145633357, + 0.9824999654437843, + 0.9295888075476517, + 0.6942195909151134, + 0.6392271159962603, + 0.6622046687372564, + 0.5739498407816973, + 0.6102779288338825, + 0.725372269896936, + 0.7725414351920691, + 0.5151773567621832, + 0.6691928665493493, + 0.6340666932552725, + 0.5067672404198594, + 0.8579891398085714, + 1.216818523393502, + 1.4001261788695554, + 1.56522304857102, + 1.4389835581991173, + 1.8374458775865694, + 2.199543673756739, + 1.6451073151800026, + 2.0248581159734593, + 1.9770863776726855, + 1.3258318688286215, + 1.2936306481919198 + ], + "peak_layer": 27, + "peak_normalized": 2.199543673756739, + "population_cv": 0.5384151707957241, + "rest_mean": 3.857702406801562e-06, + "spike_contrast": 1.3709318196595643, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.288646980261546e-06, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 6.5368176365154795e-06, + 2.295420017617289e-06, + 1.5478350405828678e-06, + 2.6415507363708457e-06, + 1.5614085668858024e-06, + 1.2647229823414818e-06, + 1.7110453427449102e-06, + 4.009864824183751e-06, + 3.793919177041971e-06, + 2.833309736161027e-06, + 2.6088696358783636e-06, + 2.7026476345781703e-06, + 2.3424543087458005e-06, + 2.4907196802814724e-06, + 2.960452775369049e-06, + 3.1529636999039212e-06, + 2.1025869045843137e-06, + 2.7311684789310675e-06, + 2.587808467069408e-06, + 2.068262801913079e-06, + 3.501700348351733e-06, + 4.966186224919511e-06, + 5.7143174672091845e-06, + 6.388125257217325e-06, + 5.872905603609979e-06, + 7.499144885514397e-06, + 8.976970093499403e-06, + 6.714155915688025e-06, + 8.264028110716026e-06, + 8.06905791250756e-06, + 5.411101028585108e-06, + 5.279678589431569e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.0839275830676343e-05, + "first_to_last_ratio": 0.38713025500084114, + "imbalance_abs_log_ratio": 0.9489940663518645, + "last_quartile_mean": 2.7999040867143776e-05, + "mean": 1.6318590368769037e-05, + "normalized": [ + 1.6422297380818347, + 0.577857691983563, + 0.3782290994995378, + 0.6502872006236952, + 0.3822780102727073, + 0.31178153988993046, + 0.41970720095668634, + 0.9514590030920431, + 0.920750817583778, + 0.6902360262942715, + 0.6326124977684876, + 0.6666642010329482, + 0.573439016269761, + 0.6073224038931471, + 0.7243939579070396, + 0.7687525976401163, + 0.5168423803555005, + 0.6726938023393403, + 0.6400255756173949, + 0.5037402338382693, + 0.8679547154047897, + 1.2120933210153013, + 1.403510673963058, + 1.5589326681275335, + 1.4338206424993105, + 1.8383886875461704, + 2.207829035366123, + 1.646625228594747, + 2.022240419628589, + 1.965711109970758, + 1.3282014630518373, + 1.283389039891731 + ], + "peak_layer": 27, + "peak_normalized": 2.207829035366123, + "population_cv": 0.5389668648883295, + "rest_mean": 1.542631884553379e-05, + "spike_contrast": 1.3701814934519443, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.1136856594239363e-05, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.6798874387168325e-05, + 9.429822966922075e-06, + 6.1721657402813435e-06, + 1.061177044903161e-05, + 6.238238256628392e-06, + 5.087835234007798e-06, + 6.849029887234792e-06, + 1.5526469724136405e-05, + 1.5025355423858855e-05, + 1.1263678970863111e-05, + 1.0323344213247765e-05, + 1.0879020010179374e-05, + 9.357716407976113e-06, + 9.910645530908369e-06, + 1.18210882646963e-05, + 1.254495873581618e-05, + 8.434139090240933e-06, + 1.097741460398538e-05, + 1.044431519403588e-05, + 8.220330528274644e-06, + 1.4163797459332272e-05, + 1.9779654394369572e-05, + 2.2903315766598098e-05, + 2.5439583623665385e-05, + 2.339793172723148e-05, + 2.9999911930644885e-05, + 3.602865763241425e-05, + 2.687060259631835e-05, + 3.300011303508654e-05, + 3.2077634386951104e-05, + 2.1674375602742657e-05, + 2.094310002576094e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.009612377849407494, + "first_to_last_ratio": 0.38713024225946485, + "imbalance_abs_log_ratio": 0.9489940992642404, + "last_quartile_mean": 0.024829829344525933, + "mean": 0.014471489033894613, + "normalized": [ + 1.6422296302631012, + 0.5778576663317349, + 0.37822910199946935, + 0.6502872213280481, + 0.38227800767762105, + 0.3117815365380894, + 0.4197072091098399, + 0.9514589510980404, + 0.9207507998752875, + 0.6902360121427672, + 0.6326125119311165, + 0.6666641983580384, + 0.573439005071423, + 0.607322401384804, + 0.7243939498038978, + 0.7687525851502272, + 0.5168424285086333, + 0.6726938110990978, + 0.6400255788101445, + 0.5037402548123486, + 0.8679547482284273, + 1.2120933573235502, + 1.4035107155843847, + 1.5589326526958214, + 1.4338205694791435, + 1.8383887153176988, + 2.2078290833379297, + 1.6466252521885574, + 2.022240548084217, + 1.9657110301441079, + 1.3282014367981623, + 1.28338902952427 + ], + "peak_layer": 27, + "peak_normalized": 2.2078290833379297, + "population_cv": 0.5389668665215603, + "rest_mean": 0.01368021371739882, + "spike_contrast": 1.3701814993673933, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.018744375742971896, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 0.023765508085489273, + 0.008362460881471634, + 0.005473538301885128, + 0.009410624392330647, + 0.0055321319960057735, + 0.004511943086981773, + 0.006073788274079561, + 0.013769027777016163, + 0.013324635103344917, + 0.009988742880523205, + 0.009154845029115677, + 0.009647623635828495, + 0.008298516273498535, + 0.008788859471678734, + 0.010483059100806713, + 0.011124994605779648, + 0.007479479536414146, + 0.009734881110489368, + 0.00926212314516306, + 0.007289871573448181, + 0.012560597620904446, + 0.017540795728564262, + 0.020310889929533005, + 0.022560076788067818, + 0.020749518647789955, + 0.026604222133755684, + 0.03195057436823845, + 0.02382911927998066, + 0.02926483191549778, + 0.02844676561653614, + 0.019221052527427673, + 0.018572550266981125 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 6.596527093449822e-07, + "first_to_last_ratio": 0.3487620963761673, + "imbalance_abs_log_ratio": 1.053365261498487, + "last_quartile_mean": 1.8914116992618801e-06, + "mean": 1.0618384331806396e-06, + "normalized": [ + 1.9623284636711533, + 0.5051262881892306, + 0.3546331093387114, + 0.6524291934355505, + 0.33242707700757573, + 0.26547608492472724, + 0.31949906663212685, + 0.5779720956168373, + 0.775638331440552, + 0.5998773794374831, + 0.5826887294747466, + 0.6213115726756598, + 0.5572960370183113, + 0.6050525236179888, + 0.7278412668111575, + 0.7634285200712856, + 0.5211325543229903, + 0.6528362049921996, + 0.6569013419877625, + 0.48939266573007467, + 0.8730686968435768, + 1.2548058690120942, + 1.4927712810480218, + 1.6059753116097122, + 1.4417731559107498, + 1.8108704859544356, + 2.3560676855554643, + 1.7166277442710838, + 2.056050650260877, + 2.0539911278871554, + 1.42644194116846, + 1.3882675440822447 + ], + "peak_layer": 27, + "peak_normalized": 2.3560676855554643, + "population_cv": 0.600205151665452, + "rest_mean": 9.962249070785792e-07, + "spike_contrast": 1.4215178360523206, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.416151474131766e-06, + "top_five_layers": [ + 27, + 29, + 30, + 1, + 26 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 2.0836757812503492e-06, + 5.363625064092048e-07, + 3.765630651741958e-07, + 6.927743925189134e-07, + 3.52983846596544e-07, + 2.818927100634028e-07, + 3.3925638831533433e-07, + 6.137129844319134e-07, + 8.236025905716815e-07, + 6.369728566824051e-07, + 6.187212875374826e-07, + 6.597325068469218e-07, + 5.917583507653035e-07, + 6.424680236705171e-07, + 7.728498303549713e-07, + 8.106377435979084e-07, + 5.533585749617487e-07, + 6.932065730325121e-07, + 6.975230917305453e-07, + 5.19655941388919e-07, + 9.270578971154464e-07, + 1.332401097897673e-06, + 1.5850819181650877e-06, + 1.7052863086064463e-06, + 1.5309301488741767e-06, + 1.9228518794989213e-06, + 2.50176321969775e-06, + 1.8227813143312233e-06, + 2.183193601013045e-06, + 2.181006721002632e-06, + 1.5146508758334676e-06, + 1.4741158338438254e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 7.227550952393358e-06, + "first_to_last_ratio": 0.3980164784812622, + "imbalance_abs_log_ratio": 0.9212618713355372, + "last_quartile_mean": 1.8158923921873793e-05, + "mean": 1.0911241304256691e-05, + "normalized": [ + 1.6660127430322156, + 0.5876271935840073, + 0.29483709953854387, + 0.5894374739528777, + 0.41635452404712836, + 0.33657065533584085, + 0.43430341472813805, + 0.9740163595711268, + 0.9875415328473797, + 0.7356703350916547, + 0.6756170470367248, + 0.7197599061272355, + 0.6198568191216928, + 0.6512295663928718, + 0.7665848858812202, + 0.8116271754128533, + 0.5305907755231244, + 0.6901913960659463, + 0.6428575406488269, + 0.5176364098703187, + 0.8723657576961961, + 1.2199632610317463, + 1.415147460346549, + 1.5302808917079092, + 1.4588334149411595, + 1.8140206258329823, + 2.1102391081606, + 1.5869337150428715, + 1.998597539089171, + 1.88124956086161, + 1.2686189428642605, + 1.195426868615217 + ], + "peak_layer": 27, + "peak_normalized": 2.1102391081606, + "population_cv": 0.5159943803714161, + "rest_mean": 1.0306438963636066e-05, + "spike_contrast": 1.375564731293613, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.4177173943608068e-05, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 1 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.81782670551911e-05, + 6.411742106138263e-06, + 3.2170387385122012e-06, + 6.431494512071367e-06, + 4.542944679997163e-06, + 3.672403636301169e-06, + 4.738789357361384e-06, + 1.0627727533574216e-05, + 1.0775303962873295e-05, + 8.027076546568424e-06, + 7.371820629487047e-06, + 7.85347401688341e-06, + 6.763407327525783e-06, + 7.105722943379078e-06, + 8.364392670046072e-06, + 8.855859960021917e-06, + 5.789403985545505e-06, + 7.530844868597342e-06, + 7.014373750280356e-06, + 5.648055775964167e-06, + 9.51859328779392e-06, + 1.3311313523445278e-05, + 1.5441015420947224e-05, + 1.66972640727181e-05, + 1.591768341313582e-05, + 1.979321677936241e-05, + 2.3025328118819743e-05, + 1.7315416698693298e-05, + 2.180718001909554e-05, + 2.0526767912087962e-05, + 1.384220740874298e-05, + 1.3043591025052592e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.632263923554092e-06, + "first_to_last_ratio": 0.3802213248794799, + "imbalance_abs_log_ratio": 0.9670017619171267, + "last_quartile_mean": 1.218307238559646e-05, + "mean": 7.265074810192118e-06, + "normalized": [ + 1.6008181275970341, + 0.5739917816715392, + 0.235460636588888, + 0.49956890329522624, + 0.4370066400923714, + 0.3438728691875936, + 0.4329980193133239, + 0.9771405297973998, + 0.9929176905310494, + 0.7133552305353121, + 0.6552622145802756, + 0.7166704391631107, + 0.6237781231575423, + 0.6649176564292614, + 0.7618691062673217, + 0.8379753775300484, + 0.5325892299976221, + 0.6974715354559842, + 0.6405465790924313, + 0.531438133277865, + 0.87605151631772, + 1.2349821890311432, + 1.4485483745005174, + 1.55527350554067, + 1.5475592171159964, + 1.891436311373116, + 2.0687037039687666, + 1.6168445987295306, + 1.977149264052817, + 1.9131038449622708, + 1.2696779592298815, + 1.1310206916163696 + ], + "peak_layer": 27, + "peak_normalized": 2.0687037039687666, + "population_cv": 0.5243879132228086, + "rest_mean": 6.817757480348519e-06, + "spike_contrast": 1.4199080004313516, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.680588391347555e-06, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 1.1630063454504125e-05, + 4.170093234279193e-06, + 1.7106391396737308e-06, + 3.6294054552854504e-06, + 3.1748859328217804e-06, + 2.4982621198432753e-06, + 3.1457630029763095e-06, + 7.098999049048871e-06, + 7.21362130207126e-06, + 5.182579116080888e-06, + 4.760529009217862e-06, + 5.206664354773238e-06, + 4.531794729700778e-06, + 4.8306765165762044e-06, + 5.535036052606301e-06, + 6.0879538068547845e-06, + 3.869300599035341e-06, + 5.067182883067289e-06, + 4.653618816519156e-06, + 3.860937795252539e-06, + 6.364579803630477e-06, + 8.972237992566079e-06, + 1.0523812306928448e-05, + 1.1299178368062712e-05, + 1.1243133485550061e-05, + 1.3741426300839521e-05, + 1.502928716945462e-05, + 1.1746496966225095e-05, + 1.4364137314260006e-05, + 1.3898842553317081e-05, + 9.224305358657148e-06, + 8.216949936468154e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.248317332487204e-05, + "first_to_last_ratio": 0.36732053671673337, + "imbalance_abs_log_ratio": 1.001520415068512, + "last_quartile_mean": 6.120859325164929e-05, + "mean": 3.520552513691655e-05, + "normalized": [ + 1.6113007958889953, + 0.5636281288577828, + 0.3099745506097867, + 0.6128259302643165, + 0.3776724202532394, + 0.30228081296349946, + 0.41660627301599373, + 0.9147212761628488, + 0.9143135144743936, + 0.6818048236562128, + 0.6378236024718279, + 0.669448797145322, + 0.5777616284561341, + 0.6047361595032282, + 0.735292441295308, + 0.776547208280745, + 0.5187662829625476, + 0.6862775953042356, + 0.6442540642333291, + 0.503974019865995, + 0.8786895478652295, + 1.1802818108868638, + 1.3947424485633424, + 1.5774151382314772, + 1.403240136016062, + 1.8387857304360302, + 2.291845133717591, + 1.6909973322209617, + 2.0609348356508104, + 1.9921347125886224, + 1.3276799111046451, + 1.3032429370526224 + ], + "peak_layer": 27, + "peak_normalized": 2.291845133717591, + "population_cv": 0.5527267740898714, + "rest_mean": 3.3335239331117247e-05, + "spike_contrast": 1.3590743428664134, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.530506848823279e-05, + "top_five_layers": [ + 27, + 29, + 30, + 26, + 28 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 5.672669067280367e-05, + 1.9842824258375913e-05, + 1.0912816833297256e-05, + 2.1574858692474663e-05, + 1.329615588474553e-05, + 1.064195475919405e-05, + 1.4666842616861686e-05, + 3.220324288122356e-05, + 3.2188887416850775e-05, + 2.4003296857699752e-05, + 2.2454914869740605e-05, + 2.356829645577818e-05, + 2.0340401533758268e-05, + 2.1290054064593278e-05, + 2.5886356525006704e-05, + 2.733875226113014e-05, + 1.826343941502273e-05, + 2.416076313238591e-05, + 2.2681302652927116e-05, + 1.7742670024745166e-05, + 3.093472696491517e-05, + 4.155244096182287e-05, + 4.910264033242129e-05, + 5.553372830036096e-05, + 4.940180588164367e-05, + 6.473541725426912e-05, + 8.068561146501452e-05, + 5.953244908596389e-05, + 7.255629316205159e-05, + 7.013414870016277e-05, + 4.6741668484173715e-05, + 4.588135197991505e-05 + ] + } + } + } + } + } + }, + "step": 8000 + } + ], + "environment": { + "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", + "compile": false, + "compute_capability": [ + 12, + 0 + ], + "cublas_workspace_config": ":4096:8", + "cuda": "12.8", + "deterministic_algorithms": true, + "gpu": "NVIDIA GeForce RTX 5090", + "python": "3.10.14", + "torch": "2.11.0+cu128" + }, + "evaluations": [ + { + "bits_per_byte": 7.9557801867581395, + "cross_entropy_nats": 5.514526605606079, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "cross_entropy_nats": 3.5801993310451508, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "cross_entropy_nats": 2.155670017004013, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "cross_entropy_nats": 1.358029618859291, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "cross_entropy_nats": 1.249566838145256, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "cross_entropy_nats": 1.1869123727083206, + "step": 8000 + } + ], + "hashes": { + "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", + "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", + "initial_public_parameter_elements": 18985152, + "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", + "initial_public_parameter_tensors": 227, + "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" + }, + "manifest": { + "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", + "file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", + "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", + "input_gate_tensor_hashes": { + "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", + "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", + "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" + }, + "parent_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", + "parent_path": "experiments/k3/attnres_gradient/manifest.json", + "path": "experiments/k3/attnres_spike/manifest.json", + "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" + }, + "model": { + "aggregation_groups": 8, + "blocks_per_group": 4, + "d_ff": 768, + "d_model": 192, + "heads": 6, + "layers": 32, + "parameters": { + "core": 18985152, + "embedding": 98304, + "mixer": 24960, + "total": 19010112 + } + }, + "optimizer": { + "betas": [ + 0.9, + 0.95 + ], + "epsilon": 1e-08, + "grad_clip": 1.0, + "min_lr": 3e-05, + "name": "AdamW", + "peak_lr": 0.0003, + "warmup_steps": 400, + "weight_decay_ndim_ge_2": 0.1 + }, + "parent_protocol_id": "llm-atlas-k3-attnres-gradient-scale-v1", + "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", + "round05_equivalence": { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true + }, + "run_kind": "replay", + "schema_version": 1, + "seed": 2026073001, + "steps": 8000, + "target_bytes_seen": 65536000, + "timing": { + "mean_ms": 224.04817768403524, + "measured_steps": 7980, + "median_ms": 227.48453948588576, + "p95_ms": 230.01583862304688, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + }, + "training_history": [ + { + "bits_per_byte": 7.958777844362678, + "learning_rate": 7.499999999999999e-07, + "loss_nats": 5.516604423522949, + "step": 1, + "unclipped_grad_norm": 21.323171615600586 + }, + { + "bits_per_byte": 7.247870358756172, + "learning_rate": 7.499999999999999e-06, + "loss_nats": 5.02384090423584, + "step": 10, + "unclipped_grad_norm": 11.153014183044434 + }, + { + "bits_per_byte": 6.7336346654397365, + "learning_rate": 1.4999999999999999e-05, + "loss_nats": 4.667399883270264, + "step": 20, + "unclipped_grad_norm": 3.4888553619384766 + }, + { + "bits_per_byte": 6.611855126224988, + "learning_rate": 2.2499999999999998e-05, + "loss_nats": 4.582988739013672, + "step": 30, + "unclipped_grad_norm": 3.075686454772949 + }, + { + "bits_per_byte": 6.3915955694037825, + "learning_rate": 2.9999999999999997e-05, + "loss_nats": 4.43031644821167, + "step": 40, + "unclipped_grad_norm": 3.2318925857543945 + }, + { + "bits_per_byte": 6.093941755902502, + "learning_rate": 3.75e-05, + "loss_nats": 4.223998546600342, + "step": 50, + "unclipped_grad_norm": 2.8704335689544678 + }, + { + "bits_per_byte": 5.8858000966610895, + "learning_rate": 4.4999999999999996e-05, + "loss_nats": 4.079725742340088, + "step": 60, + "unclipped_grad_norm": 2.669306755065918 + }, + { + "bits_per_byte": 5.715495996470702, + "learning_rate": 5.2499999999999995e-05, + "loss_nats": 3.9616799354553223, + "step": 70, + "unclipped_grad_norm": 3.1737496852874756 + }, + { + "bits_per_byte": 5.565897919513593, + "learning_rate": 5.9999999999999995e-05, + "loss_nats": 3.8579864501953125, + "step": 80, + "unclipped_grad_norm": 2.5544967651367188 + }, + { + "bits_per_byte": 5.354669171689194, + "learning_rate": 6.749999999999999e-05, + "loss_nats": 3.711573839187622, + "step": 90, + "unclipped_grad_norm": 2.425950050354004 + }, + { + "bits_per_byte": 5.180876231443403, + "learning_rate": 7.5e-05, + "loss_nats": 3.5911097526550293, + "step": 100, + "unclipped_grad_norm": 2.2783501148223877 + }, + { + "bits_per_byte": 4.992272489270609, + "learning_rate": 8.249999999999999e-05, + "loss_nats": 3.4603796005249023, + "step": 110, + "unclipped_grad_norm": 2.1105899810791016 + }, + { + "bits_per_byte": 4.775783480086919, + "learning_rate": 8.999999999999999e-05, + "loss_nats": 3.3103208541870117, + "step": 120, + "unclipped_grad_norm": 1.8969556093215942 + }, + { + "bits_per_byte": 4.582375918510225, + "learning_rate": 9.75e-05, + "loss_nats": 3.1762609481811523, + "step": 130, + "unclipped_grad_norm": 1.7015109062194824 + }, + { + "bits_per_byte": 4.427903165297061, + "learning_rate": 0.00010499999999999999, + "loss_nats": 3.0691885948181152, + "step": 140, + "unclipped_grad_norm": 1.446854829788208 + }, + { + "bits_per_byte": 4.271120341117507, + "learning_rate": 0.0001125, + "loss_nats": 2.960515022277832, + "step": 150, + "unclipped_grad_norm": 1.2307006120681763 + }, + { + "bits_per_byte": 4.166711049639903, + "learning_rate": 0.00011999999999999999, + "loss_nats": 2.888144016265869, + "step": 160, + "unclipped_grad_norm": 1.0598198175430298 + }, + { + "bits_per_byte": 4.026318516052877, + "learning_rate": 0.00012749999999999998, + "loss_nats": 2.7908313274383545, + "step": 170, + "unclipped_grad_norm": 0.9654788374900818 + }, + { + "bits_per_byte": 3.94755837315965, + "learning_rate": 0.00013499999999999997, + "loss_nats": 2.736238956451416, + "step": 180, + "unclipped_grad_norm": 0.8419675230979919 + }, + { + "bits_per_byte": 3.838975750676209, + "learning_rate": 0.0001425, + "loss_nats": 2.660975217819214, + "step": 190, + "unclipped_grad_norm": 0.817121684551239 + }, + { + "bits_per_byte": 3.774983758012572, + "learning_rate": 0.00015, + "loss_nats": 2.616619348526001, + "step": 200, + "unclipped_grad_norm": 0.955824613571167 + }, + { + "bits_per_byte": 3.7344016998190783, + "learning_rate": 0.0001575, + "loss_nats": 2.5884900093078613, + "step": 210, + "unclipped_grad_norm": 0.8557147979736328 + }, + { + "bits_per_byte": 3.63166201591605, + "learning_rate": 0.00016499999999999997, + "loss_nats": 2.5172762870788574, + "step": 220, + "unclipped_grad_norm": 0.6891153454780579 + }, + { + "bits_per_byte": 3.6208855830129867, + "learning_rate": 0.0001725, + "loss_nats": 2.5098066329956055, + "step": 230, + "unclipped_grad_norm": 1.036860466003418 + }, + { + "bits_per_byte": 3.4987607026353063, + "learning_rate": 0.00017999999999999998, + "loss_nats": 2.4251561164855957, + "step": 240, + "unclipped_grad_norm": 0.6406477689743042 + }, + { + "bits_per_byte": 3.5387748741156506, + "learning_rate": 0.0001875, + "loss_nats": 2.4528918266296387, + "step": 250, + "unclipped_grad_norm": 1.106186866760254 + }, + { + "bits_per_byte": 3.5260464381252943, + "learning_rate": 0.000195, + "loss_nats": 2.4440691471099854, + "step": 260, + "unclipped_grad_norm": 0.8556521534919739 + }, + { + "bits_per_byte": 3.5219934949750615, + "learning_rate": 0.00020249999999999996, + "loss_nats": 2.4412598609924316, + "step": 270, + "unclipped_grad_norm": 0.9917709827423096 + }, + { + "bits_per_byte": 3.474740573719901, + "learning_rate": 0.00020999999999999998, + "loss_nats": 2.4085066318511963, + "step": 280, + "unclipped_grad_norm": 1.2901222705841064 + }, + { + "bits_per_byte": 3.363479149466621, + "learning_rate": 0.00021749999999999997, + "loss_nats": 2.331386089324951, + "step": 290, + "unclipped_grad_norm": 0.7895627617835999 + }, + { + "bits_per_byte": 3.4037392561376993, + "learning_rate": 0.000225, + "loss_nats": 2.3592922687530518, + "step": 300, + "unclipped_grad_norm": 1.5281883478164673 + }, + { + "bits_per_byte": 3.3541442751430894, + "learning_rate": 0.00023249999999999996, + "loss_nats": 2.324915647506714, + "step": 310, + "unclipped_grad_norm": 0.8648361563682556 + }, + { + "bits_per_byte": 3.3434108379027414, + "learning_rate": 0.00023999999999999998, + "loss_nats": 2.3174757957458496, + "step": 320, + "unclipped_grad_norm": 1.4613573551177979 + }, + { + "bits_per_byte": 3.3325662998699364, + "learning_rate": 0.0002475, + "loss_nats": 2.3099589347839355, + "step": 330, + "unclipped_grad_norm": 1.1215400695800781 + }, + { + "bits_per_byte": 3.4180712583647006, + "learning_rate": 0.00025499999999999996, + "loss_nats": 2.3692264556884766, + "step": 340, + "unclipped_grad_norm": 1.1166281700134277 + }, + { + "bits_per_byte": 3.3577222022116393, + "learning_rate": 0.0002625, + "loss_nats": 2.3273956775665283, + "step": 350, + "unclipped_grad_norm": 1.0810235738754272 + }, + { + "bits_per_byte": 3.2923828974611427, + "learning_rate": 0.00026999999999999995, + "loss_nats": 2.2821059226989746, + "step": 360, + "unclipped_grad_norm": 1.128420114517212 + }, + { + "bits_per_byte": 3.291989401155969, + "learning_rate": 0.00027749999999999997, + "loss_nats": 2.2818331718444824, + "step": 370, + "unclipped_grad_norm": 1.0846341848373413 + }, + { + "bits_per_byte": 3.3389695579268603, + "learning_rate": 0.000285, + "loss_nats": 2.3143973350524902, + "step": 380, + "unclipped_grad_norm": 1.0868512392044067 + }, + { + "bits_per_byte": 3.256548592327368, + "learning_rate": 0.0002925, + "loss_nats": 2.257267475128174, + "step": 390, + "unclipped_grad_norm": 1.1476690769195557 + }, + { + "bits_per_byte": 3.2058776878689605, + "learning_rate": 0.0003, + "loss_nats": 2.2221450805664062, + "step": 400, + "unclipped_grad_norm": 1.1263198852539062 + }, + { + "bits_per_byte": 3.195324144481415, + "learning_rate": 0.0002999988466112388, + "loss_nats": 2.214829921722412, + "step": 410, + "unclipped_grad_norm": 1.1377208232879639 + }, + { + "bits_per_byte": 3.227808915506986, + "learning_rate": 0.00029999538646466355, + "loss_nats": 2.237346649169922, + "step": 420, + "unclipped_grad_norm": 1.2929860353469849 + }, + { + "bits_per_byte": 3.1876103786249166, + "learning_rate": 0.0002999896196193985, + "loss_nats": 2.2094831466674805, + "step": 430, + "unclipped_grad_norm": 1.071694016456604 + }, + { + "bits_per_byte": 3.2374495749837515, + "learning_rate": 0.00029998154617398326, + "loss_nats": 2.2440290451049805, + "step": 440, + "unclipped_grad_norm": 0.9089367985725403 + }, + { + "bits_per_byte": 3.155426921203657, + "learning_rate": 0.0002999711662663707, + "loss_nats": 2.1871752738952637, + "step": 450, + "unclipped_grad_norm": 1.1129212379455566 + }, + { + "bits_per_byte": 3.188086770567806, + "learning_rate": 0.0002999584800739247, + "loss_nats": 2.209813356399536, + "step": 460, + "unclipped_grad_norm": 1.3905333280563354 + }, + { + "bits_per_byte": 3.128408790718412, + "learning_rate": 0.0002999434878134173, + "loss_nats": 2.168447732925415, + "step": 470, + "unclipped_grad_norm": 1.059533953666687 + }, + { + "bits_per_byte": 3.136691131218663, + "learning_rate": 0.000299926189741025, + "loss_nats": 2.1741886138916016, + "step": 480, + "unclipped_grad_norm": 1.0161854028701782 + }, + { + "bits_per_byte": 3.0887341129908705, + "learning_rate": 0.000299906586152324, + "loss_nats": 2.1409473419189453, + "step": 490, + "unclipped_grad_norm": 1.385905385017395 + }, + { + "bits_per_byte": 3.033835874974255, + "learning_rate": 0.0002998846773822856, + "loss_nats": 2.1028947830200195, + "step": 500, + "unclipped_grad_norm": 1.1716443300247192 + }, + { + "bits_per_byte": 3.0607071322756636, + "learning_rate": 0.0002998604638052702, + "loss_nats": 2.121520519256592, + "step": 510, + "unclipped_grad_norm": 0.947066605091095 + }, + { + "bits_per_byte": 2.997100036784657, + "learning_rate": 0.0002998339458350211, + "loss_nats": 2.0774314403533936, + "step": 520, + "unclipped_grad_norm": 1.352820634841919 + }, + { + "bits_per_byte": 3.0643050093317137, + "learning_rate": 0.0002998051239246572, + "loss_nats": 2.124014377593994, + "step": 530, + "unclipped_grad_norm": 1.2646857500076294 + }, + { + "bits_per_byte": 3.008989197438803, + "learning_rate": 0.00029977399856666537, + "loss_nats": 2.085672378540039, + "step": 540, + "unclipped_grad_norm": 1.0451104640960693 + }, + { + "bits_per_byte": 2.978105584892765, + "learning_rate": 0.0002997405702928922, + "loss_nats": 2.064265489578247, + "step": 550, + "unclipped_grad_norm": 1.2619587182998657 + }, + { + "bits_per_byte": 2.968300166036473, + "learning_rate": 0.0002997048396745345, + "loss_nats": 2.057468891143799, + "step": 560, + "unclipped_grad_norm": 0.9861598610877991 + }, + { + "bits_per_byte": 2.9221661639078262, + "learning_rate": 0.0002996668073221302, + "loss_nats": 2.025491237640381, + "step": 570, + "unclipped_grad_norm": 1.1261875629425049 + }, + { + "bits_per_byte": 2.96999178739036, + "learning_rate": 0.000299626473885547, + "loss_nats": 2.0586414337158203, + "step": 580, + "unclipped_grad_norm": 0.8640331625938416 + }, + { + "bits_per_byte": 2.883698804389342, + "learning_rate": 0.00029958384005397226, + "loss_nats": 1.9988276958465576, + "step": 590, + "unclipped_grad_norm": 1.0034456253051758 + }, + { + "bits_per_byte": 2.9404706214044567, + "learning_rate": 0.0002995389065559004, + "loss_nats": 2.0381789207458496, + "step": 600, + "unclipped_grad_norm": 0.8714175224304199 + }, + { + "bits_per_byte": 2.8380133330141457, + "learning_rate": 0.000299491674159121, + "loss_nats": 1.967160940170288, + "step": 610, + "unclipped_grad_norm": 0.9427422881126404 + }, + { + "bits_per_byte": 2.894644124255553, + "learning_rate": 0.00029944214367070535, + "loss_nats": 2.0064144134521484, + "step": 620, + "unclipped_grad_norm": 1.0238139629364014 + }, + { + "bits_per_byte": 2.81883675149499, + "learning_rate": 0.00029939031593699284, + "loss_nats": 1.9538687467575073, + "step": 630, + "unclipped_grad_norm": 0.9875481128692627 + }, + { + "bits_per_byte": 2.7787291934352267, + "learning_rate": 0.00029933619184357654, + "loss_nats": 1.9260683059692383, + "step": 640, + "unclipped_grad_norm": 1.0125378370285034 + }, + { + "bits_per_byte": 2.728423580127252, + "learning_rate": 0.00029927977231528774, + "loss_nats": 1.8911991119384766, + "step": 650, + "unclipped_grad_norm": 1.5507988929748535 + }, + { + "bits_per_byte": 2.649404087396524, + "learning_rate": 0.00029922105831618057, + "loss_nats": 1.8364269733428955, + "step": 660, + "unclipped_grad_norm": 0.9040125012397766 + }, + { + "bits_per_byte": 2.7024973674923345, + "learning_rate": 0.0002991600508495153, + "loss_nats": 1.8732284307479858, + "step": 670, + "unclipped_grad_norm": 1.205861210823059 + }, + { + "bits_per_byte": 2.641403626461037, + "learning_rate": 0.00029909675095774114, + "loss_nats": 1.8308814764022827, + "step": 680, + "unclipped_grad_norm": 1.3370649814605713 + }, + { + "bits_per_byte": 2.7325929555321187, + "learning_rate": 0.0002990311597224787, + "loss_nats": 1.8940891027450562, + "step": 690, + "unclipped_grad_norm": 1.6124995946884155 + }, + { + "bits_per_byte": 2.6130706886099455, + "learning_rate": 0.00029896327826450103, + "loss_nats": 1.8112425804138184, + "step": 700, + "unclipped_grad_norm": 1.08383309841156 + }, + { + "bits_per_byte": 2.628106443844277, + "learning_rate": 0.0002988931077437152, + "loss_nats": 1.821664571762085, + "step": 710, + "unclipped_grad_norm": 1.0767779350280762 + }, + { + "bits_per_byte": 2.6725746220166666, + "learning_rate": 0.00029882064935914164, + "loss_nats": 1.852487564086914, + "step": 720, + "unclipped_grad_norm": 1.296352505683899 + }, + { + "bits_per_byte": 2.608218370098502, + "learning_rate": 0.0002987459043488943, + "loss_nats": 1.8078792095184326, + "step": 730, + "unclipped_grad_norm": 1.0864766836166382 + }, + { + "bits_per_byte": 2.555699340073923, + "learning_rate": 0.0002986688739901591, + "loss_nats": 1.7714757919311523, + "step": 740, + "unclipped_grad_norm": 1.0339913368225098 + }, + { + "bits_per_byte": 2.515936435246489, + "learning_rate": 0.0002985895595991723, + "loss_nats": 1.743914246559143, + "step": 750, + "unclipped_grad_norm": 1.1017143726348877 + }, + { + "bits_per_byte": 2.4657125136976736, + "learning_rate": 0.00029850796253119824, + "loss_nats": 1.709101676940918, + "step": 760, + "unclipped_grad_norm": 1.7173939943313599 + }, + { + "bits_per_byte": 2.601046349592221, + "learning_rate": 0.00029842408418050544, + "loss_nats": 1.802907943725586, + "step": 770, + "unclipped_grad_norm": 0.9487742781639099 + }, + { + "bits_per_byte": 2.4794518637098015, + "learning_rate": 0.0002983379259803436, + "loss_nats": 1.7186250686645508, + "step": 780, + "unclipped_grad_norm": 0.9247332811355591 + }, + { + "bits_per_byte": 2.466098098800908, + "learning_rate": 0.0002982494894029186, + "loss_nats": 1.7093689441680908, + "step": 790, + "unclipped_grad_norm": 1.9121034145355225 + }, + { + "bits_per_byte": 2.4025570446478333, + "learning_rate": 0.0002981587759593675, + "loss_nats": 1.66532564163208, + "step": 800, + "unclipped_grad_norm": 1.1767630577087402 + }, + { + "bits_per_byte": 2.455517898102479, + "learning_rate": 0.00029806578719973283, + "loss_nats": 1.7020353078842163, + "step": 810, + "unclipped_grad_norm": 1.1299240589141846 + }, + { + "bits_per_byte": 2.400107839716896, + "learning_rate": 0.00029797052471293596, + "loss_nats": 1.6636279821395874, + "step": 820, + "unclipped_grad_norm": 1.0006046295166016 + }, + { + "bits_per_byte": 2.4500607166079513, + "learning_rate": 0.0002978729901267499, + "loss_nats": 1.6982526779174805, + "step": 830, + "unclipped_grad_norm": 1.0554603338241577 + }, + { + "bits_per_byte": 2.5153210813217015, + "learning_rate": 0.0002977731851077716, + "loss_nats": 1.7434877157211304, + "step": 840, + "unclipped_grad_norm": 1.0736713409423828 + }, + { + "bits_per_byte": 2.399556119372928, + "learning_rate": 0.0002976711113613935, + "loss_nats": 1.6632455587387085, + "step": 850, + "unclipped_grad_norm": 0.8279580473899841 + }, + { + "bits_per_byte": 2.414819407654351, + "learning_rate": 0.0002975667706317743, + "loss_nats": 1.6738252639770508, + "step": 860, + "unclipped_grad_norm": 1.2342168092727661 + }, + { + "bits_per_byte": 2.3222158652452247, + "learning_rate": 0.00029746016470180915, + "loss_nats": 1.6096373796463013, + "step": 870, + "unclipped_grad_norm": 1.0648512840270996 + }, + { + "bits_per_byte": 2.3629336177502487, + "learning_rate": 0.00029735129539309926, + "loss_nats": 1.6378607749938965, + "step": 880, + "unclipped_grad_norm": 0.9258757829666138 + }, + { + "bits_per_byte": 2.2961121665146167, + "learning_rate": 0.0002972401645659207, + "loss_nats": 1.5915436744689941, + "step": 890, + "unclipped_grad_norm": 1.0717874765396118 + }, + { + "bits_per_byte": 2.3158979105831285, + "learning_rate": 0.00029712677411919283, + "loss_nats": 1.6052581071853638, + "step": 900, + "unclipped_grad_norm": 0.9598665833473206 + }, + { + "bits_per_byte": 2.3301258633063577, + "learning_rate": 0.0002970111259904454, + "loss_nats": 1.6151201725006104, + "step": 910, + "unclipped_grad_norm": 1.0995051860809326 + }, + { + "bits_per_byte": 2.3123376977276173, + "learning_rate": 0.0002968932221557859, + "loss_nats": 1.602790355682373, + "step": 920, + "unclipped_grad_norm": 1.1837356090545654 + }, + { + "bits_per_byte": 2.3340250539667196, + "learning_rate": 0.0002967730646298655, + "loss_nats": 1.6178228855133057, + "step": 930, + "unclipped_grad_norm": 1.1502224206924438 + }, + { + "bits_per_byte": 2.330418921743428, + "learning_rate": 0.0002966506554658449, + "loss_nats": 1.6153233051300049, + "step": 940, + "unclipped_grad_norm": 1.0088714361190796 + }, + { + "bits_per_byte": 2.2995397807463123, + "learning_rate": 0.00029652599675535876, + "loss_nats": 1.5939195156097412, + "step": 950, + "unclipped_grad_norm": 0.9483612179756165 + }, + { + "bits_per_byte": 2.3536873144919115, + "learning_rate": 0.0002963990906284808, + "loss_nats": 1.6314517259597778, + "step": 960, + "unclipped_grad_norm": 0.9582105278968811 + }, + { + "bits_per_byte": 2.2526236015215417, + "learning_rate": 0.0002962699392536863, + "loss_nats": 1.5613996982574463, + "step": 970, + "unclipped_grad_norm": 1.2002545595169067 + }, + { + "bits_per_byte": 2.3230659754884395, + "learning_rate": 0.0002961385448378161, + "loss_nats": 1.6102266311645508, + "step": 980, + "unclipped_grad_norm": 0.9191399812698364 + }, + { + "bits_per_byte": 2.285470911975131, + "learning_rate": 0.00029600490962603817, + "loss_nats": 1.584167718887329, + "step": 990, + "unclipped_grad_norm": 1.053741216659546 + }, + { + "bits_per_byte": 2.248061417742104, + "learning_rate": 0.00029586903590180956, + "loss_nats": 1.5582374334335327, + "step": 1000, + "unclipped_grad_norm": 0.9024814963340759 + }, + { + "bits_per_byte": 2.2402749032242903, + "learning_rate": 0.00029573092598683735, + "loss_nats": 1.552840232849121, + "step": 1010, + "unclipped_grad_norm": 1.048907995223999 + }, + { + "bits_per_byte": 2.2041726491204714, + "learning_rate": 0.00029559058224103895, + "loss_nats": 1.5278160572052002, + "step": 1020, + "unclipped_grad_norm": 1.0486118793487549 + }, + { + "bits_per_byte": 2.216881995036582, + "learning_rate": 0.0002954480070625017, + "loss_nats": 1.5366255044937134, + "step": 1030, + "unclipped_grad_norm": 0.9243784546852112 + }, + { + "bits_per_byte": 2.2610579746851553, + "learning_rate": 0.00029530320288744205, + "loss_nats": 1.5672459602355957, + "step": 1040, + "unclipped_grad_norm": 1.102870225906372 + }, + { + "bits_per_byte": 2.2528270569975124, + "learning_rate": 0.0002951561721901638, + "loss_nats": 1.5615407228469849, + "step": 1050, + "unclipped_grad_norm": 1.0764544010162354 + }, + { + "bits_per_byte": 2.14889054582713, + "learning_rate": 0.00029500691748301594, + "loss_nats": 1.489497423171997, + "step": 1060, + "unclipped_grad_norm": 0.971049427986145 + }, + { + "bits_per_byte": 2.2350196293618754, + "learning_rate": 0.00029485544131634963, + "loss_nats": 1.5491975545883179, + "step": 1070, + "unclipped_grad_norm": 1.0203380584716797 + }, + { + "bits_per_byte": 2.217042282867186, + "learning_rate": 0.0002947017462784745, + "loss_nats": 1.5367366075515747, + "step": 1080, + "unclipped_grad_norm": 1.023221492767334 + }, + { + "bits_per_byte": 2.186512266651322, + "learning_rate": 0.00029454583499561476, + "loss_nats": 1.5155748128890991, + "step": 1090, + "unclipped_grad_norm": 1.111006498336792 + }, + { + "bits_per_byte": 2.2323391077655277, + "learning_rate": 0.00029438771013186413, + "loss_nats": 1.5473395586013794, + "step": 1100, + "unclipped_grad_norm": 0.9547863602638245 + }, + { + "bits_per_byte": 2.116467344590568, + "learning_rate": 0.0002942273743891401, + "loss_nats": 1.4670233726501465, + "step": 1110, + "unclipped_grad_norm": 1.1085364818572998 + }, + { + "bits_per_byte": 2.143238164023873, + "learning_rate": 0.00029406483050713824, + "loss_nats": 1.485579490661621, + "step": 1120, + "unclipped_grad_norm": 0.901618242263794 + }, + { + "bits_per_byte": 2.077939275196329, + "learning_rate": 0.00029390008126328504, + "loss_nats": 1.4403177499771118, + "step": 1130, + "unclipped_grad_norm": 0.8683149814605713 + }, + { + "bits_per_byte": 2.224320932617002, + "learning_rate": 0.00029373312947269045, + "loss_nats": 1.5417817831039429, + "step": 1140, + "unclipped_grad_norm": 1.030465006828308 + }, + { + "bits_per_byte": 2.0642131678483175, + "learning_rate": 0.00029356397798809995, + "loss_nats": 1.4308035373687744, + "step": 1150, + "unclipped_grad_norm": 0.827274739742279 + }, + { + "bits_per_byte": 2.103388407957753, + "learning_rate": 0.0002933926296998457, + "loss_nats": 1.4579577445983887, + "step": 1160, + "unclipped_grad_norm": 0.9158841967582703 + }, + { + "bits_per_byte": 2.1566975262803965, + "learning_rate": 0.0002932190875357972, + "loss_nats": 1.4949088096618652, + "step": 1170, + "unclipped_grad_norm": 0.9397322535514832 + }, + { + "bits_per_byte": 2.1322636071070153, + "learning_rate": 0.00029304335446131126, + "loss_nats": 1.4779725074768066, + "step": 1180, + "unclipped_grad_norm": 0.8510220050811768 + }, + { + "bits_per_byte": 2.068760561119781, + "learning_rate": 0.00029286543347918116, + "loss_nats": 1.4339555501937866, + "step": 1190, + "unclipped_grad_norm": 0.9270830154418945 + }, + { + "bits_per_byte": 2.0310528489701576, + "learning_rate": 0.0002926853276295856, + "loss_nats": 1.4078185558319092, + "step": 1200, + "unclipped_grad_norm": 0.8815417885780334 + }, + { + "bits_per_byte": 2.1008361854189492, + "learning_rate": 0.0002925030399900368, + "loss_nats": 1.456188678741455, + "step": 1210, + "unclipped_grad_norm": 1.2037686109542847 + }, + { + "bits_per_byte": 2.1123913557651197, + "learning_rate": 0.00029231857367532726, + "loss_nats": 1.464198112487793, + "step": 1220, + "unclipped_grad_norm": 0.8964646458625793 + }, + { + "bits_per_byte": 2.0711254945517954, + "learning_rate": 0.0002921319318374776, + "loss_nats": 1.4355947971343994, + "step": 1230, + "unclipped_grad_norm": 0.8661414980888367 + }, + { + "bits_per_byte": 2.0159510523978943, + "learning_rate": 0.00029194311766568174, + "loss_nats": 1.397350788116455, + "step": 1240, + "unclipped_grad_norm": 0.9392534494400024 + }, + { + "bits_per_byte": 2.104896867788471, + "learning_rate": 0.000291752134386253, + "loss_nats": 1.4590033292770386, + "step": 1250, + "unclipped_grad_norm": 0.8430818915367126 + }, + { + "bits_per_byte": 2.08180441329182, + "learning_rate": 0.00029155898526256856, + "loss_nats": 1.442996859550476, + "step": 1260, + "unclipped_grad_norm": 0.8917548656463623 + }, + { + "bits_per_byte": 2.087352401626003, + "learning_rate": 0.0002913636735950141, + "loss_nats": 1.4468424320220947, + "step": 1270, + "unclipped_grad_norm": 0.9866762161254883 + }, + { + "bits_per_byte": 1.9607164163161914, + "learning_rate": 0.0002911662027209272, + "loss_nats": 1.359065055847168, + "step": 1280, + "unclipped_grad_norm": 0.8596939444541931 + }, + { + "bits_per_byte": 2.088993460080536, + "learning_rate": 0.00029096657601454013, + "loss_nats": 1.4479799270629883, + "step": 1290, + "unclipped_grad_norm": 1.007126808166504 + }, + { + "bits_per_byte": 2.034349412421896, + "learning_rate": 0.00029076479688692266, + "loss_nats": 1.4101035594940186, + "step": 1300, + "unclipped_grad_norm": 0.8474438786506653 + }, + { + "bits_per_byte": 2.0247273270714237, + "learning_rate": 0.00029056086878592323, + "loss_nats": 1.4034340381622314, + "step": 1310, + "unclipped_grad_norm": 0.8226730823516846 + }, + { + "bits_per_byte": 2.020511000402871, + "learning_rate": 0.00029035479519611045, + "loss_nats": 1.4005115032196045, + "step": 1320, + "unclipped_grad_norm": 0.9414357542991638 + }, + { + "bits_per_byte": 2.085571349293668, + "learning_rate": 0.0002901465796387134, + "loss_nats": 1.4456079006195068, + "step": 1330, + "unclipped_grad_norm": 0.9917200207710266 + }, + { + "bits_per_byte": 2.0422444479924042, + "learning_rate": 0.00028993622567156154, + "loss_nats": 1.4155759811401367, + "step": 1340, + "unclipped_grad_norm": 0.8823419213294983 + }, + { + "bits_per_byte": 2.0447034559344406, + "learning_rate": 0.0002897237368890237, + "loss_nats": 1.4172804355621338, + "step": 1350, + "unclipped_grad_norm": 0.8861486315727234 + }, + { + "bits_per_byte": 2.0988803987133373, + "learning_rate": 0.0002895091169219469, + "loss_nats": 1.4548330307006836, + "step": 1360, + "unclipped_grad_norm": 0.9090827107429504 + }, + { + "bits_per_byte": 2.082760980795919, + "learning_rate": 0.00028929236943759425, + "loss_nats": 1.4436599016189575, + "step": 1370, + "unclipped_grad_norm": 0.8366561532020569 + }, + { + "bits_per_byte": 2.0317923743688677, + "learning_rate": 0.00028907349813958225, + "loss_nats": 1.4083311557769775, + "step": 1380, + "unclipped_grad_norm": 0.8379060626029968 + }, + { + "bits_per_byte": 2.0432778917414387, + "learning_rate": 0.0002888525067678175, + "loss_nats": 1.4162923097610474, + "step": 1390, + "unclipped_grad_norm": 0.8665340542793274 + }, + { + "bits_per_byte": 1.9746810636009497, + "learning_rate": 0.00028862939909843273, + "loss_nats": 1.3687446117401123, + "step": 1400, + "unclipped_grad_norm": 0.8112426996231079 + }, + { + "bits_per_byte": 2.0034977886489185, + "learning_rate": 0.0002884041789437225, + "loss_nats": 1.388718843460083, + "step": 1410, + "unclipped_grad_norm": 0.8745980262756348 + }, + { + "bits_per_byte": 2.019574210903621, + "learning_rate": 0.0002881768501520778, + "loss_nats": 1.3998621702194214, + "step": 1420, + "unclipped_grad_norm": 0.8596643209457397 + }, + { + "bits_per_byte": 2.0775151659793014, + "learning_rate": 0.0002879474166079205, + "loss_nats": 1.4400237798690796, + "step": 1430, + "unclipped_grad_norm": 0.913244903087616 + }, + { + "bits_per_byte": 2.027370700415185, + "learning_rate": 0.00028771588223163694, + "loss_nats": 1.405266284942627, + "step": 1440, + "unclipped_grad_norm": 1.0067864656448364 + }, + { + "bits_per_byte": 1.9631205617926026, + "learning_rate": 0.00028748225097951054, + "loss_nats": 1.3607314825057983, + "step": 1450, + "unclipped_grad_norm": 0.8125852942466736 + }, + { + "bits_per_byte": 1.9959419028659118, + "learning_rate": 0.000287246526843655, + "loss_nats": 1.383481502532959, + "step": 1460, + "unclipped_grad_norm": 0.9857028126716614 + }, + { + "bits_per_byte": 1.9364156117145783, + "learning_rate": 0.00028700871385194533, + "loss_nats": 1.3422210216522217, + "step": 1470, + "unclipped_grad_norm": 0.8698804974555969 + }, + { + "bits_per_byte": 1.996789089404062, + "learning_rate": 0.00028676881606794945, + "loss_nats": 1.3840687274932861, + "step": 1480, + "unclipped_grad_norm": 0.8397032022476196 + }, + { + "bits_per_byte": 2.017818268038312, + "learning_rate": 0.00028652683759085854, + "loss_nats": 1.398645043373108, + "step": 1490, + "unclipped_grad_norm": 0.8181726932525635 + }, + { + "bits_per_byte": 1.996446843928845, + "learning_rate": 0.0002862827825554171, + "loss_nats": 1.38383150100708, + "step": 1500, + "unclipped_grad_norm": 0.8242432475090027 + }, + { + "bits_per_byte": 2.0162594172908914, + "learning_rate": 0.00028603665513185245, + "loss_nats": 1.3975645303726196, + "step": 1510, + "unclipped_grad_norm": 0.9299814701080322 + }, + { + "bits_per_byte": 1.980743108078562, + "learning_rate": 0.000285788459525803, + "loss_nats": 1.3729465007781982, + "step": 1520, + "unclipped_grad_norm": 0.8446369171142578 + }, + { + "bits_per_byte": 2.047233664693934, + "learning_rate": 0.00028553819997824707, + "loss_nats": 1.4190342426300049, + "step": 1530, + "unclipped_grad_norm": 0.9717140197753906 + }, + { + "bits_per_byte": 1.993557363411699, + "learning_rate": 0.00028528588076542964, + "loss_nats": 1.3818286657333374, + "step": 1540, + "unclipped_grad_norm": 0.8417568802833557 + }, + { + "bits_per_byte": 2.049996737962696, + "learning_rate": 0.00028503150619878996, + "loss_nats": 1.4209494590759277, + "step": 1550, + "unclipped_grad_norm": 0.8546327352523804 + }, + { + "bits_per_byte": 1.966114779744124, + "learning_rate": 0.0002847750806248876, + "loss_nats": 1.3628069162368774, + "step": 1560, + "unclipped_grad_norm": 0.9258830547332764 + }, + { + "bits_per_byte": 2.0427641795633114, + "learning_rate": 0.000284516608425328, + "loss_nats": 1.4159362316131592, + "step": 1570, + "unclipped_grad_norm": 0.8567822575569153 + }, + { + "bits_per_byte": 1.9830621221427962, + "learning_rate": 0.000284256094016688, + "loss_nats": 1.374553918838501, + "step": 1580, + "unclipped_grad_norm": 0.8522606492042542 + }, + { + "bits_per_byte": 1.9876789964052082, + "learning_rate": 0.0002839935418504399, + "loss_nats": 1.3777540922164917, + "step": 1590, + "unclipped_grad_norm": 0.8731296062469482 + }, + { + "bits_per_byte": 2.0122188568915207, + "learning_rate": 0.000283728956412876, + "loss_nats": 1.3947638273239136, + "step": 1600, + "unclipped_grad_norm": 0.8627029657363892 + }, + { + "bits_per_byte": 1.9498011934138815, + "learning_rate": 0.00028346234222503136, + "loss_nats": 1.3514991998672485, + "step": 1610, + "unclipped_grad_norm": 0.7740172147750854 + }, + { + "bits_per_byte": 1.9095543294069197, + "learning_rate": 0.00028319370384260675, + "loss_nats": 1.3236021995544434, + "step": 1620, + "unclipped_grad_norm": 0.7910023331642151 + }, + { + "bits_per_byte": 1.9407452749500496, + "learning_rate": 0.00028292304585589105, + "loss_nats": 1.3452221155166626, + "step": 1630, + "unclipped_grad_norm": 0.8484107851982117 + }, + { + "bits_per_byte": 1.9911346438089947, + "learning_rate": 0.0002826503728896824, + "loss_nats": 1.3801493644714355, + "step": 1640, + "unclipped_grad_norm": 0.8722788691520691 + }, + { + "bits_per_byte": 2.0368601871418424, + "learning_rate": 0.0002823756896032095, + "loss_nats": 1.4118438959121704, + "step": 1650, + "unclipped_grad_norm": 0.8482462763786316 + }, + { + "bits_per_byte": 1.9665676100638458, + "learning_rate": 0.0002820990006900518, + "loss_nats": 1.3631207942962646, + "step": 1660, + "unclipped_grad_norm": 0.8444085121154785 + }, + { + "bits_per_byte": 1.9945896032821775, + "learning_rate": 0.00028182031087805947, + "loss_nats": 1.3825441598892212, + "step": 1670, + "unclipped_grad_norm": 0.8560684323310852 + }, + { + "bits_per_byte": 2.0061820938635853, + "learning_rate": 0.0002815396249292725, + "loss_nats": 1.3905794620513916, + "step": 1680, + "unclipped_grad_norm": 0.8526912927627563 + }, + { + "bits_per_byte": 2.0514603103215387, + "learning_rate": 0.0002812569476398391, + "loss_nats": 1.4219639301300049, + "step": 1690, + "unclipped_grad_norm": 0.8917502760887146 + }, + { + "bits_per_byte": 1.9422809080396035, + "learning_rate": 0.00028097228383993446, + "loss_nats": 1.3462865352630615, + "step": 1700, + "unclipped_grad_norm": 0.7474871873855591 + }, + { + "bits_per_byte": 1.878015978909047, + "learning_rate": 0.0002806856383936774, + "loss_nats": 1.3017414808273315, + "step": 1710, + "unclipped_grad_norm": 0.8072315454483032 + }, + { + "bits_per_byte": 1.8905051870320533, + "learning_rate": 0.00028039701619904774, + "loss_nats": 1.3103983402252197, + "step": 1720, + "unclipped_grad_norm": 0.845556914806366 + }, + { + "bits_per_byte": 1.8762399140735866, + "learning_rate": 0.0002801064221878024, + "loss_nats": 1.3005104064941406, + "step": 1730, + "unclipped_grad_norm": 0.9344940185546875 + }, + { + "bits_per_byte": 1.9632232354351677, + "learning_rate": 0.00027981386132539134, + "loss_nats": 1.3608026504516602, + "step": 1740, + "unclipped_grad_norm": 0.7860697507858276 + }, + { + "bits_per_byte": 1.989613629244764, + "learning_rate": 0.0002795193386108724, + "loss_nats": 1.3790950775146484, + "step": 1750, + "unclipped_grad_norm": 0.8348115086555481 + }, + { + "bits_per_byte": 1.9869124697303127, + "learning_rate": 0.00027922285907682625, + "loss_nats": 1.3772227764129639, + "step": 1760, + "unclipped_grad_norm": 0.8862095475196838 + }, + { + "bits_per_byte": 1.9928130224987648, + "learning_rate": 0.00027892442778927005, + "loss_nats": 1.3813127279281616, + "step": 1770, + "unclipped_grad_norm": 0.816246509552002 + }, + { + "bits_per_byte": 1.8514405317671223, + "learning_rate": 0.00027862404984757116, + "loss_nats": 1.2833207845687866, + "step": 1780, + "unclipped_grad_norm": 0.8118871450424194 + }, + { + "bits_per_byte": 1.9588221993995885, + "learning_rate": 0.0002783217303843597, + "loss_nats": 1.3577520847320557, + "step": 1790, + "unclipped_grad_norm": 0.839783251285553 + }, + { + "bits_per_byte": 1.8827697513615458, + "learning_rate": 0.00027801747456544134, + "loss_nats": 1.3050365447998047, + "step": 1800, + "unclipped_grad_norm": 0.8949951529502869 + }, + { + "bits_per_byte": 1.92937567388416, + "learning_rate": 0.00027771128758970843, + "loss_nats": 1.33734130859375, + "step": 1810, + "unclipped_grad_norm": 0.7295374274253845 + }, + { + "bits_per_byte": 1.8974312723476003, + "learning_rate": 0.00027740317468905165, + "loss_nats": 1.3151991367340088, + "step": 1820, + "unclipped_grad_norm": 0.7755441069602966 + }, + { + "bits_per_byte": 1.8886371114783813, + "learning_rate": 0.0002770931411282704, + "loss_nats": 1.3091034889221191, + "step": 1830, + "unclipped_grad_norm": 0.7666956186294556 + }, + { + "bits_per_byte": 1.8876932706904463, + "learning_rate": 0.0002767811922049827, + "loss_nats": 1.3084492683410645, + "step": 1840, + "unclipped_grad_norm": 0.8784642219543457 + }, + { + "bits_per_byte": 1.8964055678178544, + "learning_rate": 0.00027646733324953503, + "loss_nats": 1.314488172531128, + "step": 1850, + "unclipped_grad_norm": 0.8603965044021606 + }, + { + "bits_per_byte": 1.8927124123732264, + "learning_rate": 0.00027615156962491097, + "loss_nats": 1.3119282722473145, + "step": 1860, + "unclipped_grad_norm": 0.8056252002716064 + }, + { + "bits_per_byte": 1.8818489562488263, + "learning_rate": 0.0002758339067266396, + "loss_nats": 1.3043982982635498, + "step": 1870, + "unclipped_grad_norm": 0.8151036500930786 + }, + { + "bits_per_byte": 1.8811881989042114, + "learning_rate": 0.0002755143499827035, + "loss_nats": 1.3039402961730957, + "step": 1880, + "unclipped_grad_norm": 0.79389888048172 + }, + { + "bits_per_byte": 1.8949089747900463, + "learning_rate": 0.0002751929048534456, + "loss_nats": 1.313450813293457, + "step": 1890, + "unclipped_grad_norm": 0.8192400932312012 + }, + { + "bits_per_byte": 1.9233574849825177, + "learning_rate": 0.0002748695768314763, + "loss_nats": 1.3331698179244995, + "step": 1900, + "unclipped_grad_norm": 0.7950363159179688 + }, + { + "bits_per_byte": 1.8611783614416253, + "learning_rate": 0.00027454437144157947, + "loss_nats": 1.2900705337524414, + "step": 1910, + "unclipped_grad_norm": 0.8037525415420532 + }, + { + "bits_per_byte": 1.855479888287937, + "learning_rate": 0.00027421729424061787, + "loss_nats": 1.2861206531524658, + "step": 1920, + "unclipped_grad_norm": 0.7507341504096985 + }, + { + "bits_per_byte": 1.9602173226633874, + "learning_rate": 0.0002738883508174385, + "loss_nats": 1.3587191104888916, + "step": 1930, + "unclipped_grad_norm": 0.8329893350601196 + }, + { + "bits_per_byte": 1.8066889261861108, + "learning_rate": 0.0002735575467927768, + "loss_nats": 1.2523013353347778, + "step": 1940, + "unclipped_grad_norm": 0.7595474123954773 + }, + { + "bits_per_byte": 1.9033319970987033, + "learning_rate": 0.0002732248878191609, + "loss_nats": 1.319289207458496, + "step": 1950, + "unclipped_grad_norm": 0.7690061330795288 + }, + { + "bits_per_byte": 1.816397862775269, + "learning_rate": 0.0002728903795808148, + "loss_nats": 1.259031057357788, + "step": 1960, + "unclipped_grad_norm": 0.7758741974830627 + }, + { + "bits_per_byte": 1.909067618504978, + "learning_rate": 0.0002725540277935614, + "loss_nats": 1.3232648372650146, + "step": 1970, + "unclipped_grad_norm": 0.7503845691680908 + }, + { + "bits_per_byte": 1.9405849871194454, + "learning_rate": 0.00027221583820472456, + "loss_nats": 1.3451110124588013, + "step": 1980, + "unclipped_grad_norm": 0.8655099868774414 + }, + { + "bits_per_byte": 1.9436471382180576, + "learning_rate": 0.0002718758165930314, + "loss_nats": 1.347233533859253, + "step": 1990, + "unclipped_grad_norm": 0.8436980247497559 + }, + { + "bits_per_byte": 1.9015801818170146, + "learning_rate": 0.0002715339687685131, + "loss_nats": 1.3180749416351318, + "step": 2000, + "unclipped_grad_norm": 0.8021923303604126 + }, + { + "bits_per_byte": 1.8746163978494417, + "learning_rate": 0.00027119030057240576, + "loss_nats": 1.2993850708007812, + "step": 2010, + "unclipped_grad_norm": 0.7083144187927246 + }, + { + "bits_per_byte": 1.9072580170525995, + "learning_rate": 0.0002708448178770506, + "loss_nats": 1.3220105171203613, + "step": 2020, + "unclipped_grad_norm": 0.7678324580192566 + }, + { + "bits_per_byte": 1.983610230851096, + "learning_rate": 0.00027049752658579376, + "loss_nats": 1.3749338388442993, + "step": 2030, + "unclipped_grad_norm": 0.8317387104034424 + }, + { + "bits_per_byte": 1.909348466173837, + "learning_rate": 0.00027014843263288514, + "loss_nats": 1.323459506034851, + "step": 2040, + "unclipped_grad_norm": 0.8302199840545654 + }, + { + "bits_per_byte": 1.961244747019642, + "learning_rate": 0.00026979754198337724, + "loss_nats": 1.359431266784668, + "step": 2050, + "unclipped_grad_norm": 0.7990598082542419 + }, + { + "bits_per_byte": 1.9220292629699043, + "learning_rate": 0.00026944486063302305, + "loss_nats": 1.3322491645812988, + "step": 2060, + "unclipped_grad_norm": 0.8404503464698792 + }, + { + "bits_per_byte": 1.8663830724046866, + "learning_rate": 0.0002690903946081738, + "loss_nats": 1.2936781644821167, + "step": 2070, + "unclipped_grad_norm": 0.7720745801925659 + }, + { + "bits_per_byte": 1.8672120287818452, + "learning_rate": 0.00026873414996567596, + "loss_nats": 1.2942527532577515, + "step": 2080, + "unclipped_grad_norm": 0.8028935790061951 + }, + { + "bits_per_byte": 1.857879218250124, + "learning_rate": 0.00026837613279276736, + "loss_nats": 1.2877837419509888, + "step": 2090, + "unclipped_grad_norm": 0.7713416814804077 + }, + { + "bits_per_byte": 1.8603779541845098, + "learning_rate": 0.0002680163492069739, + "loss_nats": 1.289515733718872, + "step": 2100, + "unclipped_grad_norm": 0.7443708181381226 + }, + { + "bits_per_byte": 1.897931741861611, + "learning_rate": 0.00026765480535600414, + "loss_nats": 1.3155460357666016, + "step": 2110, + "unclipped_grad_norm": 0.7534405589103699 + }, + { + "bits_per_byte": 1.8710701155886507, + "learning_rate": 0.0002672915074176452, + "loss_nats": 1.2969269752502441, + "step": 2120, + "unclipped_grad_norm": 0.7505779266357422 + }, + { + "bits_per_byte": 1.8349618420920515, + "learning_rate": 0.00026692646159965624, + "loss_nats": 1.271898627281189, + "step": 2130, + "unclipped_grad_norm": 0.7835680246353149 + }, + { + "bits_per_byte": 1.9616896661374241, + "learning_rate": 0.0002665596741396633, + "loss_nats": 1.3597396612167358, + "step": 2140, + "unclipped_grad_norm": 0.7786996960639954 + }, + { + "bits_per_byte": 1.8168395142226847, + "learning_rate": 0.000266191151305052, + "loss_nats": 1.2593371868133545, + "step": 2150, + "unclipped_grad_norm": 0.7194197773933411 + }, + { + "bits_per_byte": 1.8739360344826286, + "learning_rate": 0.00026582089939286075, + "loss_nats": 1.2989134788513184, + "step": 2160, + "unclipped_grad_norm": 0.8012002110481262 + }, + { + "bits_per_byte": 1.9634492206384013, + "learning_rate": 0.0002654489247296732, + "loss_nats": 1.3609592914581299, + "step": 2170, + "unclipped_grad_norm": 0.8270084261894226 + }, + { + "bits_per_byte": 1.8175344961148212, + "learning_rate": 0.00026507523367151007, + "loss_nats": 1.2598189115524292, + "step": 2180, + "unclipped_grad_norm": 0.7824540734291077 + }, + { + "bits_per_byte": 1.8458584908680682, + "learning_rate": 0.00026469983260372037, + "loss_nats": 1.279451608657837, + "step": 2190, + "unclipped_grad_norm": 0.7973959445953369 + }, + { + "bits_per_byte": 1.8557901449900935, + "learning_rate": 0.0002643227279408727, + "loss_nats": 1.2863357067108154, + "step": 2200, + "unclipped_grad_norm": 0.7451788783073425 + }, + { + "bits_per_byte": 1.8380872828061812, + "learning_rate": 0.0002639439261266453, + "loss_nats": 1.2740650177001953, + "step": 2210, + "unclipped_grad_norm": 0.7804975509643555 + }, + { + "bits_per_byte": 1.8298283319464475, + "learning_rate": 0.000263563433633716, + "loss_nats": 1.2683403491973877, + "step": 2220, + "unclipped_grad_norm": 0.7694668173789978 + }, + { + "bits_per_byte": 1.9031531351418058, + "learning_rate": 0.00026318125696365156, + "loss_nats": 1.3191652297973633, + "step": 2230, + "unclipped_grad_norm": 0.8418469429016113 + }, + { + "bits_per_byte": 1.8400618156207371, + "learning_rate": 0.000262797402646797, + "loss_nats": 1.2754336595535278, + "step": 2240, + "unclipped_grad_norm": 0.7987614870071411 + }, + { + "bits_per_byte": 1.852736936989326, + "learning_rate": 0.00026241187724216334, + "loss_nats": 1.2842193841934204, + "step": 2250, + "unclipped_grad_norm": 0.7795421481132507 + }, + { + "bits_per_byte": 1.8339105121473271, + "learning_rate": 0.0002620246873373162, + "loss_nats": 1.271169900894165, + "step": 2260, + "unclipped_grad_norm": 0.7681040167808533 + }, + { + "bits_per_byte": 1.825916414569922, + "learning_rate": 0.00026163583954826264, + "loss_nats": 1.2656288146972656, + "step": 2270, + "unclipped_grad_norm": 0.7933830618858337 + }, + { + "bits_per_byte": 1.8451832869807807, + "learning_rate": 0.0002612453405193386, + "loss_nats": 1.2789835929870605, + "step": 2280, + "unclipped_grad_norm": 0.7935850024223328 + }, + { + "bits_per_byte": 1.8711039961708706, + "learning_rate": 0.000260853196923095, + "loss_nats": 1.2969504594802856, + "step": 2290, + "unclipped_grad_norm": 0.7818295955657959 + }, + { + "bits_per_byte": 1.7944778140095496, + "learning_rate": 0.0002604594154601839, + "loss_nats": 1.2438372373580933, + "step": 2300, + "unclipped_grad_norm": 0.7245032787322998 + }, + { + "bits_per_byte": 1.8221434591752936, + "learning_rate": 0.0002600640028592441, + "loss_nats": 1.2630136013031006, + "step": 2310, + "unclipped_grad_norm": 0.7396021485328674 + }, + { + "bits_per_byte": 1.7928696042413317, + "learning_rate": 0.0002596669658767859, + "loss_nats": 1.242722511291504, + "step": 2320, + "unclipped_grad_norm": 0.7618010640144348 + }, + { + "bits_per_byte": 1.8702191454321815, + "learning_rate": 0.000259268311297076, + "loss_nats": 1.2963371276855469, + "step": 2330, + "unclipped_grad_norm": 0.7263991236686707 + }, + { + "bits_per_byte": 1.7994940319872652, + "learning_rate": 0.00025886804593202114, + "loss_nats": 1.247314214706421, + "step": 2340, + "unclipped_grad_norm": 0.7788637280464172 + }, + { + "bits_per_byte": 1.849551646312696, + "learning_rate": 0.00025846617662105193, + "loss_nats": 1.2820115089416504, + "step": 2350, + "unclipped_grad_norm": 0.7732409238815308 + }, + { + "bits_per_byte": 1.8613248906601605, + "learning_rate": 0.00025806271023100637, + "loss_nats": 1.2901721000671387, + "step": 2360, + "unclipped_grad_norm": 0.8177725672721863 + }, + { + "bits_per_byte": 1.8822589628884834, + "learning_rate": 0.00025765765365601165, + "loss_nats": 1.3046824932098389, + "step": 2370, + "unclipped_grad_norm": 0.7931574583053589 + }, + { + "bits_per_byte": 1.8382422391746087, + "learning_rate": 0.00025725101381736707, + "loss_nats": 1.2741724252700806, + "step": 2380, + "unclipped_grad_norm": 0.8604201674461365 + }, + { + "bits_per_byte": 1.8797817247854554, + "learning_rate": 0.00025684279766342567, + "loss_nats": 1.3029654026031494, + "step": 2390, + "unclipped_grad_norm": 0.7636454105377197 + }, + { + "bits_per_byte": 1.817860919186159, + "learning_rate": 0.00025643301216947504, + "loss_nats": 1.2600451707839966, + "step": 2400, + "unclipped_grad_norm": 0.779283881187439 + }, + { + "bits_per_byte": 1.8354538844561699, + "learning_rate": 0.00025602166433761866, + "loss_nats": 1.2722396850585938, + "step": 2410, + "unclipped_grad_norm": 0.7781111001968384 + }, + { + "bits_per_byte": 1.8069649583407457, + "learning_rate": 0.0002556087611966561, + "loss_nats": 1.2524926662445068, + "step": 2420, + "unclipped_grad_norm": 0.7548633813858032 + }, + { + "bits_per_byte": 1.8169932667125561, + "learning_rate": 0.00025519430980196277, + "loss_nats": 1.259443759918213, + "step": 2430, + "unclipped_grad_norm": 0.7562755942344666 + }, + { + "bits_per_byte": 1.8760644917697067, + "learning_rate": 0.0002547783172353695, + "loss_nats": 1.3003888130187988, + "step": 2440, + "unclipped_grad_norm": 0.7478408813476562 + }, + { + "bits_per_byte": 1.838183937055966, + "learning_rate": 0.0002543607906050415, + "loss_nats": 1.2741320133209229, + "step": 2450, + "unclipped_grad_norm": 0.760445237159729 + }, + { + "bits_per_byte": 1.8195946762895068, + "learning_rate": 0.0002539417370453568, + "loss_nats": 1.261246919631958, + "step": 2460, + "unclipped_grad_norm": 0.7331485748291016 + }, + { + "bits_per_byte": 1.8163858239897086, + "learning_rate": 0.00025352116371678464, + "loss_nats": 1.2590227127075195, + "step": 2470, + "unclipped_grad_norm": 0.7682369351387024 + }, + { + "bits_per_byte": 1.8266683227194942, + "learning_rate": 0.0002530990778057627, + "loss_nats": 1.2661499977111816, + "step": 2480, + "unclipped_grad_norm": 0.7132375240325928 + }, + { + "bits_per_byte": 1.8042156436840528, + "learning_rate": 0.00025267548652457475, + "loss_nats": 1.250586986541748, + "step": 2490, + "unclipped_grad_norm": 0.7770085334777832 + }, + { + "bits_per_byte": 1.8313916542427904, + "learning_rate": 0.00025225039711122696, + "loss_nats": 1.2694239616394043, + "step": 2500, + "unclipped_grad_norm": 0.7582318782806396 + }, + { + "bits_per_byte": 1.7788244690759694, + "learning_rate": 0.00025182381682932464, + "loss_nats": 1.2329871654510498, + "step": 2510, + "unclipped_grad_norm": 0.8006963133811951 + }, + { + "bits_per_byte": 1.8448090527325032, + "learning_rate": 0.0002513957529679477, + "loss_nats": 1.278724193572998, + "step": 2520, + "unclipped_grad_norm": 0.7802237272262573 + }, + { + "bits_per_byte": 1.855150369528884, + "learning_rate": 0.00025096621284152656, + "loss_nats": 1.2858922481536865, + "step": 2530, + "unclipped_grad_norm": 0.7603861689567566 + }, + { + "bits_per_byte": 1.7974587892969547, + "learning_rate": 0.0002505352037897168, + "loss_nats": 1.245903491973877, + "step": 2540, + "unclipped_grad_norm": 0.7829954624176025 + }, + { + "bits_per_byte": 1.8048358131230642, + "learning_rate": 0.0002501027331772739, + "loss_nats": 1.2510168552398682, + "step": 2550, + "unclipped_grad_norm": 0.8207260966300964 + }, + { + "bits_per_byte": 1.8410618947355044, + "learning_rate": 0.00024966880839392746, + "loss_nats": 1.2761268615722656, + "step": 2560, + "unclipped_grad_norm": 0.8064310550689697 + }, + { + "bits_per_byte": 1.8072459779922554, + "learning_rate": 0.0002492334368542548, + "loss_nats": 1.2526874542236328, + "step": 2570, + "unclipped_grad_norm": 0.7300699353218079 + }, + { + "bits_per_byte": 1.8352736466380657, + "learning_rate": 0.00024879662599755416, + "loss_nats": 1.2721147537231445, + "step": 2580, + "unclipped_grad_norm": 0.7476873993873596 + }, + { + "bits_per_byte": 1.793617556789934, + "learning_rate": 0.00024835838328771803, + "loss_nats": 1.2432409524917603, + "step": 2590, + "unclipped_grad_norm": 0.7210070490837097 + }, + { + "bits_per_byte": 1.797604974550188, + "learning_rate": 0.0002479187162131051, + "loss_nats": 1.2460048198699951, + "step": 2600, + "unclipped_grad_norm": 0.7853913903236389 + }, + { + "bits_per_byte": 1.7385629865436842, + "learning_rate": 0.00024747763228641264, + "loss_nats": 1.2050800323486328, + "step": 2610, + "unclipped_grad_norm": 0.7092626094818115 + }, + { + "bits_per_byte": 1.8079149905041119, + "learning_rate": 0.00024703513904454795, + "loss_nats": 1.2531511783599854, + "step": 2620, + "unclipped_grad_norm": 0.7522015571594238 + }, + { + "bits_per_byte": 1.8198622812942493, + "learning_rate": 0.00024659124404849956, + "loss_nats": 1.261432409286499, + "step": 2630, + "unclipped_grad_norm": 0.7675716876983643 + }, + { + "bits_per_byte": 1.680096624469618, + "learning_rate": 0.00024614595488320833, + "loss_nats": 1.164554238319397, + "step": 2640, + "unclipped_grad_norm": 0.6825466156005859 + }, + { + "bits_per_byte": 1.7638260340768748, + "learning_rate": 0.00024569927915743745, + "loss_nats": 1.2225910425186157, + "step": 2650, + "unclipped_grad_norm": 0.7630164623260498 + }, + { + "bits_per_byte": 1.8201589513669878, + "learning_rate": 0.0002452512245036427, + "loss_nats": 1.2616380453109741, + "step": 2660, + "unclipped_grad_norm": 0.7772946953773499 + }, + { + "bits_per_byte": 1.811244230659514, + "learning_rate": 0.00024480179857784177, + "loss_nats": 1.2554588317871094, + "step": 2670, + "unclipped_grad_norm": 0.7582975625991821 + }, + { + "bits_per_byte": 1.74257860945868, + "learning_rate": 0.00024435100905948387, + "loss_nats": 1.207863450050354, + "step": 2680, + "unclipped_grad_norm": 0.7818799614906311 + }, + { + "bits_per_byte": 1.9030688636428832, + "learning_rate": 0.00024389886365131793, + "loss_nats": 1.3191068172454834, + "step": 2690, + "unclipped_grad_norm": 0.7075229287147522 + }, + { + "bits_per_byte": 1.8200497423836899, + "learning_rate": 0.00024344537007926163, + "loss_nats": 1.2615623474121094, + "step": 2700, + "unclipped_grad_norm": 0.8272333741188049 + }, + { + "bits_per_byte": 1.7354167359288, + "learning_rate": 0.00024299053609226878, + "loss_nats": 1.2028992176055908, + "step": 2710, + "unclipped_grad_norm": 0.7310962677001953 + }, + { + "bits_per_byte": 1.728331394678554, + "learning_rate": 0.00024253436946219733, + "loss_nats": 1.1979880332946777, + "step": 2720, + "unclipped_grad_norm": 0.7193791270256042 + }, + { + "bits_per_byte": 1.76859700479446, + "learning_rate": 0.0002420768779836764, + "loss_nats": 1.225898027420044, + "step": 2730, + "unclipped_grad_norm": 0.773657500743866 + }, + { + "bits_per_byte": 1.7790022991369616, + "learning_rate": 0.00024161806947397305, + "loss_nats": 1.2331104278564453, + "step": 2740, + "unclipped_grad_norm": 0.7759174704551697 + }, + { + "bits_per_byte": 1.7909771071512373, + "learning_rate": 0.00024115795177285886, + "loss_nats": 1.241410732269287, + "step": 2750, + "unclipped_grad_norm": 0.770866870880127 + }, + { + "bits_per_byte": 1.81886375002334, + "learning_rate": 0.0002406965327424758, + "loss_nats": 1.2607402801513672, + "step": 2760, + "unclipped_grad_norm": 0.7793042659759521 + }, + { + "bits_per_byte": 1.704546881994831, + "learning_rate": 0.00024023382026720197, + "loss_nats": 1.181501865386963, + "step": 2770, + "unclipped_grad_norm": 0.7179723978042603 + }, + { + "bits_per_byte": 1.7222445846992174, + "learning_rate": 0.00023976982225351698, + "loss_nats": 1.1937689781188965, + "step": 2780, + "unclipped_grad_norm": 0.711125910282135 + }, + { + "bits_per_byte": 1.8073385046584196, + "learning_rate": 0.0002393045466298666, + "loss_nats": 1.2527515888214111, + "step": 2790, + "unclipped_grad_norm": 0.7564381957054138 + }, + { + "bits_per_byte": 1.8564130661515186, + "learning_rate": 0.00023883800134652763, + "loss_nats": 1.2867674827575684, + "step": 2800, + "unclipped_grad_norm": 0.7982874512672424 + }, + { + "bits_per_byte": 1.7924368958917607, + "learning_rate": 0.0002383701943754715, + "loss_nats": 1.2424225807189941, + "step": 2810, + "unclipped_grad_norm": 0.827074408531189 + }, + { + "bits_per_byte": 1.781587886310033, + "learning_rate": 0.00023790113371022884, + "loss_nats": 1.2349026203155518, + "step": 2820, + "unclipped_grad_norm": 0.6945767998695374 + }, + { + "bits_per_byte": 1.862601517877515, + "learning_rate": 0.0002374308273657521, + "loss_nats": 1.2910569906234741, + "step": 2830, + "unclipped_grad_norm": 0.7473636269569397 + }, + { + "bits_per_byte": 1.7520161574247766, + "learning_rate": 0.00023695928337827911, + "loss_nats": 1.2144050598144531, + "step": 2840, + "unclipped_grad_norm": 0.7269325852394104 + }, + { + "bits_per_byte": 1.7726036846116109, + "learning_rate": 0.00023648650980519564, + "loss_nats": 1.2286752462387085, + "step": 2850, + "unclipped_grad_norm": 0.7537488341331482 + }, + { + "bits_per_byte": 1.7375121725469125, + "learning_rate": 0.00023601251472489757, + "loss_nats": 1.2043516635894775, + "step": 2860, + "unclipped_grad_norm": 0.7839256525039673 + }, + { + "bits_per_byte": 1.836773851301542, + "learning_rate": 0.00023553730623665305, + "loss_nats": 1.273154616355896, + "step": 2870, + "unclipped_grad_norm": 0.7615643739700317 + }, + { + "bits_per_byte": 1.8235200083127996, + "learning_rate": 0.00023506089246046403, + "loss_nats": 1.263967752456665, + "step": 2880, + "unclipped_grad_norm": 0.7500733137130737 + }, + { + "bits_per_byte": 1.7346435019305209, + "learning_rate": 0.00023458328153692736, + "loss_nats": 1.2023632526397705, + "step": 2890, + "unclipped_grad_norm": 0.7335168123245239 + }, + { + "bits_per_byte": 1.710140445731493, + "learning_rate": 0.0002341044816270961, + "loss_nats": 1.1853790283203125, + "step": 2900, + "unclipped_grad_norm": 0.7237004041671753 + }, + { + "bits_per_byte": 1.7640040361205178, + "learning_rate": 0.00023362450091233953, + "loss_nats": 1.2227144241333008, + "step": 2910, + "unclipped_grad_norm": 0.7653093338012695 + }, + { + "bits_per_byte": 1.7372325287566095, + "learning_rate": 0.00023314334759420388, + "loss_nats": 1.204157829284668, + "step": 2920, + "unclipped_grad_norm": 0.7592934966087341 + }, + { + "bits_per_byte": 1.7605243111456106, + "learning_rate": 0.0002326610298942717, + "loss_nats": 1.2203024625778198, + "step": 2930, + "unclipped_grad_norm": 0.7871323227882385 + }, + { + "bits_per_byte": 1.7773966691085064, + "learning_rate": 0.00023217755605402175, + "loss_nats": 1.2319974899291992, + "step": 2940, + "unclipped_grad_norm": 0.7994735240936279 + }, + { + "bits_per_byte": 1.761262116717812, + "learning_rate": 0.00023169293433468802, + "loss_nats": 1.2208138704299927, + "step": 2950, + "unclipped_grad_norm": 0.7672474384307861 + }, + { + "bits_per_byte": 1.8009139207527887, + "learning_rate": 0.00023120717301711857, + "loss_nats": 1.2482984066009521, + "step": 2960, + "unclipped_grad_norm": 0.7721540927886963 + }, + { + "bits_per_byte": 1.7577087831683356, + "learning_rate": 0.00023072028040163408, + "loss_nats": 1.218350887298584, + "step": 2970, + "unclipped_grad_norm": 0.7659915685653687 + }, + { + "bits_per_byte": 1.7511909846659368, + "learning_rate": 0.00023023226480788593, + "loss_nats": 1.2138330936431885, + "step": 2980, + "unclipped_grad_norm": 0.7457980513572693 + }, + { + "bits_per_byte": 1.7677185174138528, + "learning_rate": 0.00022974313457471416, + "loss_nats": 1.2252891063690186, + "step": 2990, + "unclipped_grad_norm": 0.7551211714744568 + }, + { + "bits_per_byte": 1.7804380103063644, + "learning_rate": 0.0002292528980600049, + "loss_nats": 1.2341055870056152, + "step": 3000, + "unclipped_grad_norm": 0.7369211316108704 + }, + { + "bits_per_byte": 1.77081558099059, + "learning_rate": 0.00022876156364054758, + "loss_nats": 1.227435827255249, + "step": 3010, + "unclipped_grad_norm": 0.7526711225509644 + }, + { + "bits_per_byte": 1.726040929734363, + "learning_rate": 0.0002282691397118918, + "loss_nats": 1.1964004039764404, + "step": 3020, + "unclipped_grad_norm": 0.8113403916358948 + }, + { + "bits_per_byte": 1.8045116258261877, + "learning_rate": 0.00022777563468820376, + "loss_nats": 1.250792145729065, + "step": 3030, + "unclipped_grad_norm": 0.7493385672569275 + }, + { + "bits_per_byte": 1.7229166928987893, + "learning_rate": 0.00022728105700212282, + "loss_nats": 1.194234848022461, + "step": 3040, + "unclipped_grad_norm": 0.7606903314590454 + }, + { + "bits_per_byte": 1.7403560776615798, + "learning_rate": 0.00022678541510461698, + "loss_nats": 1.2063229084014893, + "step": 3050, + "unclipped_grad_norm": 0.7690193057060242 + }, + { + "bits_per_byte": 1.759874388708, + "learning_rate": 0.00022628871746483876, + "loss_nats": 1.2198519706726074, + "step": 3060, + "unclipped_grad_norm": 0.7480750679969788 + }, + { + "bits_per_byte": 1.6873952242069339, + "learning_rate": 0.0002257909725699804, + "loss_nats": 1.169613242149353, + "step": 3070, + "unclipped_grad_norm": 0.7001752853393555 + }, + { + "bits_per_byte": 1.737272084766308, + "learning_rate": 0.00022529218892512886, + "loss_nats": 1.2041852474212646, + "step": 3080, + "unclipped_grad_norm": 0.8322204351425171 + }, + { + "bits_per_byte": 1.8015143121869506, + "learning_rate": 0.00022479237505312034, + "loss_nats": 1.248714566230774, + "step": 3090, + "unclipped_grad_norm": 0.8183287382125854 + }, + { + "bits_per_byte": 1.7001176408045102, + "learning_rate": 0.00022429153949439496, + "loss_nats": 1.178431749343872, + "step": 3100, + "unclipped_grad_norm": 0.7750749588012695 + }, + { + "bits_per_byte": 1.7770231227908324, + "learning_rate": 0.00022378969080685053, + "loss_nats": 1.231738567352295, + "step": 3110, + "unclipped_grad_norm": 0.7082787752151489 + }, + { + "bits_per_byte": 1.749435729731231, + "learning_rate": 0.0002232868375656965, + "loss_nats": 1.2126164436340332, + "step": 3120, + "unclipped_grad_norm": 0.736611545085907 + }, + { + "bits_per_byte": 1.7966217497352055, + "learning_rate": 0.00022278298836330733, + "loss_nats": 1.2453233003616333, + "step": 3130, + "unclipped_grad_norm": 0.7366204857826233 + }, + { + "bits_per_byte": 1.7636881039908827, + "learning_rate": 0.00022227815180907572, + "loss_nats": 1.222495436668396, + "step": 3140, + "unclipped_grad_norm": 0.7583194375038147 + }, + { + "bits_per_byte": 1.7359698321339747, + "learning_rate": 0.00022177233652926546, + "loss_nats": 1.2032825946807861, + "step": 3150, + "unclipped_grad_norm": 0.7600930333137512 + }, + { + "bits_per_byte": 1.7415977924008093, + "learning_rate": 0.00022126555116686416, + "loss_nats": 1.207183599472046, + "step": 3160, + "unclipped_grad_norm": 0.7639689445495605 + }, + { + "bits_per_byte": 1.654884311818432, + "learning_rate": 0.00022075780438143532, + "loss_nats": 1.1470783948898315, + "step": 3170, + "unclipped_grad_norm": 0.7011478543281555 + }, + { + "bits_per_byte": 1.7584833930278216, + "learning_rate": 0.00022024910484897065, + "loss_nats": 1.2188878059387207, + "step": 3180, + "unclipped_grad_norm": 0.7645173072814941 + }, + { + "bits_per_byte": 1.7305350083840592, + "learning_rate": 0.00021973946126174158, + "loss_nats": 1.199515461921692, + "step": 3190, + "unclipped_grad_norm": 0.7665988206863403 + }, + { + "bits_per_byte": 1.7003714871971838, + "learning_rate": 0.00021922888232815086, + "loss_nats": 1.178607702255249, + "step": 3200, + "unclipped_grad_norm": 0.7379894852638245 + }, + { + "bits_per_byte": 1.6959066455981344, + "learning_rate": 0.0002187173767725837, + "loss_nats": 1.1755129098892212, + "step": 3210, + "unclipped_grad_norm": 0.8013631701469421 + }, + { + "bits_per_byte": 1.7098007799960389, + "learning_rate": 0.0002182049533352588, + "loss_nats": 1.1851435899734497, + "step": 3220, + "unclipped_grad_norm": 0.7296364903450012 + }, + { + "bits_per_byte": 1.7987765203678656, + "learning_rate": 0.00021769162077207884, + "loss_nats": 1.246816873550415, + "step": 3230, + "unclipped_grad_norm": 0.7610434889793396 + }, + { + "bits_per_byte": 1.6862874839527267, + "learning_rate": 0.00021717738785448102, + "loss_nats": 1.1688454151153564, + "step": 3240, + "unclipped_grad_norm": 0.723595917224884 + }, + { + "bits_per_byte": 1.8345631863073515, + "learning_rate": 0.00021666226336928708, + "loss_nats": 1.2716223001480103, + "step": 3250, + "unclipped_grad_norm": 0.8353017568588257 + }, + { + "bits_per_byte": 1.6872093109613513, + "learning_rate": 0.0002161462561185532, + "loss_nats": 1.1694843769073486, + "step": 3260, + "unclipped_grad_norm": 0.7602720260620117 + }, + { + "bits_per_byte": 1.7198954736710823, + "learning_rate": 0.00021562937491941957, + "loss_nats": 1.1921406984329224, + "step": 3270, + "unclipped_grad_norm": 0.7516167163848877 + }, + { + "bits_per_byte": 1.7692164863028679, + "learning_rate": 0.00021511162860395975, + "loss_nats": 1.2263274192810059, + "step": 3280, + "unclipped_grad_norm": 0.7487898468971252 + }, + { + "bits_per_byte": 1.7624494849393688, + "learning_rate": 0.0002145930260190298, + "loss_nats": 1.2216368913650513, + "step": 3290, + "unclipped_grad_norm": 0.7708044052124023 + }, + { + "bits_per_byte": 1.7002387165907176, + "learning_rate": 0.00021407357602611698, + "loss_nats": 1.1785156726837158, + "step": 3300, + "unclipped_grad_norm": 0.7703046202659607 + }, + { + "bits_per_byte": 1.7332599015043295, + "learning_rate": 0.0002135532875011885, + "loss_nats": 1.2014042139053345, + "step": 3310, + "unclipped_grad_norm": 0.8132672905921936 + }, + { + "bits_per_byte": 1.7262779218272521, + "learning_rate": 0.0002130321693345397, + "loss_nats": 1.1965646743774414, + "step": 3320, + "unclipped_grad_norm": 0.7545798420906067 + }, + { + "bits_per_byte": 1.7272033604715449, + "learning_rate": 0.00021251023043064224, + "loss_nats": 1.1972061395645142, + "step": 3330, + "unclipped_grad_norm": 0.7775130867958069 + }, + { + "bits_per_byte": 1.6975052243379043, + "learning_rate": 0.00021198747970799188, + "loss_nats": 1.1766209602355957, + "step": 3340, + "unclipped_grad_norm": 0.7105540633201599 + }, + { + "bits_per_byte": 1.7083408192728646, + "learning_rate": 0.00021146392609895618, + "loss_nats": 1.1841316223144531, + "step": 3350, + "unclipped_grad_norm": 0.7163253426551819 + }, + { + "bits_per_byte": 1.732425613664994, + "learning_rate": 0.00021093957854962178, + "loss_nats": 1.2008259296417236, + "step": 3360, + "unclipped_grad_norm": 0.7626740336418152 + }, + { + "bits_per_byte": 1.6756070173688444, + "learning_rate": 0.00021041444601964157, + "loss_nats": 1.1614422798156738, + "step": 3370, + "unclipped_grad_norm": 0.8061453700065613 + }, + { + "bits_per_byte": 1.7092863798873084, + "learning_rate": 0.00020988853748208164, + "loss_nats": 1.1847870349884033, + "step": 3380, + "unclipped_grad_norm": 0.7406735420227051 + }, + { + "bits_per_byte": 1.7539413312185348, + "learning_rate": 0.00020936186192326782, + "loss_nats": 1.2157394886016846, + "step": 3390, + "unclipped_grad_norm": 0.7695116400718689 + }, + { + "bits_per_byte": 1.8092177590843976, + "learning_rate": 0.00020883442834263225, + "loss_nats": 1.2540541887283325, + "step": 3400, + "unclipped_grad_norm": 0.7858016490936279 + }, + { + "bits_per_byte": 1.8255297975707825, + "learning_rate": 0.0002083062457525596, + "loss_nats": 1.2653608322143555, + "step": 3410, + "unclipped_grad_norm": 0.7603964805603027 + }, + { + "bits_per_byte": 1.6857877023693193, + "learning_rate": 0.00020777732317823306, + "loss_nats": 1.1684989929199219, + "step": 3420, + "unclipped_grad_norm": 0.7168062329292297 + }, + { + "bits_per_byte": 1.6176447364444573, + "learning_rate": 0.00020724766965748005, + "loss_nats": 1.1212658882141113, + "step": 3430, + "unclipped_grad_norm": 0.7431134581565857 + }, + { + "bits_per_byte": 1.711189711884407, + "learning_rate": 0.00020671729424061788, + "loss_nats": 1.1861063241958618, + "step": 3440, + "unclipped_grad_norm": 0.7753933668136597 + }, + { + "bits_per_byte": 1.77535626693867, + "learning_rate": 0.00020618620599029906, + "loss_nats": 1.2305831909179688, + "step": 3450, + "unclipped_grad_norm": 0.8042635321617126 + }, + { + "bits_per_byte": 1.7083845028661837, + "learning_rate": 0.00020565441398135649, + "loss_nats": 1.184161901473999, + "step": 3460, + "unclipped_grad_norm": 0.7639697194099426 + }, + { + "bits_per_byte": 1.6937178224006033, + "learning_rate": 0.00020512192730064827, + "loss_nats": 1.1739957332611084, + "step": 3470, + "unclipped_grad_norm": 0.748127281665802 + }, + { + "bits_per_byte": 1.6501064617948125, + "learning_rate": 0.00020458875504690273, + "loss_nats": 1.1437666416168213, + "step": 3480, + "unclipped_grad_norm": 0.7506517767906189 + }, + { + "bits_per_byte": 1.8358463488654388, + "learning_rate": 0.00020405490633056245, + "loss_nats": 1.2725117206573486, + "step": 3490, + "unclipped_grad_norm": 0.836082398891449 + }, + { + "bits_per_byte": 1.714104301868579, + "learning_rate": 0.00020352039027362917, + "loss_nats": 1.188126564025879, + "step": 3500, + "unclipped_grad_norm": 0.7544166445732117 + }, + { + "bits_per_byte": 1.744907598516664, + "learning_rate": 0.0002029852160095075, + "loss_nats": 1.2094777822494507, + "step": 3510, + "unclipped_grad_norm": 0.7650617361068726 + }, + { + "bits_per_byte": 1.738600994709525, + "learning_rate": 0.000202449392682849, + "loss_nats": 1.2051063776016235, + "step": 3520, + "unclipped_grad_norm": 0.7794367671012878 + }, + { + "bits_per_byte": 1.7051804660806096, + "learning_rate": 0.0002019129294493959, + "loss_nats": 1.181941032409668, + "step": 3530, + "unclipped_grad_norm": 0.7445787787437439 + }, + { + "bits_per_byte": 1.7158172490711725, + "learning_rate": 0.00020137583547582478, + "loss_nats": 1.1893138885498047, + "step": 3540, + "unclipped_grad_norm": 0.7629107236862183 + }, + { + "bits_per_byte": 1.7173026632266746, + "learning_rate": 0.00020083811993958973, + "loss_nats": 1.1903434991836548, + "step": 3550, + "unclipped_grad_norm": 0.7763495445251465 + }, + { + "bits_per_byte": 1.6560372835098163, + "learning_rate": 0.00020029979202876563, + "loss_nats": 1.14787757396698, + "step": 3560, + "unclipped_grad_norm": 0.7450886964797974 + }, + { + "bits_per_byte": 1.6838996768281473, + "learning_rate": 0.00019976086094189118, + "loss_nats": 1.1671903133392334, + "step": 3570, + "unclipped_grad_norm": 0.7487124800682068 + }, + { + "bits_per_byte": 1.6946102403759304, + "learning_rate": 0.00019922133588781175, + "loss_nats": 1.1746143102645874, + "step": 3580, + "unclipped_grad_norm": 0.757170557975769 + }, + { + "bits_per_byte": 1.7307594457434352, + "learning_rate": 0.0001986812260855219, + "loss_nats": 1.1996710300445557, + "step": 3590, + "unclipped_grad_norm": 0.7700981497764587 + }, + { + "bits_per_byte": 1.7491381997452384, + "learning_rate": 0.00019814054076400787, + "loss_nats": 1.2124102115631104, + "step": 3600, + "unclipped_grad_norm": 0.7877120971679688 + }, + { + "bits_per_byte": 1.688025540622346, + "learning_rate": 0.00019759928916209004, + "loss_nats": 1.1700501441955566, + "step": 3610, + "unclipped_grad_norm": 0.780867874622345 + }, + { + "bits_per_byte": 1.6436277033541595, + "learning_rate": 0.00019705748052826493, + "loss_nats": 1.1392759084701538, + "step": 3620, + "unclipped_grad_norm": 0.7423334717750549 + }, + { + "bits_per_byte": 1.6950637586262558, + "learning_rate": 0.00019651512412054723, + "loss_nats": 1.1749286651611328, + "step": 3630, + "unclipped_grad_norm": 0.7513464689254761 + }, + { + "bits_per_byte": 1.6758522646289749, + "learning_rate": 0.00019597222920631144, + "loss_nats": 1.1616122722625732, + "step": 3640, + "unclipped_grad_norm": 0.7453336715698242 + }, + { + "bits_per_byte": 1.736063734661346, + "learning_rate": 0.0001954288050621338, + "loss_nats": 1.2033476829528809, + "step": 3650, + "unclipped_grad_norm": 0.7682862877845764 + }, + { + "bits_per_byte": 1.6360068081291268, + "learning_rate": 0.00019488486097363363, + "loss_nats": 1.1339935064315796, + "step": 3660, + "unclipped_grad_norm": 0.7531437277793884 + }, + { + "bits_per_byte": 1.7183529612754938, + "learning_rate": 0.00019434040623531452, + "loss_nats": 1.1910715103149414, + "step": 3670, + "unclipped_grad_norm": 0.7578480243682861 + }, + { + "bits_per_byte": 1.6583936178092877, + "learning_rate": 0.0001937954501504058, + "loss_nats": 1.1495108604431152, + "step": 3680, + "unclipped_grad_norm": 0.7224898934364319 + }, + { + "bits_per_byte": 1.6372521345040243, + "learning_rate": 0.00019325000203070333, + "loss_nats": 1.1348567008972168, + "step": 3690, + "unclipped_grad_norm": 0.7246464490890503 + }, + { + "bits_per_byte": 1.6659130432703113, + "learning_rate": 0.0001927040711964106, + "loss_nats": 1.1547229290008545, + "step": 3700, + "unclipped_grad_norm": 0.7878638505935669 + }, + { + "bits_per_byte": 1.694639821391879, + "learning_rate": 0.00019215766697597923, + "loss_nats": 1.1746348142623901, + "step": 3710, + "unclipped_grad_norm": 0.7455682754516602 + }, + { + "bits_per_byte": 1.6891824679147003, + "learning_rate": 0.00019161079870594978, + "loss_nats": 1.1708520650863647, + "step": 3720, + "unclipped_grad_norm": 0.7515487670898438 + }, + { + "bits_per_byte": 1.6091924770851538, + "learning_rate": 0.0001910634757307922, + "loss_nats": 1.1154072284698486, + "step": 3730, + "unclipped_grad_norm": 0.7353127598762512 + }, + { + "bits_per_byte": 1.6633680440028435, + "learning_rate": 0.0001905157074027459, + "loss_nats": 1.152958869934082, + "step": 3740, + "unclipped_grad_norm": 0.7253232002258301 + }, + { + "bits_per_byte": 1.6922951819126664, + "learning_rate": 0.00018996750308166037, + "loss_nats": 1.1730096340179443, + "step": 3750, + "unclipped_grad_norm": 0.7686370611190796 + }, + { + "bits_per_byte": 1.7713081393026613, + "learning_rate": 0.00018941887213483482, + "loss_nats": 1.2277772426605225, + "step": 3760, + "unclipped_grad_norm": 0.7601476907730103 + }, + { + "bits_per_byte": 1.7305630415561497, + "learning_rate": 0.00018886982393685845, + "loss_nats": 1.1995348930358887, + "step": 3770, + "unclipped_grad_norm": 0.7877833247184753 + }, + { + "bits_per_byte": 1.5644979697442287, + "learning_rate": 0.00018832036786945008, + "loss_nats": 1.0844273567199707, + "step": 3780, + "unclipped_grad_norm": 0.7386898994445801 + }, + { + "bits_per_byte": 1.6644766441703052, + "learning_rate": 0.0001877705133212979, + "loss_nats": 1.1537272930145264, + "step": 3790, + "unclipped_grad_norm": 0.7420179843902588 + }, + { + "bits_per_byte": 1.6653939996300076, + "learning_rate": 0.00018722026968789906, + "loss_nats": 1.1543631553649902, + "step": 3800, + "unclipped_grad_norm": 0.7838358283042908 + }, + { + "bits_per_byte": 1.6952983429620325, + "learning_rate": 0.00018666964637139907, + "loss_nats": 1.17509126663208, + "step": 3810, + "unclipped_grad_norm": 0.8048396110534668 + }, + { + "bits_per_byte": 1.6744323758634516, + "learning_rate": 0.00018611865278043115, + "loss_nats": 1.160628080368042, + "step": 3820, + "unclipped_grad_norm": 0.7709245681762695 + }, + { + "bits_per_byte": 1.720191799778519, + "learning_rate": 0.00018556729832995555, + "loss_nats": 1.1923460960388184, + "step": 3830, + "unclipped_grad_norm": 0.7672807574272156 + }, + { + "bits_per_byte": 1.6467005173771254, + "learning_rate": 0.00018501559244109864, + "loss_nats": 1.1414058208465576, + "step": 3840, + "unclipped_grad_norm": 0.7319741249084473 + }, + { + "bits_per_byte": 1.5964396195583748, + "learning_rate": 0.00018446354454099178, + "loss_nats": 1.106567621231079, + "step": 3850, + "unclipped_grad_norm": 0.7756319642066956 + }, + { + "bits_per_byte": 1.6360811046342996, + "learning_rate": 0.00018391116406261048, + "loss_nats": 1.1340450048446655, + "step": 3860, + "unclipped_grad_norm": 0.766754150390625 + }, + { + "bits_per_byte": 1.7433616464680584, + "learning_rate": 0.00018335846044461305, + "loss_nats": 1.2084062099456787, + "step": 3870, + "unclipped_grad_norm": 0.7916337847709656 + }, + { + "bits_per_byte": 1.6358671582166262, + "learning_rate": 0.0001828054431311793, + "loss_nats": 1.1338967084884644, + "step": 3880, + "unclipped_grad_norm": 0.7497678399085999 + }, + { + "bits_per_byte": 1.600699113872294, + "learning_rate": 0.00018225212157184934, + "loss_nats": 1.1095200777053833, + "step": 3890, + "unclipped_grad_norm": 0.7555440664291382 + }, + { + "bits_per_byte": 1.6778517349279058, + "learning_rate": 0.00018169850522136192, + "loss_nats": 1.1629981994628906, + "step": 3900, + "unclipped_grad_norm": 0.826478123664856 + }, + { + "bits_per_byte": 1.6805711246033486, + "learning_rate": 0.000181144603539493, + "loss_nats": 1.1648831367492676, + "step": 3910, + "unclipped_grad_norm": 0.7849931120872498 + }, + { + "bits_per_byte": 1.6375147520118918, + "learning_rate": 0.00018059042599089412, + "loss_nats": 1.1350387334823608, + "step": 3920, + "unclipped_grad_norm": 0.7588120102882385 + }, + { + "bits_per_byte": 1.673005091843941, + "learning_rate": 0.00018003598204493048, + "loss_nats": 1.15963876247406, + "step": 3930, + "unclipped_grad_norm": 0.742434024810791 + }, + { + "bits_per_byte": 1.6996766772876981, + "learning_rate": 0.00017948128117551936, + "loss_nats": 1.1781260967254639, + "step": 3940, + "unclipped_grad_norm": 0.7588658928871155 + }, + { + "bits_per_byte": 1.7339400928884918, + "learning_rate": 0.00017892633286096816, + "loss_nats": 1.2018756866455078, + "step": 3950, + "unclipped_grad_norm": 0.7937244176864624 + }, + { + "bits_per_byte": 1.713467450112434, + "learning_rate": 0.00017837114658381247, + "loss_nats": 1.1876851320266724, + "step": 3960, + "unclipped_grad_norm": 0.7884036302566528 + }, + { + "bits_per_byte": 1.6516962694193882, + "learning_rate": 0.00017781573183065386, + "loss_nats": 1.1448686122894287, + "step": 3970, + "unclipped_grad_norm": 0.8004661798477173 + }, + { + "bits_per_byte": 1.6740500584305835, + "learning_rate": 0.0001772600980919981, + "loss_nats": 1.1603630781173706, + "step": 3980, + "unclipped_grad_norm": 0.7692103385925293 + }, + { + "bits_per_byte": 1.691941929547794, + "learning_rate": 0.0001767042548620927, + "loss_nats": 1.172764778137207, + "step": 3990, + "unclipped_grad_norm": 0.7891059517860413 + }, + { + "bits_per_byte": 1.5813677479673616, + "learning_rate": 0.00017614821163876486, + "loss_nats": 1.0961205959320068, + "step": 4000, + "unclipped_grad_norm": 0.7436221837997437 + }, + { + "bits_per_byte": 1.6793110077204767, + "learning_rate": 0.00017559197792325906, + "loss_nats": 1.164009690284729, + "step": 4010, + "unclipped_grad_norm": 0.8044730424880981 + }, + { + "bits_per_byte": 1.6584589712166156, + "learning_rate": 0.0001750355632200748, + "loss_nats": 1.1495561599731445, + "step": 4020, + "unclipped_grad_norm": 0.8183668851852417 + }, + { + "bits_per_byte": 1.6850787898824628, + "learning_rate": 0.00017447897703680418, + "loss_nats": 1.1680076122283936, + "step": 4030, + "unclipped_grad_norm": 0.793128252029419 + }, + { + "bits_per_byte": 1.617821878574846, + "learning_rate": 0.0001739222288839694, + "loss_nats": 1.1213886737823486, + "step": 4040, + "unclipped_grad_norm": 0.7582541108131409 + }, + { + "bits_per_byte": 1.6450140555027646, + "learning_rate": 0.00017336532827486026, + "loss_nats": 1.1402368545532227, + "step": 4050, + "unclipped_grad_norm": 0.7559582591056824 + }, + { + "bits_per_byte": 1.6581645369183384, + "learning_rate": 0.00017280828472537163, + "loss_nats": 1.1493520736694336, + "step": 4060, + "unclipped_grad_norm": 0.743245005607605 + }, + { + "bits_per_byte": 1.680131880913045, + "learning_rate": 0.0001722511077538408, + "loss_nats": 1.1645786762237549, + "step": 4070, + "unclipped_grad_norm": 0.7624887824058533 + }, + { + "bits_per_byte": 1.7564567494700543, + "learning_rate": 0.00017169380688088497, + "loss_nats": 1.2174830436706543, + "step": 4080, + "unclipped_grad_norm": 0.8132302761077881 + }, + { + "bits_per_byte": 1.6669108866106175, + "learning_rate": 0.0001711363916292384, + "loss_nats": 1.1554145812988281, + "step": 4090, + "unclipped_grad_norm": 0.8276432752609253 + }, + { + "bits_per_byte": 1.638365894151012, + "learning_rate": 0.00017057887152358974, + "loss_nats": 1.1356287002563477, + "step": 4100, + "unclipped_grad_norm": 0.7751885652542114 + }, + { + "bits_per_byte": 1.6303674970073352, + "learning_rate": 0.0001700212560904194, + "loss_nats": 1.1300846338272095, + "step": 4110, + "unclipped_grad_norm": 0.7898787260055542 + }, + { + "bits_per_byte": 1.6896222275529564, + "learning_rate": 0.00016946355485783656, + "loss_nats": 1.171156883239746, + "step": 4120, + "unclipped_grad_norm": 0.8148247599601746 + }, + { + "bits_per_byte": 1.6213521664491068, + "learning_rate": 0.0001689057773554166, + "loss_nats": 1.1238356828689575, + "step": 4130, + "unclipped_grad_norm": 0.8133904933929443 + }, + { + "bits_per_byte": 1.6881359534842002, + "learning_rate": 0.00016834793311403806, + "loss_nats": 1.1701266765594482, + "step": 4140, + "unclipped_grad_norm": 0.8217440247535706 + }, + { + "bits_per_byte": 1.6129334437067215, + "learning_rate": 0.00016779003166571981, + "loss_nats": 1.1180002689361572, + "step": 4150, + "unclipped_grad_norm": 0.7947918176651001 + }, + { + "bits_per_byte": 1.651558683298698, + "learning_rate": 0.00016723208254345838, + "loss_nats": 1.144773244857788, + "step": 4160, + "unclipped_grad_norm": 0.7860805988311768 + }, + { + "bits_per_byte": 1.6828089628563754, + "learning_rate": 0.00016667409528106481, + "loss_nats": 1.1664342880249023, + "step": 4170, + "unclipped_grad_norm": 0.8207855224609375 + }, + { + "bits_per_byte": 1.7311515661874024, + "learning_rate": 0.00016611607941300184, + "loss_nats": 1.1999428272247314, + "step": 4180, + "unclipped_grad_norm": 0.8373809456825256 + }, + { + "bits_per_byte": 1.6433404923272186, + "learning_rate": 0.0001655580444742211, + "loss_nats": 1.139076828956604, + "step": 4190, + "unclipped_grad_norm": 0.7621071338653564 + }, + { + "bits_per_byte": 1.682679287937625, + "learning_rate": 0.00016499999999999997, + "loss_nats": 1.166344404220581, + "step": 4200, + "unclipped_grad_norm": 0.7672834396362305 + }, + { + "bits_per_byte": 1.661693964879345, + "learning_rate": 0.0001644419555257789, + "loss_nats": 1.1517984867095947, + "step": 4210, + "unclipped_grad_norm": 0.7330533862113953 + }, + { + "bits_per_byte": 1.6972419188994332, + "learning_rate": 0.0001638839205869981, + "loss_nats": 1.1764384508132935, + "step": 4220, + "unclipped_grad_norm": 0.7866858243942261 + }, + { + "bits_per_byte": 1.6670219874030747, + "learning_rate": 0.0001633259047189352, + "loss_nats": 1.155491590499878, + "step": 4230, + "unclipped_grad_norm": 0.7902593612670898 + }, + { + "bits_per_byte": 1.6114956687455084, + "learning_rate": 0.0001627679174565416, + "loss_nats": 1.1170036792755127, + "step": 4240, + "unclipped_grad_norm": 0.740251362323761 + }, + { + "bits_per_byte": 1.6335233786506678, + "learning_rate": 0.00016220996833428016, + "loss_nats": 1.1322721242904663, + "step": 4250, + "unclipped_grad_norm": 0.7777352333068848 + }, + { + "bits_per_byte": 1.6115895712728796, + "learning_rate": 0.00016165206688596191, + "loss_nats": 1.1170687675476074, + "step": 4260, + "unclipped_grad_norm": 0.774424135684967 + }, + { + "bits_per_byte": 1.619611874005026, + "learning_rate": 0.00016109422264458335, + "loss_nats": 1.1226294040679932, + "step": 4270, + "unclipped_grad_norm": 0.7803477048873901 + }, + { + "bits_per_byte": 1.6299772683725275, + "learning_rate": 0.00016053644514216336, + "loss_nats": 1.1298141479492188, + "step": 4280, + "unclipped_grad_norm": 0.7580373287200928 + }, + { + "bits_per_byte": 1.6795242662075467, + "learning_rate": 0.00015997874390958058, + "loss_nats": 1.164157509803772, + "step": 4290, + "unclipped_grad_norm": 0.7638103365898132 + }, + { + "bits_per_byte": 1.600082040120998, + "learning_rate": 0.00015942112847641018, + "loss_nats": 1.109092354774475, + "step": 4300, + "unclipped_grad_norm": 0.7825598120689392 + }, + { + "bits_per_byte": 1.6771565810531184, + "learning_rate": 0.00015886360837076156, + "loss_nats": 1.1625163555145264, + "step": 4310, + "unclipped_grad_norm": 0.7940689921379089 + }, + { + "bits_per_byte": 1.6577068910843924, + "learning_rate": 0.00015830619311911495, + "loss_nats": 1.149034857749939, + "step": 4320, + "unclipped_grad_norm": 0.7976187467575073 + }, + { + "bits_per_byte": 1.6195226150092283, + "learning_rate": 0.00015774889224615915, + "loss_nats": 1.1225675344467163, + "step": 4330, + "unclipped_grad_norm": 0.7892199158668518 + }, + { + "bits_per_byte": 1.655279183984813, + "learning_rate": 0.00015719171527462834, + "loss_nats": 1.1473520994186401, + "step": 4340, + "unclipped_grad_norm": 0.8144169449806213 + }, + { + "bits_per_byte": 1.6394932404274176, + "learning_rate": 0.0001566346717251397, + "loss_nats": 1.136410117149353, + "step": 4350, + "unclipped_grad_norm": 0.8119622468948364 + }, + { + "bits_per_byte": 1.6287904160989233, + "learning_rate": 0.00015607777111603056, + "loss_nats": 1.1289914846420288, + "step": 4360, + "unclipped_grad_norm": 0.7554351091384888 + }, + { + "bits_per_byte": 1.6321829458698431, + "learning_rate": 0.00015552102296319576, + "loss_nats": 1.1313430070877075, + "step": 4370, + "unclipped_grad_norm": 0.808085024356842 + }, + { + "bits_per_byte": 1.6820371047193032, + "learning_rate": 0.00015496443677992516, + "loss_nats": 1.1658992767333984, + "step": 4380, + "unclipped_grad_norm": 0.7708722949028015 + }, + { + "bits_per_byte": 1.6522423143358775, + "learning_rate": 0.0001544080220767409, + "loss_nats": 1.1452471017837524, + "step": 4390, + "unclipped_grad_norm": 0.8123121857643127 + }, + { + "bits_per_byte": 1.6025224739367414, + "learning_rate": 0.00015385178836123511, + "loss_nats": 1.1107839345932007, + "step": 4400, + "unclipped_grad_norm": 0.7717524766921997 + }, + { + "bits_per_byte": 1.6853694405624209, + "learning_rate": 0.00015329574513790726, + "loss_nats": 1.1682090759277344, + "step": 4410, + "unclipped_grad_norm": 0.81525057554245 + }, + { + "bits_per_byte": 1.6142167782474597, + "learning_rate": 0.00015273990190800186, + "loss_nats": 1.1188898086547852, + "step": 4420, + "unclipped_grad_norm": 0.7956154346466064 + }, + { + "bits_per_byte": 1.6755413199962148, + "learning_rate": 0.0001521842681693461, + "loss_nats": 1.1613967418670654, + "step": 4430, + "unclipped_grad_norm": 0.7907513380050659 + }, + { + "bits_per_byte": 1.6011154838700328, + "learning_rate": 0.00015162885341618752, + "loss_nats": 1.1098086833953857, + "step": 4440, + "unclipped_grad_norm": 0.7607184648513794 + }, + { + "bits_per_byte": 1.5990663105850023, + "learning_rate": 0.00015107366713903178, + "loss_nats": 1.1083883047103882, + "step": 4450, + "unclipped_grad_norm": 0.7442869544029236 + }, + { + "bits_per_byte": 1.5859970049806358, + "learning_rate": 0.0001505187188244806, + "loss_nats": 1.0993293523788452, + "step": 4460, + "unclipped_grad_norm": 0.7537500262260437 + }, + { + "bits_per_byte": 1.6451918855637568, + "learning_rate": 0.00014996401795506952, + "loss_nats": 1.1403601169586182, + "step": 4470, + "unclipped_grad_norm": 0.7931647896766663 + }, + { + "bits_per_byte": 1.6505943765753102, + "learning_rate": 0.00014940957400910585, + "loss_nats": 1.1441048383712769, + "step": 4480, + "unclipped_grad_norm": 0.9106901288032532 + }, + { + "bits_per_byte": 1.6301040195862135, + "learning_rate": 0.00014885539646050699, + "loss_nats": 1.1299020051956177, + "step": 4490, + "unclipped_grad_norm": 0.7690432667732239 + }, + { + "bits_per_byte": 1.6520481459230536, + "learning_rate": 0.00014830149477863805, + "loss_nats": 1.1451125144958496, + "step": 4500, + "unclipped_grad_norm": 0.7990603446960449 + }, + { + "bits_per_byte": 1.5968599451570833, + "learning_rate": 0.00014774787842815066, + "loss_nats": 1.1068589687347412, + "step": 4510, + "unclipped_grad_norm": 0.8130239248275757 + }, + { + "bits_per_byte": 1.6430763269754933, + "learning_rate": 0.00014719455686882066, + "loss_nats": 1.138893723487854, + "step": 4520, + "unclipped_grad_norm": 0.7745712399482727 + }, + { + "bits_per_byte": 1.660709880151108, + "learning_rate": 0.00014664153955538695, + "loss_nats": 1.1511163711547852, + "step": 4530, + "unclipped_grad_norm": 0.7695059776306152 + }, + { + "bits_per_byte": 1.707295852686222, + "learning_rate": 0.00014608883593738946, + "loss_nats": 1.1834073066711426, + "step": 4540, + "unclipped_grad_norm": 0.8436031937599182 + }, + { + "bits_per_byte": 1.5623325361872151, + "learning_rate": 0.00014553645545900817, + "loss_nats": 1.0829263925552368, + "step": 4550, + "unclipped_grad_norm": 0.7549967169761658 + }, + { + "bits_per_byte": 1.6257084869954619, + "learning_rate": 0.0001449844075589013, + "loss_nats": 1.1268552541732788, + "step": 4560, + "unclipped_grad_norm": 0.8449703454971313 + }, + { + "bits_per_byte": 1.6175644205465043, + "learning_rate": 0.0001444327016700444, + "loss_nats": 1.1212102174758911, + "step": 4570, + "unclipped_grad_norm": 0.7528937458992004 + }, + { + "bits_per_byte": 1.570274179056107, + "learning_rate": 0.0001438813472195688, + "loss_nats": 1.0884311199188232, + "step": 4580, + "unclipped_grad_norm": 0.790579617023468 + }, + { + "bits_per_byte": 1.609637052237634, + "learning_rate": 0.00014333035362860093, + "loss_nats": 1.1157153844833374, + "step": 4590, + "unclipped_grad_norm": 0.7935193181037903 + }, + { + "bits_per_byte": 1.6088013885370918, + "learning_rate": 0.00014277973031210088, + "loss_nats": 1.1151361465454102, + "step": 4600, + "unclipped_grad_norm": 0.7915687561035156 + }, + { + "bits_per_byte": 1.6615916352020816, + "learning_rate": 0.00014222948667870207, + "loss_nats": 1.151727557182312, + "step": 4610, + "unclipped_grad_norm": 0.8493216633796692 + }, + { + "bits_per_byte": 1.6675298521710726, + "learning_rate": 0.00014167963213054991, + "loss_nats": 1.1558436155319214, + "step": 4620, + "unclipped_grad_norm": 0.8179065585136414 + }, + { + "bits_per_byte": 1.6286588493710132, + "learning_rate": 0.00014113017606314152, + "loss_nats": 1.1289002895355225, + "step": 4630, + "unclipped_grad_norm": 0.8027536273002625 + }, + { + "bits_per_byte": 1.7031861552612044, + "learning_rate": 0.00014058112786516518, + "loss_nats": 1.180558681488037, + "step": 4640, + "unclipped_grad_norm": 0.8179929256439209 + }, + { + "bits_per_byte": 1.6059254946493637, + "learning_rate": 0.0001400324969183396, + "loss_nats": 1.113142728805542, + "step": 4650, + "unclipped_grad_norm": 0.8228299021720886 + }, + { + "bits_per_byte": 1.5739184914278899, + "learning_rate": 0.00013948429259725406, + "loss_nats": 1.0909571647644043, + "step": 4660, + "unclipped_grad_norm": 0.820624589920044 + }, + { + "bits_per_byte": 1.7237126286069824, + "learning_rate": 0.00013893652426920776, + "loss_nats": 1.194786548614502, + "step": 4670, + "unclipped_grad_norm": 0.8582949638366699 + }, + { + "bits_per_byte": 1.7009104808249877, + "learning_rate": 0.00013838920129405016, + "loss_nats": 1.1789813041687012, + "step": 4680, + "unclipped_grad_norm": 0.8158925175666809 + }, + { + "bits_per_byte": 1.6679322915740917, + "learning_rate": 0.0001378423330240207, + "loss_nats": 1.1561225652694702, + "step": 4690, + "unclipped_grad_norm": 0.849236249923706 + }, + { + "bits_per_byte": 1.589747430648001, + "learning_rate": 0.00013729592880358936, + "loss_nats": 1.101928949356079, + "step": 4700, + "unclipped_grad_norm": 0.7403190732002258 + }, + { + "bits_per_byte": 1.6742342518496576, + "learning_rate": 0.00013674999796929659, + "loss_nats": 1.1604907512664795, + "step": 4710, + "unclipped_grad_norm": 0.8582251071929932 + }, + { + "bits_per_byte": 1.607032031025015, + "learning_rate": 0.00013620454984959417, + "loss_nats": 1.1139097213745117, + "step": 4720, + "unclipped_grad_norm": 0.8400667905807495 + }, + { + "bits_per_byte": 1.5176990826873464, + "learning_rate": 0.00013565959376468542, + "loss_nats": 1.0519888401031494, + "step": 4730, + "unclipped_grad_norm": 0.7827285528182983 + }, + { + "bits_per_byte": 1.5868581221135059, + "learning_rate": 0.00013511513902636634, + "loss_nats": 1.099926233291626, + "step": 4740, + "unclipped_grad_norm": 0.7815871238708496 + }, + { + "bits_per_byte": 1.6377919880450826, + "learning_rate": 0.0001345711949378662, + "loss_nats": 1.1352308988571167, + "step": 4750, + "unclipped_grad_norm": 0.7846289873123169 + }, + { + "bits_per_byte": 1.6125860387519786, + "learning_rate": 0.00013402777079368853, + "loss_nats": 1.1177594661712646, + "step": 4760, + "unclipped_grad_norm": 0.8307337164878845 + }, + { + "bits_per_byte": 1.5936545325103024, + "learning_rate": 0.0001334848758794528, + "loss_nats": 1.1046371459960938, + "step": 4770, + "unclipped_grad_norm": 0.8213374614715576 + }, + { + "bits_per_byte": 1.6121588338472355, + "learning_rate": 0.00013294251947173504, + "loss_nats": 1.1174633502960205, + "step": 4780, + "unclipped_grad_norm": 0.8224621415138245 + }, + { + "bits_per_byte": 1.5705015401205475, + "learning_rate": 0.00013240071083790996, + "loss_nats": 1.0885887145996094, + "step": 4790, + "unclipped_grad_norm": 0.8122685551643372 + }, + { + "bits_per_byte": 1.5722774329733569, + "learning_rate": 0.0001318594592359921, + "loss_nats": 1.0898196697235107, + "step": 4800, + "unclipped_grad_norm": 0.8455460667610168 + }, + { + "bits_per_byte": 1.6480244398234671, + "learning_rate": 0.0001313187739144781, + "loss_nats": 1.1423234939575195, + "step": 4810, + "unclipped_grad_norm": 0.8817458748817444 + }, + { + "bits_per_byte": 1.574972745077679, + "learning_rate": 0.0001307786641121882, + "loss_nats": 1.0916879177093506, + "step": 4820, + "unclipped_grad_norm": 0.7954387664794922 + }, + { + "bits_per_byte": 1.608869665649484, + "learning_rate": 0.00013023913905810876, + "loss_nats": 1.1151834726333618, + "step": 4830, + "unclipped_grad_norm": 0.8687117695808411 + }, + { + "bits_per_byte": 1.5985519104762718, + "learning_rate": 0.00012970020797123432, + "loss_nats": 1.1080317497253418, + "step": 4840, + "unclipped_grad_norm": 0.8505465388298035 + }, + { + "bits_per_byte": 1.6228387844831649, + "learning_rate": 0.00012916188006041027, + "loss_nats": 1.1248661279678345, + "step": 4850, + "unclipped_grad_norm": 0.7912812232971191 + }, + { + "bits_per_byte": 1.5432041098103009, + "learning_rate": 0.00012862416452417516, + "loss_nats": 1.0696675777435303, + "step": 4860, + "unclipped_grad_norm": 0.7933070063591003 + }, + { + "bits_per_byte": 1.635918581029234, + "learning_rate": 0.00012808707055060405, + "loss_nats": 1.13393235206604, + "step": 4870, + "unclipped_grad_norm": 0.8805916905403137 + }, + { + "bits_per_byte": 1.4766798445953597, + "learning_rate": 0.000127550607317151, + "loss_nats": 1.0235564708709717, + "step": 4880, + "unclipped_grad_norm": 0.8066675066947937 + }, + { + "bits_per_byte": 1.5344280071194227, + "learning_rate": 0.00012701478399049248, + "loss_nats": 1.0635844469070435, + "step": 4890, + "unclipped_grad_norm": 0.8088182806968689 + }, + { + "bits_per_byte": 1.6101516243290155, + "learning_rate": 0.0001264796097263708, + "loss_nats": 1.1160720586776733, + "step": 4900, + "unclipped_grad_norm": 0.8653903603553772 + }, + { + "bits_per_byte": 1.6779198400576476, + "learning_rate": 0.0001259450936694375, + "loss_nats": 1.1630454063415527, + "step": 4910, + "unclipped_grad_norm": 0.8391038179397583 + }, + { + "bits_per_byte": 1.6284904783558185, + "learning_rate": 0.00012541124495309727, + "loss_nats": 1.1287835836410522, + "step": 4920, + "unclipped_grad_norm": 0.8723531365394592 + }, + { + "bits_per_byte": 1.5831391692712484, + "learning_rate": 0.00012487807269935167, + "loss_nats": 1.0973484516143799, + "step": 4930, + "unclipped_grad_norm": 0.7553638815879822 + }, + { + "bits_per_byte": 1.5755332365368409, + "learning_rate": 0.00012434558601864349, + "loss_nats": 1.0920764207839966, + "step": 4940, + "unclipped_grad_norm": 0.8158645033836365 + }, + { + "bits_per_byte": 1.530142715407874, + "learning_rate": 0.0001238137940097009, + "loss_nats": 1.0606141090393066, + "step": 4950, + "unclipped_grad_norm": 0.7773033380508423 + }, + { + "bits_per_byte": 1.5220387209165676, + "learning_rate": 0.0001232827057593821, + "loss_nats": 1.0549968481063843, + "step": 4960, + "unclipped_grad_norm": 0.8325244784355164 + }, + { + "bits_per_byte": 1.6031579498316795, + "learning_rate": 0.00012275233034251992, + "loss_nats": 1.1112244129180908, + "step": 4970, + "unclipped_grad_norm": 0.8591333627700806 + }, + { + "bits_per_byte": 1.6532000857185325, + "learning_rate": 0.0001222226768217669, + "loss_nats": 1.1459109783172607, + "step": 4980, + "unclipped_grad_norm": 0.8351151347160339 + }, + { + "bits_per_byte": 1.585272098107249, + "learning_rate": 0.00012169375424744034, + "loss_nats": 1.0988268852233887, + "step": 4990, + "unclipped_grad_norm": 0.8779996037483215 + }, + { + "bits_per_byte": 1.5394700224947675, + "learning_rate": 0.00012116557165736772, + "loss_nats": 1.0670793056488037, + "step": 5000, + "unclipped_grad_norm": 0.7928086519241333 + }, + { + "bits_per_byte": 1.5552827953456976, + "learning_rate": 0.00012063813807673214, + "loss_nats": 1.0780398845672607, + "step": 5010, + "unclipped_grad_norm": 0.8201308846473694 + }, + { + "bits_per_byte": 1.5898205232746179, + "learning_rate": 0.00012011146251791832, + "loss_nats": 1.1019796133041382, + "step": 5020, + "unclipped_grad_norm": 0.7774789929389954 + }, + { + "bits_per_byte": 1.5697566832596608, + "learning_rate": 0.0001195855539803584, + "loss_nats": 1.088072419166565, + "step": 5030, + "unclipped_grad_norm": 0.7855319380760193 + }, + { + "bits_per_byte": 1.539816567536256, + "learning_rate": 0.00011906042145037817, + "loss_nats": 1.0673195123672485, + "step": 5040, + "unclipped_grad_norm": 0.7920348048210144 + }, + { + "bits_per_byte": 1.5092143186070297, + "learning_rate": 0.0001185360739010438, + "loss_nats": 1.0461076498031616, + "step": 5050, + "unclipped_grad_norm": 0.8045132160186768 + }, + { + "bits_per_byte": 1.5339026001210367, + "learning_rate": 0.00011801252029200808, + "loss_nats": 1.0632202625274658, + "step": 5060, + "unclipped_grad_norm": 0.8044583797454834 + }, + { + "bits_per_byte": 1.6138325690054323, + "learning_rate": 0.00011748976956935777, + "loss_nats": 1.1186234951019287, + "step": 5070, + "unclipped_grad_norm": 0.8723475337028503 + }, + { + "bits_per_byte": 1.574335549356232, + "learning_rate": 0.00011696783066546024, + "loss_nats": 1.091246247291565, + "step": 5080, + "unclipped_grad_norm": 0.8741123080253601 + }, + { + "bits_per_byte": 1.5594206979254568, + "learning_rate": 0.00011644671249881147, + "loss_nats": 1.0809080600738525, + "step": 5090, + "unclipped_grad_norm": 0.8465037941932678 + }, + { + "bits_per_byte": 1.5640659493252613, + "learning_rate": 0.00011592642397388293, + "loss_nats": 1.0841279029846191, + "step": 5100, + "unclipped_grad_norm": 0.783299446105957 + }, + { + "bits_per_byte": 1.488738924108558, + "learning_rate": 0.00011540697398097015, + "loss_nats": 1.0319151878356934, + "step": 5110, + "unclipped_grad_norm": 0.8348572254180908 + }, + { + "bits_per_byte": 1.5847128105266433, + "learning_rate": 0.00011488837139604016, + "loss_nats": 1.0984392166137695, + "step": 5120, + "unclipped_grad_norm": 0.8110998868942261 + }, + { + "bits_per_byte": 1.590178763136365, + "learning_rate": 0.00011437062508058039, + "loss_nats": 1.1022279262542725, + "step": 5130, + "unclipped_grad_norm": 0.8208310604095459 + }, + { + "bits_per_byte": 1.5494388248693793, + "learning_rate": 0.00011385374388144673, + "loss_nats": 1.0739891529083252, + "step": 5140, + "unclipped_grad_norm": 0.8678956627845764 + }, + { + "bits_per_byte": 1.6273239200350162, + "learning_rate": 0.00011333773663071286, + "loss_nats": 1.1279749870300293, + "step": 5150, + "unclipped_grad_norm": 0.848126232624054 + }, + { + "bits_per_byte": 1.5085432423033631, + "learning_rate": 0.00011282261214551897, + "loss_nats": 1.0456424951553345, + "step": 5160, + "unclipped_grad_norm": 0.7922812104225159 + }, + { + "bits_per_byte": 1.6700107019097687, + "learning_rate": 0.00011230837922792112, + "loss_nats": 1.1575632095336914, + "step": 5170, + "unclipped_grad_norm": 0.8759046196937561 + }, + { + "bits_per_byte": 1.5869253673299932, + "learning_rate": 0.0001117950466647412, + "loss_nats": 1.0999728441238403, + "step": 5180, + "unclipped_grad_norm": 0.8647724390029907 + }, + { + "bits_per_byte": 1.5237559676854329, + "learning_rate": 0.00011128262322741627, + "loss_nats": 1.0561871528625488, + "step": 5190, + "unclipped_grad_norm": 0.813337504863739 + }, + { + "bits_per_byte": 1.556985939537192, + "learning_rate": 0.00011077111767184913, + "loss_nats": 1.0792204141616821, + "step": 5200, + "unclipped_grad_norm": 0.8388134241104126 + }, + { + "bits_per_byte": 1.5602121620847276, + "learning_rate": 0.00011026053873825837, + "loss_nats": 1.0814566612243652, + "step": 5210, + "unclipped_grad_norm": 0.8162683248519897 + }, + { + "bits_per_byte": 1.6432956048553435, + "learning_rate": 0.0001097508951510293, + "loss_nats": 1.1390457153320312, + "step": 5220, + "unclipped_grad_norm": 0.8667425513267517 + }, + { + "bits_per_byte": 1.5501931407760636, + "learning_rate": 0.0001092421956185646, + "loss_nats": 1.074512004852295, + "step": 5230, + "unclipped_grad_norm": 0.8668702840805054 + }, + { + "bits_per_byte": 1.5510165937083948, + "learning_rate": 0.00010873444883313581, + "loss_nats": 1.075082778930664, + "step": 5240, + "unclipped_grad_norm": 0.8049055337905884 + }, + { + "bits_per_byte": 1.5393878147876552, + "learning_rate": 0.00010822766347073447, + "loss_nats": 1.0670223236083984, + "step": 5250, + "unclipped_grad_norm": 0.8209428191184998 + }, + { + "bits_per_byte": 1.540701762240247, + "learning_rate": 0.00010772184819092424, + "loss_nats": 1.0679330825805664, + "step": 5260, + "unclipped_grad_norm": 0.8524174690246582 + }, + { + "bits_per_byte": 1.5376201771020872, + "learning_rate": 0.0001072170116366926, + "loss_nats": 1.0657970905303955, + "step": 5270, + "unclipped_grad_norm": 0.8064137697219849 + }, + { + "bits_per_byte": 1.5550475230793173, + "learning_rate": 0.00010671316243430345, + "loss_nats": 1.0778768062591553, + "step": 5280, + "unclipped_grad_norm": 0.8169121146202087 + }, + { + "bits_per_byte": 1.5423493560355128, + "learning_rate": 0.00010621030919314944, + "loss_nats": 1.069075107574463, + "step": 5290, + "unclipped_grad_norm": 0.8263237476348877 + }, + { + "bits_per_byte": 1.5542627662434303, + "learning_rate": 0.000105708460505605, + "loss_nats": 1.077332854270935, + "step": 5300, + "unclipped_grad_norm": 0.8492859601974487 + }, + { + "bits_per_byte": 1.5509104804128124, + "learning_rate": 0.00010520762494687964, + "loss_nats": 1.0750092267990112, + "step": 5310, + "unclipped_grad_norm": 0.8885292410850525 + }, + { + "bits_per_byte": 1.5517654061702515, + "learning_rate": 0.00010470781107487111, + "loss_nats": 1.0756018161773682, + "step": 5320, + "unclipped_grad_norm": 0.8326203227043152 + }, + { + "bits_per_byte": 1.5477100552629064, + "learning_rate": 0.00010420902743001955, + "loss_nats": 1.0727908611297607, + "step": 5330, + "unclipped_grad_norm": 0.8806997537612915 + }, + { + "bits_per_byte": 1.503639328996661, + "learning_rate": 0.00010371128253516117, + "loss_nats": 1.0422433614730835, + "step": 5340, + "unclipped_grad_norm": 0.8234722018241882 + }, + { + "bits_per_byte": 1.5869049013945407, + "learning_rate": 0.000103214584895383, + "loss_nats": 1.0999586582183838, + "step": 5350, + "unclipped_grad_norm": 0.8706242442131042 + }, + { + "bits_per_byte": 1.5072631754329913, + "learning_rate": 0.00010271894299787715, + "loss_nats": 1.044755220413208, + "step": 5360, + "unclipped_grad_norm": 0.8708282709121704 + }, + { + "bits_per_byte": 1.471025742965594, + "learning_rate": 0.00010222436531179622, + "loss_nats": 1.0196373462677002, + "step": 5370, + "unclipped_grad_norm": 0.8303158283233643 + }, + { + "bits_per_byte": 1.5968417149960918, + "learning_rate": 0.0001017308602881082, + "loss_nats": 1.1068463325500488, + "step": 5380, + "unclipped_grad_norm": 0.883955180644989 + }, + { + "bits_per_byte": 1.531507741707772, + "learning_rate": 0.00010123843635945242, + "loss_nats": 1.0615602731704712, + "step": 5390, + "unclipped_grad_norm": 0.8507125377655029 + }, + { + "bits_per_byte": 1.5483677169198058, + "learning_rate": 0.00010074710193999505, + "loss_nats": 1.073246717453003, + "step": 5400, + "unclipped_grad_norm": 0.8522453904151917 + }, + { + "bits_per_byte": 1.571479089508052, + "learning_rate": 0.0001002568654252858, + "loss_nats": 1.089266300201416, + "step": 5410, + "unclipped_grad_norm": 0.8313266038894653 + }, + { + "bits_per_byte": 1.5408727129952047, + "learning_rate": 9.976773519211404e-05, + "loss_nats": 1.0680515766143799, + "step": 5420, + "unclipped_grad_norm": 0.8902517557144165 + }, + { + "bits_per_byte": 1.5778336764747818, + "learning_rate": 9.92797195983659e-05, + "loss_nats": 1.0936709642410278, + "step": 5430, + "unclipped_grad_norm": 0.8956695795059204 + }, + { + "bits_per_byte": 1.513927503153925, + "learning_rate": 9.879282698288142e-05, + "loss_nats": 1.0493745803833008, + "step": 5440, + "unclipped_grad_norm": 0.897573709487915 + }, + { + "bits_per_byte": 1.5078418970531444, + "learning_rate": 9.830706566531197e-05, + "loss_nats": 1.0451563596725464, + "step": 5450, + "unclipped_grad_norm": 0.8440007567405701 + }, + { + "bits_per_byte": 1.5030907043404085, + "learning_rate": 9.782244394597826e-05, + "loss_nats": 1.0418630838394165, + "step": 5460, + "unclipped_grad_norm": 0.8436890244483948 + }, + { + "bits_per_byte": 1.5464133060754008, + "learning_rate": 9.733897010572829e-05, + "loss_nats": 1.0718920230865479, + "step": 5470, + "unclipped_grad_norm": 0.8681005239486694 + }, + { + "bits_per_byte": 1.553998600891705, + "learning_rate": 9.685665240579612e-05, + "loss_nats": 1.077149748802185, + "step": 5480, + "unclipped_grad_norm": 0.8523774147033691 + }, + { + "bits_per_byte": 1.543775436176467, + "learning_rate": 9.637549908766041e-05, + "loss_nats": 1.070063591003418, + "step": 5490, + "unclipped_grad_norm": 0.8906207084655762 + }, + { + "bits_per_byte": 1.5444891641775478, + "learning_rate": 9.589551837290389e-05, + "loss_nats": 1.0705583095550537, + "step": 5500, + "unclipped_grad_norm": 0.8658390045166016 + }, + { + "bits_per_byte": 1.6370650173798855, + "learning_rate": 9.541671846307257e-05, + "loss_nats": 1.1347270011901855, + "step": 5510, + "unclipped_grad_norm": 0.9161565899848938 + }, + { + "bits_per_byte": 1.5520985365649729, + "learning_rate": 9.493910753953597e-05, + "loss_nats": 1.075832724571228, + "step": 5520, + "unclipped_grad_norm": 0.845907986164093 + }, + { + "bits_per_byte": 1.5215033389244317, + "learning_rate": 9.446269376334689e-05, + "loss_nats": 1.0546257495880127, + "step": 5530, + "unclipped_grad_norm": 0.8585748672485352 + }, + { + "bits_per_byte": 1.5943249208833656, + "learning_rate": 9.39874852751024e-05, + "loss_nats": 1.1051018238067627, + "step": 5540, + "unclipped_grad_norm": 0.9360165596008301 + }, + { + "bits_per_byte": 1.5176287417831436, + "learning_rate": 9.351349019480433e-05, + "loss_nats": 1.0519400835037231, + "step": 5550, + "unclipped_grad_norm": 0.8308430910110474 + }, + { + "bits_per_byte": 1.5164790377621256, + "learning_rate": 9.304071662172086e-05, + "loss_nats": 1.0511431694030762, + "step": 5560, + "unclipped_grad_norm": 0.894058346748352 + }, + { + "bits_per_byte": 1.4905304673825959, + "learning_rate": 9.256917263424787e-05, + "loss_nats": 1.0331569910049438, + "step": 5570, + "unclipped_grad_norm": 0.8399901390075684 + }, + { + "bits_per_byte": 1.5655759569998369, + "learning_rate": 9.209886628977114e-05, + "loss_nats": 1.085174560546875, + "step": 5580, + "unclipped_grad_norm": 0.8579773902893066 + }, + { + "bits_per_byte": 1.482571282283316, + "learning_rate": 9.162980562452843e-05, + "loss_nats": 1.0276401042938232, + "step": 5590, + "unclipped_grad_norm": 0.8161733746528625 + }, + { + "bits_per_byte": 1.5496305855250914, + "learning_rate": 9.116199865347238e-05, + "loss_nats": 1.0741220712661743, + "step": 5600, + "unclipped_grad_norm": 0.8440633416175842 + }, + { + "bits_per_byte": 1.5466313800766949, + "learning_rate": 9.069545337013333e-05, + "loss_nats": 1.0720431804656982, + "step": 5610, + "unclipped_grad_norm": 0.8384022116661072 + }, + { + "bits_per_byte": 1.55366271877457, + "learning_rate": 9.0230177746483e-05, + "loss_nats": 1.0769169330596924, + "step": 5620, + "unclipped_grad_norm": 0.8643615245819092 + }, + { + "bits_per_byte": 1.5359743031333302, + "learning_rate": 8.976617973279797e-05, + "loss_nats": 1.0646562576293945, + "step": 5630, + "unclipped_grad_norm": 0.8559057712554932 + }, + { + "bits_per_byte": 1.5154779267514533, + "learning_rate": 8.930346725752419e-05, + "loss_nats": 1.050449252128601, + "step": 5640, + "unclipped_grad_norm": 0.8564271330833435 + }, + { + "bits_per_byte": 1.5399061704973558, + "learning_rate": 8.88420482271411e-05, + "loss_nats": 1.0673816204071045, + "step": 5650, + "unclipped_grad_norm": 0.899230420589447 + }, + { + "bits_per_byte": 1.6017691899259623, + "learning_rate": 8.838193052602691e-05, + "loss_nats": 1.1102617979049683, + "step": 5660, + "unclipped_grad_norm": 0.8796955943107605 + }, + { + "bits_per_byte": 1.5203328250026595, + "learning_rate": 8.792312201632352e-05, + "loss_nats": 1.05381441116333, + "step": 5670, + "unclipped_grad_norm": 0.8509314060211182 + }, + { + "bits_per_byte": 1.5301421994599214, + "learning_rate": 8.746563053780261e-05, + "loss_nats": 1.060613751411438, + "step": 5680, + "unclipped_grad_norm": 0.8624072074890137 + }, + { + "bits_per_byte": 1.6028342784827556, + "learning_rate": 8.700946390773114e-05, + "loss_nats": 1.1110000610351562, + "step": 5690, + "unclipped_grad_norm": 0.8914096355438232 + }, + { + "bits_per_byte": 1.526354281574668, + "learning_rate": 8.655462992073833e-05, + "loss_nats": 1.057988166809082, + "step": 5700, + "unclipped_grad_norm": 0.8579604625701904 + }, + { + "bits_per_byte": 1.5347007716036911, + "learning_rate": 8.610113634868199e-05, + "loss_nats": 1.063773512840271, + "step": 5710, + "unclipped_grad_norm": 0.8810272216796875 + }, + { + "bits_per_byte": 1.5491378552303696, + "learning_rate": 8.564899094051613e-05, + "loss_nats": 1.0737805366516113, + "step": 5720, + "unclipped_grad_norm": 0.8394830226898193 + }, + { + "bits_per_byte": 1.5456280332915613, + "learning_rate": 8.519820142215816e-05, + "loss_nats": 1.071347713470459, + "step": 5730, + "unclipped_grad_norm": 0.865943968296051 + }, + { + "bits_per_byte": 1.5433466834278662, + "learning_rate": 8.474877549635728e-05, + "loss_nats": 1.0697664022445679, + "step": 5740, + "unclipped_grad_norm": 0.886944591999054 + }, + { + "bits_per_byte": 1.5488365416260579, + "learning_rate": 8.430072084256247e-05, + "loss_nats": 1.0735716819763184, + "step": 5750, + "unclipped_grad_norm": 0.9145897626876831 + }, + { + "bits_per_byte": 1.5636783004302166, + "learning_rate": 8.385404511679161e-05, + "loss_nats": 1.0838592052459717, + "step": 5760, + "unclipped_grad_norm": 0.9009037017822266 + }, + { + "bits_per_byte": 1.511110599315443, + "learning_rate": 8.340875595150034e-05, + "loss_nats": 1.0474220514297485, + "step": 5770, + "unclipped_grad_norm": 0.844070315361023 + }, + { + "bits_per_byte": 1.494012772097266, + "learning_rate": 8.296486095545203e-05, + "loss_nats": 1.035570740699768, + "step": 5780, + "unclipped_grad_norm": 0.8943845629692078 + }, + { + "bits_per_byte": 1.5135893852623288, + "learning_rate": 8.252236771358729e-05, + "loss_nats": 1.049140214920044, + "step": 5790, + "unclipped_grad_norm": 0.8841454386711121 + }, + { + "bits_per_byte": 1.4905120652389536, + "learning_rate": 8.208128378689484e-05, + "loss_nats": 1.033144235610962, + "step": 5800, + "unclipped_grad_norm": 0.8580894470214844 + }, + { + "bits_per_byte": 1.5735937881830608, + "learning_rate": 8.16416167122819e-05, + "loss_nats": 1.0907320976257324, + "step": 5810, + "unclipped_grad_norm": 0.933175265789032 + }, + { + "bits_per_byte": 1.5042643139498963, + "learning_rate": 8.120337400244578e-05, + "loss_nats": 1.042676568031311, + "step": 5820, + "unclipped_grad_norm": 0.828717052936554 + }, + { + "bits_per_byte": 1.5156406223391694, + "learning_rate": 8.076656314574521e-05, + "loss_nats": 1.0505620241165161, + "step": 5830, + "unclipped_grad_norm": 0.9511673450469971 + }, + { + "bits_per_byte": 1.5092205099824607, + "learning_rate": 8.03311916060725e-05, + "loss_nats": 1.0461119413375854, + "step": 5840, + "unclipped_grad_norm": 0.8395604491233826 + }, + { + "bits_per_byte": 1.477936693807865, + "learning_rate": 7.989726682272606e-05, + "loss_nats": 1.0244276523590088, + "step": 5850, + "unclipped_grad_norm": 0.9143134355545044 + }, + { + "bits_per_byte": 1.5336375748560571, + "learning_rate": 7.946479621028318e-05, + "loss_nats": 1.063036561012268, + "step": 5860, + "unclipped_grad_norm": 0.9014923572540283 + }, + { + "bits_per_byte": 1.5402322496033916, + "learning_rate": 7.903378715847343e-05, + "loss_nats": 1.0676076412200928, + "step": 5870, + "unclipped_grad_norm": 0.8728002905845642 + }, + { + "bits_per_byte": 1.555285031120159, + "learning_rate": 7.860424703205224e-05, + "loss_nats": 1.078041434288025, + "step": 5880, + "unclipped_grad_norm": 0.8936865925788879 + }, + { + "bits_per_byte": 1.564054942435606, + "learning_rate": 7.817618317067534e-05, + "loss_nats": 1.084120273590088, + "step": 5890, + "unclipped_grad_norm": 0.8752698302268982 + }, + { + "bits_per_byte": 1.5458715607251832, + "learning_rate": 7.774960288877295e-05, + "loss_nats": 1.071516513824463, + "step": 5900, + "unclipped_grad_norm": 0.8957662582397461 + }, + { + "bits_per_byte": 1.5568008862048637, + "learning_rate": 7.732451347542522e-05, + "loss_nats": 1.0790921449661255, + "step": 5910, + "unclipped_grad_norm": 0.8808028101921082 + }, + { + "bits_per_byte": 1.4896764015384112, + "learning_rate": 7.690092219423722e-05, + "loss_nats": 1.0325649976730347, + "step": 5920, + "unclipped_grad_norm": 0.8660398721694946 + }, + { + "bits_per_byte": 1.5908711652887386, + "learning_rate": 7.647883628321536e-05, + "loss_nats": 1.102707862854004, + "step": 5930, + "unclipped_grad_norm": 0.9604227542877197 + }, + { + "bits_per_byte": 1.4574508083676907, + "learning_rate": 7.605826295464316e-05, + "loss_nats": 1.010227918624878, + "step": 5940, + "unclipped_grad_norm": 0.8380697965621948 + }, + { + "bits_per_byte": 1.5131755950043528, + "learning_rate": 7.563920939495851e-05, + "loss_nats": 1.0488533973693848, + "step": 5950, + "unclipped_grad_norm": 0.8392192125320435 + }, + { + "bits_per_byte": 1.5452194025131114, + "learning_rate": 7.522168276463047e-05, + "loss_nats": 1.0710644721984863, + "step": 5960, + "unclipped_grad_norm": 0.932554304599762 + }, + { + "bits_per_byte": 1.5213482105733536, + "learning_rate": 7.480569019803719e-05, + "loss_nats": 1.054518222808838, + "step": 5970, + "unclipped_grad_norm": 0.927780270576477 + }, + { + "bits_per_byte": 1.5601101763727658, + "learning_rate": 7.439123880334388e-05, + "loss_nats": 1.0813859701156616, + "step": 5980, + "unclipped_grad_norm": 0.8920019268989563 + }, + { + "bits_per_byte": 1.5382338112003657, + "learning_rate": 7.39783356623813e-05, + "loss_nats": 1.0662224292755127, + "step": 5990, + "unclipped_grad_norm": 0.9537093639373779 + }, + { + "bits_per_byte": 1.4675097296513546, + "learning_rate": 7.356698783052497e-05, + "loss_nats": 1.017200231552124, + "step": 6000, + "unclipped_grad_norm": 0.8773758411407471 + }, + { + "bits_per_byte": 1.506333093257125, + "learning_rate": 7.315720233657431e-05, + "loss_nats": 1.0441105365753174, + "step": 6010, + "unclipped_grad_norm": 0.8785988092422485 + }, + { + "bits_per_byte": 1.4794742187065786, + "learning_rate": 7.274898618263286e-05, + "loss_nats": 1.0254933834075928, + "step": 6020, + "unclipped_grad_norm": 0.8875186443328857 + }, + { + "bits_per_byte": 1.5245675538148544, + "learning_rate": 7.234234634398831e-05, + "loss_nats": 1.056749701499939, + "step": 6030, + "unclipped_grad_norm": 0.88789963722229 + }, + { + "bits_per_byte": 1.5628784091210537, + "learning_rate": 7.193728976899362e-05, + "loss_nats": 1.083304762840271, + "step": 6040, + "unclipped_grad_norm": 0.9149079322814941 + }, + { + "bits_per_byte": 1.4881958028971334, + "learning_rate": 7.153382337894797e-05, + "loss_nats": 1.031538724899292, + "step": 6050, + "unclipped_grad_norm": 0.9299342632293701 + }, + { + "bits_per_byte": 1.4785307218839452, + "learning_rate": 7.113195406797887e-05, + "loss_nats": 1.0248394012451172, + "step": 6060, + "unclipped_grad_norm": 0.8783867359161377 + }, + { + "bits_per_byte": 1.5509557118499895, + "learning_rate": 7.073168870292397e-05, + "loss_nats": 1.075040578842163, + "step": 6070, + "unclipped_grad_norm": 0.9337034821510315 + }, + { + "bits_per_byte": 1.4914187577743023, + "learning_rate": 7.033303412321404e-05, + "loss_nats": 1.0337727069854736, + "step": 6080, + "unclipped_grad_norm": 0.9420101046562195 + }, + { + "bits_per_byte": 1.4968499698885498, + "learning_rate": 6.993599714075586e-05, + "loss_nats": 1.0375373363494873, + "step": 6090, + "unclipped_grad_norm": 0.8664546012878418 + }, + { + "bits_per_byte": 1.4906561867003767, + "learning_rate": 6.954058453981609e-05, + "loss_nats": 1.0332441329956055, + "step": 6100, + "unclipped_grad_norm": 0.8687640428543091 + }, + { + "bits_per_byte": 1.5538661742505406, + "learning_rate": 6.914680307690504e-05, + "loss_nats": 1.077057957649231, + "step": 6110, + "unclipped_grad_norm": 0.8986173868179321 + }, + { + "bits_per_byte": 1.5476213122150613, + "learning_rate": 6.875465948066139e-05, + "loss_nats": 1.0727293491363525, + "step": 6120, + "unclipped_grad_norm": 0.9189077019691467 + }, + { + "bits_per_byte": 1.4789527673091625, + "learning_rate": 6.836416045173733e-05, + "loss_nats": 1.0251319408416748, + "step": 6130, + "unclipped_grad_norm": 0.8851786851882935 + }, + { + "bits_per_byte": 1.4768556108645414, + "learning_rate": 6.797531266268377e-05, + "loss_nats": 1.0236783027648926, + "step": 6140, + "unclipped_grad_norm": 0.9043483138084412 + }, + { + "bits_per_byte": 1.552988202817886, + "learning_rate": 6.758812275783663e-05, + "loss_nats": 1.0764493942260742, + "step": 6150, + "unclipped_grad_norm": 0.9005390405654907 + }, + { + "bits_per_byte": 1.3994871515820966, + "learning_rate": 6.720259735320298e-05, + "loss_nats": 0.970050573348999, + "step": 6160, + "unclipped_grad_norm": 0.8554620742797852 + }, + { + "bits_per_byte": 1.5302097886417105, + "learning_rate": 6.681874303634842e-05, + "loss_nats": 1.0606606006622314, + "step": 6170, + "unclipped_grad_norm": 0.8954014778137207 + }, + { + "bits_per_byte": 1.4938230752333643, + "learning_rate": 6.643656636628402e-05, + "loss_nats": 1.0354392528533936, + "step": 6180, + "unclipped_grad_norm": 0.892278790473938 + }, + { + "bits_per_byte": 1.516749394489282, + "learning_rate": 6.605607387335466e-05, + "loss_nats": 1.05133056640625, + "step": 6190, + "unclipped_grad_norm": 0.9185703992843628 + }, + { + "bits_per_byte": 1.5086576107661869, + "learning_rate": 6.567727205912724e-05, + "loss_nats": 1.0457217693328857, + "step": 6200, + "unclipped_grad_norm": 0.8927735090255737 + }, + { + "bits_per_byte": 1.5088584865023946, + "learning_rate": 6.530016739627959e-05, + "loss_nats": 1.045861005783081, + "step": 6210, + "unclipped_grad_norm": 0.8906724452972412 + }, + { + "bits_per_byte": 1.5118191678369979, + "learning_rate": 6.492476632848993e-05, + "loss_nats": 1.0479131937026978, + "step": 6220, + "unclipped_grad_norm": 0.9077256917953491 + }, + { + "bits_per_byte": 1.4471287536282067, + "learning_rate": 6.455107527032678e-05, + "loss_nats": 1.0030732154846191, + "step": 6230, + "unclipped_grad_norm": 0.8839235305786133 + }, + { + "bits_per_byte": 1.5076786855174757, + "learning_rate": 6.417910060713926e-05, + "loss_nats": 1.0450432300567627, + "step": 6240, + "unclipped_grad_norm": 0.9515005946159363 + }, + { + "bits_per_byte": 1.5408861276419719, + "learning_rate": 6.380884869494799e-05, + "loss_nats": 1.0680608749389648, + "step": 6250, + "unclipped_grad_norm": 0.901561975479126 + }, + { + "bits_per_byte": 1.4200397662908057, + "learning_rate": 6.34403258603367e-05, + "loss_nats": 0.9842965602874756, + "step": 6260, + "unclipped_grad_norm": 0.889621376991272 + }, + { + "bits_per_byte": 1.5216213190229235, + "learning_rate": 6.30735384003437e-05, + "loss_nats": 1.0547075271606445, + "step": 6270, + "unclipped_grad_norm": 0.9411186575889587 + }, + { + "bits_per_byte": 1.5224643779774532, + "learning_rate": 6.270849258235484e-05, + "loss_nats": 1.0552918910980225, + "step": 6280, + "unclipped_grad_norm": 0.9406285881996155 + }, + { + "bits_per_byte": 1.452789390598705, + "learning_rate": 6.234519464399581e-05, + "loss_nats": 1.0069968700408936, + "step": 6290, + "unclipped_grad_norm": 0.8839805126190186 + }, + { + "bits_per_byte": 1.5074908804627336, + "learning_rate": 6.198365079302614e-05, + "loss_nats": 1.0449130535125732, + "step": 6300, + "unclipped_grad_norm": 0.9184122681617737 + }, + { + "bits_per_byte": 1.478450577968643, + "learning_rate": 6.16238672072326e-05, + "loss_nats": 1.0247838497161865, + "step": 6310, + "unclipped_grad_norm": 0.8948619365692139 + }, + { + "bits_per_byte": 1.4214500240278807, + "learning_rate": 6.126585003432406e-05, + "loss_nats": 0.985274076461792, + "step": 6320, + "unclipped_grad_norm": 0.8749227523803711 + }, + { + "bits_per_byte": 1.5185149683830395, + "learning_rate": 6.090960539182614e-05, + "loss_nats": 1.0525543689727783, + "step": 6330, + "unclipped_grad_norm": 0.9816636443138123 + }, + { + "bits_per_byte": 1.4759477144506365, + "learning_rate": 6.0555139366976935e-05, + "loss_nats": 1.023048996925354, + "step": 6340, + "unclipped_grad_norm": 0.8950188159942627 + }, + { + "bits_per_byte": 1.587707716408768, + "learning_rate": 6.020245801662274e-05, + "loss_nats": 1.1005151271820068, + "step": 6350, + "unclipped_grad_norm": 1.01009202003479 + }, + { + "bits_per_byte": 1.4638088349874379, + "learning_rate": 5.985156736711483e-05, + "loss_nats": 1.0146349668502808, + "step": 6360, + "unclipped_grad_norm": 0.9816907644271851 + }, + { + "bits_per_byte": 1.4382974445064014, + "learning_rate": 5.950247341420618e-05, + "loss_nats": 0.9969518184661865, + "step": 6370, + "unclipped_grad_norm": 0.8970034122467041 + }, + { + "bits_per_byte": 1.4818757844432269, + "learning_rate": 5.915518212294935e-05, + "loss_nats": 1.0271580219268799, + "step": 6380, + "unclipped_grad_norm": 0.9058489799499512 + }, + { + "bits_per_byte": 1.527050123380059, + "learning_rate": 5.880969942759425e-05, + "loss_nats": 1.0584704875946045, + "step": 6390, + "unclipped_grad_norm": 0.929339587688446 + }, + { + "bits_per_byte": 1.5156593684481134, + "learning_rate": 5.846603123148688e-05, + "loss_nats": 1.0505750179290771, + "step": 6400, + "unclipped_grad_norm": 0.9009133577346802 + }, + { + "bits_per_byte": 1.4969940913499729, + "learning_rate": 5.812418340696858e-05, + "loss_nats": 1.0376372337341309, + "step": 6410, + "unclipped_grad_norm": 0.957128643989563 + }, + { + "bits_per_byte": 1.4783625228514012, + "learning_rate": 5.778416179527541e-05, + "loss_nats": 1.0247228145599365, + "step": 6420, + "unclipped_grad_norm": 0.9414587616920471 + }, + { + "bits_per_byte": 1.523227980947284, + "learning_rate": 5.7445972206438635e-05, + "loss_nats": 1.055821180343628, + "step": 6430, + "unclipped_grad_norm": 0.9234302639961243 + }, + { + "bits_per_byte": 1.4623229048839832, + "learning_rate": 5.710962041918516e-05, + "loss_nats": 1.013604998588562, + "step": 6440, + "unclipped_grad_norm": 0.9177728295326233 + }, + { + "bits_per_byte": 1.5306256426914968, + "learning_rate": 5.677511218083908e-05, + "loss_nats": 1.0609488487243652, + "step": 6450, + "unclipped_grad_norm": 0.9453822374343872 + }, + { + "bits_per_byte": 1.5847898587542297, + "learning_rate": 5.644245320722316e-05, + "loss_nats": 1.0984926223754883, + "step": 6460, + "unclipped_grad_norm": 0.9647909998893738 + }, + { + "bits_per_byte": 1.5690988496201106, + "learning_rate": 5.61116491825615e-05, + "loss_nats": 1.0876164436340332, + "step": 6470, + "unclipped_grad_norm": 0.9592546820640564 + }, + { + "bits_per_byte": 1.4544314809491432, + "learning_rate": 5.5782705759382104e-05, + "loss_nats": 1.0081350803375244, + "step": 6480, + "unclipped_grad_norm": 0.9334172010421753 + }, + { + "bits_per_byte": 1.5573938823850386, + "learning_rate": 5.5455628558420546e-05, + "loss_nats": 1.0795031785964966, + "step": 6490, + "unclipped_grad_norm": 0.967822790145874 + }, + { + "bits_per_byte": 1.5280063469188563, + "learning_rate": 5.513042316852368e-05, + "loss_nats": 1.0591332912445068, + "step": 6500, + "unclipped_grad_norm": 0.9625112414360046 + }, + { + "bits_per_byte": 1.4687796495453256, + "learning_rate": 5.480709514655443e-05, + "loss_nats": 1.018080472946167, + "step": 6510, + "unclipped_grad_norm": 0.9510544538497925 + }, + { + "bits_per_byte": 1.4284500618832987, + "learning_rate": 5.448565001729653e-05, + "loss_nats": 0.9901261329650879, + "step": 6520, + "unclipped_grad_norm": 0.8985839486122131 + }, + { + "bits_per_byte": 1.4546447394362132, + "learning_rate": 5.41660932733604e-05, + "loss_nats": 1.0082828998565674, + "step": 6530, + "unclipped_grad_norm": 0.907886803150177 + }, + { + "bits_per_byte": 1.4970680438898438, + "learning_rate": 5.384843037508902e-05, + "loss_nats": 1.0376884937286377, + "step": 6540, + "unclipped_grad_norm": 0.9381445050239563 + }, + { + "bits_per_byte": 1.4280130539674563, + "learning_rate": 5.353266675046498e-05, + "loss_nats": 0.9898232221603394, + "step": 6550, + "unclipped_grad_norm": 0.9031082391738892 + }, + { + "bits_per_byte": 1.5210671909218436, + "learning_rate": 5.321880779501729e-05, + "loss_nats": 1.054323434829712, + "step": 6560, + "unclipped_grad_norm": 0.9433532357215881 + }, + { + "bits_per_byte": 1.479243074023819, + "learning_rate": 5.290685887172962e-05, + "loss_nats": 1.0253331661224365, + "step": 6570, + "unclipped_grad_norm": 0.9311387538909912 + }, + { + "bits_per_byte": 1.5051876888023787, + "learning_rate": 5.2596825310948295e-05, + "loss_nats": 1.0433166027069092, + "step": 6580, + "unclipped_grad_norm": 0.9569925665855408 + }, + { + "bits_per_byte": 1.494823842278735, + "learning_rate": 5.228871241029154e-05, + "loss_nats": 1.0361329317092896, + "step": 6590, + "unclipped_grad_norm": 0.8958868384361267 + }, + { + "bits_per_byte": 1.4302109062541577, + "learning_rate": 5.198252543455865e-05, + "loss_nats": 0.9913466572761536, + "step": 6600, + "unclipped_grad_norm": 0.9136191010475159 + }, + { + "bits_per_byte": 1.436879103584736, + "learning_rate": 5.167826961564027e-05, + "loss_nats": 0.9959686994552612, + "step": 6610, + "unclipped_grad_norm": 0.8710504770278931 + }, + { + "bits_per_byte": 1.4278971376607748, + "learning_rate": 5.137595015242884e-05, + "loss_nats": 0.9897428750991821, + "step": 6620, + "unclipped_grad_norm": 0.9145034551620483 + }, + { + "bits_per_byte": 1.4582168190946336, + "learning_rate": 5.1075572210729925e-05, + "loss_nats": 1.010758876800537, + "step": 6630, + "unclipped_grad_norm": 0.8999127149581909 + }, + { + "bits_per_byte": 1.5240701799885592, + "learning_rate": 5.07771409231737e-05, + "loss_nats": 1.056404948234558, + "step": 6640, + "unclipped_grad_norm": 0.9801992774009705 + }, + { + "bits_per_byte": 1.5169175935218258, + "learning_rate": 5.048066138912756e-05, + "loss_nats": 1.0514471530914307, + "step": 6650, + "unclipped_grad_norm": 0.8882671594619751 + }, + { + "bits_per_byte": 1.5152221885496202, + "learning_rate": 5.018613867460863e-05, + "loss_nats": 1.050271987915039, + "step": 6660, + "unclipped_grad_norm": 0.9029646515846252 + }, + { + "bits_per_byte": 1.4896082964086694, + "learning_rate": 4.989357781219755e-05, + "loss_nats": 1.0325177907943726, + "step": 6670, + "unclipped_grad_norm": 0.9797500371932983 + }, + { + "bits_per_byte": 1.4764526555135697, + "learning_rate": 4.960298380095222e-05, + "loss_nats": 1.023398995399475, + "step": 6680, + "unclipped_grad_norm": 0.9356769919395447 + }, + { + "bits_per_byte": 1.4958203097578342, + "learning_rate": 4.931436160632256e-05, + "loss_nats": 1.0368236303329468, + "step": 6690, + "unclipped_grad_norm": 0.939789891242981 + }, + { + "bits_per_byte": 1.5480882451121538, + "learning_rate": 4.9027716160065485e-05, + "loss_nats": 1.073053002357483, + "step": 6700, + "unclipped_grad_norm": 1.0080333948135376 + }, + { + "bits_per_byte": 1.525887864625528, + "learning_rate": 4.874305236016084e-05, + "loss_nats": 1.0576648712158203, + "step": 6710, + "unclipped_grad_norm": 0.9197862148284912 + }, + { + "bits_per_byte": 1.534906462854123, + "learning_rate": 4.846037507072752e-05, + "loss_nats": 1.0639160871505737, + "step": 6720, + "unclipped_grad_norm": 0.9543269872665405 + }, + { + "bits_per_byte": 1.4820493149379474, + "learning_rate": 4.817968912194049e-05, + "loss_nats": 1.0272783041000366, + "step": 6730, + "unclipped_grad_norm": 0.9817740321159363 + }, + { + "bits_per_byte": 1.4384499931177168, + "learning_rate": 4.790099930994813e-05, + "loss_nats": 0.9970575571060181, + "step": 6740, + "unclipped_grad_norm": 0.9710240364074707 + }, + { + "bits_per_byte": 1.511069151496585, + "learning_rate": 4.762431039679047e-05, + "loss_nats": 1.0473933219909668, + "step": 6750, + "unclipped_grad_norm": 0.9612263441085815 + }, + { + "bits_per_byte": 1.4780942299160553, + "learning_rate": 4.734962711031755e-05, + "loss_nats": 1.0245368480682373, + "step": 6760, + "unclipped_grad_norm": 0.9238441586494446 + }, + { + "bits_per_byte": 1.3767874194686909, + "learning_rate": 4.707695414410892e-05, + "loss_nats": 0.9543163180351257, + "step": 6770, + "unclipped_grad_norm": 0.9215719699859619 + }, + { + "bits_per_byte": 1.4785716537548506, + "learning_rate": 4.680629615739322e-05, + "loss_nats": 1.0248677730560303, + "step": 6780, + "unclipped_grad_norm": 0.9403936266899109 + }, + { + "bits_per_byte": 1.4459957319243224, + "learning_rate": 4.653765777496863e-05, + "loss_nats": 1.0022878646850586, + "step": 6790, + "unclipped_grad_norm": 0.9651554822921753 + }, + { + "bits_per_byte": 1.5175859181030786, + "learning_rate": 4.6271043587123986e-05, + "loss_nats": 1.051910400390625, + "step": 6800, + "unclipped_grad_norm": 0.9926899075508118 + }, + { + "bits_per_byte": 1.5110634760691066, + "learning_rate": 4.600645814956006e-05, + "loss_nats": 1.0473893880844116, + "step": 6810, + "unclipped_grad_norm": 0.9793192148208618 + }, + { + "bits_per_byte": 1.5052492585913877, + "learning_rate": 4.574390598331204e-05, + "loss_nats": 1.0433592796325684, + "step": 6820, + "unclipped_grad_norm": 1.0081095695495605 + }, + { + "bits_per_byte": 1.466343515295854, + "learning_rate": 4.548339157467199e-05, + "loss_nats": 1.0163918733596802, + "step": 6830, + "unclipped_grad_norm": 0.960361659526825 + }, + { + "bits_per_byte": 1.4376153613130795, + "learning_rate": 4.52249193751124e-05, + "loss_nats": 0.9964790344238281, + "step": 6840, + "unclipped_grad_norm": 0.9702624678611755 + }, + { + "bits_per_byte": 1.4922177891702109, + "learning_rate": 4.4968493801209985e-05, + "loss_nats": 1.0343265533447266, + "step": 6850, + "unclipped_grad_norm": 0.9893983006477356 + }, + { + "bits_per_byte": 1.4659078832412185, + "learning_rate": 4.471411923457034e-05, + "loss_nats": 1.016089916229248, + "step": 6860, + "unclipped_grad_norm": 0.9693014621734619 + }, + { + "bits_per_byte": 1.413224953750367, + "learning_rate": 4.446180002175291e-05, + "loss_nats": 0.9795728921890259, + "step": 6870, + "unclipped_grad_norm": 0.9252057075500488 + }, + { + "bits_per_byte": 1.4200107012228098, + "learning_rate": 4.421154047419693e-05, + "loss_nats": 0.9842764139175415, + "step": 6880, + "unclipped_grad_norm": 0.8884434700012207 + }, + { + "bits_per_byte": 1.4366857090938405, + "learning_rate": 4.396334486814753e-05, + "loss_nats": 0.9958346486091614, + "step": 6890, + "unclipped_grad_norm": 0.9679457545280457 + }, + { + "bits_per_byte": 1.5416700245646047, + "learning_rate": 4.371721744458285e-05, + "loss_nats": 1.0686042308807373, + "step": 6900, + "unclipped_grad_norm": 0.9985603094100952 + }, + { + "bits_per_byte": 1.4893800754309745, + "learning_rate": 4.347316240914145e-05, + "loss_nats": 1.0323596000671387, + "step": 6910, + "unclipped_grad_norm": 0.9936915636062622 + }, + { + "bits_per_byte": 1.4499423897963222, + "learning_rate": 4.3231183932050546e-05, + "loss_nats": 1.00502347946167, + "step": 6920, + "unclipped_grad_norm": 0.923648476600647 + }, + { + "bits_per_byte": 1.4406846496917027, + "learning_rate": 4.299128614805464e-05, + "loss_nats": 0.9986065030097961, + "step": 6930, + "unclipped_grad_norm": 0.9748247861862183 + }, + { + "bits_per_byte": 1.408974058568991, + "learning_rate": 4.2753473156344965e-05, + "loss_nats": 0.9766263961791992, + "step": 6940, + "unclipped_grad_norm": 0.9357670545578003 + }, + { + "bits_per_byte": 1.434796565665438, + "learning_rate": 4.251774902048942e-05, + "loss_nats": 0.9945251941680908, + "step": 6950, + "unclipped_grad_norm": 0.9624595642089844 + }, + { + "bits_per_byte": 1.49675830313564, + "learning_rate": 4.228411776836306e-05, + "loss_nats": 1.0374737977981567, + "step": 6960, + "unclipped_grad_norm": 0.9387544989585876 + }, + { + "bits_per_byte": 1.4929531869853003, + "learning_rate": 4.2052583392079427e-05, + "loss_nats": 1.0348362922668457, + "step": 6970, + "unclipped_grad_norm": 0.9343914985656738 + }, + { + "bits_per_byte": 1.4660633555575984, + "learning_rate": 4.182314984792214e-05, + "loss_nats": 1.016197681427002, + "step": 6980, + "unclipped_grad_norm": 0.9326976537704468 + }, + { + "bits_per_byte": 1.492015881538098, + "learning_rate": 4.159582105627748e-05, + "loss_nats": 1.034186601638794, + "step": 6990, + "unclipped_grad_norm": 1.0069916248321533 + }, + { + "bits_per_byte": 1.52302985693349, + "learning_rate": 4.137060090156724e-05, + "loss_nats": 1.0556838512420654, + "step": 7000, + "unclipped_grad_norm": 0.9719612002372742 + }, + { + "bits_per_byte": 1.512785538352196, + "learning_rate": 4.114749323218249e-05, + "loss_nats": 1.0485830307006836, + "step": 7010, + "unclipped_grad_norm": 0.9873834848403931 + }, + { + "bits_per_byte": 1.4473323810868282, + "learning_rate": 4.0926501860417706e-05, + "loss_nats": 1.0032143592834473, + "step": 7020, + "unclipped_grad_norm": 0.939332127571106 + }, + { + "bits_per_byte": 1.5166639191118032, + "learning_rate": 4.070763056240571e-05, + "loss_nats": 1.0512713193893433, + "step": 7030, + "unclipped_grad_norm": 0.9837853312492371 + }, + { + "bits_per_byte": 1.5072451172546506, + "learning_rate": 4.049088307805306e-05, + "loss_nats": 1.0447427034378052, + "step": 7040, + "unclipped_grad_norm": 0.9734936952590942 + }, + { + "bits_per_byte": 1.4796651194490362, + "learning_rate": 4.027626311097629e-05, + "loss_nats": 1.0256257057189941, + "step": 7050, + "unclipped_grad_norm": 1.005554437637329 + }, + { + "bits_per_byte": 1.4927048440374544, + "learning_rate": 4.006377432843846e-05, + "loss_nats": 1.0346641540527344, + "step": 7060, + "unclipped_grad_norm": 0.9903765320777893 + }, + { + "bits_per_byte": 1.4561590466770602, + "learning_rate": 3.985342036128656e-05, + "loss_nats": 1.009332537651062, + "step": 7070, + "unclipped_grad_norm": 1.009922981262207 + }, + { + "bits_per_byte": 1.4353391709289103, + "learning_rate": 3.964520480388956e-05, + "loss_nats": 0.9949012994766235, + "step": 7080, + "unclipped_grad_norm": 0.9552373290061951 + }, + { + "bits_per_byte": 1.4780041110070032, + "learning_rate": 3.943913121407678e-05, + "loss_nats": 1.0244743824005127, + "step": 7090, + "unclipped_grad_norm": 1.0046122074127197 + }, + { + "bits_per_byte": 1.4728850474040718, + "learning_rate": 3.923520311307735e-05, + "loss_nats": 1.0209261178970337, + "step": 7100, + "unclipped_grad_norm": 0.9868813157081604 + }, + { + "bits_per_byte": 1.4862149067244954, + "learning_rate": 3.9033423985459816e-05, + "loss_nats": 1.030165672302246, + "step": 7110, + "unclipped_grad_norm": 0.9593554735183716 + }, + { + "bits_per_byte": 1.5328834309320238, + "learning_rate": 3.88337972790728e-05, + "loss_nats": 1.062513828277588, + "step": 7120, + "unclipped_grad_norm": 1.0129024982452393 + }, + { + "bits_per_byte": 1.4811555211014134, + "learning_rate": 3.8636326404985845e-05, + "loss_nats": 1.0266587734222412, + "step": 7130, + "unclipped_grad_norm": 1.0546875 + }, + { + "bits_per_byte": 1.4724865636020226, + "learning_rate": 3.844101473743143e-05, + "loss_nats": 1.0206499099731445, + "step": 7140, + "unclipped_grad_norm": 0.9776664972305298 + }, + { + "bits_per_byte": 1.4357171887955067, + "learning_rate": 3.824786561374701e-05, + "loss_nats": 0.9951633214950562, + "step": 7150, + "unclipped_grad_norm": 0.9291411638259888 + }, + { + "bits_per_byte": 1.5232358921492237, + "learning_rate": 3.805688233431824e-05, + "loss_nats": 1.0558266639709473, + "step": 7160, + "unclipped_grad_norm": 1.0189123153686523 + }, + { + "bits_per_byte": 1.4618840051589814, + "learning_rate": 3.786806816252236e-05, + "loss_nats": 1.0133007764816284, + "step": 7170, + "unclipped_grad_norm": 0.9592419266700745 + }, + { + "bits_per_byte": 1.4762450724539784, + "learning_rate": 3.768142632467269e-05, + "loss_nats": 1.0232551097869873, + "step": 7180, + "unclipped_grad_norm": 0.9680334329605103 + }, + { + "bits_per_byte": 1.4441101141402626, + "learning_rate": 3.7496960009963236e-05, + "loss_nats": 1.0009808540344238, + "step": 7190, + "unclipped_grad_norm": 1.0148857831954956 + }, + { + "bits_per_byte": 1.472608843266786, + "learning_rate": 3.7314672370414325e-05, + "loss_nats": 1.0207346677780151, + "step": 7200, + "unclipped_grad_norm": 0.9935786128044128 + }, + { + "bits_per_byte": 1.5299951542934338, + "learning_rate": 3.713456652081882e-05, + "loss_nats": 1.060511827468872, + "step": 7210, + "unclipped_grad_norm": 0.9725626111030579 + }, + { + "bits_per_byte": 1.5238232129019202, + "learning_rate": 3.695664553868873e-05, + "loss_nats": 1.0562337636947632, + "step": 7220, + "unclipped_grad_norm": 1.0141370296478271 + }, + { + "bits_per_byte": 1.4390708504873315, + "learning_rate": 3.678091246420277e-05, + "loss_nats": 0.9974879026412964, + "step": 7230, + "unclipped_grad_norm": 1.0297694206237793 + }, + { + "bits_per_byte": 1.4632025961431465, + "learning_rate": 3.660737030015427e-05, + "loss_nats": 1.0142147541046143, + "step": 7240, + "unclipped_grad_norm": 0.9345177412033081 + }, + { + "bits_per_byte": 1.5031187375124992, + "learning_rate": 3.6436022011900056e-05, + "loss_nats": 1.0418825149536133, + "step": 7250, + "unclipped_grad_norm": 1.0408201217651367 + }, + { + "bits_per_byte": 1.3820238612308362, + "learning_rate": 3.626687052730954e-05, + "loss_nats": 0.9579459428787231, + "step": 7260, + "unclipped_grad_norm": 0.9544695615768433 + }, + { + "bits_per_byte": 1.4274823155068936, + "learning_rate": 3.6099918736714955e-05, + "loss_nats": 0.9894553422927856, + "step": 7270, + "unclipped_grad_norm": 0.9940797686576843 + }, + { + "bits_per_byte": 1.4990891840027836, + "learning_rate": 3.5935169492861716e-05, + "loss_nats": 1.0390894412994385, + "step": 7280, + "unclipped_grad_norm": 1.013985514640808 + }, + { + "bits_per_byte": 1.4370805812602216, + "learning_rate": 3.5772625610859886e-05, + "loss_nats": 0.99610835313797, + "step": 7290, + "unclipped_grad_norm": 0.9820091724395752 + }, + { + "bits_per_byte": 1.4451605841717328, + "learning_rate": 3.5612289868135887e-05, + "loss_nats": 1.001708984375, + "step": 7300, + "unclipped_grad_norm": 0.9996907114982605 + }, + { + "bits_per_byte": 1.4502488628801598, + "learning_rate": 3.545416500438522e-05, + "loss_nats": 1.0052359104156494, + "step": 7310, + "unclipped_grad_norm": 0.9723894000053406 + }, + { + "bits_per_byte": 1.4141343980081293, + "learning_rate": 3.52982537215255e-05, + "loss_nats": 0.9802032709121704, + "step": 7320, + "unclipped_grad_norm": 0.9815637469291687 + }, + { + "bits_per_byte": 1.429772436485783, + "learning_rate": 3.5144558683650396e-05, + "loss_nats": 0.9910427331924438, + "step": 7330, + "unclipped_grad_norm": 1.0089058876037598 + }, + { + "bits_per_byte": 1.4033304478809279, + "learning_rate": 3.499308251698404e-05, + "loss_nats": 0.9727145433425903, + "step": 7340, + "unclipped_grad_norm": 0.9830491542816162 + }, + { + "bits_per_byte": 1.4800763299672492, + "learning_rate": 3.484382780983616e-05, + "loss_nats": 1.02591073513031, + "step": 7350, + "unclipped_grad_norm": 0.9842677712440491 + }, + { + "bits_per_byte": 1.4409101189469837, + "learning_rate": 3.469679711255795e-05, + "loss_nats": 0.9987627863883972, + "step": 7360, + "unclipped_grad_norm": 1.0266367197036743 + }, + { + "bits_per_byte": 1.4700960047550296, + "learning_rate": 3.4551992937498284e-05, + "loss_nats": 1.0189929008483887, + "step": 7370, + "unclipped_grad_norm": 0.9935671091079712 + }, + { + "bits_per_byte": 1.4111584102175994, + "learning_rate": 3.440941775896104e-05, + "loss_nats": 0.9781404733657837, + "step": 7380, + "unclipped_grad_norm": 0.9791932702064514 + }, + { + "bits_per_byte": 1.4805810990475317, + "learning_rate": 3.4269074013162596e-05, + "loss_nats": 1.0262606143951416, + "step": 7390, + "unclipped_grad_norm": 0.9806758761405945 + }, + { + "bits_per_byte": 1.422500752033327, + "learning_rate": 3.41309640981904e-05, + "loss_nats": 0.9860023856163025, + "step": 7400, + "unclipped_grad_norm": 1.014562726020813 + }, + { + "bits_per_byte": 1.419007526420327, + "learning_rate": 3.3995090373961795e-05, + "loss_nats": 0.9835810661315918, + "step": 7410, + "unclipped_grad_norm": 0.9609370231628418 + }, + { + "bits_per_byte": 1.483305992167802, + "learning_rate": 3.3861455162183876e-05, + "loss_nats": 1.0281493663787842, + "step": 7420, + "unclipped_grad_norm": 0.9924190640449524 + }, + { + "bits_per_byte": 1.4892452410326982, + "learning_rate": 3.3730060746313664e-05, + "loss_nats": 1.0322661399841309, + "step": 7430, + "unclipped_grad_norm": 1.0244429111480713 + }, + { + "bits_per_byte": 1.50888548777858, + "learning_rate": 3.3600909371519225e-05, + "loss_nats": 1.0458797216415405, + "step": 7440, + "unclipped_grad_norm": 1.0394790172576904 + }, + { + "bits_per_byte": 1.4553271665948369, + "learning_rate": 3.3474003244641194e-05, + "loss_nats": 1.0087559223175049, + "step": 7450, + "unclipped_grad_norm": 0.99248868227005 + }, + { + "bits_per_byte": 1.4425173828106221, + "learning_rate": 3.334934453415514e-05, + "loss_nats": 0.999876856803894, + "step": 7460, + "unclipped_grad_norm": 1.0004469156265259 + }, + { + "bits_per_byte": 1.437790267669007, + "learning_rate": 3.3226935370134486e-05, + "loss_nats": 0.9966002702713013, + "step": 7470, + "unclipped_grad_norm": 1.0109968185424805 + }, + { + "bits_per_byte": 1.4682612938356252, + "learning_rate": 3.31067778442141e-05, + "loss_nats": 1.017721176147461, + "step": 7480, + "unclipped_grad_norm": 1.0299010276794434 + }, + { + "bits_per_byte": 1.5222762289574092, + "learning_rate": 3.298887400955457e-05, + "loss_nats": 1.055161476135254, + "step": 7490, + "unclipped_grad_norm": 1.0327476263046265 + }, + { + "bits_per_byte": 1.4747042798848986, + "learning_rate": 3.287322588080714e-05, + "loss_nats": 1.0221871137619019, + "step": 7500, + "unclipped_grad_norm": 0.9886637926101685 + }, + { + "bits_per_byte": 1.4352613487793948, + "learning_rate": 3.275983543407924e-05, + "loss_nats": 0.9948473572731018, + "step": 7510, + "unclipped_grad_norm": 0.9710773825645447 + }, + { + "bits_per_byte": 1.4570311706995855, + "learning_rate": 3.2648704606900735e-05, + "loss_nats": 1.009937047958374, + "step": 7520, + "unclipped_grad_norm": 1.0144232511520386 + }, + { + "bits_per_byte": 1.502727820947088, + "learning_rate": 3.253983529819086e-05, + "loss_nats": 1.0416115522384644, + "step": 7530, + "unclipped_grad_norm": 1.0307852029800415 + }, + { + "bits_per_byte": 1.4572943041554056, + "learning_rate": 3.243322936822571e-05, + "loss_nats": 1.0101194381713867, + "step": 7540, + "unclipped_grad_norm": 0.9732069969177246 + }, + { + "bits_per_byte": 1.4747278415080667, + "learning_rate": 3.232888863860648e-05, + "loss_nats": 1.0222034454345703, + "step": 7550, + "unclipped_grad_norm": 1.0250486135482788 + }, + { + "bits_per_byte": 1.4513507557242378, + "learning_rate": 3.222681489222838e-05, + "loss_nats": 1.0059996843338013, + "step": 7560, + "unclipped_grad_norm": 1.0031380653381348 + }, + { + "bits_per_byte": 1.4127096077370564, + "learning_rate": 3.212700987325009e-05, + "loss_nats": 0.979215681552887, + "step": 7570, + "unclipped_grad_norm": 0.9720339775085449 + }, + { + "bits_per_byte": 1.587541409185384, + "learning_rate": 3.202947528706403e-05, + "loss_nats": 1.1003998517990112, + "step": 7580, + "unclipped_grad_norm": 1.080464482307434 + }, + { + "bits_per_byte": 1.4421111597892842, + "learning_rate": 3.1934212800267116e-05, + "loss_nats": 0.9995952844619751, + "step": 7590, + "unclipped_grad_norm": 0.9815735220909119 + }, + { + "bits_per_byte": 1.4267507013101233, + "learning_rate": 3.1841224040632484e-05, + "loss_nats": 0.9889482259750366, + "step": 7600, + "unclipped_grad_norm": 1.029758334159851 + }, + { + "bits_per_byte": 1.4539179407536669, + "learning_rate": 3.175051059708143e-05, + "loss_nats": 1.0077791213989258, + "step": 7610, + "unclipped_grad_norm": 1.031221628189087 + }, + { + "bits_per_byte": 1.4028547438686412, + "learning_rate": 3.1662074019656414e-05, + "loss_nats": 0.9723848104476929, + "step": 7620, + "unclipped_grad_norm": 0.9562389850616455 + }, + { + "bits_per_byte": 1.3938100042771153, + "learning_rate": 3.157591581949455e-05, + "loss_nats": 0.9661154747009277, + "step": 7630, + "unclipped_grad_norm": 1.023301124572754 + }, + { + "bits_per_byte": 1.4184527103886437, + "learning_rate": 3.149203746880175e-05, + "loss_nats": 0.983196496963501, + "step": 7640, + "unclipped_grad_norm": 0.9981763958930969 + }, + { + "bits_per_byte": 1.5608753271864544, + "learning_rate": 3.141044040082763e-05, + "loss_nats": 1.081916332244873, + "step": 7650, + "unclipped_grad_norm": 1.0295648574829102 + }, + { + "bits_per_byte": 1.4665378556913289, + "learning_rate": 3.133112600984091e-05, + "loss_nats": 1.0165265798568726, + "step": 7660, + "unclipped_grad_norm": 1.0587892532348633 + }, + { + "bits_per_byte": 1.4931012640476933, + "learning_rate": 3.125409565110571e-05, + "loss_nats": 1.034938931465149, + "step": 7670, + "unclipped_grad_norm": 1.0342050790786743 + }, + { + "bits_per_byte": 1.4465245785757257, + "learning_rate": 3.117935064085835e-05, + "loss_nats": 1.0026544332504272, + "step": 7680, + "unclipped_grad_norm": 1.0567405223846436 + }, + { + "bits_per_byte": 1.4541416901824393, + "learning_rate": 3.110689225628479e-05, + "loss_nats": 1.0079342126846313, + "step": 7690, + "unclipped_grad_norm": 1.0190099477767944 + }, + { + "bits_per_byte": 1.3949941048283057, + "learning_rate": 3.1036721735498926e-05, + "loss_nats": 0.9669362306594849, + "step": 7700, + "unclipped_grad_norm": 1.002891182899475 + }, + { + "bits_per_byte": 1.4762163513512843, + "learning_rate": 3.096884027752132e-05, + "loss_nats": 1.0232352018356323, + "step": 7710, + "unclipped_grad_norm": 0.9984713792800903 + }, + { + "bits_per_byte": 1.480895139368007, + "learning_rate": 3.090324904225885e-05, + "loss_nats": 1.0264782905578613, + "step": 7720, + "unclipped_grad_norm": 1.0342158079147339 + }, + { + "bits_per_byte": 1.486845911070511, + "learning_rate": 3.083994915048472e-05, + "loss_nats": 1.030603051185608, + "step": 7730, + "unclipped_grad_norm": 1.027107834815979 + }, + { + "bits_per_byte": 1.4387832095037634, + "learning_rate": 3.077894168381942e-05, + "loss_nats": 0.9972885251045227, + "step": 7740, + "unclipped_grad_norm": 1.0455139875411987 + }, + { + "bits_per_byte": 1.4381068877292456, + "learning_rate": 3.0720227684712254e-05, + "loss_nats": 0.9968197345733643, + "step": 7750, + "unclipped_grad_norm": 1.0159157514572144 + }, + { + "bits_per_byte": 1.4538223183997872, + "learning_rate": 3.0663808156423456e-05, + "loss_nats": 1.0077128410339355, + "step": 7760, + "unclipped_grad_norm": 0.9867706894874573 + }, + { + "bits_per_byte": 1.3986834766546148, + "learning_rate": 3.060968406300711e-05, + "loss_nats": 0.9694935083389282, + "step": 7770, + "unclipped_grad_norm": 0.9943742156028748 + }, + { + "bits_per_byte": 1.4760859885019304, + "learning_rate": 3.055785632929463e-05, + "loss_nats": 1.0231448411941528, + "step": 7780, + "unclipped_grad_norm": 1.0422483682632446 + }, + { + "bits_per_byte": 1.4162891686407895, + "learning_rate": 3.0508325840878984e-05, + "loss_nats": 0.9816968441009521, + "step": 7790, + "unclipped_grad_norm": 0.9937946796417236 + }, + { + "bits_per_byte": 1.45477768202533, + "learning_rate": 3.046109344409957e-05, + "loss_nats": 1.0083750486373901, + "step": 7800, + "unclipped_grad_norm": 1.0428839921951294 + }, + { + "bits_per_byte": 1.4292601001688625, + "learning_rate": 3.0416159946027727e-05, + "loss_nats": 0.9906876087188721, + "step": 7810, + "unclipped_grad_norm": 1.028254508972168 + }, + { + "bits_per_byte": 1.4070698666586379, + "learning_rate": 3.0373526114452946e-05, + "loss_nats": 0.975306510925293, + "step": 7820, + "unclipped_grad_norm": 1.0253064632415771 + }, + { + "bits_per_byte": 1.4530960356651936, + "learning_rate": 3.0333192677869803e-05, + "loss_nats": 1.0072094202041626, + "step": 7830, + "unclipped_grad_norm": 1.0885143280029297 + }, + { + "bits_per_byte": 1.4330722676078875, + "learning_rate": 3.029516032546544e-05, + "loss_nats": 0.9933300018310547, + "step": 7840, + "unclipped_grad_norm": 1.0074925422668457 + }, + { + "bits_per_byte": 1.3858685333908742, + "learning_rate": 3.02594297071078e-05, + "loss_nats": 0.9606108665466309, + "step": 7850, + "unclipped_grad_norm": 0.9880329966545105 + }, + { + "bits_per_byte": 1.4517502714221922, + "learning_rate": 3.0226001433334603e-05, + "loss_nats": 1.0062766075134277, + "step": 7860, + "unclipped_grad_norm": 1.0082415342330933 + }, + { + "bits_per_byte": 1.4675547891058807, + "learning_rate": 3.019487607534278e-05, + "loss_nats": 1.0172314643859863, + "step": 7870, + "unclipped_grad_norm": 1.023923397064209 + }, + { + "bits_per_byte": 1.3867994754799946, + "learning_rate": 3.016605416497886e-05, + "loss_nats": 0.9612561464309692, + "step": 7880, + "unclipped_grad_norm": 1.0151392221450806 + }, + { + "bits_per_byte": 1.4568015738606837, + "learning_rate": 3.0139536194729734e-05, + "loss_nats": 1.0097779035568237, + "step": 7890, + "unclipped_grad_norm": 1.0505098104476929 + }, + { + "bits_per_byte": 1.446776189193938, + "learning_rate": 3.0115322617714362e-05, + "loss_nats": 1.00282883644104, + "step": 7900, + "unclipped_grad_norm": 1.0279992818832397 + }, + { + "bits_per_byte": 1.461830518554563, + "learning_rate": 3.0093413847675965e-05, + "loss_nats": 1.0132637023925781, + "step": 7910, + "unclipped_grad_norm": 1.0125948190689087 + }, + { + "bits_per_byte": 1.4593548282953928, + "learning_rate": 3.0073810258974985e-05, + "loss_nats": 1.0115476846694946, + "step": 7920, + "unclipped_grad_norm": 1.0411789417266846 + }, + { + "bits_per_byte": 1.4317973602170415, + "learning_rate": 3.0056512186582658e-05, + "loss_nats": 0.9924463033676147, + "step": 7930, + "unclipped_grad_norm": 1.0330348014831543 + }, + { + "bits_per_byte": 1.431196452834927, + "learning_rate": 3.0041519926075316e-05, + "loss_nats": 0.9920297861099243, + "step": 7940, + "unclipped_grad_norm": 1.0142537355422974 + }, + { + "bits_per_byte": 1.49434246283897, + "learning_rate": 3.0028833733629313e-05, + "loss_nats": 1.035799264907837, + "step": 7950, + "unclipped_grad_norm": 1.0716009140014648 + }, + { + "bits_per_byte": 1.4186909923514142, + "learning_rate": 3.0018453826016686e-05, + "loss_nats": 0.9833616614341736, + "step": 7960, + "unclipped_grad_norm": 1.0115852355957031 + }, + { + "bits_per_byte": 1.4420433126335188, + "learning_rate": 3.001038038060143e-05, + "loss_nats": 0.9995482563972473, + "step": 7970, + "unclipped_grad_norm": 1.0335451364517212 + }, + { + "bits_per_byte": 1.4872165336831205, + "learning_rate": 3.0004613535336433e-05, + "loss_nats": 1.0308599472045898, + "step": 7980, + "unclipped_grad_norm": 1.0748283863067627 + }, + { + "bits_per_byte": 1.5234063269562288, + "learning_rate": 3.000115338876117e-05, + "loss_nats": 1.055944800376892, + "step": 7990, + "unclipped_grad_norm": 1.076164722442627 + }, + { + "bits_per_byte": 1.4309937712808851, + "learning_rate": 3e-05, + "loss_nats": 0.9918892979621887, + "step": 8000, + "unclipped_grad_norm": 1.0322442054748535 + } + ] +} diff --git a/package.json b/package.json index 142a2f1..f7f8ff7 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "check:data:deepseek-chat-task-bootstrap": "node scripts/check-deepseek-chat-task-bootstrap-crn-data.mjs", "check:data:k3-attnres": "node scripts/check-k3-attnres-data.mjs", "check:data:k3-attnres-gradient": "node scripts/check-k3-attnres-gradient-data.mjs", + "check:data:k3-attnres-spike": "node scripts/check-k3-attnres-spike-data.mjs", "check:site": "node scripts/check-site.mjs", "check:moe-browser": "node scripts/check-moe-browser.mjs", "check:reasoning-browser": "node scripts/check-reasoning-browser.mjs", diff --git a/research/K3_ATTNRES_SPIKE_AUDIT.md b/research/K3_ATTNRES_SPIKE_AUDIT.md new file mode 100644 index 0000000..c8de6e6 --- /dev/null +++ b/research/K3_ATTNRES_SPIKE_AUDIT.md @@ -0,0 +1,503 @@ +# K3 Attention Residuals 局部梯度尖峰与反向路由审计 + +研究日期：2026-07-30 +协议 ID：`llm-atlas-k3-attnres-spike-path-v1` +父协议：`llm-atlas-k3-attnres-gradient-scale-v1` +研究身份：**Round 05 已知异常驱动的定向机制追踪，不是盲发现** + +## 0. 一句话结果 + +在与 Round 05 **model / optimizer / BPC / post-MLP gradient 全部 exact** 的 +depth-32 Block AttnRes 三 seed 重放中： + +1. layer 21–25 的局部峰在六个捕获位置最终都 3 / 3 可见； +2. 四种预注册 Token-gradient reductions 的 12 / 12 单元全部过闸； +3. 全局切断 64 个 depth mixers 与 output mixer 的 softmax / query / key + source-gradient path，峰没有下降，反而在 3 / 3 seed 略增； +4. 保持 learned forward 完全不变、只把所有 mixer 的 value backward coefficients + 从 learned `w` 改为 `1/N` 后，spike contrast 平均下降 **70.2%**、peak normalized + 平均下降 **37.0%**，两项都在 3 / 3 seed 超过预注册 20% 阈值； +5. seed-1 从初始化完整重训 8,000 steps 的 replay，连六位置、七 reductions、三模式和 + 65 个 mixer quantile summaries 都 exact。 + +最窄、最准确的结论是： + +> 在这个已训练的缩小 Block AttnRes 图上，最终 layer 21–25 峰对 learned mixer 的 +> **value-backward routing coefficients** 有预注册阈值下的强全局敏感性；它不是由 +> softmax/key source-gradient derivative path 单独解释的。 + +这不等于： + +- uniform backward 是更好的训练方法； +- learned value weights 让模型训练变差； +- 某一个 layer 或同层 MLP mixer 是唯一原因； +- Kimi K3 真实 checkpoint 存在同样尖峰。 + +## 1. 研究问题从哪里来 + +Round 05 已知： + +- Block 的首/末四分位失衡在 6 / 6 配对中改善； +- 全层 CV 却在 6 / 6 配对中恶化； +- depth-32 三 seed 平均的相对高值集中在 layer 21–25； +- post-MLP gradient 与 MLP latest-source mean weight 在 96 个 + `seed × layer` 点上 Pearson `r≈0.651`，在 layer 19–28 的 30 点上 `r≈0.690`。 + +这些 observational correlations 不能回答： + +- 是 softmax / key 的导数路径重要，还是 value coefficients 重要？ +- 峰是否只是 element RMS 的 reduction 产物？ +- 峰在一个 block 内到哪个位置才出现？ + +因此 Round 06 在看见新结果前固定： + +- 六个 activation 位置； +- 四个主 reductions、两个 cancellation-sensitive reductions 和一个代数控制； +- 三种 same-forward backward rules； +- fixed spike set `S={21,22,23,24,25}`； +- `contrast≥1.5`、top-5 overlap、Spearman、20% material drop； +- 三 formal seeds 和一个完整 replay。 + +前置 scoping、协议与 runner 分别在提交 `204f599`、`54c80fb` 前后进入公开仓库， +formal 训练随后才开始。 + +## 2. 对抗审查如何改变协议 + +本机 Grok Headless 只用于一次只读、对抗性方法审查，不提供事实来源，也没有修改文件。 +主代理采纳并在结果前修正的关键问题包括： + +- 把“最早生成位置”降级为“最早可见 tensor”； +- 主判定时点固定为 step 8,000； +- intervention 明确作用于全部 64 个 depth mixers 加 output mixer； +- 结论从局部因果分解降级为全局 backward-rule sensitivity； +- material 判定固定只用 `element_rms`； +- median / p95 固定 Hyndman–Fan Type 7； +- top-5 ties、Spearman ties、正分母与 invalid 条件全部写死； +- 增加六位置 loss×2、optimizer isolation 和训练不得进入 custom autograd 的闸门。 + +这次审查的价值是发现协议漏洞，不是为结果背书。 + +## 3. 第一次 smoke 为什么被拦住 + +最初 intervention custom backward 直接用 elementwise multiplication 分配 +`weights × grad_output`。forward identity 已经 exact，但 step-0 uniform negative +control 失败： + +```text +六位置 max raw relative difference：约 0.0037–0.0046 +六位置 max normalized difference：约 0.0011–0.0020 +``` + +这说明虽然代数公式一致，手写 backward 没有复用父 runner autocast einsum 的实际数值 +路径，不能满足预注册的“query=0 时三模式等价”。 + +正式训练尚未开始。实现随后改为： + +1. forward 直接返回父 `DepthMixer.forward` 已算出的 `y_parent`； +2. backward 在 `torch.enable_grad()` 下，用冻结的 learned / uniform weights + 重新走同一个 BF16-autocast einsum surrogate； +3. 只把 surrogate 对 source values 的梯度返回； +4. 不向 `y_parent`、query、key path 回传。 + +修正后： + +```text +三模式 logits / loss / 六位置 activations：byte-exact +65 个 mixer summaries：exact +step-0 六位置 raw gradient spectra：max difference 0 +step-0 normalized spectra：max difference 0 +六位置 × 七 reductions loss×2：scale / shape error 0 +两个全新进程非计时字段：exact +``` + +这个失败不是需要隐藏的“调试噪声”；它正是负控制在正式结果前发挥作用的证据。 + +## 4. 训练合同与工作量 + +Round 06 没有创建新的训练分布： + +| 字段 | 值 | +|---|---:| +| architecture | Block AttnRes | +| depth | 32 Transformer blocks | +| groups | 8，每组 4 blocks | +| width / heads / FFN | 192 / 6 / 768 | +| context / batch | 256 / 32 | +| steps | 8,000 | +| formal seeds | 2026073001 / 2 / 3 | +| replay | seed-1，从初始化完整 8,000 steps | +| formal target bytes | 196,608,000 | +| 含 replay target bytes | 262,144,000 | +| diagnostic tensor | 固定 16 × 256 | +| diagnostic steps | 0 / 100 / 500 / 2,000 / 4,000 / 8,000 | +| intervention steps | 0 / 8,000 | + +数据与父协议 exact，完整值为： + +```text +formal schedule: +5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e + +validation: +f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338 + +diagnostic: +21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716 +``` + +## 5. 父协议训练等价闸门 + +三个 formal seed 全部满足： + +| 字段 | seed-1 | seed-2 | seed-3 | +|---|---:|---:|---:| +| final model-state hash | exact | exact | exact | +| final optimizer-state hash | exact | exact | exact | +| 六个 validation BPC | exact | exact | exact | +| 六条 post-MLP element-RMS arrays | exact | exact | exact | +| training history | exact | exact | exact | + +最终 BPC： + +| seed | Round 06 | Round 05 | +|---:|---:|---:| +| 2026073001 | 1.7123525940760473 | exact | +| 2026073002 | 1.7093240655856785 | exact | +| 2026073003 | 1.7030966812870931 | exact | + +因此新增 capture / diagnostics 没有改变 optimizer trajectory。Round 06 不是一组“看起来 +相近”的新模型，而是同一冻结训练的独立数值重放。 + +## 6. 峰在训练中形成，不是初始化常数 + +最终主指标是 post-MLP element-RMS： + +```text +spike_contrast = +mean(layers 21–25) / mean(other 27 layers) +``` + +时间轨迹： + +| step | seed-1 | seed-2 | seed-3 | +|---:|---:|---:|---:| +| 0 | 0.782 | 0.691 | 0.797 | +| 100 | 0.560 | 0.556 | 0.575 | +| 500 | 0.522 | 0.573 | 0.478 | +| 2,000 | 2.497 | 2.740 | **1.461** | +| 4,000 | 2.851 | 3.157 | 1.773 | +| 8,000 | **3.046** | **3.333** | **1.881** | + +step 500 时三个 seed 的目标层都低于其他层均值；到 step 2,000，seed-1/2 已形成明显峰， +seed-3 仍略低于预注册 1.5 阈值；seed-3 到 step 4,000 才跨过。 + +所以允许说： + +> 尖峰在这个训练过程中发展出来，且 onset 有 seed 差异。 + +不允许说： + +> aggregation boundary 本身在初始化就必然产生 layer 21 峰。 + +## 7. 六个位置：处处可见，不等于找到 origin + +最终 element-RMS spike contrast： + +| 位置 | seed-1 | seed-2 | seed-3 | 三 seed mean | 3/3 ≥1.5 | +|---|---:|---:|---:|---:|---:| +| pre-attention input | 3.724 | 3.189 | 2.040 | 2.984 | 是 | +| attention branch output | 3.200 | 3.662 | 2.005 | 2.955 | 是 | +| post-attention state | 3.200 | 3.662 | 2.005 | 2.955 | 是 | +| pre-MLP input | 3.829 | 4.527 | 2.978 | **3.778** | 是 | +| MLP branch output | 3.046 | 3.333 | 1.881 | 2.754 | 是 | +| post-MLP state | 3.046 | 3.333 | 1.881 | 2.754 | 是 | + +协议顺序中的第一个可见 tensor 是 `pre_attention_input`。但 activation gradient +本来就汇总全部下游路径，所以这只说明： + +> 在进入该层 attention 之前，固定深度模式已经能在损失对该 tensor 的敏感性中看到。 + +它不说明峰在 pre-attention mixer “生成”。六个位置全部可见，反而证明单靠位置 trace +无法定位唯一 origin。 + +值得注意的是，`pre_mlp_input` 的平均 contrast 最高，但这仍是读数，不是“MLP 是唯一 +原因”的消融证据。 + +## 8. reduction robustness：12 / 12 过闸 + +主 family： + +1. element RMS； +2. mean token RMS； +3. median token RMS； +4. p95 token RMS。 + +每个 seed / reduction 必须同时满足： + +- contrast ≥1.5； +- top-5 与固定集合至少重合 3； +- 与 element-RMS 32-layer Spearman ≥0.8。 + +结果： + +| reduction | seed contrasts | 最低 top-5 overlap | 最低 Spearman | verdict | +|---|---|---:|---:|---:| +| element RMS | 3.046 / 3.333 / 1.881 | 3 | 1.000 | pass | +| token RMS mean | 2.869 / 3.182 / 1.808 | 3 | 0.958 | pass | +| token RMS median | 2.646 / 2.905 / 1.666 | 3 | **0.895** | pass | +| token RMS p95 | 3.143 / 3.532 / 2.012 | 3 | 0.993 | pass | + +总判定： + +```text +12 / 12 cells pass +robust within the preregistered reduction family +``` + +`global_l2` 与 element RMS 的固定 shape 代数控制也通过。batch-mean / token-mean +reduction 同样保存，但它们允许正负梯度先抵消，按协议只作探索性读数，不进入 robustness +判定。 + +这个结果支持： + +> layer 21–25 峰不是只由“先平方再平均”的单一 reduction 选择制造的。 + +它不支持： + +> 论文所有可能的 gradient magnitude 定义都会给出同一曲线。 + +## 9. same-forward interventions 到底改了什么 + +三模式： + +### learned + +原计算图： + +```text +w = softmax(qᵀ RMSNorm(sources)) +y = Σ w_i source_i +``` + +### detached learned + +forward 的 `w`、`y`、logits、loss 完全相同；backward 对 sources 保留 learned +value coefficients，但切断 softmax / query / key 的 source-gradient path。 + +### uniform value backward + +forward 仍逐字节返回 learned `y`；backward 对每个 source 返回 `grad_y/N`，不向 +softmax / query / key 回传。 + +作用域是全部 64 个 depth mixers 加 output mixer。它不是只改 layer 21 附近，也不是一次 +训练消融。 + +三个 seed 的最终 forward identity 均满足： + +```text +logits hash exact +loss exact +六位置 activation hashes exact +65 mixer summary hash exact +``` + +所以三模式的梯度差不能归因于不同 forward activation 或不同预测。 + +## 10. softmax / key path：没有降低峰 + +learned → detached learned： + +| seed | contrast | detached | relative drop | peak relative drop | +|---:|---:|---:|---:|---:| +| 2026073001 | 3.046 | 3.093 | −1.54% | −0.67% | +| 2026073002 | 3.333 | 3.397 | −1.93% | −0.38% | +| 2026073003 | 1.881 | 1.929 | −2.53% | −1.47% | +| mean | — | — | **−2.00%** | **−0.84%** | + +“relative drop”为负表示 detaching 后峰略高。预注册规则要求 contrast 与 peak 都下降至少 +20%、3 / 3 seed 同向；实际是两项在 3 / 3 seed 都朝相反方向。 + +结论： + +> 全局移除 softmax / query / key 的 source-gradient derivative path，没有在本阈值下 +> 降低固定峰；它在三个 seed 中都使峰略增。 + +这不证明该路径处处为零，也不证明它对训练没有作用。它只否定本轮预注册的 +“material peak reduction”方向。 + +## 11. learned value coefficients：3 / 3 过 material 闸门 + +detached learned → uniform value backward： + +| seed | detached contrast | uniform contrast | contrast drop | peak drop | +|---:|---:|---:|---:|---:| +| 2026073001 | 3.093 | 0.780 | **74.8%** | **39.3%** | +| 2026073002 | 3.397 | 0.786 | **76.9%** | **43.2%** | +| 2026073003 | 1.929 | 0.792 | **58.9%** | **28.6%** | +| mean | — | — | **70.2%** | **37.0%** | + +六个相对下降全部超过预注册 20%。uniform value backward 下： + +- 固定 target-layer contrast 都低于 0.8； +- peak layer 从 learned / detached 的 layer 21 移到 layer 2； +- forward logits、loss 与 activation 完全不变。 + +预注册判定： + +```text +material sensitivity at the preregistered threshold +3 / 3 seeds +``` + +允许说： + +> 在最终已训练计算图上，learned source value-backward coefficients 是该局部梯度形状的 +> material carrier / amplifier。 + +禁止说： + +- “把训练改成 uniform backward 一定更好”； +- “learned mixer 导致 BPC 下降或上升”； +- “峰完全由同层 MLP latest weight造成”； +- “70.2% 是因果贡献率”。 + +相对下降是对一个**全局反向规则干预**的指标变化，不是可加的路径归因百分比。 + +## 12. mixer association 与 intervention 如何一起读 + +最终 observational association： + +| scope | metric | Pearson | Spearman | +|---|---|---:|---:| +| 96 seed×layer | MLP latest mass | 0.651 | 0.692 | +| 96 seed×layer | MLP normalized entropy | −0.274 | −0.417 | +| layer 19–28 | MLP latest mass | 0.690 | 0.693 | +| layer 19–28 | MLP max mass | 0.702 | 0.705 | +| layer 19–28 | MLP normalized entropy | −0.636 | −0.640 | +| layer 19–28 | attention latest mass | 0.158 | 0.339 | + +Round 06 的 intervention 让“value coefficients 值得关注”从单纯相关线索升级为 +same-forward backward sensitivity。但它仍然是**全部 mixers 同时改写**，所以不能把 +全局结果定向归因给表中同层 MLP latest mass。 + +更窄的下一步需要： + +- 逐 group / 逐 mixer intervention； +- output mixer 与 64 个 depth mixers 分开； +- current partial、completed group、embedding source 分开； +- 保持 forward exact 的局部 backward path ablation。 + +这些没有在本轮结果后追加，以免事后挑选 layer。 + +## 13. replay 与完整复现 + +seed-1 replay 从初始化完整训练 8,000 steps，不加载 formal checkpoint。 + +exact 字段： + +```text +architecture / depth / seed / steps / batch / target bytes +manifest / model / optimizer contracts +model and optimizer state hashes +six validation evaluations +all training-history checkpoints +six diagnostic steps +six positions +seven reductions +three intervention modes at step 0 / 8,000 +65 mixer summaries with quantiles +forward identity / negative control / loss-scale gates +environment / artifact identities +Round 05 equivalence record +``` + +冻结 compare SHA-256： + +```text +05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87 +``` + +排除计时、run kind 和输出路径。replay verdict：`exact`。 + +## 14. 成本数字的正确解释 + +formal 三个进程的 mean step time 都约 224 ms；peak allocated 约 12.84 GB， +reserved 约 13.92 GB。 + +它们与 Round 05 单进程约 109 ms 不可直接比较，因为本轮两进程并行竞争同一 RTX 5090， +GPU 利用率 100%。训练 step 本身走父 runner exact 路径，新增诊断只发生在冻结时点。 + +因此： + +- 计时证明实验真实执行； +- 计时不用于评价新诊断 overhead； +- 更不能外推到 K3 训练系统。 + +## 15. 工件与哈希 + +核心源码： + +| 工件 | SHA-256 | +|---|---| +| manifest | `d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef` | +| protocol | `6cb101b8760d9f1c81caeb2f16880b16152da103867224a06761a75a12984a16` | +| scoping | `590166bd62580bb8238293823cfcc39bc0a465fec4c697025343f3f1138abd27` | +| runner | `77298081d3c491d2e88e4705995174b9879ef377f520eb5fe5ea107e7a1da084` | +| analyzer | `b9dd7f723e8c5765ecffd9ae3050ff7a95643f95853adaeeaa2c2c323d0ca057` | + +raw physical hashes： + +| 文件 | SHA-256 | +|---|---| +| formal seed-1 | `e39e93b7a7fce3c56f5f14f95cfdc04afdce53628affee1202fe62bd1bdb7f71` | +| formal seed-2 | `1c6f6c731030ec0adb2a8e7a4d586e0c4005cc3319568a7ac83c08c2a4b8eaf8` | +| formal seed-3 | `115f8245577ece6dfaaa8ada68445c186e6523a7f3b26efcc3eb4c0c4ce82406` | +| replay seed-1 | `4f91c7a2698662ea69af9aa3425e144107834fd9b6b717fb82b84f153b867dc5` | + +聚合工件： + +| 工件 | physical SHA-256 | canonical SHA-256 | +|---|---|---| +| full aggregate | `75058f9e…c1e90` | `1d0faef2…fc8b4` | +| compact | `7b3a3749…8ef6c` | `a430dd26…26cd` | +| reproduction | `86ac20ee…879a` | `7c522314…3515` | + +完整值保存在文件自身和 `reproduction.json`，缩写只用于阅读。 + +## 16. 允许与禁止的最终表述 + +### 允许 + +> 固定 layer 21–25 峰在四种预注册 Token-gradient reductions 中 12 / 12 过闸， +> 所以它在本轮 reduction family 内稳健。 + +> 在同一个 learned forward 上，全局 value-backward coefficient uniformization 使 +> contrast / peak 在 3 / 3 seed 超过 20% 下降阈值。 + +> 全局移除 softmax/key source-gradient path 没有降低峰，反而在 3 / 3 seed 略增。 + +> 峰从训练中发展，且 seed-3 的形成更晚、更弱。 + +### 禁止 + +- “找到了梯度尖峰的唯一来源”； +- “pre-attention 是尖峰生成点”； +- “论文 Figure 5(c) 已复现”； +- “Kimi K3 checkpoint 有 layer 21 尖峰”； +- “uniform backward 会提高训练质量”； +- “learned mixer 的 70.2% 梯度是坏的”； +- “softmax/key path 没有任何作用”； +- 从 3 seeds 推断总体显著性； +- 从 width 192 / depth 32 外推到 2.8T 参数。 + +## 17. 下一轮最有价值的实验 + +1. 把全局 intervention 拆成 output mixer、group 6、group 7 和其他 mixers 四组； +2. 对 completed-group value path 与 current-partial value path 分开做 same-forward backward + ablation； +3. 预先固定 local intervention grid，不根据本轮最大的单层事后选择； +4. 比较“只改诊断 backward”与“从初始化按替代 backward 训练”的身份差异； +5. 若 Attention Residuals 官方发布 Figure 5 telemetry，再开独立 protocol 对齐其 tensor、 + norm、reduction、batch 与 AMP 时点。 + +真实 K3 forward 仍受 `A_log [128]→[96]` 官方转换合同缺失阻断；本轮不绕过该边界。 diff --git a/scripts/check-k3-attnres-spike-data.mjs b/scripts/check-k3-attnres-spike-data.mjs new file mode 100644 index 0000000..339906b --- /dev/null +++ b/scripts/check-k3-attnres-spike-data.mjs @@ -0,0 +1,87 @@ +import { createHash } from "node:crypto"; +import { readdirSync, readFileSync } from "node:fs"; + +const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const read = (path) => { + const bytes = readFileSync(new URL(path, import.meta.url)); + return { bytes, json: JSON.parse(bytes), sha256: hash(bytes) }; +}; +const aggregate = read("../src/data/k3-attnres-spike.json"); +const compact = read("../src/data/k3-attnres-spike-compact.json"); +const reproduction = read("../experiments/k3/attnres_spike/reproduction.json"); +const manifest = read("../experiments/k3/attnres_spike/manifest.json"); +const rawDirectory = new URL("../experiments/k3/attnres_spike/results/raw/", import.meta.url); +const failures = []; +const expect = (condition, message) => { + if (!condition) failures.push(message); +}; + +expect(aggregate.sha256 === "75058f9ef1f89593436960428378ab920569017f1758f9c15a2a0b1e081c1e90", "aggregate physical SHA-256 changed"); +expect(compact.sha256 === "7b3a3749d1344e56fb61039d2678a4e2aefe11a0a1e279516d6f27ff3bb8ef6c", "compact physical SHA-256 changed"); +expect(reproduction.sha256 === "86ac20ee06189a924eed2b2614e535660b536447f5ad019f4c1fd99cf4aa879a", "reproduction physical SHA-256 changed"); +expect(manifest.sha256 === "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "manifest physical SHA-256 changed"); + +expect(aggregate.json.canonical_sha256_without_self === "1d0faef297682d61d35ce2ab54da3868e18a84e9ebc6e44c617d08016e4fc8b4", "aggregate canonical SHA-256 changed"); +expect(compact.json.canonical_sha256_without_self === "a430dd26f512db7d64c933df65d9a97e6899028099333fe0a1212728f13126cd", "compact canonical SHA-256 changed"); +expect(reproduction.json.canonical_sha256_without_self === "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515", "reproduction canonical SHA-256 changed"); + +expect(compact.json.protocol_id === "llm-atlas-k3-attnres-spike-path-v1", "protocol identity mismatch"); +expect(compact.json.study.seeds.length === 3, "formal seed count changed"); +expect(compact.json.study.steps === 8000, "formal step budget changed"); +expect(compact.json.study.spike_layers.join(",") === "21,22,23,24,25", "fixed spike set changed"); +expect(readdirSync(rawDirectory).filter((name) => name.endsWith(".json")).length === 4, "raw run count is not four"); + +for (const [name, expected] of Object.entries(reproduction.json.raw_files)) { + const raw = read(`../experiments/k3/attnres_spike/results/raw/${name}`); + expect(raw.sha256 === expected.file_sha256, `${name} physical hash mismatch`); + expect(raw.json.canonical_sha256_without_self === expected.canonical_sha256, `${name} canonical hash mismatch`); +} + +expect(compact.json.hashes.aggregate_canonical_sha256 === aggregate.json.canonical_sha256_without_self, "compact→aggregate canonical link mismatch"); +expect(compact.json.hashes.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "compact→reproduction canonical link mismatch"); +expect(aggregate.json.reproduction_canonical_sha256 === reproduction.json.canonical_sha256_without_self, "aggregate→reproduction canonical link mismatch"); + +expect(reproduction.json.replay_gate.passed, "full replay is not exact"); +expect(reproduction.json.replay_gate.frozen_compare_sha256 === "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87", "replay compare hash changed"); +expect(compact.json.gates.round05_equivalence.every((item) => item.passed), "Round 05 equivalence gate failed"); +expect(compact.json.gates.forward_identity.every((item) => item.passed), "forward identity gate failed"); +expect(compact.json.gates.initialization_negative_control.every((item) => item.passed), "initialization negative control failed"); +expect(compact.json.gates.loss_scale.every((item) => item.passed), "loss-scale gate failed"); + +const verdicts = compact.json.verdicts; +expect(verdicts.reduction_robustness.verdict === "robust within the preregistered reduction family", "reduction robustness verdict changed"); +expect(verdicts.reduction_robustness.passed_cells === 12, "reduction pass count changed"); +expect(verdicts.visible_positions.visible_positions.length === 6, "visible-position count changed"); +expect(verdicts.visible_positions.earliest_observed_tensor === "pre_attention_input", "earliest observed tensor changed"); +expect(verdicts.interventions.softmax_key_path.opposite_direction_3_of_3, "softmax/key opposite-direction result changed"); +expect(!verdicts.interventions.softmax_key_path.passed_3_of_3, "softmax/key path unexpectedly passed material threshold"); +expect(verdicts.interventions.value_coefficients.passed_3_of_3, "value-coefficient material threshold failed"); +expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_contrast - 0.701998806119879) < 1e-15, "value contrast drop changed"); +expect(Math.abs(verdicts.interventions.value_coefficients.mean_relative_drop_peak - 0.3702754647109951) < 1e-15, "value peak drop changed"); + +for (const run of compact.json.final_arrays) { + expect(Object.keys(run.position_element_rms).length === 6, `seed ${run.seed} position count changed`); + expect(Object.keys(run.post_mlp_reductions).length === 7, `seed ${run.seed} reduction count changed`); + expect(Object.keys(run.interventions).length === 3, `seed ${run.seed} intervention count changed`); +} + +if (failures.length) { + console.error(`FAIL K3 AttnRes spike data\n- ${failures.join("\n- ")}`); + process.exit(1); +} + +console.log(JSON.stringify({ + protocol: compact.json.protocol_id, + formalRuns: compact.json.study.seeds.length, + replayExact: reproduction.json.replay_gate.passed, + reduction: verdicts.reduction_robustness.verdict, + visiblePositions: verdicts.visible_positions.visible_positions.length, + softmaxKey: verdicts.interventions.softmax_key_path.verdict, + valueCoefficients: verdicts.interventions.value_coefficients.verdict, + hashes: { + aggregate: aggregate.sha256, + compact: compact.sha256, + reproduction: reproduction.sha256, + }, +}, null, 2)); +console.log("PASS K3 AttnRes spike frozen data"); diff --git a/src/data/k3-attnres-spike-compact.json b/src/data/k3-attnres-spike-compact.json new file mode 100644 index 0000000..e378a00 --- /dev/null +++ b/src/data/k3-attnres-spike-compact.json @@ -0,0 +1,5838 @@ +{ + "canonical_sha256_without_self": "a430dd26f512db7d64c933df65d9a97e6899028099333fe0a1212728f13126cd", + "final_arrays": [ + { + "interventions": { + "detached_learned": { + "statistics": { + "first_quartile_mean": 5.772373378931661e-05, + "first_to_last_ratio": 0.7709541968266947, + "imbalance_abs_log_ratio": 0.26012631467157654, + "last_quartile_mean": 7.487310404030723e-05, + "mean": 7.085559036568156e-05, + "normalized": [ + 0.767456541889667, + 0.8910969874796977, + 0.7949526654342541, + 0.7325741840266811, + 0.943544133913626, + 0.868390294625501, + 0.8054913948148298, + 0.7138323441642012, + 0.5978064063949543, + 0.5556378314836203, + 0.5261854153191788, + 0.505768953649729, + 0.715108334510866, + 0.6608161569684796, + 0.612256437426044, + 0.5785464585899268, + 0.739960161627241, + 0.6892568994732284, + 0.697060043562065, + 0.8113764458982461, + 3.2405349047340315, + 2.5249733605503017, + 1.9894958890200054, + 1.5842775624032772, + 2.3150907133984013, + 1.7369124395330813, + 1.1174777605817472, + 0.8878712764898554, + 0.8457898829566253, + 0.5666580094993431, + 0.5093321459106411, + 0.474467963670652 + ], + "peak_layer": 21, + "peak_normalized": 3.2405349047340315, + "population_cv": 0.6532808754860918, + "rest_mean": 5.3392646400930564e-05, + "spike_contrast": 3.093225357948515, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016515548777533694, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.437858635559678e-05, + 6.313920312095433e-05, + 5.632684042211622e-05, + 5.1906976295867935e-05, + 6.685537664452568e-05, + 6.153030699351802e-05, + 5.707356831408106e-05, + 5.057901216787286e-05, + 4.2357925849501044e-05, + 3.9370046579279006e-05, + 3.728317824425176e-05, + 3.583655779948458e-05, + 5.0669423217186704e-05, + 4.682251892518252e-05, + 4.338179132901132e-05, + 4.099325087736361e-05, + 5.2430314099183306e-05, + 4.8837704525794834e-05, + 4.9390600906917825e-05, + 5.749055708292872e-05, + 0.00022961001377552748, + 0.00017890847811941057, + 0.00014096690574660897, + 0.00011225492198718712, + 0.0001640371192479506, + 0.00012306995631661266, + 7.917954644653946e-05, + 6.291064346441999e-05, + 5.9928941482212394e-05, + 4.015088779851794e-05, + 3.608902989071794e-05, + 3.36187076754868e-05 + ] + }, + "learned": { + "statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "rest_mean": 5.7697108584559625e-05, + "spike_contrast": 3.04626897714722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017576091195223854, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ] + }, + "uniform_value_backward": { + "statistics": { + "first_quartile_mean": 3.269887292844942e-05, + "first_to_last_ratio": 2.3568639793204684, + "imbalance_abs_log_ratio": 0.8573319130483905, + "last_quartile_mean": 1.3873890566173941e-05, + "mean": 2.12624314031018e-05, + "normalized": [ + 1.6816001107908207, + 1.968609461712434, + 1.7075260440141944, + 1.4905139409933708, + 1.4972595154646264, + 1.4034185225085787, + 1.3238781474315933, + 1.2301608589864579, + 1.121553773570522, + 1.0499501635664128, + 1.001131281922829, + 0.9633193641679035, + 0.9560899219831485, + 0.9203740514079343, + 0.8930538331784411, + 0.8703809179546808, + 0.8841528402061924, + 0.8558056863273277, + 0.8550540488218812, + 0.8483642525948266, + 0.8350365031100208, + 0.8178976965352276, + 0.8078428125234317, + 0.7969682791900239, + 0.7797478584358322, + 0.7532962238158083, + 0.7097589943815636, + 0.6713244476982704, + 0.629948240254275, + 0.5812766747709892, + 0.5573943041371163, + 0.5373112275432649 + ], + "peak_layer": 2, + "peak_normalized": 1.968609461712434, + "population_cv": 0.35388972104860866, + "rest_mean": 2.2020403102268808e-05, + "spike_contrast": 0.7797034481094922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.716938422759995e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5754907003138214e-05, + 4.185742363915779e-05, + 3.630615537986159e-05, + 3.169195042573847e-05, + 3.183537774020806e-05, + 2.9840090064681135e-05, + 2.8148868295829743e-05, + 2.6156210878980346e-05, + 2.3846960175433196e-05, + 2.2324493329506367e-05, + 2.128648520738352e-05, + 2.048251189989969e-05, + 2.0328796381363645e-05, + 1.9569390133256093e-05, + 1.8988495867233723e-05, + 1.8506414562580176e-05, + 1.8799239114741795e-05, + 1.819650969991926e-05, + 1.8180528059019707e-05, + 1.8038286725641228e-05, + 1.775490636646282e-05, + 1.739049366733525e-05, + 1.7176702385768294e-05, + 1.6945483366725966e-05, + 1.6579335351707414e-05, + 1.6016909285099246e-05, + 1.5091201930772513e-05, + 1.4273990018409677e-05, + 1.3394231245911215e-05, + 1.2359355423541274e-05, + 1.1851558156195097e-05, + 1.1424543117755093e-05 + ] + } + }, + "position_element_rms": { + "attention_branch_output": { + "statistics": { + "first_quartile_mean": 6.422064188882359e-05, + "first_to_last_ratio": 0.6376814356469364, + "imbalance_abs_log_ratio": 0.4499164375054873, + "last_quartile_mean": 0.00010070959933727863, + "mean": 8.629182207187114e-05, + "normalized": [ + 0.6940773401510585, + 0.8062528427202595, + 0.7180374505283312, + 0.6647119372719499, + 0.8588689882759897, + 0.7941431696191276, + 0.7473948010342778, + 0.6703237246625169, + 0.591781846497542, + 0.522694550559361, + 0.4893812082436792, + 0.4670959071660659, + 0.6619658672285874, + 0.6164719403371215, + 0.5787012511245688, + 0.5393739243504363, + 0.6707710740427385, + 0.62668399370843, + 0.6317390712162502, + 0.6712190136492653, + 3.138964807406113, + 2.6795820619127526, + 2.136435916928567, + 1.6866743661161174, + 2.264472697470994, + 1.978410075097681, + 1.4778942897678682, + 0.9605324338748193, + 1.022040338716437, + 0.7239267286003249, + 0.47869752189559756, + 0.4306788598251707 + ], + "peak_layer": 21, + "peak_normalized": 3.138964807406113, + "population_cv": 0.6905024564046187, + "rest_mean": 6.421987658271703e-05, + "spike_contrast": 3.1996375366532934, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020548032771330326, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.989319834043272e-05, + 6.957302684895694e-05, + 6.196075992193073e-05, + 5.7359204220119864e-05, + 7.411336991935968e-05, + 6.852806109236553e-05, + 6.449405918829143e-05, + 5.784345557913184e-05, + 5.1065933803329244e-05, + 4.5104265154805034e-05, + 4.222959614708088e-05, + 4.03065569116734e-05, + 5.712224083254114e-05, + 5.319648698787205e-05, + 4.993718539481051e-05, + 4.654355871025473e-05, + 5.788205817225389e-05, + 5.4077703680377454e-05, + 5.4513915529241785e-05, + 5.792071169707924e-05, + 0.0002708669926505536, + 0.00023122601851355284, + 0.00018435694801155478, + 0.00014554620429407805, + 0.00019540547509677708, + 0.0001707206101855263, + 0.00012753019109368324, + 8.288609387818724e-05, + 8.81937230587937e-05, + 6.246895645745099e-05, + 4.1307681385660544e-05, + 3.716406354214996e-05 + ] + }, + "mlp_branch_output": { + "statistics": { + "first_quartile_mean": 6.365280069076107e-05, + "first_to_last_ratio": 0.8123684077728273, + "imbalance_abs_log_ratio": 0.2078013375593398, + "last_quartile_mean": 7.83545988269907e-05, + "mean": 7.614439766712167e-05, + "normalized": [ + 0.7854660713099145, + 0.9130496523819581, + 0.8147298517264512, + 0.752236196131495, + 0.9689924576782674, + 0.8892579571020611, + 0.8242433765322099, + 0.7396130336387524, + 0.6125904066287632, + 0.5670744382794471, + 0.5359292329408729, + 0.5145351492524995, + 0.7290151059417195, + 0.673146260009013, + 0.621425875046763, + 0.5883337897938153, + 0.7535116607120764, + 0.6934400787203782, + 0.6978162466203766, + 0.8168476312813977, + 3.2190719364475022, + 2.5076839554965473, + 1.973830744692153, + 1.575947332799018, + 2.2647184639164535, + 1.6880790923847124, + 1.0924366554795215, + 0.8732736568641918, + 0.8206255787733402, + 0.5459821164389519, + 0.48994786526283923, + 0.4571481297165359 + ], + "peak_layer": 21, + "peak_normalized": 3.2190719364475022, + "population_cv": 0.642032487569562, + "rest_mean": 5.76970003883751e-05, + "spike_contrast": 3.0462648281411457, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001757603429723531, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.9808840887853876e-05, + 6.952361582079902e-05, + 6.203711382113397e-05, + 5.727857205783948e-05, + 7.378334703389555e-05, + 6.771201151423156e-05, + 6.276151543715969e-05, + 5.6317388953175396e-05, + 4.664532752940431e-05, + 4.3179541535209864e-05, + 4.0808008634485304e-05, + 3.917896901839413e-05, + 5.551041613216512e-05, + 5.125631651026197e-05, + 4.731809895019978e-05, + 4.479832205106504e-05, + 5.7375691540073603e-05, + 5.280157711240463e-05, + 5.3134797781240195e-05, + 6.219837086973712e-05, + 0.00024511429364793, + 0.00019094608433078974, + 0.0001502961531514302, + 0.00011999956041108817, + 0.00017244562332052737, + 0.00012853776570409536, + 8.318293112097308e-05, + 6.649489660048857e-05, + 6.248604040592909e-05, + 4.157347939326428e-05, + 3.7306785088730976e-05, + 3.480926898191683e-05 + ] + }, + "post_attention_state": { + "statistics": { + "first_quartile_mean": 6.422088154067751e-05, + "first_to_last_ratio": 0.637682021407296, + "imbalance_abs_log_ratio": 0.44991551892756126, + "last_quartile_mean": 0.00010070988264487823, + "mean": 8.629204910448607e-05, + "normalized": [ + 0.6940783808537224, + 0.8062729813904731, + 0.7180303336815479, + 0.6647219507645815, + 0.8588589713696609, + 0.794136864353616, + 0.7473889560340541, + 0.6703283692089702, + 0.5917748510321142, + 0.5226947352403994, + 0.4893764215021349, + 0.4670989784552037, + 0.661952700545094, + 0.6164859593679979, + 0.5786990118719374, + 0.539368710966703, + 0.6707742418503443, + 0.6266861813765591, + 0.6317438172781369, + 0.6712283354793382, + 3.1389231589965445, + 2.679577878793134, + 2.1364533990925567, + 1.686690164787602, + 2.264487313238182, + 1.9784244316759128, + 1.477878596963198, + 0.9605191983702851, + 1.022047261980904, + 0.7239260887312458, + 0.4786954192739982, + 0.4306763354738471 + ], + "peak_layer": 21, + "peak_normalized": 3.1389231589965445, + "population_cv": 0.690500620274875, + "rest_mean": 6.42200389443638e-05, + "spike_contrast": 3.1996384204494492, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002054809039691463, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.98934457229916e-05, + 6.957494770176709e-05, + 6.196030881255865e-05, + 5.736021921620704e-05, + 7.411270053125918e-05, + 6.852769729448482e-05, + 6.449372449424118e-05, + 5.784400855191052e-05, + 5.106546450406313e-05, + 4.51043997600209e-05, + 4.2229294194839895e-05, + 4.030692798551172e-05, + 5.7121254940284416e-05, + 5.319783667800948e-05, + 4.993712354917079e-05, + 4.654323129216209e-05, + 5.788248381577432e-05, + 5.40780347364489e-05, + 5.451446850202046e-05, + 5.7921668485505506e-05, + 0.00027086411137133837, + 0.00023122626589611173, + 0.00018435894162394106, + 0.00014554795052390546, + 0.00019540725043043494, + 0.0001707222982076928, + 0.00012752917245961726, + 8.288516983157024e-05, + 8.819455251796171e-05, + 6.24690655968152e-05, + 4.13076086260844e-05, + 3.716394348884933e-05 + ] + }, + "post_mlp_state": { + "statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "rest_mean": 5.7697108584559625e-05, + "spike_contrast": 3.04626897714722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017576091195223854, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ] + }, + "pre_attention_input": { + "statistics": { + "first_quartile_mean": 3.944479362871789e-05, + "first_to_last_ratio": 1.1135282861542686, + "imbalance_abs_log_ratio": 0.1075336103151947, + "last_quartile_mean": 3.54232524841791e-05, + "mean": 5.533300526394669e-05, + "normalized": [ + 0.5280207224829799, + 0.8999098868607942, + 0.6284748654319019, + 0.5778568458898549, + 0.34988006202074273, + 0.7764713506345448, + 0.9190909808587969, + 1.023190827540606, + 0.6226417920568781, + 0.5519145939388094, + 0.4568960180345979, + 0.4213932325876586, + 0.4074494455716704, + 0.7465319443901276, + 0.6474109861274324, + 0.5764712279753077, + 0.7979703549327781, + 1.2369697451324837, + 1.2324489817728508, + 1.5808985864286207, + 2.5860289117498976, + 4.232714993086349, + 2.835646318943954, + 2.2422528664695807, + 1.1646676435019565, + 1.2175941057769888, + 0.783401544111113, + 0.5289102464485725, + 0.46735876466607296, + 0.2966943922188728, + 0.3266155865457417, + 0.3362221758114643 + ], + "peak_layer": 22, + "peak_normalized": 4.232714993086349, + "population_cv": 0.8496980089222349, + "rest_mean": 3.881239232820614e-05, + "spike_contrast": 3.7241794809927478, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014454431511694564, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.921697341662366e-05, + 4.9794718506745994e-05, + 3.477540303720161e-05, + 3.197455589543097e-05, + 1.9359915313543752e-05, + 4.296449333196506e-05, + 5.085606608190574e-05, + 5.661622344632633e-05, + 3.4452641557436436e-05, + 3.053909313166514e-05, + 2.5281429770984687e-05, + 2.3316953956964426e-05, + 2.2545402316609398e-05, + 4.1307856008643284e-05, + 3.582319550332613e-05, + 3.189788549207151e-05, + 4.415409784996882e-05, + 6.844525341875851e-05, + 6.819510599598289e-05, + 8.747586980462074e-05, + 0.0001430927513865754, + 0.00023420884099323303, + 0.00015690483269281685, + 0.00012407058966346085, + 6.444456084864214e-05, + 6.737314106430858e-05, + 4.3347961764084175e-05, + 2.9266193450894207e-05, + 2.5860364985419437e-05, + 1.641699236643035e-05, + 1.807262196962256e-05, + 1.8604183424031362e-05 + ] + }, + "pre_mlp_input": { + "statistics": { + "first_quartile_mean": 1.8878105947806034e-05, + "first_to_last_ratio": 0.1437388462831867, + "imbalance_abs_log_ratio": 1.939757193388504, + "last_quartile_mean": 0.00013133614493199275, + "mean": 7.144442986373178e-05, + "normalized": [ + 0.543944422065902, + 0.20498362677674017, + 0.14117605251679072, + 0.2675823635170904, + 0.19429426405696296, + 0.15941273479469137, + 0.21634544427316113, + 0.386139638870723, + 0.42865458135412043, + 0.3129364237141443, + 0.2788876084512782, + 0.2815826978190512, + 0.3736543391719335, + 0.3909795972451435, + 0.4389936626557141, + 0.44232523153890874, + 0.3688366584716598, + 0.45947150441297413, + 0.42734480687548726, + 0.5800824758005463, + 2.6187902433353907, + 2.987231504916195, + 2.580721746390286, + 2.2092452217225382, + 2.879951595578818, + 2.8406783220158585, + 2.397028447242086, + 1.5906421267626043, + 1.8247593500794728, + 1.4018234858691145, + 0.9198949147470885, + 0.8516049069575247 + ], + "peak_layer": 22, + "peak_normalized": 2.987231504916195, + "population_cv": 0.9601858610516129, + "rest_mean": 4.954554700547036e-05, + "spike_contrast": 3.828767846228416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00018969839729834347, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.886179911205545e-05, + 1.4644938346464187e-05, + 1.008624258247437e-05, + 1.911726940306835e-05, + 1.3881242921343073e-05, + 1.1389151950425003e-05, + 1.5456676919711754e-05, + 2.7587526346906088e-05, + 3.062498217332177e-05, + 2.2357564375852235e-05, + 1.9924966181861237e-05, + 2.0117515305173583e-05, + 2.6695521228248253e-05, + 2.7933314413530752e-05, + 3.13636519422289e-05, + 3.160167398164049e-05, + 2.6351324777351692e-05, + 3.282667967141606e-05, + 3.0531406082445756e-05, + 4.144366175751202e-05, + 0.00018709797586780041, + 0.00021342105173971504, + 0.0001843781938077882, + 0.00015783826529514045, + 0.0002057564997812733, + 0.0002029506431426853, + 0.00017125433078035712, + 0.00011364251986378804, + 0.00013036889140494168, + 0.00010015247971750796, + 6.572136771865189e-05, + 6.08424270467367e-05 + ] + } + }, + "post_mlp_reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5986864127626177e-05, + "first_to_last_ratio": 0.8170274391577745, + "imbalance_abs_log_ratio": 0.20208259942567244, + "last_quartile_mean": 1.956710798367567e-05, + "mean": 1.9039307829871177e-05, + "normalized": [ + 0.7853167979867789, + 0.9097665173048055, + 0.808777694116954, + 0.752402497635975, + 0.9858540561935626, + 0.9044938358034963, + 0.8288447236919866, + 0.7419578801196516, + 0.6090850929460484, + 0.5628647453138417, + 0.5328979550345672, + 0.5131403750850531, + 0.7265824660927316, + 0.6757007408890954, + 0.6225826951824261, + 0.5898791530514104, + 0.7532772493405897, + 0.6927950381940383, + 0.6897848547806317, + 0.8094505726944553, + 3.2076147584918284, + 2.5002121502474965, + 1.982484105644761, + 1.5924612114390058, + 2.273889158865966, + 1.6940625327896233, + 1.0968336406315982, + 0.8719471681044353, + 0.815653609383569, + 0.5388945327422838, + 0.4819664542042268, + 0.448525735997107 + ], + "peak_layer": 21, + "peak_normalized": 3.2076147584918284, + "population_cv": 0.6428608089466793, + "rest_mean": 1.4415815443229534e-05, + "spike_contrast": 3.052631111367605, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.400616671773605e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4951888260839041e-05, + 1.7321324776276015e-05, + 1.5398567484226078e-05, + 1.4325222764455248e-05, + 1.8769978851196356e-05, + 1.7220936570083722e-05, + 1.5780629837536253e-05, + 1.4126364476396702e-05, + 1.1596558579185512e-05, + 1.0716555152612273e-05, + 1.0146008207811974e-05, + 9.769837561179884e-06, + 1.3833627235726453e-05, + 1.286487440665951e-05, + 1.1853543583129067e-05, + 1.1230890777369495e-05, + 1.4341877431434114e-05, + 1.3190337995183654e-05, + 1.3133026186551433e-05, + 1.541137862659525e-05, + 6.107076478656381e-05, + 4.760230876854621e-05, + 3.7745125155197456e-05, + 3.03193592117168e-05, + 4.329327566665597e-05, + 3.225377804483287e-05, + 2.0882953322143294e-05, + 1.6601270544924773e-05, + 1.5529480151599273e-05, + 1.0260178896714933e-05, + 9.176307685265783e-06, + 8.539619557268452e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "rest_mean": 5.7697108584559625e-05, + "spike_contrast": 3.04626897714722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017576091195223854, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05644787335768342, + "first_to_last_ratio": 0.8123632868395458, + "imbalance_abs_log_ratio": 0.2078076412872703, + "last_quartile_mean": 0.06948599754832685, + "mean": 0.06752577290171757, + "normalized": [ + 0.7854618348329956, + 0.9130540359301327, + 0.8147131249195937, + 0.7522410113175043, + 0.9690071730110875, + 0.88924395208314, + 0.8242353599678512, + 0.7396083806676593, + 0.6126026750590019, + 0.5670643980854334, + 0.5359283892166958, + 0.5145269586503932, + 0.7290174229589911, + 0.6731434525628229, + 0.6214223402600404, + 0.5883307763560162, + 0.7535191533057549, + 0.6934459281737582, + 0.6978155981634587, + 0.8168473704719704, + 3.2190670201887848, + 2.507709505453073, + 1.9738347607250095, + 1.5759251284181852, + 2.264726151347608, + 1.68807248858185, + 1.0924542395938426, + 0.8732750663142743, + 0.820625855657368, + 0.5459870819951741, + 0.48994655813463567, + 0.45714680759589343 + ], + "peak_layer": 21, + "peak_normalized": 3.2190670201887848, + "population_cv": 0.6420341436220799, + "rest_mean": 0.05116637251167386, + "spike_contrast": 3.0462690113979045, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15586653500795364, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05303891748189926, + 0.06165467947721481, + 0.05501413345336914, + 0.05079565569758415, + 0.06543295830488205, + 0.060046885162591934, + 0.05565712973475456, + 0.04994262754917145, + 0.041366469115018845, + 0.038291461765766144, + 0.03618897870182991, + 0.03474383056163788, + 0.04922746494412422, + 0.04545453190803528, + 0.041962023824453354, + 0.03972749039530754, + 0.05088196322321892, + 0.04682547226548195, + 0.04712053760886192, + 0.05515825003385544, + 0.21736998856067657, + 0.1693350225687027, + 0.13328471779823303, + 0.10641556233167648, + 0.15292738378047943, + 0.11398839950561523, + 0.07376881688833237, + 0.058968573808670044, + 0.055413395166397095, + 0.036868199706077576, + 0.033084020018577576, + 0.030869191512465477 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.579253447265728e-06, + "first_to_last_ratio": 0.47590496457778414, + "imbalance_abs_log_ratio": 0.7425370989326324, + "last_quartile_mean": 5.419681741614113e-06, + "mean": 4.2137082871818166e-06, + "normalized": [ + 0.6771144196047112, + 0.6338426021784607, + 0.6121020910961615, + 0.6258737201152189, + 0.6050362914422536, + 0.5852204423685657, + 0.5784631886764509, + 0.5792281322851218, + 0.4697078599262646, + 0.4522117174548025, + 0.44580914592548354, + 0.4558904724294877, + 0.5773213311917524, + 0.5667240346830847, + 0.5644321177223985, + 0.5681434108598962, + 0.6503548834184826, + 0.6631947770191623, + 0.6837887360559592, + 0.7582966501774209, + 3.020973835679826, + 2.627473806530212, + 2.3058644468737186, + 2.00331262981993, + 2.819406877957528, + 2.1402761714678284, + 1.4825745701937771, + 1.1227288221975413, + 0.9855094000056028, + 0.6429466506282437, + 0.5709200007532207, + 0.5252567632614316 + ], + "peak_layer": 21, + "peak_normalized": 3.020973835679826, + "population_cv": 0.7577358346351384, + "rest_mean": 2.999999306094078e-06, + "spike_contrast": 3.589246425218318, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0767736785055605e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8531626412586775e-06, + 2.670827825568267e-06, + 2.5792196538532153e-06, + 2.6372492811788106e-06, + 2.5494464352959767e-06, + 2.465948227836634e-06, + 2.43747513195558e-06, + 2.440698381178663e-06, + 1.979211901925737e-06, + 1.9054882614000235e-06, + 1.8785096926876577e-06, + 1.920989461723366e-06, + 2.4326636776095256e-06, + 2.3880097614892293e-06, + 2.3783522919984534e-06, + 2.393990598648088e-06, + 2.7404057618696243e-06, + 2.7945093279413413e-06, + 2.8812862638005754e-06, + 3.1952408789948095e-06, + 1.2729502486763522e-05, + 1.1071408152929507e-05, + 9.716240128909703e-06, + 8.441375030088238e-06, + 1.1880158126587048e-05, + 9.018499440571759e-06, + 6.247136752790539e-06, + 4.7308517423516605e-06, + 4.152649125899188e-06, + 2.709189629968023e-06, + 2.405690338491695e-06, + 2.2132787762529915e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.826905797017389e-05, + "first_to_last_ratio": 0.9723593605402814, + "imbalance_abs_log_ratio": 0.028029830358518852, + "last_quartile_mean": 4.964117169947713e-05, + "mean": 5.395593586854375e-05, + "normalized": [ + 0.8546740011831107, + 0.9667023865336807, + 0.8780594173977533, + 0.8215201862718362, + 1.028762848378348, + 0.9492707323927138, + 0.8750383055629682, + 0.7827836490446655, + 0.6672270484578459, + 0.6202534690629704, + 0.5854228619045188, + 0.5620710233383177, + 0.781592991024926, + 0.7155185842665527, + 0.6597031604820425, + 0.621567177701999, + 0.8222052916650392, + 0.7616961441483728, + 0.7641199379358804, + 0.8753280307745216, + 3.236716477578425, + 2.479496544122304, + 1.8830004075567146, + 1.4470156791415152, + 2.05665793704697, + 1.5065644924277526, + 1.0009572516438536, + 0.8155596816984735, + 0.6595753226879756, + 0.4759799891964506, + 0.4318790517932509, + 0.413079917578251 + ], + "peak_layer": 21, + "peak_normalized": 3.236716477578425, + "population_cv": 0.6107674157639074, + "rest_mean": 4.176012171902349e-05, + "spike_contrast": 2.869084843241085, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011981333227595314, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.61147355963476e-05, + 5.2159331971779466e-05, + 4.737651761388406e-05, + 4.4325890485197306e-05, + 5.550786227104254e-05, + 5.121879075886682e-05, + 4.72135106974747e-05, + 4.2235824366798624e-05, + 3.6000859836349264e-05, + 3.3466356399003416e-05, + 3.158703839289956e-05, + 3.0327068088809028e-05, + 4.217158129904419e-05, + 3.860647484543733e-05, + 3.559490141924471e-05, + 3.353723877808079e-05, + 4.4362855987856165e-05, + 4.1098028304986656e-05, + 4.122880636714399e-05, + 4.7229143092408776e-05, + 0.00017464006668888032, + 0.00013378355652093887, + 0.00010159904923057184, + 7.807508518453687e-05, + 0.00011096890375483781, + 8.128809713525698e-05, + 5.400758527684957e-05, + 4.4004285882692784e-05, + 3.558800381142646e-05, + 2.5681945771793835e-05, + 2.330243842152413e-05, + 2.228811354143545e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.626825309765991e-05, + "first_to_last_ratio": 1.1971952602722011, + "imbalance_abs_log_ratio": 0.17998153797771219, + "last_quartile_mean": 3.0294350722215313e-05, + "mean": 3.794959516767449e-05, + "normalized": [ + 0.9342317076920968, + 1.032527492584725, + 0.9483990781630639, + 0.8885231558783968, + 1.1008949985426397, + 1.0148066560537, + 0.9027782413985963, + 0.8234017777861798, + 0.7333936750261243, + 0.6823073758552121, + 0.6464481216407498, + 0.616777331910669, + 0.8501710687237772, + 0.7702178906455867, + 0.7042577074377577, + 0.660638930723233, + 0.914060026408416, + 0.847914539167618, + 0.8394293783449699, + 0.9442746494437388, + 3.300220154812763, + 2.4489522165880904, + 1.7640184700183157, + 1.245126339879113, + 1.7643693302152461, + 1.271277790934411, + 0.9095623054467832, + 0.780451313056054, + 0.5159478762516835, + 0.40747033934370974, + 0.37633772798986104, + 0.36081233203671814 + ], + "peak_layer": 21, + "peak_normalized": 3.300220154812763, + "population_cv": 0.5858778602410362, + "rest_mean": 3.018723526582579e-05, + "spike_contrast": 2.6456990159702416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.986633863765746e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5453715099720284e-05, + 3.9184000343084335e-05, + 3.599136107368395e-05, + 3.371909406268969e-05, + 4.1778519516810775e-05, + 3.85115017706994e-05, + 3.4260068787261844e-05, + 3.124776412732899e-05, + 2.783199306577444e-05, + 2.5893288693623617e-05, + 2.453244451317005e-05, + 2.3406450054608285e-05, + 3.226364788133651e-05, + 2.9229457140900195e-05, + 2.6726294890977442e-05, + 2.5070979972952046e-05, + 3.468820796115324e-05, + 3.217801349819638e-05, + 3.185600508004427e-05, + 3.5834840673487633e-05, + 0.00012524201883934438, + 9.293674520449713e-05, + 6.694378680549562e-05, + 4.725204053102061e-05, + 6.695710180792958e-05, + 4.8244477511616424e-05, + 3.451752127148211e-05, + 2.9617811378557235e-05, + 1.95800130313728e-05, + 1.546333442092873e-05, + 1.4281864423537627e-05, + 1.3692681932298001e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012846494701079791, + "first_to_last_ratio": 0.7741545804954827, + "imbalance_abs_log_ratio": 0.25598370893064326, + "last_quartile_mean": 0.00016594224234722788, + "mean": 0.000157460202672155, + "normalized": [ + 0.7615886502877881, + 0.869859991005712, + 0.7823534284439785, + 0.735201115807512, + 0.9596786286759813, + 0.8813992955944876, + 0.8128285893567467, + 0.7239431731448671, + 0.5977864740547744, + 0.5606030792342883, + 0.5318673348078621, + 0.5130567490014816, + 0.6987134321316841, + 0.6407269063222457, + 0.6075335461477536, + 0.5704889146486948, + 0.7180428901779269, + 0.6784140170824181, + 0.6819219612414157, + 0.7908928021790416, + 3.2176296651190497, + 2.545498211708922, + 2.0288816729118655, + 1.6601468184271124, + 2.321862751456626, + 1.7738386620365922, + 1.1711907122839438, + 0.9284857615000071, + 0.7896970255450443, + 0.5236030848875767, + 0.47341239614817127, + 0.4488522586284289 + ], + "peak_layer": 21, + "peak_normalized": 3.2176296651190497, + "population_cv": 0.6625973254291921, + "rest_mean": 0.0001179550758765631, + "spike_contrast": 3.143466990402101, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037078788736835124, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011991990322712809, + 0.00013696833048015833, + 0.00012318952940404415, + 0.00011576491669984534, + 0.0001511111913714558, + 0.00013878531171940267, + 0.00012798815441783518, + 0.00011399223876651376, + 9.412757935933769e-05, + 8.82726744748652e-05, + 8.374793833354488e-05, + 8.078601968009025e-05, + 0.000110019558633212, + 0.00010088898852700368, + 9.566235530655831e-05, + 8.982930012280121e-05, + 0.0001130631790147163, + 0.00010682320862542838, + 0.00010737557022366673, + 0.00012453414092306048, + 0.0005066486191935837, + 0.00040081466431729496, + 0.00031946811941452324, + 0.0002614070544950664, + 0.0003656009794212878, + 0.00027930899523198605, + 0.00018441592692397535, + 0.00014619955618400127, + 0.00012434585369192064, + 8.24466478661634e-05, + 7.454361184500158e-05, + 7.067636761348695e-05 + ] + } + }, + "seed": 2026073001 + }, + { + "interventions": { + "detached_learned": { + "statistics": { + "first_quartile_mean": 4.0602767057862366e-05, + "first_to_last_ratio": 0.6220527400735582, + "imbalance_abs_log_ratio": 0.47473039872247574, + "last_quartile_mean": 6.52722260383598e-05, + "mean": 6.584079096683126e-05, + "normalized": [ + 0.6772375898692375, + 0.7419156446131505, + 0.6265276809050203, + 0.5983548462080928, + 0.6909826192767372, + 0.5826528795658069, + 0.521809962087311, + 0.49396705008933894, + 0.6208428547650034, + 0.583552196572707, + 0.5417821373953783, + 0.5073870191655783, + 0.43782941267698894, + 0.4086714236070873, + 0.40807611508949443, + 0.4979465665227966, + 1.2379953934660572, + 1.244289507430791, + 1.1312943811704865, + 1.027048864560996, + 3.6326329119144347, + 2.8432062989929086, + 2.2290691465672023, + 1.784011106006968, + 1.8683393699616757, + 1.5302817088353375, + 1.1091498325862283, + 0.8793222326330687, + 0.8937044532253541, + 0.6693159640810682, + 0.5098515212827301, + 0.47095130887496306 + ], + "peak_layer": 21, + "peak_normalized": 3.6326329119144347, + "population_cv": 0.7406534740263053, + "rest_mean": 4.7899763524549565e-05, + "spike_contrast": 3.3971428496041334, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016272233915515244, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.458985858946107e-05, + 4.884831287199631e-05, + 4.1251078073401004e-05, + 3.93961563531775e-05, + 4.549484219751321e-05, + 3.83623264497146e-05, + 3.435638063820079e-05, + 3.252318128943443e-05, + 4.087678462383337e-05, + 3.8421538192778826e-05, + 3.567136445781216e-05, + 3.340676266816445e-05, + 2.882703483919613e-05, + 2.6907249775831588e-05, + 2.686805419216398e-05, + 3.278519579907879e-05, + 8.15105959190987e-05, + 8.192500536097214e-05, + 7.448531687259674e-05, + 6.762170960428193e-05, + 0.00023917542421258986, + 0.00018719895160757005, + 0.00014676367572974414, + 0.00011746070231311023, + 0.00012301294191274792, + 0.0001007549581117928, + 7.302730227820575e-05, + 5.789527131128125e-05, + 5.8842208090936765e-05, + 4.4068292481824756e-05, + 3.356902743689716e-05, + 3.100780668319203e-05 + ] + }, + "learned": { + "statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "rest_mean": 5.15456125598306e-05, + "spike_contrast": 3.3328477735441013, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179368005599828, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ] + }, + "uniform_value_backward": { + "statistics": { + "first_quartile_mean": 3.093152440669655e-05, + "first_to_last_ratio": 2.2774685844866087, + "imbalance_abs_log_ratio": 0.8230645561980515, + "last_quartile_mean": 1.3581537245954678e-05, + "mean": 2.01113496416383e-05, + "normalized": [ + 1.8305028390484899, + 2.063895445405066, + 1.6144389462797186, + 1.4292254312672816, + 1.5742789705777873, + 1.395222470726611, + 1.2383859090556808, + 1.15815685579556, + 1.0870603073810894, + 1.0303295548521039, + 0.9686496018996943, + 0.9150613022126579, + 0.8858369611012855, + 0.866010221809998, + 0.8620422691094124, + 0.8950413704487017, + 0.898565866832108, + 0.8961986259205167, + 0.8726168436106015, + 0.8416436403028231, + 0.8273619591175169, + 0.8203659676564082, + 0.8150936944275661, + 0.8114785712381606, + 0.790107287070579, + 0.7656015997474095, + 0.7288914121037795, + 0.6928588452491169, + 0.6522528807444096, + 0.6198966689630461, + 0.586304513764539, + 0.566623166280281 + ], + "peak_layer": 2, + "peak_normalized": 2.063895445405066, + "population_cv": 0.3585699713300157, + "rest_mean": 2.0808239578737238e-05, + "spike_contrast": 0.7856572354159773, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.634814398130402e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.6813882616115734e-05, + 4.1507722926326096e-05, + 3.246854612370953e-05, + 2.874365236493759e-05, + 3.166087481076829e-05, + 2.805980693665333e-05, + 2.4905612008296885e-05, + 2.329209746676497e-05, + 2.186224992328789e-05, + 2.072131792374421e-05, + 1.9480850824038498e-05, + 1.8403117792331614e-05, + 1.7815376850194298e-05, + 1.7416634364053607e-05, + 1.7336833479930647e-05, + 1.800048994482495e-05, + 1.8071372323902324e-05, + 1.8023763914243318e-05, + 1.7549502445035614e-05, + 1.6926589523791336e-05, + 1.6639365640003234e-05, + 1.6498666809638962e-05, + 1.6392634279327467e-05, + 1.631992927286774e-05, + 1.5890123904682696e-05, + 1.539728145871777e-05, + 1.4658990039606579e-05, + 1.3934326489106752e-05, + 1.3117685739416629e-05, + 1.246695865120273e-05, + 1.1791375072789378e-05, + 1.1395556612114888e-05 + ] + } + }, + "position_element_rms": { + "attention_branch_output": { + "statistics": { + "first_quartile_mean": 4.418448133947095e-05, + "first_to_last_ratio": 0.515514370401758, + "imbalance_abs_log_ratio": 0.662590099285622, + "last_quartile_mean": 8.570950467401417e-05, + "mean": 8.019432914352365e-05, + "normalized": [ + 0.5977618675800617, + 0.6560769396425044, + 0.5549086183167843, + 0.5332641078022274, + 0.6187504975834317, + 0.5235180355140238, + 0.4739614950807707, + 0.44949966520899015, + 0.5901273331445138, + 0.5434806557289369, + 0.5105162863240122, + 0.47519844997761185, + 0.40525254510695746, + 0.3803726356397213, + 0.3632907544081655, + 0.4413534187685878, + 1.1120243122062796, + 1.1101422281940985, + 1.048076642833664, + 0.9240879167666287, + 3.7925106998539326, + 3.0713745688935163, + 2.3834694147362985, + 1.8907998104142179, + 1.7924928574135153, + 1.575834720390749, + 1.2856009256033192, + 0.946613406637061, + 1.179458790809137, + 0.7622280566154953, + 0.5700461300235112, + 0.43790621278127495 + ], + "peak_layer": 21, + "peak_normalized": 3.7925106998539326, + "population_cv": 0.799556889668146, + "rest_mean": 5.663903492084636e-05, + "spike_contrast": 3.6616605179769537, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020739291794598104, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.793711195816286e-05, + 5.261365004116669e-05, + 4.450052438187413e-05, + 4.27647573815193e-05, + 4.9620281060924754e-05, + 4.1983177652582526e-05, + 3.800902413786389e-05, + 3.604732410167344e-05, + 4.7324865590780973e-05, + 4.358406658866443e-05, + 4.09405110985972e-05, + 3.810822090599686e-05, + 3.249895598855801e-05, + 3.0503728339681402e-05, + 2.913385833380744e-05, + 3.539404133334756e-05, + 8.917804370867088e-05, + 8.902711124392226e-05, + 8.404980326304212e-05, + 7.410661055473611e-05, + 0.0003041378513444215, + 0.0002463068231008947, + 0.00019114073074888438, + 0.0001516314223408699, + 0.00014374776219483465, + 0.00012637300824280828, + 0.00010309790377505124, + 7.591302710352466e-05, + 9.458590648137033e-05, + 6.11263676546514e-05, + 4.5714466978097335e-05, + 3.511759496177547e-05 + ] + }, + "mlp_branch_output": { + "statistics": { + "first_quartile_mean": 4.3983808154735016e-05, + "first_to_last_ratio": 0.6452579224313538, + "imbalance_abs_log_ratio": 0.43810516237842645, + "last_quartile_mean": 6.816469294790295e-05, + "mean": 7.033422900803998e-05, + "normalized": [ + 0.6809924600708206, + 0.7475303658027369, + 0.6323190441913445, + 0.6077859654839745, + 0.7056348319250383, + 0.5942536325242065, + 0.530225745030374, + 0.5040918553425093, + 0.6368908391102093, + 0.5978963063844708, + 0.5559498242106791, + 0.5233025181432952, + 0.44344812436490716, + 0.41411157823288425, + 0.41744328645242573, + 0.5192458958427938, + 1.2557243607052013, + 1.2676253654309775, + 1.1574430802993265, + 1.060137110625566, + 3.6189237818679025, + 2.8123811167698722, + 2.2018107363871295, + 1.7616009064115667, + 1.8179427908533603, + 1.4871270732826793, + 1.0868453970932423, + 0.8727044844737096, + 0.8717769668640218, + 0.6523346900704022, + 0.4999162674775265, + 0.4645835982748448 + ], + "peak_layer": 21, + "peak_normalized": 3.6189237818679025, + "population_cv": 0.7299683580686596, + "rest_mean": 5.154545740677147e-05, + "spike_contrast": 3.3328561680843984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179359565488994, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789707963936962e-05, + 5.2576971938833594e-05, + 4.4473672460298985e-05, + 4.274815728422254e-05, + 4.963028186466545e-05, + 4.1796371078817174e-05, + 3.729301897692494e-05, + 3.545491199474782e-05, + 4.479522613110021e-05, + 4.205257573630661e-05, + 3.910230225301348e-05, + 3.680607915157452e-05, + 3.1189581932267174e-05, + 2.9126218578312546e-05, + 2.9360551707213745e-05, + 3.6520759749691933e-05, + 8.832040475681424e-05, + 8.915745274862275e-05, + 8.140786667354405e-05, + 7.456392631866038e-05, + 0.0002545342140365392, + 0.00019780665752477944, + 0.00015486266056541353, + 0.00012390084157232195, + 0.00012786360457539558, + 0.00010459593613632023, + 7.644243305549026e-05, + 6.138099706731737e-05, + 6.13157608313486e-05, + 4.588145748130046e-05, + 3.516122524160892e-05, + 3.267612919444218e-05 + ] + }, + "post_attention_state": { + "statistics": { + "first_quartile_mean": 4.4184340367792174e-05, + "first_to_last_ratio": 0.5155127283755759, + "imbalance_abs_log_ratio": 0.6625932845097232, + "last_quartile_mean": 8.570950421926682e-05, + "mean": 8.019429117211985e-05, + "normalized": [ + 0.5977553912963086, + 0.6560764337279869, + 0.5549115575710053, + 0.5332632715493767, + 0.618731238431461, + 0.5235220032899371, + 0.4739610390296197, + 0.44950831585208795, + 0.5901295178770004, + 0.5434863568103182, + 0.5105118554999238, + 0.4751981306058326, + 0.4052536442822589, + 0.3803760366271764, + 0.36329074496540736, + 0.44135367311072854, + 1.112029465926781, + 1.1101484697729092, + 1.04807895367264, + 0.9240971550402413, + 3.7925041485004933, + 3.0713698535863756, + 2.383447135178006, + 1.890810504439103, + 1.7925006015588865, + 1.5758247605004587, + 1.2855904653729862, + 0.9466104071448787, + 1.1794713255182339, + 0.7622315023149855, + 0.5700532953493974, + 0.43790274559719394 + ], + "peak_layer": 21, + "peak_normalized": 3.7925041485004933, + "population_cv": 0.7995548542275265, + "rest_mean": 5.6639052976000435e-05, + "spike_contrast": 3.6616533387138865, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002073925774311647, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.79365698993206e-05, + 5.2613584557548165e-05, + 4.450073902262375e-05, + 4.276467007002793e-05, + 4.96187130920589e-05, + 4.198347596684471e-05, + 3.800896956818178e-05, + 3.604800076573156e-05, + 4.732501838589087e-05, + 4.358450314612128e-05, + 4.094013638678007e-05, + 3.8108177250251174e-05, + 3.249902874813415e-05, + 3.0503986636176705e-05, + 2.913384378189221e-05, + 3.5394044971326366e-05, + 8.917841478250921e-05, + 8.902756962925196e-05, + 8.40499487821944e-05, + 7.410731632262468e-05, + 0.000304137181956321, + 0.0002463063283357769, + 0.00019113885355181992, + 0.00015163220814429224, + 0.00014374831516761333, + 0.0001263721496798098, + 0.0001030970161082223, + 7.591275061713532e-05, + 9.45868669077754e-05, + 6.112661503721029e-05, + 4.571501995087601e-05, + 3.511730028549209e-05 + ] + }, + "post_mlp_state": { + "statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "rest_mean": 5.15456125598306e-05, + "spike_contrast": 3.3328477735441013, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179368005599828, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ] + }, + "pre_attention_input": { + "statistics": { + "first_quartile_mean": 2.7120979211758822e-05, + "first_to_last_ratio": 0.8594772224120714, + "imbalance_abs_log_ratio": 0.15143095548529256, + "last_quartile_mean": 3.1555204145661264e-05, + "mean": 4.800072150601409e-05, + "normalized": [ + 0.509760479963835, + 0.6173326836717695, + 0.7324351629463104, + 0.39842116793036353, + 0.4759789060227602, + 0.7458666061827044, + 0.6096831903000134, + 0.4306170616938511, + 0.4047810550951731, + 0.5961537125874838, + 0.6210341956779442, + 0.5605157741769637, + 0.27837672031341976, + 0.3443893546178484, + 0.25310408037089555, + 1.0527443019909122, + 0.7465088513707779, + 1.4090569908205048, + 2.369608868078584, + 2.622524932193837, + 3.51885122958336, + 3.2980052855840127, + 2.541370727154349, + 1.6037570225598534, + 0.918310576031736, + 1.0270328185033981, + 1.0005888996255676, + 0.5708475165844191, + 0.6850494580580762, + 0.5307877571911549, + 0.29761830732641337, + 0.22888630579170677 + ], + "peak_layer": 21, + "peak_normalized": 3.51885122958336, + "population_cv": 0.8786708090610466, + "rest_mean": 3.5768902374979075e-05, + "spike_contrast": 3.1885950432011234, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011405254481360317, + "top_five_layers": [ + 21, + 22, + 20, + 23, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.446887083351612e-05, + 2.96324142254889e-05, + 3.515741627779789e-05, + 1.912450352392625e-05, + 2.2847330910735764e-05, + 3.580213524401188e-05, + 2.9265233024489135e-05, + 2.0669929654104635e-05, + 1.9429782696533948e-05, + 2.8615808332688175e-05, + 2.9810089472448453e-05, + 2.6905161575996317e-05, + 1.3362283425522037e-05, + 1.653093750064727e-05, + 1.2149178473919164e-05, + 5.0532486056908965e-05, + 3.5832963476423174e-05, + 6.76357522024773e-05, + 0.00011374293535482138, + 0.00012588308891281486, + 0.00016890739789232612, + 0.00015830663323868066, + 0.00012198762851767242, + 7.698149420320988e-05, + 4.407957021612674e-05, + 4.929831629851833e-05, + 4.802898911293596e-05, + 2.7401092665968463e-05, + 3.288286825409159e-05, + 2.5478195311734453e-05, + 1.4285893485066481e-05, + 1.0986707820848096e-05 + ] + }, + "pre_mlp_input": { + "statistics": { + "first_quartile_mean": 1.4786839983571554e-05, + "first_to_last_ratio": 0.1368590004352419, + "imbalance_abs_log_ratio": 1.9888040770410698, + "last_quartile_mean": 0.00010804433713929029, + "mean": 6.53327875852483e-05, + "normalized": [ + 0.4303813973935663, + 0.1954661274427069, + 0.34854762874244455, + 0.22114030543306437, + 0.10883074349044422, + 0.12627252893821267, + 0.16836982659974983, + 0.21164003420329566, + 0.27785076214706256, + 0.25390299121217014, + 0.3323483997673342, + 0.40930824768453605, + 0.28788144954928885, + 0.2920379121787492, + 0.30710829102986115, + 0.24768611608853355, + 0.33445308089382414, + 0.5081445956986791, + 0.6162106751311494, + 0.7807238268363051, + 3.828274701557073, + 3.4515650116091967, + 2.7664213382241543, + 2.2654046605337452, + 2.2812415827715613, + 2.1356021257971247, + 2.0106696974531495, + 1.537539473929636, + 2.0088419319348305, + 1.3422522168191928, + 1.0598743605751482, + 0.8540079583342102 + ], + "peak_layer": 21, + "peak_normalized": 3.828274701557073, + "population_cv": 1.0279395854576847, + "rest_mean": 4.2120514451569225e-05, + "spike_contrast": 4.52698797699541, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001906790625071153, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.8118016416556202e-05, + 1.2770346984325442e-05, + 2.2771588191972114e-05, + 1.4447712601395324e-05, + 7.110215847205836e-06, + 8.249736310972366e-06, + 1.1000070117006544e-05, + 1.38270333991386e-05, + 1.8152764823753387e-05, + 1.6588190192123875e-05, + 2.1713247406296432e-05, + 2.6741248802863993e-05, + 1.8808097593137063e-05, + 1.9079650883213617e-05, + 2.0064240743522532e-05, + 1.6182024410227314e-05, + 2.1850752091268077e-05, + 3.3198502933373675e-05, + 4.0258761146105826e-05, + 5.1006863941438496e-05, + 0.00025011185789480805, + 0.0002255003637401387, + 0.000180738017661497, + 0.0001480052014812827, + 0.00014903987175785005, + 0.00013952484005130827, + 0.00013136265624780208, + 0.00010045173985417932, + 0.0001312432432314381, + 8.769307896727696e-05, + 6.924454646650702e-05, + 5.579472053796053e-05 + ] + } + }, + "post_mlp_reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1220713531656656e-05, + "first_to_last_ratio": 0.6555413366591524, + "imbalance_abs_log_ratio": 0.4222939165017446, + "last_quartile_mean": 1.7116713934228756e-05, + "mean": 1.7638015336274293e-05, + "normalized": [ + 0.6920608231884311, + 0.7659177824262653, + 0.6390754033749174, + 0.6123424804403663, + 0.7313911494807722, + 0.6060429021656846, + 0.5358717188243214, + 0.5066293084101811, + 0.6361394778273108, + 0.5935071698997143, + 0.5532047504443424, + 0.5203003869133469, + 0.4427001574083234, + 0.41555518288954896, + 0.42010731902783316, + 0.5249828006548773, + 1.2555950689969066, + 1.2709301325611273, + 1.1358461030518159, + 1.0637808540376092, + 3.605678554956403, + 2.784492491030135, + 2.1780228929741425, + 1.7462695606216458, + 1.8163019324419567, + 1.491254439501435, + 1.0911345614825427, + 0.8818559802285882, + 0.8715835151550508, + 0.6503123816361532, + 0.4986363533333601, + 0.46247636461489006 + ], + "peak_layer": 21, + "peak_normalized": 3.605678554956403, + "population_cv": 0.7234183193184713, + "rest_mean": 1.2979772741851371e-05, + "spike_contrast": 3.2968624487684486, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2792525346158075e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.220657941303216e-05, + 1.3509269592759665e-05, + 1.1272021765762474e-05, + 1.0800506061059423e-05, + 1.2900288311357144e-05, + 1.0689394002838526e-05, + 9.451713594899047e-06, + 8.935935511544812e-06, + 1.122023786592763e-05, + 1.0468288564879913e-05, + 9.757433872437105e-06, + 9.177066203847062e-06, + 7.808352165739052e-06, + 7.329568688874133e-06, + 7.409859335893998e-06, + 9.259654689230956e-06, + 2.214620508311782e-05, + 2.2416685169446282e-05, + 2.003407098527532e-05, + 1.876298301795032e-05, + 6.359701364999637e-05, + 4.911292126053013e-05, + 3.841600118903443e-05, + 3.080072929151356e-05, + 3.203596133971587e-05, + 2.630276867421344e-05, + 1.9245448129368015e-05, + 1.555418930365704e-05, + 1.5373003407148644e-05, + 1.1470219760667533e-05, + 8.794955647317693e-06, + 8.157165211741813e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "rest_mean": 5.15456125598306e-05, + "spike_contrast": 3.3328477735441013, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179368005599828, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.039005559869110584, + "first_to_last_ratio": 0.6452634722813873, + "imbalance_abs_log_ratio": 0.43809656143611553, + "last_quartile_mean": 0.06044904375448823, + "mean": 0.06237322464585304, + "normalized": [ + 0.6809932518812878, + 0.7475633220322179, + 0.6323153569646397, + 0.6077851973803072, + 0.7056447145510248, + 0.5942445836918125, + 0.530223252027616, + 0.5040898132583496, + 0.6368974600710677, + 0.5978889321237267, + 0.5559539597752757, + 0.523297269904143, + 0.4434509324241475, + 0.4141123687040429, + 0.4174418427557242, + 0.5192489959347639, + 1.2557404102027183, + 1.2676085225122937, + 1.157434741358947, + 1.0601387967701486, + 3.61892435638739, + 2.812397521737949, + 2.201832924858681, + 1.7615672286963322, + 1.8179181559541986, + 1.487118729202359, + 1.086854143939758, + 0.8727033118250799, + 0.8717668711180101, + 0.6523365178130685, + 0.4999226849610752, + 0.46458382918184327 + ], + "peak_layer": 21, + "peak_normalized": 3.61892435638739, + "population_cv": 0.7299683256836244, + "rest_mean": 0.045711164397222025, + "spike_contrast": 3.3328477188762906, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15234834998846053, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04247574508190155, + 0.04662793502211571, + 0.039439547806978226, + 0.03790952265262604, + 0.044013336300849915, + 0.03706495091319084, + 0.03307173401117325, + 0.03144170716404915, + 0.039725348353385925, + 0.03729226067662239, + 0.03467664122581482, + 0.03263973817229271, + 0.02765946462750435, + 0.02582952380180359, + 0.026037193834781647, + 0.03238723427057266, + 0.0783245787024498, + 0.07906483113765717, + 0.07219293713569641, + 0.06612427532672882, + 0.2257239818572998, + 0.17541830241680145, + 0.1373354196548462, + 0.10987462848424911, + 0.11338941752910614, + 0.09275639057159424, + 0.06779059767723083, + 0.054433319717645645, + 0.05437491089105606, + 0.04068833217024803, + 0.031181789934635162, + 0.028977591544389725 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.896312141980161e-06, + "first_to_last_ratio": 0.40447227453095713, + "imbalance_abs_log_ratio": 0.905172087401449, + "last_quartile_mean": 4.688361258331497e-06, + "mean": 3.997668517996544e-06, + "normalized": [ + 0.6172058310645722, + 0.5425329228673286, + 0.5226213455478163, + 0.5130242205418598, + 0.4179738021592733, + 0.4022397236307717, + 0.38715722309433015, + 0.3920811131204843, + 0.47573503087335345, + 0.47091681751525505, + 0.4692356598433531, + 0.46858709630953144, + 0.435518404119361, + 0.43348810961922396, + 0.457344361488258, + 0.4834240606909693, + 0.943243320831244, + 0.9556851843891272, + 0.9328847891755226, + 0.9144352241016374, + 3.593133434897055, + 3.0161686306352355, + 2.572714523011844, + 2.200458051372808, + 2.189693136620607, + 1.8429744726585007, + 1.3739587829516955, + 1.1483183935808434, + 1.0092761091983693, + 0.7452413615169265, + 0.5556410385724699, + 0.5170878240003723 + ], + "peak_layer": 21, + "peak_normalized": 3.593133434897055, + "population_cv": 0.8307777929041875, + "rest_mean": 2.728457953135485e-06, + "spike_contrast": 3.9771203202072667, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0851405568246264e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4673843199707335e-06, + 2.1688667857233668e-06, + 2.0892668999294983e-06, + 2.050900775429909e-06, + 1.670920710239443e-06, + 1.6080210798463668e-06, + 1.5477262422791682e-06, + 1.567410322422802e-06, + 1.901830955830519e-06, + 1.8825693359758588e-06, + 1.8758486248771078e-06, + 1.8732558828560286e-06, + 1.741058213156066e-06, + 1.7329417687506066e-06, + 1.8283111558048404e-06, + 1.9325691482663387e-06, + 3.770774128497578e-06, + 3.820512574748136e-06, + 3.7293641526048305e-06, + 3.6556089071382303e-06, + 1.4364156413648743e-05, + 1.2057642379659228e-05, + 1.0284859854436945e-05, + 8.796701877145097e-06, + 8.753667316341307e-06, + 7.3676010288181715e-06, + 5.49263177163084e-06, + 4.590596290654503e-06, + 4.0347513277083635e-06, + 2.9792279292450985e-06, + 2.2212686872080667e-06, + 2.0671457150456263e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.3950954957617796e-05, + "first_to_last_ratio": 0.7815786467251918, + "imbalance_abs_log_ratio": 0.24643949857917652, + "last_quartile_mean": 4.343894897829159e-05, + "mean": 4.974799179535694e-05, + "normalized": [ + 0.7500094412446059, + 0.8083859646655934, + 0.6989360803110847, + 0.6730394265060721, + 0.7487882742003255, + 0.6472039809659893, + 0.5806034653418889, + 0.5527037948336986, + 0.6849833016487895, + 0.641463932771085, + 0.5923270784846439, + 0.5570934219720602, + 0.5025383930607131, + 0.4685801281186291, + 0.467958282860767, + 0.5622422290415673, + 1.3163199357516149, + 1.3145043099431595, + 1.1718295263393368, + 1.089421848322846, + 3.6786851702332073, + 2.7746307606104894, + 2.0966820217393844, + 1.6356296326692725, + 1.680361539469081, + 1.3632870777077486, + 0.9862895783238333, + 0.8095320290956769, + 0.7283886637343489, + 0.5544846482049088, + 0.4452099057793211, + 0.417886156048257 + ], + "peak_layer": 21, + "peak_normalized": 3.6786851702332073, + "population_cv": 0.710922698103497, + "rest_mean": 3.709728177151798e-05, + "spike_contrast": 3.18249263251226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011806182592408732, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.7311463529476896e-05, + 4.021557833766565e-05, + 3.477066638879478e-05, + 3.348235986777581e-05, + 3.725071292137727e-05, + 3.219709833501838e-05, + 2.8883856430184096e-05, + 2.7495903850649484e-05, + 3.407654367038049e-05, + 3.191154246451333e-05, + 2.946708264062181e-05, + 2.7714278985513374e-05, + 2.5000275854836218e-05, + 2.3310920369112864e-05, + 2.327998481632676e-05, + 2.7970421797363088e-05, + 6.548427336383611e-05, + 6.539394962601364e-05, + 5.8296165661886334e-05, + 5.4196549172047526e-05, + 0.00018300719966646284, + 0.0001380323083139956, + 0.0001043057200149633, + 8.136928954627365e-05, + 8.359461207874119e-05, + 6.782079435652122e-05, + 4.906592585030012e-05, + 4.027259274153039e-05, + 3.623587326728739e-05, + 2.7584497729549184e-05, + 2.21482987399213e-05, + 2.078899706248194e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.600453626655508e-05, + "first_to_last_ratio": 1.032180910721196, + "imbalance_abs_log_ratio": 0.031673952782361334, + "last_quartile_mean": 2.5193777560161834e-05, + "mean": 3.414015603198095e-05, + "normalized": [ + 0.8380570158080917, + 0.8932999084125006, + 0.7929620023886231, + 0.7615443521416788, + 0.8187638282400099, + 0.7186815599217726, + 0.6527123798513835, + 0.6175741300582618, + 0.7513411682592698, + 0.705872928051971, + 0.6475562024934578, + 0.6094685286755085, + 0.5930459106503145, + 0.5494388493370179, + 0.5439464215496852, + 0.6333820560847124, + 1.4124654565316987, + 1.425994860990475, + 1.2380543674563398, + 1.1313103339756791, + 3.756835524951107, + 2.7186060870583466, + 1.87468091251249, + 1.410793634954463, + 1.4329939421834126, + 1.1438971076949418, + 0.8490265267268121, + 0.7287288463621843, + 0.5779352074283672, + 0.4639524923457935, + 0.36814597511060676, + 0.33893148179302335 + ], + "peak_layer": 21, + "peak_normalized": 3.756835524951107, + "population_cv": 0.6885594503102112, + "rest_mean": 2.6308265020176163e-05, + "spike_contrast": 2.9052606637917697, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.643236749572679e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8611397283384576e-05, + 3.0497398256557062e-05, + 2.7071846488979645e-05, + 2.599924300739076e-05, + 2.795272484945599e-05, + 2.4535900593036786e-05, + 2.228370249213185e-05, + 2.1084077161503956e-05, + 2.5650904717622325e-05, + 2.4098611902445555e-05, + 2.21076697926037e-05, + 2.0807350665563717e-05, + 2.024667992372997e-05, + 1.8757928046397865e-05, + 1.8570415704743937e-05, + 2.1623762222588994e-05, + 4.82217910757754e-05, + 4.86836870550178e-05, + 4.226736928103492e-05, + 3.8623111322522163e-05, + 0.00012825895100831985, + 9.281363600166515e-05, + 6.400189886335284e-05, + 4.816471482627094e-05, + 4.89226367790252e-05, + 3.905282574123703e-05, + 2.898589809774421e-05, + 2.4878916519810446e-05, + 1.9730798157979734e-05, + 1.5839410480111837e-05, + 1.256856103282189e-05, + 1.1571173672564328e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.909050666261464e-05, + "first_to_last_ratio": 0.6062699294291527, + "imbalance_abs_log_ratio": 0.5004299639855584, + "last_quartile_mean": 0.00014694858236907749, + "mean": 0.00014725223013556388, + "normalized": [ + 0.6636255714147841, + 0.7248917149077715, + 0.6138660879695628, + 0.5985975801010156, + 0.6806345015153643, + 0.5652332367621573, + 0.5042378190281714, + 0.4890714447869022, + 0.6294477660992072, + 0.5850399957826026, + 0.5478904329704705, + 0.518038310415621, + 0.40778029155663986, + 0.3853440981019431, + 0.3845833582250337, + 0.48007518680371525, + 1.178544482689773, + 1.1716070054101007, + 1.1012637666749874, + 1.0246953672401915, + 3.668461219741745, + 2.9054304267727518, + 2.326469432391579, + 1.8616676451976302, + 1.8927854440285927, + 1.5414183922061209, + 1.1508159193509495, + 0.9341585266360799, + 0.8717798331906053, + 0.6455697098701952, + 0.49047863567768574, + 0.45649679648005004 + ], + "peak_layer": 21, + "peak_normalized": 3.668461219741745, + "population_cv": 0.7615887971484168, + "rest_mean": 0.00010550450948627528, + "spike_contrast": 3.5324549012779802, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003726899216417223, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.772034536581486e-05, + 0.00010674192162696272, + 9.039315045811236e-05, + 8.814482862362638e-05, + 0.00010022494825534523, + 8.323185465997085e-05, + 7.42501433705911e-05, + 7.201686094049364e-05, + 9.268758731195703e-05, + 8.614844409748912e-05, + 8.067808812484145e-05, + 7.628229650435969e-05, + 6.004655733704567e-05, + 5.674277781508863e-05, + 5.663075717166066e-05, + 7.06921418895945e-05, + 0.00017354330339003354, + 0.00017252174438908696, + 0.00016216354561038315, + 0.00015088867803569883, + 0.0005401890957728028, + 0.0004278311098460108, + 0.00034257781226187944, + 0.00027413471252657473, + 0.0002787168778013438, + 0.00022697729582432657, + 0.00016946021059993654, + 0.0001375569263473153, + 0.0001283715246245265, + 9.506157948635519e-05, + 7.222407293738797e-05, + 6.722017133142799e-05 + ] + } + }, + "seed": 2026073002 + }, + { + "interventions": { + "detached_learned": { + "statistics": { + "first_quartile_mean": 7.388767562588328e-05, + "first_to_last_ratio": 1.316792081076884, + "imbalance_abs_log_ratio": 0.2751985371595874, + "last_quartile_mean": 5.611187725662603e-05, + "mean": 8.168265617314319e-05, + "normalized": [ + 0.7533431897136043, + 0.7827542764436777, + 0.7759396123840813, + 0.7316445186865006, + 1.1365162577009855, + 1.1353975532420753, + 1.004206629730762, + 0.9167575048500237, + 1.0078390566063014, + 0.9220130731102486, + 0.8550917608474935, + 0.8013922543773514, + 0.7721726808717908, + 0.7274660564313035, + 0.7065479131389877, + 0.6779112969812525, + 1.6398021444587536, + 1.5049103837665865, + 1.3267399301434915, + 1.1597912607176666, + 2.3220162536772593, + 1.9302044249737353, + 1.584049170842242, + 1.3298950882483613, + 1.2567261692224045, + 0.9579394397382498, + 0.801457903329196, + 0.6752085554658249, + 0.5811903488324587, + 0.45264756329860717, + 0.3896296913930569, + 0.38079803677566654 + ], + "peak_layer": 21, + "peak_normalized": 2.3220162536772593, + "population_cv": 0.4315618932878077, + "rest_mean": 7.132743997283763e-05, + "spike_contrast": 1.9291428895694742, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001376008236547932, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.153507274575531e-05, + 6.39374484308064e-05, + 6.338080856949091e-05, + 5.976266766083427e-05, + 9.283366671297699e-05, + 9.274228796130046e-05, + 8.202626486308873e-05, + 7.488318806281313e-05, + 8.232297113863751e-05, + 7.531247683800757e-05, + 6.984616629779339e-05, + 6.54598479741253e-05, + 6.30731155979447e-05, + 5.942135976511054e-05, + 5.771271025878377e-05, + 5.537359538720921e-05, + 0.00013394339475780725, + 0.00012292507744859904, + 0.00010837164154509082, + 9.473483078181744e-05, + 0.0001896684552775696, + 0.00015766422438900918, + 0.0001293893437832594, + 0.00010862936323974282, + 0.00010265273158438504, + 7.824703789083287e-05, + 6.546521035488695e-05, + 5.515282828127965e-05, + 4.7473171434830874e-05, + 3.69734552805312e-05, + 3.1825988116906956e-05, + 3.1104595109354705e-05 + ] + }, + "learned": { + "statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "rest_mean": 7.697225831819836e-05, + "spike_contrast": 1.8814916302334765, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001448226597858593, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ] + }, + "uniform_value_backward": { + "statistics": { + "first_quartile_mean": 3.5575103993323864e-05, + "first_to_last_ratio": 2.443213468991071, + "imbalance_abs_log_ratio": 0.893314168315378, + "last_quartile_mean": 1.456078416595119e-05, + "mean": 2.3831100747884193e-05, + "normalized": [ + 1.614905261758675, + 1.6568148874352646, + 1.6283515751571318, + 1.4605379751384047, + 1.4717695441533916, + 1.4559185813526447, + 1.358376097224691, + 1.295738430337987, + 1.1776907075521434, + 1.1141057290944703, + 1.0640572841848701, + 1.0269673379730528, + 1.0041631647243947, + 0.9733861101490422, + 0.9597079874490758, + 0.9349675920893245, + 0.9158525986134215, + 0.9066797587654788, + 0.890357239209801, + 0.8734858422464332, + 0.8579185157197757, + 0.8414715045701692, + 0.8239471946120328, + 0.8048352542714806, + 0.7658980848440458, + 0.7173626211575344, + 0.6794957283273223, + 0.6347225670052176, + 0.5854538165358278, + 0.5328910383364602, + 0.49362355782826073, + 0.47854641218217414 + ], + "peak_layer": 2, + "peak_normalized": 1.6568148874352646, + "population_cv": 0.33383138748240165, + "rest_mean": 2.4630704299650257e-05, + "spike_contrast": 0.7922323832463255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.951324156834744e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.848496999125928e-05, + 3.94837225030642e-05, + 3.880541044054553e-05, + 3.48062276316341e-05, + 3.507388828438707e-05, + 3.4696142392931506e-05, + 3.237159762647934e-05, + 3.087887307628989e-05, + 2.8065665901522152e-05, + 2.6550365873845294e-05, + 2.535765634092968e-05, + 2.4473762096022256e-05, + 2.393031354586128e-05, + 2.3196862457552925e-05, + 2.2870897737448104e-05, + 2.2281306883087382e-05, + 2.182577554776799e-05, + 2.1607176677207462e-05, + 2.1218193069216795e-05, + 2.081612910842523e-05, + 2.0445142581593245e-05, + 2.0053192201885395e-05, + 1.9635568605735898e-05, + 1.9180110029992647e-05, + 1.825219442253001e-05, + 1.7095540897571482e-05, + 1.6193131159525365e-05, + 1.5126137441257015e-05, + 1.3952008885098621e-05, + 1.2699380022240803e-05, + 1.176359273813432e-05, + 1.1404287761251908e-05 + ] + } + }, + "position_element_rms": { + "attention_branch_output": { + "statistics": { + "first_quartile_mean": 8.209008865378564e-05, + "first_to_last_ratio": 1.1473326374136463, + "imbalance_abs_log_ratio": 0.13743980256699795, + "last_quartile_mean": 7.154863897085306e-05, + "mean": 9.638685185109352e-05, + "normalized": [ + 0.7014672692081079, + 0.730745142012726, + 0.7197152784749434, + 0.6836365522681692, + 1.073826467208612, + 1.0803379033203697, + 0.9514146538604249, + 0.8722415119444402, + 1.0096074578076277, + 0.8908657485913869, + 0.8272757726491637, + 0.7688961382914924, + 0.7419179025068791, + 0.6953651992846211, + 0.6657440869339828, + 0.6349379036282153, + 1.6060213837194506, + 1.5018800747743897, + 1.362846846779478, + 1.1942969804897212, + 2.2656295070514414, + 2.0104706485090547, + 1.6756941049043201, + 1.3967091226783026, + 1.3149112210845808, + 1.1000992011684827, + 0.8430071951587476, + 0.7025118587919716, + 0.7359466516142374, + 0.5075845580254474, + 0.3944123577040649, + 0.33998329955514606 + ], + "peak_layer": 21, + "peak_normalized": 2.2656295070514414, + "population_cv": 0.4413317448728873, + "rest_mean": 8.330888886121102e-05, + "spike_contrast": 2.004682264754327, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016700785199645906, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761222175555304e-05, + 7.043422374408692e-05, + 6.93710899213329e-05, + 6.589357508346438e-05, + 0.00010350275260861963, + 0.00010413036943646148, + 9.17038632906042e-05, + 8.407261339016259e-05, + 9.731288446346298e-05, + 8.586774492869154e-05, + 7.973850733833387e-05, + 7.411147817038e-05, + 7.15111309546046e-05, + 6.70240624458529e-05, + 6.416897667804733e-05, + 6.119966565165669e-05, + 0.0001547993451822549, + 0.00014476149226538837, + 0.0001313605171162635, + 0.0001151145261246711, + 0.00021837689564563334, + 0.00019378293654881418, + 0.00016151487943716347, + 0.00013462439528666437, + 0.00012674015306401998, + 0.00010603509872453287, + 8.125480962917209e-05, + 6.771290645701811e-05, + 7.093558087944984e-05, + 4.8924477596301585e-05, + 3.801616549026221e-05, + 3.2769919926067814e-05 + ] + }, + "mlp_branch_output": { + "statistics": { + "first_quartile_mean": 8.156287549354602e-05, + "first_to_last_ratio": 1.3967979973430424, + "imbalance_abs_log_ratio": 0.3341824723557829, + "last_quartile_mean": 5.839274945174111e-05, + "mean": 8.75740092851629e-05, + "normalized": [ + 0.7720751985061499, + 0.8035956764433113, + 0.7960739585260428, + 0.7524919125585545, + 1.1804156983562801, + 1.1778760010857563, + 1.0298326204965722, + 0.9385140198493713, + 1.0207128740101707, + 0.929812017339553, + 0.8640701821512353, + 0.8130222503970004, + 0.7736044339144612, + 0.7280851293348032, + 0.7061941984091654, + 0.6826155087046274, + 1.642189790371069, + 1.511740343745529, + 1.3334033796706957, + 1.1637276240936476, + 2.2882576030527964, + 1.895449319429852, + 1.5552551751392194, + 1.3067312409844312, + 1.222859749246748, + 0.9319757616478574, + 0.7801162715542103, + 0.6613743325062905, + 0.561952429248885, + 0.43456344744462927, + 0.3748431739999157, + 0.36656867778116903 + ], + "peak_layer": 21, + "peak_normalized": 2.2882576030527964, + "population_cv": 0.4268460433011898, + "rest_mean": 7.697251674943362e-05, + "spike_contrast": 1.8814776376552176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014482206897810103, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761372060282156e-05, + 7.037409523036331e-05, + 6.971538823563606e-05, + 6.589873373741284e-05, + 0.00010337373532820493, + 0.00010315132385585457, + 9.018657146953046e-05, + 8.21894354885444e-05, + 8.9387918706052e-05, + 8.142736623995006e-05, + 7.567009015474468e-05, + 7.119961810531095e-05, + 6.774764187866822e-05, + 6.376133387675509e-05, + 6.184425728861243e-05, + 5.977937689749524e-05, + 0.0001438131439499557, + 0.0001323891628999263, + 0.00011677147995214909, + 0.00010191229375777766, + 0.00020039189257659018, + 0.00016599209629930556, + 0.00013619993114843965, + 0.00011443569383118302, + 0.00010709073103498667, + 8.161685400409624e-05, + 6.831790960859507e-05, + 5.79192019358743e-05, + 4.92124272568617e-05, + 3.8056463381508365e-05, + 3.282651960034855e-05, + 3.2101888791657984e-05 + ] + }, + "post_attention_state": { + "statistics": { + "first_quartile_mean": 8.209185853047529e-05, + "first_to_last_ratio": 1.1473585117140146, + "imbalance_abs_log_ratio": 0.13746235401177884, + "last_quartile_mean": 7.154856803026632e-05, + "mean": 9.638741801154538e-05, + "normalized": [ + 0.701459601059617, + 0.7307291493400122, + 0.7197289413331719, + 0.6836538994288254, + 1.0739127818136336, + 1.080318121017968, + 0.9514366180464413, + 0.8722525427024255, + 1.0096073400424832, + 0.8908667057242938, + 0.8272733289673383, + 0.7688963021177191, + 0.7419276606217693, + 0.6953759857067083, + 0.6657591236204922, + 0.6349357593481342, + 1.6060262927227642, + 1.5018600810151583, + 1.362811062623623, + 1.194289437021534, + 2.265619973525555, + 2.010448271283051, + 1.6756913579512287, + 1.3967040891245648, + 1.314898968363092, + 1.100096740194715, + 0.8429885804297745, + 0.7025081852971004, + 0.7359520665807775, + 0.5075723672109625, + 0.3944131736981434, + 0.33998549206692247 + ], + "peak_layer": 21, + "peak_normalized": 2.265619973525555, + "population_cv": 0.44132695283622186, + "rest_mean": 8.330956363890975e-05, + "spike_contrast": 2.0046657830023333, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016700783162377776, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761187978554517e-05, + 7.043309597065672e-05, + 6.937281432328746e-05, + 6.58956341794692e-05, + 0.00010351168020861223, + 0.00010412907431600615, + 9.170651901513338e-05, + 8.407417044509202e-05, + 9.731344471219927e-05, + 8.58683415572159e-05, + 7.973874016897753e-05, + 7.411192927975208e-05, + 7.151249155867845e-05, + 6.70254958095029e-05, + 6.417080294340849e-05, + 6.119981844676659e-05, + 0.0001548007276142016, + 0.00014476041542366147, + 0.0001313578395638615, + 0.0001151144751929678, + 0.00021837725944351405, + 0.0001937819179147482, + 0.0001615155633771792, + 0.00013462470087688416, + 0.00012673971650656313, + 0.00010603548435028642, + 8.125349268084392e-05, + 6.771295011276379e-05, + 7.093651947798207e-05, + 4.8923589929472655e-05, + 3.80164674425032e-05, + 3.27703237417154e-05 + ] + }, + "post_mlp_state": { + "statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "rest_mean": 7.697225831819836e-05, + "spike_contrast": 1.8814916302334765, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001448226597858593, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ] + }, + "pre_attention_input": { + "statistics": { + "first_quartile_mean": 4.013745751763054e-05, + "first_to_last_ratio": 1.3002217470857882, + "imbalance_abs_log_ratio": 0.26253482460265976, + "last_quartile_mean": 3.0869701731717214e-05, + "mean": 5.617674656832605e-05, + "normalized": [ + 0.473654064493233, + 0.5256902444154287, + 0.3705068590661575, + 0.5387329076786941, + 0.37665460925127686, + 0.9021763170314637, + 1.3591261829588805, + 1.169340915014312, + 1.4504557484696843, + 1.2215794360380687, + 0.9103877586499164, + 0.9321134746924985, + 0.5414129154069731, + 0.8525397685215896, + 0.9808579608967221, + 0.9142526712062526, + 1.0619023556310665, + 1.8034129735144708, + 1.5888135435441055, + 1.899281403844036, + 1.9024910148779852, + 2.5497535502321647, + 2.089500431013523, + 1.18928036938163, + 1.0430326636758747, + 0.9143825787913804, + 0.5551399248346642, + 0.5581364123079678, + 0.38887965127025986, + 0.4518317596354874, + 0.24874412658654727, + 0.2359354070676856 + ], + "peak_layer": 22, + "peak_normalized": 2.5497535502321647, + "population_cv": 0.5766137085977164, + "rest_mean": 4.832436503353098e-05, + "spike_contrast": 2.0399565889343343, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.857960685621947e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.6608344342093915e-05, + 2.9531567633966915e-05, + 2.0813869923586026e-05, + 3.0264262022683397e-05, + 2.1159230527700856e-05, + 5.0681330321822315e-05, + 7.635128713445738e-05, + 6.568976823473349e-05, + 8.148188499035314e-05, + 6.862435839138925e-05, + 5.114262239658274e-05, + 5.236310244072229e-05, + 3.041481613763608e-05, + 4.7892910515656695e-05, + 5.510140908882022e-05, + 5.135974060976878e-05, + 5.9654219512594864e-05, + 0.00010130987357115373, + 8.925437578000128e-05, + 0.00010669545008568093, + 0.00010687575559131801, + 0.00014323685900308192, + 0.00011738133616745472, + 6.680990190943703e-05, + 5.859418160980567e-05, + 5.1367038395255804e-05, + 3.1185954867396504e-05, + 3.1354287784779444e-05, + 2.18459936149884e-05, + 2.538243825256359e-05, + 1.397363575961208e-05, + 1.3254083569336217e-05 + ] + }, + "pre_mlp_input": { + "statistics": { + "first_quartile_mean": 2.0563243310789403e-05, + "first_to_last_ratio": 0.184584542859635, + "imbalance_abs_log_ratio": 1.6896476935739415, + "last_quartile_mean": 0.00011140284550492652, + "mean": 8.183052003118974e-05, + "normalized": [ + 0.5514618447323059, + 0.2169408826973503, + 0.08472485593965659, + 0.28295099514537675, + 0.21989392642956848, + 0.1303640614627248, + 0.17290402482659964, + 0.35108448581118906, + 0.6388397786640535, + 0.4693907558756991, + 0.4878178928069798, + 0.4684568375678178, + 0.5735758405807379, + 0.5222431877258841, + 0.48692296371012395, + 0.47894342377930604, + 1.3828038952387727, + 1.4137992065669855, + 1.4773604231197706, + 1.4101154596735193, + 2.482879371711871, + 2.526943137473127, + 2.259311590178407, + 2.019190865749238, + 2.085961378955964, + 1.8764944434843154, + 1.4765035496268348, + 1.350239590483524, + 1.4291199659133378, + 1.0658000032658883, + 0.838995628185017, + 0.7679657326180543 + ], + "peak_layer": 22, + "peak_normalized": 2.526943137473127, + "population_cv": 0.7282772544392188, + "rest_mean": 6.251158794367695e-05, + "spike_contrast": 2.977891930556664, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001861527533037588, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.51264095318038e-05, + 1.7752385247149505e-05, + 6.933079021109734e-06, + 2.3154027076088823e-05, + 1.7994034351431765e-05, + 1.0667758942872752e-05, + 1.4148826267046388e-05, + 2.8729426048812456e-05, + 5.2276591304689646e-05, + 3.841048965114169e-05, + 3.9918391848914325e-05, + 3.833406663034111e-05, + 4.6936009312048554e-05, + 4.2735431634355336e-05, + 3.984515933552757e-05, + 3.91921894333791e-05, + 0.00011315556184854358, + 0.00011569192429305986, + 0.00012089317169738933, + 0.00011539048136910424, + 0.00020317531016189605, + 0.00020678107102867216, + 0.00018488064233679324, + 0.00016523143858648837, + 0.00017069530440494418, + 0.0001535545161459595, + 0.00012082305329386145, + 0.00011049080785596743, + 0.00011694562999764457, + 8.721496851649135e-05, + 6.865544855827466e-05, + 6.2843035266269e-05 + ] + } + }, + "post_mlp_reductions": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0823702925554244e-05, + "first_to_last_ratio": 1.4242510302785252, + "imbalance_abs_log_ratio": 0.35364608276130804, + "last_quartile_mean": 1.4620809452026151e-05, + "mean": 2.2054440051988422e-05, + "normalized": [ + 0.7841928807641645, + 0.8183408510588258, + 0.809986731151413, + 0.7637938668182616, + 1.192276091281044, + 1.1966830153099992, + 1.039215154291241, + 0.9490753634193776, + 1.0189872806238243, + 0.9331490903420598, + 0.8658120301757168, + 0.81301306861575, + 0.7774052511262896, + 0.7266654164061733, + 0.7030327996851689, + 0.6829495498627286, + 1.6322510246428148, + 1.499422250137046, + 1.3169815172043178, + 1.1572342326306069, + 2.2864238933994527, + 1.8918479809103845, + 1.543069371898747, + 1.294657154506133, + 1.2171810844243176, + 0.9287685590307506, + 0.7771677166674748, + 0.6578712720326597, + 0.5567773486562734, + 0.43254113576640124, + 0.3708732313504376, + 0.3623537858101448 + ], + "peak_layer": 21, + "peak_normalized": 2.2864238933994527, + "population_cv": 0.4245664907667088, + "rest_mean": 1.9413478454495175e-05, + "spike_contrast": 1.8706401721656991, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6315632678451947e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.729493487800937e-05, + 1.804804924177006e-05, + 1.78638038050849e-05, + 1.684504604781978e-05, + 2.629498158057686e-05, + 2.639217382238712e-05, + 2.2919308321434073e-05, + 2.093132570735179e-05, + 2.2473193894256838e-05, + 2.0580080672516488e-05, + 1.9094999515800737e-05, + 1.7930547983269207e-05, + 1.7145237507065758e-05, + 1.6026198863983154e-05, + 1.5504994735238142e-05, + 1.5062069905980024e-05, + 3.5998382372781634e-05, + 3.306891812826507e-05, + 2.9045289920759387e-05, + 2.5522153009660542e-05, + 5.0425798690412194e-05, + 4.172364788246341e-05, + 3.403153095860034e-05, + 2.855293860193342e-05, + 2.6844247258850373e-05, + 2.048347050731536e-05, + 1.7139998817583546e-05, + 1.450898253096966e-05, + 1.2279412658244837e-05, + 9.539452548779082e-06, + 8.179401447705459e-06, + 7.991509846760891e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "rest_mean": 7.697225831819836e-05, + "spike_contrast": 1.8814916302334765, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001448226597858593, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07233011117205024, + "first_to_last_ratio": 1.3967870631582553, + "imbalance_abs_log_ratio": 0.3341746442893017, + "last_quartile_mean": 0.051783205242827535, + "mean": 0.07766139676095918, + "normalized": [ + 0.7720675426341356, + 0.8035859729306402, + 0.7960780122636479, + 0.7524732326455996, + 1.1804674279966365, + 1.1778079658527474, + 1.0298294903648106, + 0.9385077949910019, + 1.020698424099846, + 0.9298230264026498, + 0.8640760490713295, + 0.8130195756795249, + 0.773607039370292, + 0.7280910561484961, + 0.7061969508238138, + 0.6826252958058289, + 1.64216899408666, + 1.511744542110794, + 1.3334176327397504, + 1.1637251251728324, + 2.288267386572505, + 1.8954517517435265, + 1.5552435035136007, + 1.3067718770227776, + 1.2228641722871045, + 0.9319785328940787, + 0.7801153392913037, + 0.6613715681663117, + 0.5619466726726858, + 0.43456725256605405, + 0.37483694737332934, + 0.36657384470568466 + ], + "peak_layer": 21, + "peak_normalized": 2.288267386572505, + "population_cv": 0.4268470914878033, + "rest_mean": 0.0682597693607763, + "spike_contrast": 1.8814916300573643, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12843018472194673, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05995984375476837, + 0.062407609075307846, + 0.061824530363082886, + 0.058438122272491455, + 0.0916767492890358, + 0.09147021174430847, + 0.07997799664735794, + 0.07288582623004913, + 0.07926886528730392, + 0.072211354970932, + 0.06710535287857056, + 0.06314023584127426, + 0.06007940322160721, + 0.05654456838965416, + 0.054844241589307785, + 0.0530136339366436, + 0.12753313779830933, + 0.11740419268608093, + 0.10355507582426071, + 0.09037651866674423, + 0.17771004140377045, + 0.14720343053340912, + 0.12078238278627396, + 0.1014857292175293, + 0.09496933966875076, + 0.07237875461578369, + 0.06058484688401222, + 0.05136303976178169, + 0.04364156350493431, + 0.03374909982085228, + 0.029110360890626907, + 0.02846863679587841 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.1000530213987076e-06, + "first_to_last_ratio": 0.7825005490137864, + "imbalance_abs_log_ratio": 0.2452606549530124, + "last_quartile_mean": 3.961726321222159e-06, + "mean": 4.656059921614997e-06, + "normalized": [ + 0.7040313055728622, + 0.6663442620495864, + 0.6544029514954961, + 0.6681637652166335, + 0.6743674803950376, + 0.6670078174062475, + 0.6466284443239834, + 0.6455370061269436, + 0.7660325155889827, + 0.7325145706427032, + 0.7131598309497137, + 0.7280186266136806, + 0.64870437442493, + 0.656793469733594, + 0.6699169519151534, + 0.7314456448845376, + 1.53365232903732, + 1.4601427212016398, + 1.3940973786195396, + 1.306558663485338, + 2.534675981169938, + 2.2580403975857246, + 1.9597194264610087, + 1.773041790596292, + 1.6211455309545355, + 1.2094494759638503, + 1.0219317012739941, + 0.8604275624593173, + 0.6846808632651675, + 0.5134649266877133, + 0.4503974279453127, + 0.4455048059532228 + ], + "peak_layer": 21, + "peak_normalized": 2.534675981169938, + "population_cv": 0.5361438958058841, + "rest_mean": 3.768541933755774e-06, + "spike_contrast": 2.5072447705625382, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.4486570560548e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.2780119454400847e-06, + 3.1025388125272e-06, + 3.046939355044742e-06, + 3.1110105283005396e-06, + 3.1398953979078215e-06, + 3.105628366029123e-06, + 3.0107407837931532e-06, + 3.005658982146997e-06, + 3.566693294487777e-06, + 3.4106317343685078e-06, + 3.320514906590688e-06, + 3.389698349565151e-06, + 3.020406438736245e-06, + 3.058069751205039e-06, + 3.119173470622627e-06, + 3.405654751986731e-06, + 7.140777142922161e-06, + 6.798512004024815e-06, + 6.4910009314189665e-06, + 6.083415428292938e-06, + 1.1801603250205517e-05, + 1.0513571396586485e-05, + 9.12457107915543e-06, + 8.255388820543885e-06, + 7.548150733782677e-06, + 5.631269232253544e-06, + 4.758175236929674e-06, + 4.006202289019711e-06, + 3.187915126545704e-06, + 2.3907234663056443e-06, + 2.0970774130546488e-06, + 2.074297071885667e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.95857286498358e-05, + "first_to_last_ratio": 1.5974327409797537, + "imbalance_abs_log_ratio": 0.4683978037101298, + "last_quartile_mean": 3.7300931126083015e-05, + "mean": 6.105390025368251e-05, + "normalized": [ + 0.8419145460820371, + 0.8656854461464137, + 0.8583401181387939, + 0.8192663123570689, + 1.1963719080724353, + 1.1751919414919585, + 1.0676149913033914, + 0.9832376272965914, + 1.0816749854357006, + 0.984736879340849, + 0.9088680283961561, + 0.8483339022774173, + 0.8441772779737997, + 0.7882426814662962, + 0.7677338358974833, + 0.7384876656990601, + 1.5920129687595903, + 1.4697744406924866, + 1.283993033566668, + 1.1290203889690138, + 2.340760926498109, + 1.860435632123778, + 1.4645539617726704, + 1.2019638482713701, + 1.15797649283655, + 0.8703237664687024, + 0.7241030451553558, + 0.6218659849233957, + 0.4685510324632258, + 0.37521378882561096, + 0.3379462835275562, + 0.3316262577704645 + ], + "peak_layer": 21, + "peak_normalized": 2.340760926498109, + "population_cv": 0.4213559298637173, + "rest_mean": 5.4212039955288036e-05, + "spike_contrast": 1.8077155175462722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.79999458650127e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.140216671861708e-05, + 5.285347288008779e-05, + 5.240501195657998e-05, + 5.0019403715850785e-05, + 7.304317114176229e-05, + 7.175005157478154e-05, + 6.518205918837339e-05, + 6.003049202263355e-05, + 6.604047666769475e-05, + 6.012202720739879e-05, + 5.548993794946e-05, + 5.1794093451462686e-05, + 5.154031532583758e-05, + 4.8125290049938485e-05, + 4.687314503826201e-05, + 4.5087552280165255e-05, + 9.7198600997217e-05, + 8.973546209745109e-05, + 7.839278259780258e-05, + 6.8931098212488e-05, + 0.000142912584124133, + 0.00011358685151208192, + 8.941673149820417e-05, + 7.338458090089262e-05, + 7.069898128975183e-05, + 5.3136660426389426e-05, + 4.420931509230286e-05, + 3.796734381467104e-05, + 2.8606867999769747e-05, + 2.2908265236765146e-05, + 2.0632938685594127e-05, + 2.0247076463419944e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.39446339441929e-05, + "first_to_last_ratio": 1.9902838030955072, + "imbalance_abs_log_ratio": 0.6882772431887607, + "last_quartile_mean": 2.207958175404201e-05, + "mean": 4.171556903997953e-05, + "normalized": [ + 0.9393219483952423, + 0.9671725353750462, + 0.9606035942825071, + 0.9165296658237206, + 1.25754251069206, + 1.233040231347365, + 1.1241638961033311, + 1.029104366181827, + 1.1530467776467852, + 1.041181174787795, + 0.9503458810810488, + 0.8874956508445306, + 0.9380795670187313, + 0.868712023499227, + 0.8459898570744635, + 0.811097306638917, + 1.57041471098841, + 1.447423664007262, + 1.250399123009104, + 1.0954141829132344, + 2.392471764387743, + 1.7753028127291355, + 1.2921083756975622, + 1.0187283101804303, + 1.065894151409315, + 0.8051193818729687, + 0.6754751497722067, + 0.6026093861097862, + 0.3198852539229254, + 0.2669904264118248, + 0.2517857094641488, + 0.2465506103313451 + ], + "peak_layer": 21, + "peak_normalized": 2.392471764387743, + "population_cv": 0.4303681174959756, + "rest_mean": 3.778425454786177e-05, + "spike_contrast": 1.6658967617763285, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.294466729741543e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.9184349589049816e-05, + 4.034615267300978e-05, + 4.007212555734441e-05, + 3.823355655185878e-05, + 5.245910142548382e-05, + 5.1436974899843335e-05, + 4.689513662015088e-05, + 4.2929674236802384e-05, + 4.810000245925039e-05, + 4.343346517998725e-05, + 3.9644219214096665e-05, + 3.702238609548658e-05, + 3.913252294296399e-05, + 3.6238816392142326e-05, + 3.5290948289912194e-05, + 3.3835385693237185e-05, + 6.551074329763651e-05, + 6.0380101785995066e-05, + 5.2161110943416134e-05, + 4.5695825974689797e-05, + 9.980332106351852e-05, + 7.40577670512721e-05, + 5.3901036153547466e-05, + 4.249683115631342e-05, + 4.446438106242567e-05, + 3.358601315994747e-05, + 2.8177830245113e-05, + 2.513819345040247e-05, + 1.3344195394893177e-05, + 1.113765756599605e-05, + 1.0503384146431927e-05, + 1.0284999007126316e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015719397015345749, + "first_to_last_ratio": 1.2580343238755638, + "imbalance_abs_log_ratio": 0.22955044238581457, + "last_quartile_mean": 0.00012495205191953573, + "mean": 0.00018078252287523355, + "normalized": [ + 0.7253227592072276, + 0.7485993006094415, + 0.7410050906139998, + 0.7104508417729404, + 1.0745241672934553, + 1.0524252536641314, + 0.9748384190355198, + 0.9289919933888637, + 1.0038768041568893, + 0.9389629508381758, + 0.8789427325469985, + 0.8198897306139241, + 0.7627520038160613, + 0.7096499260969147, + 0.6897553416925117, + 0.6787601781063402, + 1.6052869610938005, + 1.5071148227224969, + 1.3317635520239959, + 1.1792135859285877, + 2.364258320942869, + 1.9856700690456726, + 1.6414783683894763, + 1.4170804705838413, + 1.2788477375082987, + 0.9694880608375548, + 0.8072605380605845, + 0.691415531221962, + 0.5679201295495192, + 0.44599504581333255, + 0.3904354851204105, + 0.3780238277042025 + ], + "peak_layer": 21, + "peak_normalized": 2.364258320942869, + "population_cv": 0.4451626458968529, + "rest_mean": 0.0001560934222113617, + "spike_contrast": 2.0122799667677387, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00031410366646014156, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00013112567830830812, + 0.0001353336701868102, + 0.00013396076974458992, + 0.00012843709555454552, + 0.00019425518985372037, + 0.0001902600924950093, + 0.00017623374878894538, + 0.00016794551629573107, + 0.00018148338131140918, + 0.00016974809113889933, + 0.00015889748465269804, + 0.0001482217339798808, + 0.00013789223157800734, + 0.00012829230399802327, + 0.00012469571083784103, + 0.00012270797742530704, + 0.00029020782676525414, + 0.00027246001991443336, + 0.00024075957480818033, + 0.0002131812070729211, + 0.00042741658398881555, + 0.0003589744446799159, + 0.00029675060068257153, + 0.00025618338258937, + 0.0002311933203600347, + 0.00017526649753563106, + 0.00014593859668821096, + 0.0001249958440894261, + 0.00010267003381159157, + 8.062810957198963e-05, + 7.058391202008352e-05, + 6.834010127931833e-05 + ] + } + }, + "seed": 2026073003 + } + ], + "gates": { + "forward_identity": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "initialization_negative_control": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "loss_scale": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "replay": { + "checks": { + "architecture": true, + "artifacts": true, + "batch_size": true, + "depth": true, + "diagnostics": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "round05_equivalence": true, + "seed": true, + "steps": true, + "target_bytes_seen": true, + "training_history": true + }, + "frozen_compare_sha256": "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87", + "passed": true + }, + "round05_equivalence": [ + { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true, + "seed": 2026073001 + }, + { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true, + "seed": 2026073002 + }, + { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true, + "seed": 2026073003 + } + ] + }, + "hashes": { + "aggregate_canonical_sha256": "1d0faef297682d61d35ce2ab54da3868e18a84e9ebc6e44c617d08016e4fc8b4", + "reproduction_canonical_sha256": "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515" + }, + "mixer_associations": { + "all_layers": { + "attention_entropy": { + "pearson": -0.29619123279185056, + "spearman": -0.2883517139673049 + }, + "attention_latest": { + "pearson": 0.06600313295625082, + "spearman": 0.29780653565593507 + }, + "attention_max": { + "pearson": -0.024396635036849867, + "spearman": 0.20476403642736357 + }, + "mlp_entropy": { + "pearson": -0.2736346519450915, + "spearman": -0.41707813347802497 + }, + "mlp_latest": { + "pearson": 0.6507856318169407, + "spearman": 0.692444384156267 + }, + "mlp_max": { + "pearson": 0.23015916710007592, + "spearman": 0.4651112316874661 + }, + "points": 96 + }, + "identity": "observational association; not causal attribution", + "layers_19_28": { + "attention_entropy": { + "pearson": -0.05745838348637786, + "spearman": -0.11279199110122358 + }, + "attention_latest": { + "pearson": 0.15799271238887527, + "spearman": 0.3388209121245829 + }, + "attention_max": { + "pearson": 0.15799271238887527, + "spearman": 0.3388209121245829 + }, + "mlp_entropy": { + "pearson": -0.6355230712915908, + "spearman": -0.6400444938820912 + }, + "mlp_latest": { + "pearson": 0.6896916851820697, + "spearman": 0.692992213570634 + }, + "mlp_max": { + "pearson": 0.7022594570959072, + "spearman": 0.7045606229143493 + }, + "points": 30 + }, + "target_layers": [ + { + "attention_entropy": 0.9286576240857585, + "attention_latest": 0.34922462701797485, + "attention_max": 0.34922462701797485, + "group": 5, + "layer": 19, + "mlp_entropy": 0.8417470208868109, + "mlp_latest": 0.4519178469975789, + "mlp_max": 0.4519178469975789, + "normalized_gradient": 1.0628893270643442, + "offset": 3, + "per_seed_normalized_gradient": [ + 0.6978155985835179, + 1.1574347089891701, + 1.3334176736203447 + ] + }, + { + "attention_entropy": 0.9460862283961503, + "attention_latest": 0.3155268132686615, + "attention_max": 0.3155268132686615, + "group": 5, + "layer": 20, + "mlp_entropy": 0.862835637646716, + "mlp_latest": 0.4188626905282338, + "mlp_max": 0.4188626905282338, + "normalized_gradient": 1.0135704098952112, + "offset": 4, + "per_seed_normalized_gradient": [ + 0.8168473226649812, + 1.060138766241709, + 1.1637251407789435 + ] + }, + { + "attention_entropy": 0.9687891325525921, + "attention_latest": 0.2497197538614273, + "attention_max": 0.2497197538614273, + "group": 6, + "layer": 21, + "mlp_entropy": 0.7626460544111615, + "mlp_latest": 0.512785017490387, + "mlp_max": 0.512785017490387, + "normalized_gradient": 3.0420861976019666, + "offset": 1, + "per_seed_normalized_gradient": [ + 3.219066994145898, + 3.6189242296270336, + 2.2882673690329693 + ] + }, + { + "attention_entropy": 0.8507401102456327, + "attention_latest": 0.4364195068677266, + "attention_max": 0.4364195068677266, + "group": 6, + "layer": 22, + "mlp_entropy": 0.7208451982094757, + "mlp_latest": 0.563143769900004, + "mlp_max": 0.563143769900004, + "normalized_gradient": 2.405186320789127, + "offset": 2, + "per_seed_normalized_gradient": [ + 2.5077095541323673, + 2.81239769967395, + 1.8954517085610634 + ] + }, + { + "attention_entropy": 0.879444880309749, + "attention_latest": 0.40026357769966125, + "attention_max": 0.40026357769966125, + "group": 6, + "layer": 23, + "mlp_entropy": 0.7369153442658093, + "mlp_latest": 0.5514553984006246, + "mlp_max": 0.5514553984006246, + "normalized_gradient": 1.9103037349623444, + "offset": 3, + "per_seed_normalized_gradient": [ + 1.973834673592298, + 2.2018329150221976, + 1.5552436162725376 + ] + }, + { + "attention_entropy": 0.8894285781602397, + "attention_latest": 0.38436253865559894, + "attention_max": 0.38436253865559894, + "group": 6, + "layer": 24, + "mlp_entropy": 0.7713335619488723, + "mlp_latest": 0.5146652460098267, + "mlp_max": 0.5146652460098267, + "normalized_gradient": 1.548088097624808, + "offset": 4, + "per_seed_normalized_gradient": [ + 1.5759251613516239, + 1.7615672973682381, + 1.306771834154562 + ] + }, + { + "attention_entropy": 0.9336027031788025, + "attention_latest": 0.3171870708465576, + "attention_max": 0.3171870708465576, + "group": 7, + "layer": 25, + "mlp_entropy": 0.7657190825854533, + "mlp_latest": 0.3575698534647624, + "mlp_max": 0.3722212314605713, + "normalized_gradient": 1.7685028111508305, + "offset": 1, + "per_seed_normalized_gradient": [ + 2.2647260999477155, + 1.8179181698124194, + 1.2228641636923565 + ] + }, + { + "attention_entropy": 0.8756262007907392, + "attention_latest": 0.3432680070400238, + "attention_max": 0.3432680070400238, + "group": 7, + "layer": 26, + "mlp_entropy": 0.7635627832181188, + "mlp_latest": 0.4196455279986064, + "mlp_max": 0.4196455279986064, + "normalized_gradient": 1.3690565893386435, + "offset": 2, + "per_seed_normalized_gradient": [ + 1.6880724220748888, + 1.4871187506667403, + 0.9319785952743014 + ] + }, + { + "attention_entropy": 0.891179394493967, + "attention_latest": 0.32520759105682373, + "attention_max": 0.32520759105682373, + "group": 7, + "layer": 27, + "mlp_entropy": 0.7748494313187421, + "mlp_latest": 0.40247613191604614, + "mlp_max": 0.40247613191604614, + "normalized_gradient": 0.9864745886014125, + "offset": 3, + "per_seed_normalized_gradient": [ + 1.092454232265147, + 1.086854135909831, + 0.7801153976292595 + ] + }, + { + "attention_entropy": 0.9019494449297653, + "attention_latest": 0.2957511991262436, + "attention_max": 0.2957511991262436, + "group": 7, + "layer": 28, + "mlp_entropy": 0.821866239503451, + "mlp_latest": 0.30352112650871277, + "mlp_max": 0.3173763652642568, + "normalized_gradient": 0.8024500092961132, + "offset": 4, + "per_seed_normalized_gradient": [ + 0.8732751258075867, + 0.872703317202576, + 0.661371584878177 + ] + } + ] + }, + "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", + "runs": [ + { + "canonical_sha256": "76b0ccfb55c38baef50c395788ac4b351cbe0d58de70064b50702acb5c93f515", + "final_bpc": 1.7123525940760473, + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "seed": 2026073001, + "timing": { + "mean_ms": 224.20719212223366, + "measured_steps": 7980, + "median_ms": 227.38306851533707, + "p95_ms": 230.37451171875, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + } + }, + { + "canonical_sha256": "5352c74eca853b375c0e85933dafd7c5916c39fc59052e742ca14ffd6d68bc78", + "final_bpc": 1.7093240655856785, + "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1", + "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "seed": 2026073002, + "timing": { + "mean_ms": 223.82424076982844, + "measured_steps": 7980, + "median_ms": 227.36876149429008, + "p95_ms": 229.48265075683594, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + } + }, + { + "canonical_sha256": "7e764c07e90b78c4cd0acc2e99600225f16428cbb25d5188a0a5a8fe797f8766", + "final_bpc": 1.7030966812870931, + "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76", + "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "seed": 2026073003, + "timing": { + "mean_ms": 224.65581085475935, + "measured_steps": 7980, + "median_ms": 227.5267465010984, + "p95_ms": 230.52696228027344, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + } + } + ], + "schema_version": 1, + "study": { + "architecture": "block", + "depth": 32, + "identity": "targeted follow-up informed by Round 05; not blind discovery", + "seeds": [ + 2026073001, + 2026073002, + 2026073003 + ], + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "steps": 8000 + }, + "trajectory": [ + { + "points": [ + { + "bits_per_byte": 7.9557801867581395, + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "spike_contrast": 0.7816308178118705, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "peak_layer": 1, + "peak_normalized": 2.145023601546889, + "population_cv": 0.5072011380661324, + "spike_contrast": 0.5596113913680666, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "peak_layer": 2, + "peak_normalized": 2.1343785185339303, + "population_cv": 0.5868596491376813, + "spike_contrast": 0.5221542367874475, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "peak_layer": 21, + "peak_normalized": 2.863607509132394, + "population_cv": 0.5384894543542103, + "spike_contrast": 2.4966108002124767, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "peak_layer": 21, + "peak_normalized": 3.115049509935762, + "population_cv": 0.6027314599912318, + "spike_contrast": 2.8510104941924466, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "spike_contrast": 3.04626897714722, + "step": 8000 + } + ], + "seed": 2026073001 + }, + { + "points": [ + { + "bits_per_byte": 8.034013546791773, + "peak_layer": 1, + "peak_normalized": 2.16920774571012, + "population_cv": 0.39438035124793563, + "spike_contrast": 0.6908033638721538, + "step": 0 + }, + { + "bits_per_byte": 5.132431212443787, + "peak_layer": 1, + "peak_normalized": 2.254829765809383, + "population_cv": 0.5330470009230729, + "spike_contrast": 0.5561530270868602, + "step": 100 + }, + { + "bits_per_byte": 3.0911456537212687, + "peak_layer": 2, + "peak_normalized": 2.173061151786652, + "population_cv": 0.5858213540607353, + "spike_contrast": 0.5734017265344913, + "step": 500 + }, + { + "bits_per_byte": 1.947918907793682, + "peak_layer": 21, + "peak_normalized": 3.189751492495938, + "population_cv": 0.6175097196118516, + "spike_contrast": 2.740373880872814, + "step": 2000 + }, + { + "bits_per_byte": 1.7911587423283797, + "peak_layer": 21, + "peak_normalized": 3.420348943656972, + "population_cv": 0.6827037752697015, + "spike_contrast": 3.1572107254761765, + "step": 4000 + }, + { + "bits_per_byte": 1.7093240655856785, + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "spike_contrast": 3.3328477735441013, + "step": 8000 + } + ], + "seed": 2026073002 + }, + { + "points": [ + { + "bits_per_byte": 8.061194200882085, + "peak_layer": 1, + "peak_normalized": 1.7301666786506156, + "population_cv": 0.2498932502086342, + "spike_contrast": 0.7974048446155259, + "step": 0 + }, + { + "bits_per_byte": 5.191908918496252, + "peak_layer": 1, + "peak_normalized": 2.133789250541258, + "population_cv": 0.49221598973047537, + "spike_contrast": 0.5746026267549044, + "step": 100 + }, + { + "bits_per_byte": 3.0881952483500545, + "peak_layer": 2, + "peak_normalized": 2.195594858531836, + "population_cv": 0.606636113722199, + "spike_contrast": 0.47779529155407807, + "step": 500 + }, + { + "bits_per_byte": 1.948972795980338, + "peak_layer": 21, + "peak_normalized": 1.8057756999530474, + "population_cv": 0.34914284904214865, + "spike_contrast": 1.4606649819879878, + "step": 2000 + }, + { + "bits_per_byte": 1.7872753095784035, + "peak_layer": 21, + "peak_normalized": 2.1130451955668876, + "population_cv": 0.3859968358313239, + "spike_contrast": 1.7734534826874866, + "step": 4000 + }, + { + "bits_per_byte": 1.7030966812870931, + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "spike_contrast": 1.8814916302334765, + "step": 8000 + } + ], + "seed": 2026073003 + } + ], + "verdicts": { + "interventions": { + "forward_identity": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "scope": "all 64 depth mixers plus the output mixer", + "softmax_key_path": { + "label": "global removal of all mixer softmax/query/key source-gradient paths", + "mean_relative_drop_contrast": -0.0200106813233286, + "mean_relative_drop_peak": -0.008401901006156563, + "opposite_direction_3_of_3": true, + "passed_3_of_3": false, + "per_seed": [ + { + "passed": false, + "relative_drop_contrast": -0.015414390900329828, + "relative_drop_peak": -0.006668985338663224, + "seed": 2026073001, + "source_peak_normalized": 3.219066994145898, + "source_spike_contrast": 3.04626897714722, + "target_peak_normalized": 3.2405349047340315, + "target_spike_contrast": 3.093225357948515 + }, + { + "passed": false, + "relative_drop_contrast": -0.01929133294667749, + "relative_drop_peak": -0.003788054520504258, + "seed": 2026073002, + "source_peak_normalized": 3.6189242296270336, + "source_spike_contrast": 3.3328477735441013, + "target_peak_normalized": 3.6326329119144347, + "target_spike_contrast": 3.3971428496041334 + }, + { + "passed": false, + "relative_drop_contrast": -0.025326320122978477, + "relative_drop_peak": -0.014748663159302204, + "seed": 2026073003, + "source_peak_normalized": 2.2882673690329693, + "source_spike_contrast": 1.8814916302334765, + "target_peak_normalized": 2.3220162536772593, + "target_spike_contrast": 1.9291428895694742 + } + ], + "same_direction_3_of_3": false, + "source_mode": "learned", + "target_mode": "detached_learned", + "threshold": 0.2, + "verdict": "opposite direction in 3/3 seeds; no material reduction" + }, + "value_coefficients": { + "label": "global replacement of learned value-backward coefficients with 1/N", + "mean_relative_drop_contrast": 0.701998806119879, + "mean_relative_drop_peak": 0.3702754647109951, + "opposite_direction_3_of_3": false, + "passed_3_of_3": true, + "per_seed": [ + { + "passed": true, + "relative_drop_contrast": 0.7479318970064935, + "relative_drop_peak": 0.3925047809741125, + "seed": 2026073001, + "source_peak_normalized": 3.2405349047340315, + "source_spike_contrast": 3.093225357948515, + "target_peak_normalized": 1.968609461712434, + "target_spike_contrast": 0.7797034481094922 + }, + { + "passed": true, + "relative_drop_contrast": 0.7687299974719846, + "relative_drop_peak": 0.4318458552099138, + "seed": 2026073002, + "source_peak_normalized": 3.6326329119144347, + "source_spike_contrast": 3.3971428496041334, + "target_peak_normalized": 2.063895445405066, + "target_spike_contrast": 0.7856572354159773 + }, + { + "passed": true, + "relative_drop_contrast": 0.5893345238811586, + "relative_drop_peak": 0.2864757579489588, + "seed": 2026073003, + "source_peak_normalized": 2.3220162536772593, + "source_spike_contrast": 1.9291428895694742, + "target_peak_normalized": 1.6568148874352646, + "target_spike_contrast": 0.7922323832463255 + } + ], + "same_direction_3_of_3": true, + "source_mode": "detached_learned", + "target_mode": "uniform_value_backward", + "threshold": 0.2, + "verdict": "material sensitivity at the preregistered threshold" + } + }, + "reduction_robustness": { + "cells": [ + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "element_rms", + "seed": 2026073001, + "spearman_vs_element_rms": 1.0, + "spike_contrast": 3.04626897714722, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_mean", + "seed": 2026073001, + "spearman_vs_element_rms": 0.9578445747800587, + "spike_contrast": 2.869084843241085, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_median", + "seed": 2026073001, + "spearman_vs_element_rms": 0.9013929618768328, + "spike_contrast": 2.6456990159702416, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_p95", + "seed": 2026073001, + "spearman_vs_element_rms": 0.9967008797653959, + "spike_contrast": 3.143466990402101, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "element_rms", + "seed": 2026073002, + "spearman_vs_element_rms": 1.0, + "spike_contrast": 3.3328477735441013, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_mean", + "seed": 2026073002, + "spearman_vs_element_rms": 0.9732404692082112, + "spike_contrast": 3.18249263251226, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_median", + "seed": 2026073002, + "spearman_vs_element_rms": 0.8951612903225806, + "spike_contrast": 2.9052606637917697, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 18 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_p95", + "seed": 2026073002, + "spearman_vs_element_rms": 0.9970674486803519, + "spike_contrast": 3.5324549012779802, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "element_rms", + "seed": 2026073003, + "spearman_vs_element_rms": 1.0, + "spike_contrast": 1.8814916302334765, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_mean", + "seed": 2026073003, + "spearman_vs_element_rms": 0.9875366568914956, + "spike_contrast": 1.8077155175462722, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_median", + "seed": 2026073003, + "spearman_vs_element_rms": 0.9343841642228738, + "spike_contrast": 1.6658967617763285, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_p95", + "seed": 2026073003, + "spearman_vs_element_rms": 0.9926686217008798, + "spike_contrast": 2.0122799667677387, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + } + ], + "passed_cells": 12, + "total_cells": 12, + "verdict": "robust within the preregistered reduction family" + }, + "visible_positions": { + "criterion": "step 8000 element-RMS spike contrast >= 1.5 in 3/3 seeds", + "earliest_observed_tensor": "pre_attention_input", + "positions": [ + { + "mean_spike_contrast": 2.9842437043760683, + "per_seed": [ + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 4.232714993086349, + "seed": 2026073001, + "spike_contrast": 3.7241794809927478 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.51885122958336, + "seed": 2026073002, + "spike_contrast": 3.1885950432011234 + }, + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 2.5497535502321647, + "seed": 2026073003, + "spike_contrast": 2.0399565889343343 + } + ], + "position": "pre_attention_input", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.9553267731281916, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.138964807406113, + "seed": 2026073001, + "spike_contrast": 3.1996375366532934 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.7925106998539326, + "seed": 2026073002, + "spike_contrast": 3.6616605179769537 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.2656295070514414, + "seed": 2026073003, + "spike_contrast": 2.004682264754327 + } + ], + "position": "attention_branch_output", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.9553191807218897, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.1389231589965445, + "seed": 2026073001, + "spike_contrast": 3.1996384204494492 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.7925041485004933, + "seed": 2026073002, + "spike_contrast": 3.6616533387138865 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.265619973525555, + "seed": 2026073003, + "spike_contrast": 2.0046657830023333 + } + ], + "position": "post_attention_state", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 3.7778825845934967, + "per_seed": [ + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 2.987231504916195, + "seed": 2026073001, + "spike_contrast": 3.828767846228416 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.828274701557073, + "seed": 2026073002, + "spike_contrast": 4.52698797699541 + }, + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 2.526943137473127, + "seed": 2026073003, + "spike_contrast": 2.977891930556664 + } + ], + "position": "pre_mlp_input", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.7535328779602537, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.2190719364475022, + "seed": 2026073001, + "spike_contrast": 3.0462648281411457 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.6189237818679025, + "seed": 2026073002, + "spike_contrast": 3.3328561680843984 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.2882576030527964, + "seed": 2026073003, + "spike_contrast": 1.8814776376552176 + } + ], + "position": "mlp_branch_output", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.7535361269749323, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "seed": 2026073001, + "spike_contrast": 3.04626897714722 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "seed": 2026073002, + "spike_contrast": 3.3328477735441013 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "seed": 2026073003, + "spike_contrast": 1.8814916302334765 + } + ], + "position": "post_mlp_state", + "visible_3_of_3": true + } + ], + "verdict": "visible at one or more positions", + "visible_positions": [ + "pre_attention_input", + "attention_branch_output", + "post_attention_state", + "pre_mlp_input", + "mlp_branch_output", + "post_mlp_state" + ] + } + } +} diff --git a/src/data/k3-attnres-spike.json b/src/data/k3-attnres-spike.json new file mode 100644 index 0000000..25d2c40 --- /dev/null +++ b/src/data/k3-attnres-spike.json @@ -0,0 +1,24540 @@ +{ + "canonical_sha256_without_self": "1d0faef297682d61d35ce2ab54da3868e18a84e9ebc6e44c617d08016e4fc8b4", + "final_arrays": [ + { + "final_mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.22886337339878082, + "group": 1, + "latest_source_mass": 0.06350618600845337, + "layer": 1, + "max_source_mass": 0.9364938139915466, + "mixer_index": 1, + "normalized_entropy": 0.3301800538435402, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9364938139915466, + "median": 0.9452826976776123, + "p05": 0.8725647330284119, + "p95": 0.9712384939193726 + }, + { + "label": "current_group_partial", + "mean": 0.06350618600845337, + "median": 0.0547172985970974, + "p05": 0.028761589899659157, + "p95": 0.12743529677391052 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.603729784488678, + "group": 1, + "latest_source_mass": 0.4133155345916748, + "layer": 2, + "max_source_mass": 0.5866844654083252, + "mixer_index": 2, + "normalized_entropy": 0.8709979661187783, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5866844654083252, + "median": 0.5734395384788513, + "p05": 0.29813364148139954, + "p95": 0.8843642473220825 + }, + { + "label": "current_group_partial", + "mean": 0.4133155345916748, + "median": 0.4265604615211487, + "p05": 0.11563576757907867, + "p95": 0.7018663287162781 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.3967982828617096, + "group": 1, + "latest_source_mass": 0.14110636711120605, + "layer": 2, + "max_source_mass": 0.858893632888794, + "mixer_index": 3, + "normalized_entropy": 0.5724589149178446, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.858893632888794, + "median": 0.8705973029136658, + "p05": 0.7599387168884277, + "p95": 0.9270997643470764 + }, + { + "label": "current_group_partial", + "mean": 0.14110636711120605, + "median": 0.12940272688865662, + "p05": 0.07290027290582657, + "p95": 0.2400612235069275 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6852496862411499, + "group": 1, + "latest_source_mass": 0.518975019454956, + "layer": 3, + "max_source_mass": 0.518975019454956, + "mixer_index": 4, + "normalized_entropy": 0.9886063241108252, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48102498054504395, + "median": 0.48102718591690063, + "p05": 0.38786420226097107, + "p95": 0.580143928527832 + }, + { + "label": "current_group_partial", + "mean": 0.518975019454956, + "median": 0.5189728140830994, + "p05": 0.41985607147216797, + "p95": 0.6121357679367065 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5890734195709229, + "group": 1, + "latest_source_mass": 0.3123423159122467, + "layer": 3, + "max_source_mass": 0.6876576542854309, + "mixer_index": 5, + "normalized_entropy": 0.8498533011344741, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6876576542854309, + "median": 0.6909762620925903, + "p05": 0.48803940415382385, + "p95": 0.8900076746940613 + }, + { + "label": "current_group_partial", + "mean": 0.3123423159122467, + "median": 0.3090237081050873, + "p05": 0.10999235510826111, + "p95": 0.5119606256484985 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6873323321342468, + "group": 1, + "latest_source_mass": 0.4831485152244568, + "layer": 4, + "max_source_mass": 0.5168514251708984, + "mixer_index": 6, + "normalized_entropy": 0.9916109470127238, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5168514251708984, + "median": 0.5219584703445435, + "p05": 0.4251725673675537, + "p95": 0.5947866439819336 + }, + { + "label": "current_group_partial", + "mean": 0.4831485152244568, + "median": 0.47804147005081177, + "p05": 0.405213326215744, + "p95": 0.5748274326324463 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.487301766872406, + "group": 1, + "latest_source_mass": 0.2060384750366211, + "layer": 4, + "max_source_mass": 0.7939615249633789, + "mixer_index": 7, + "normalized_entropy": 0.7030278424832499, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7939615249633789, + "median": 0.7953404188156128, + "p05": 0.6590759754180908, + "p95": 0.9329437017440796 + }, + { + "label": "current_group_partial", + "mean": 0.2060384750366211, + "median": 0.20465964078903198, + "p05": 0.06705628335475922, + "p95": 0.3409239649772644 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.674697756767273, + "group": 2, + "latest_source_mass": 0.42952626943588257, + "layer": 5, + "max_source_mass": 0.5704737305641174, + "mixer_index": 8, + "normalized_entropy": 0.9733831077870527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5704737305641174, + "median": 0.5696132779121399, + "p05": 0.4695202708244324, + "p95": 0.6750621199607849 + }, + { + "label": "completed_group_1", + "mean": 0.42952626943588257, + "median": 0.4303867816925049, + "p05": 0.3249378800392151, + "p95": 0.5304797887802124 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.9691112041473389, + "group": 2, + "latest_source_mass": 0.2681158185005188, + "layer": 5, + "max_source_mass": 0.5506307482719421, + "mixer_index": 9, + "normalized_entropy": 0.8821230329784768, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5506307482719421, + "median": 0.543993353843689, + "p05": 0.41262108087539673, + "p95": 0.7161091566085815 + }, + { + "label": "completed_group_1", + "mean": 0.18125343322753906, + "median": 0.1794438362121582, + "p05": 0.11575750261545181, + "p95": 0.25031501054763794 + }, + { + "label": "current_group_partial", + "mean": 0.2681158185005188, + "median": 0.2664111256599426, + "p05": 0.1564716249704361, + "p95": 0.3817881643772125 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0745112895965576, + "group": 2, + "latest_source_mass": 0.412794291973114, + "layer": 6, + "max_source_mass": 0.412794291973114, + "mixer_index": 10, + "normalized_entropy": 0.9780623252441762, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.34262996912002563, + "median": 0.3412924110889435, + "p05": 0.29556700587272644, + "p95": 0.39125555753707886 + }, + { + "label": "completed_group_1", + "mean": 0.24457570910453796, + "median": 0.24382859468460083, + "p05": 0.22120657563209534, + "p95": 0.2734297513961792 + }, + { + "label": "current_group_partial", + "mean": 0.412794291973114, + "median": 0.4151499271392822, + "p05": 0.37735897302627563, + "p95": 0.43749260902404785 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074045658111572, + "group": 2, + "latest_source_mass": 0.3775783181190491, + "layer": 6, + "max_source_mass": 0.4399106502532959, + "mixer_index": 11, + "normalized_entropy": 0.9169791528842925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4399106502532959, + "median": 0.44856905937194824, + "p05": 0.22063353657722473, + "p95": 0.6359403133392334 + }, + { + "label": "completed_group_1", + "mean": 0.18251106142997742, + "median": 0.18562838435173035, + "p05": 0.1196163073182106, + "p95": 0.23045453429222107 + }, + { + "label": "current_group_partial", + "mean": 0.3775783181190491, + "median": 0.3675365447998047, + "p05": 0.23817461729049683, + "p95": 0.5557451844215393 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.060497522354126, + "group": 2, + "latest_source_mass": 0.4025024175643921, + "layer": 7, + "max_source_mass": 0.4025024175643921, + "mixer_index": 12, + "normalized_entropy": 0.9653064445872968, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3786153197288513, + "median": 0.3907395303249359, + "p05": 0.28039106726646423, + "p95": 0.4436405599117279 + }, + { + "label": "completed_group_1", + "mean": 0.2188822478055954, + "median": 0.21226292848587036, + "p05": 0.1885872334241867, + "p95": 0.2672030031681061 + }, + { + "label": "current_group_partial", + "mean": 0.4025024175643921, + "median": 0.39815032482147217, + "p05": 0.3533596992492676, + "p95": 0.460100919008255 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9941019415855408, + "group": 2, + "latest_source_mass": 0.4801652431488037, + "layer": 7, + "max_source_mass": 0.4801652431488037, + "mixer_index": 13, + "normalized_entropy": 0.9048705824970601, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32296955585479736, + "median": 0.2803739905357361, + "p05": 0.1784922033548355, + "p95": 0.6440245509147644 + }, + { + "label": "completed_group_1", + "mean": 0.19686520099639893, + "median": 0.2053825855255127, + "p05": 0.09813719242811203, + "p95": 0.2504696547985077 + }, + { + "label": "current_group_partial", + "mean": 0.4801652431488037, + "median": 0.5057140588760376, + "p05": 0.2529003322124481, + "p95": 0.598870575428009 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0503522157669067, + "group": 2, + "latest_source_mass": 0.4001518487930298, + "layer": 8, + "max_source_mass": 0.40130966901779175, + "mixer_index": 14, + "normalized_entropy": 0.9560717885654542, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40130966901779175, + "median": 0.396759569644928, + "p05": 0.3479611873626709, + "p95": 0.4733521044254303 + }, + { + "label": "completed_group_1", + "mean": 0.19853851199150085, + "median": 0.19769755005836487, + "p05": 0.17136667668819427, + "p95": 0.22788912057876587 + }, + { + "label": "current_group_partial", + "mean": 0.4001518487930298, + "median": 0.40291696786880493, + "p05": 0.3506356179714203, + "p95": 0.43941178917884827 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.0129402875900269, + "group": 2, + "latest_source_mass": 0.42161279916763306, + "layer": 8, + "max_source_mass": 0.42161279916763306, + "mixer_index": 15, + "normalized_entropy": 0.9220179839951123, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4093267023563385, + "median": 0.4046899676322937, + "p05": 0.2895161807537079, + "p95": 0.5762131810188293 + }, + { + "label": "completed_group_1", + "mean": 0.16906049847602844, + "median": 0.16888955235481262, + "p05": 0.11233551055192947, + "p95": 0.22477403283119202 + }, + { + "label": "current_group_partial", + "mean": 0.42161279916763306, + "median": 0.4227708578109741, + "p05": 0.3131049573421478, + "p95": 0.519075334072113 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0774019956588745, + "group": 3, + "latest_source_mass": 0.3560434877872467, + "layer": 9, + "max_source_mass": 0.38510820269584656, + "mixer_index": 16, + "normalized_entropy": 0.9806935592947451, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.38510820269584656, + "median": 0.3892287313938141, + "p05": 0.2934730052947998, + "p95": 0.4718902111053467 + }, + { + "label": "completed_group_1", + "mean": 0.25884830951690674, + "median": 0.25734007358551025, + "p05": 0.22217319905757904, + "p95": 0.29551801085472107 + }, + { + "label": "completed_group_2", + "mean": 0.3560434877872467, + "median": 0.35412120819091797, + "p05": 0.29444968700408936, + "p95": 0.41908568143844604 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2740377187728882, + "group": 3, + "latest_source_mass": 0.3633861243724823, + "layer": 9, + "max_source_mass": 0.3633861243724823, + "mixer_index": 17, + "normalized_entropy": 0.9190239493895668, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29550763964653015, + "median": 0.30360478162765503, + "p05": 0.16904757916927338, + "p95": 0.45146965980529785 + }, + { + "label": "completed_group_1", + "mean": 0.09541244804859161, + "median": 0.09486871212720871, + "p05": 0.061360105872154236, + "p95": 0.13037073612213135 + }, + { + "label": "completed_group_2", + "mean": 0.24569380283355713, + "median": 0.24333015084266663, + "p05": 0.1698218435049057, + "p95": 0.31589236855506897 + }, + { + "label": "current_group_partial", + "mean": 0.3633861243724823, + "median": 0.36472851037979126, + "p05": 0.2795379161834717, + "p95": 0.4399876892566681 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.360302448272705, + "group": 3, + "latest_source_mass": 0.28569501638412476, + "layer": 10, + "max_source_mass": 0.3109527826309204, + "mixer_index": 18, + "normalized_entropy": 0.9812507981160736, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3109527826309204, + "median": 0.32172977924346924, + "p05": 0.23520725965499878, + "p95": 0.37052470445632935 + }, + { + "label": "completed_group_1", + "mean": 0.20984819531440735, + "median": 0.20958667993545532, + "p05": 0.1891367882490158, + "p95": 0.23184512555599213 + }, + { + "label": "completed_group_2", + "mean": 0.19350400567054749, + "median": 0.19153150916099548, + "p05": 0.16607457399368286, + "p95": 0.22877082228660583 + }, + { + "label": "current_group_partial", + "mean": 0.28569501638412476, + "median": 0.28060320019721985, + "p05": 0.23824527859687805, + "p95": 0.3439165949821472 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2722704410552979, + "group": 3, + "latest_source_mass": 0.29912811517715454, + "layer": 10, + "max_source_mass": 0.35036465525627136, + "mixer_index": 19, + "normalized_entropy": 0.9177491279900463, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35036465525627136, + "median": 0.36210548877716064, + "p05": 0.12807878851890564, + "p95": 0.5797606706619263 + }, + { + "label": "completed_group_1", + "mean": 0.12631990015506744, + "median": 0.1264108121395111, + "p05": 0.06977735459804535, + "p95": 0.1755298376083374 + }, + { + "label": "completed_group_2", + "mean": 0.22418735921382904, + "median": 0.22563719749450684, + "p05": 0.1176326721906662, + "p95": 0.32001155614852905 + }, + { + "label": "current_group_partial", + "mean": 0.29912811517715454, + "median": 0.28589946031570435, + "p05": 0.193109393119812, + "p95": 0.44119665026664734 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3558036088943481, + "group": 3, + "latest_source_mass": 0.314197838306427, + "layer": 11, + "max_source_mass": 0.314197838306427, + "mixer_index": 20, + "normalized_entropy": 0.978005571485618, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298976868391037, + "median": 0.2961939573287964, + "p05": 0.24950163066387177, + "p95": 0.35835936665534973 + }, + { + "label": "completed_group_1", + "mean": 0.1941717267036438, + "median": 0.1918032318353653, + "p05": 0.16809208691120148, + "p95": 0.22895419597625732 + }, + { + "label": "completed_group_2", + "mean": 0.1926535815000534, + "median": 0.19306811690330505, + "p05": 0.16983944177627563, + "p95": 0.2155935913324356 + }, + { + "label": "current_group_partial", + "mean": 0.314197838306427, + "median": 0.31541913747787476, + "p05": 0.2673022747039795, + "p95": 0.35941022634506226 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2696614265441895, + "group": 3, + "latest_source_mass": 0.2862822115421295, + "layer": 11, + "max_source_mass": 0.3667730689048767, + "mixer_index": 21, + "normalized_entropy": 0.9158671218416545, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3667730689048767, + "median": 0.4067659378051758, + "p05": 0.10593366622924805, + "p95": 0.5672274827957153 + }, + { + "label": "completed_group_1", + "mean": 0.15310388803482056, + "median": 0.15050218999385834, + "p05": 0.08651046454906464, + "p95": 0.22459912300109863 + }, + { + "label": "completed_group_2", + "mean": 0.19384080171585083, + "median": 0.18732509016990662, + "p05": 0.1267009675502777, + "p95": 0.281107634305954 + }, + { + "label": "current_group_partial", + "mean": 0.2862822115421295, + "median": 0.26271265745162964, + "p05": 0.17024393379688263, + "p95": 0.475044310092926 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3606936931610107, + "group": 3, + "latest_source_mass": 0.3028184771537781, + "layer": 12, + "max_source_mass": 0.3028184771537781, + "mixer_index": 22, + "normalized_entropy": 0.9815330216461395, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.298227459192276, + "median": 0.2954302132129669, + "p05": 0.25287362933158875, + "p95": 0.35405808687210083 + }, + { + "label": "completed_group_1", + "mean": 0.18611131608486176, + "median": 0.1850002110004425, + "p05": 0.15901757776737213, + "p95": 0.21725836396217346 + }, + { + "label": "completed_group_2", + "mean": 0.21284276247024536, + "median": 0.21402311325073242, + "p05": 0.18819917738437653, + "p95": 0.23418807983398438 + }, + { + "label": "current_group_partial", + "mean": 0.3028184771537781, + "median": 0.3041122555732727, + "p05": 0.25661808252334595, + "p95": 0.34458184242248535 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.269316554069519, + "group": 3, + "latest_source_mass": 0.2653314769268036, + "layer": 12, + "max_source_mass": 0.4081852436065674, + "mixer_index": 23, + "normalized_entropy": 0.9156183489371815, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4081852436065674, + "median": 0.4254342317581177, + "p05": 0.20765496790409088, + "p95": 0.587238073348999 + }, + { + "label": "completed_group_1", + "mean": 0.14420127868652344, + "median": 0.1438048928976059, + "p05": 0.1064920499920845, + "p95": 0.1871720254421234 + }, + { + "label": "completed_group_2", + "mean": 0.18228203058242798, + "median": 0.17507661879062653, + "p05": 0.12183455377817154, + "p95": 0.26505547761917114 + }, + { + "label": "current_group_partial", + "mean": 0.2653314769268036, + "median": 0.24924281239509583, + "p05": 0.15609969198703766, + "p95": 0.4061311185359955 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3636136054992676, + "group": 4, + "latest_source_mass": 0.3136237859725952, + "layer": 13, + "max_source_mass": 0.3136237859725952, + "mixer_index": 24, + "normalized_entropy": 0.9836392931712563, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27427661418914795, + "median": 0.27403637766838074, + "p05": 0.22928935289382935, + "p95": 0.319773405790329 + }, + { + "label": "completed_group_1", + "mean": 0.21341991424560547, + "median": 0.20697325468063354, + "p05": 0.18552586436271667, + "p95": 0.26688331365585327 + }, + { + "label": "completed_group_2", + "mean": 0.19867971539497375, + "median": 0.197545126080513, + "p05": 0.1794913411140442, + "p95": 0.222100168466568 + }, + { + "label": "completed_group_3", + "mean": 0.3136237859725952, + "median": 0.31875866651535034, + "p05": 0.24156072735786438, + "p95": 0.3695686459541321 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.570773959159851, + "group": 4, + "latest_source_mass": 0.2294243425130844, + "layer": 13, + "max_source_mass": 0.2658740282058716, + "mixer_index": 25, + "normalized_entropy": 0.9759767351225285, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2658740282058716, + "median": 0.2747163474559784, + "p05": 0.17927880585193634, + "p95": 0.3512483835220337 + }, + { + "label": "completed_group_1", + "mean": 0.13419361412525177, + "median": 0.13374894857406616, + "p05": 0.11072897166013718, + "p95": 0.1586270034313202 + }, + { + "label": "completed_group_2", + "mean": 0.1907704919576645, + "median": 0.19074805080890656, + "p05": 0.14699944853782654, + "p95": 0.23316305875778198 + }, + { + "label": "completed_group_3", + "mean": 0.17973749339580536, + "median": 0.17616114020347595, + "p05": 0.1363743096590042, + "p95": 0.23202598094940186 + }, + { + "label": "current_group_partial", + "mean": 0.2294243425130844, + "median": 0.22995705902576447, + "p05": 0.17975632846355438, + "p95": 0.2789157032966614 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5560038089752197, + "group": 4, + "latest_source_mass": 0.32468360662460327, + "layer": 14, + "max_source_mass": 0.32468360662460327, + "mixer_index": 26, + "normalized_entropy": 0.9667995248241249, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20477107167243958, + "median": 0.206403911113739, + "p05": 0.16619175672531128, + "p95": 0.2431035190820694 + }, + { + "label": "completed_group_1", + "mean": 0.15721669793128967, + "median": 0.15658551454544067, + "p05": 0.13878680765628815, + "p95": 0.17778633534908295 + }, + { + "label": "completed_group_2", + "mean": 0.13811248540878296, + "median": 0.13743364810943604, + "p05": 0.12352626770734787, + "p95": 0.15553249418735504 + }, + { + "label": "completed_group_3", + "mean": 0.1752161681652069, + "median": 0.17309701442718506, + "p05": 0.15223677456378937, + "p95": 0.20474471151828766 + }, + { + "label": "current_group_partial", + "mean": 0.32468360662460327, + "median": 0.32497870922088623, + "p05": 0.2946181893348694, + "p95": 0.35203880071640015 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5659973621368408, + "group": 4, + "latest_source_mass": 0.26703086495399475, + "layer": 14, + "max_source_mass": 0.26703086495399475, + "mixer_index": 27, + "normalized_entropy": 0.9730088685238187, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20531430840492249, + "median": 0.20229148864746094, + "p05": 0.10698921978473663, + "p95": 0.3393132984638214 + }, + { + "label": "completed_group_1", + "mean": 0.15986156463623047, + "median": 0.15952220559120178, + "p05": 0.11969029903411865, + "p95": 0.20180393755435944 + }, + { + "label": "completed_group_2", + "mean": 0.21557100117206573, + "median": 0.21654027700424194, + "p05": 0.1673085242509842, + "p95": 0.2610440254211426 + }, + { + "label": "completed_group_3", + "mean": 0.15222224593162537, + "median": 0.1510905921459198, + "p05": 0.11800260841846466, + "p95": 0.18893754482269287 + }, + { + "label": "current_group_partial", + "mean": 0.26703086495399475, + "median": 0.258539617061615, + "p05": 0.19330690801143646, + "p95": 0.36521556973457336 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5552496910095215, + "group": 4, + "latest_source_mass": 0.31774693727493286, + "layer": 15, + "max_source_mass": 0.31774693727493286, + "mixer_index": 28, + "normalized_entropy": 0.9663309649872576, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2162160575389862, + "median": 0.2145705670118332, + "p05": 0.17727568745613098, + "p95": 0.26878347992897034 + }, + { + "label": "completed_group_1", + "mean": 0.17982497811317444, + "median": 0.18024154007434845, + "p05": 0.15453973412513733, + "p95": 0.2030555009841919 + }, + { + "label": "completed_group_2", + "mean": 0.1307673156261444, + "median": 0.13107559084892273, + "p05": 0.11463287472724915, + "p95": 0.14630235731601715 + }, + { + "label": "completed_group_3", + "mean": 0.1554446816444397, + "median": 0.15516987442970276, + "p05": 0.13627474009990692, + "p95": 0.1754152774810791 + }, + { + "label": "current_group_partial", + "mean": 0.31774693727493286, + "median": 0.319039523601532, + "p05": 0.27606821060180664, + "p95": 0.35650834441185 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5068919658660889, + "group": 4, + "latest_source_mass": 0.3298327624797821, + "layer": 15, + "max_source_mass": 0.3298327624797821, + "mixer_index": 29, + "normalized_entropy": 0.9362846209998111, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20646697282791138, + "median": 0.18775638937950134, + "p05": 0.0806753933429718, + "p95": 0.4274584949016571 + }, + { + "label": "completed_group_1", + "mean": 0.1197461187839508, + "median": 0.11989086866378784, + "p05": 0.0809398666024208, + "p95": 0.15743419528007507 + }, + { + "label": "completed_group_2", + "mean": 0.19215428829193115, + "median": 0.19620640575885773, + "p05": 0.1148109883069992, + "p95": 0.2491212636232376 + }, + { + "label": "completed_group_3", + "mean": 0.15179988741874695, + "median": 0.15212899446487427, + "p05": 0.10697861760854721, + "p95": 0.19542087614536285 + }, + { + "label": "current_group_partial", + "mean": 0.3298327624797821, + "median": 0.32837340235710144, + "p05": 0.22746729850769043, + "p95": 0.4339301884174347 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5435223579406738, + "group": 4, + "latest_source_mass": 0.3224271535873413, + "layer": 16, + "max_source_mass": 0.3224271535873413, + "mixer_index": 30, + "normalized_entropy": 0.9590443632623663, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.22410975396633148, + "median": 0.22760415077209473, + "p05": 0.14593973755836487, + "p95": 0.30093055963516235 + }, + { + "label": "completed_group_1", + "mean": 0.16986176371574402, + "median": 0.16963240504264832, + "p05": 0.13569067418575287, + "p95": 0.20651353895664215 + }, + { + "label": "completed_group_2", + "mean": 0.13080403208732605, + "median": 0.13075244426727295, + "p05": 0.10945598781108856, + "p95": 0.15187400579452515 + }, + { + "label": "completed_group_3", + "mean": 0.15279728174209595, + "median": 0.15036612749099731, + "p05": 0.12810495495796204, + "p95": 0.18468116223812103 + }, + { + "label": "current_group_partial", + "mean": 0.3224271535873413, + "median": 0.3164973258972168, + "p05": 0.2600978910923004, + "p95": 0.40530598163604736 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4949920177459717, + "group": 4, + "latest_source_mass": 0.31847020983695984, + "layer": 16, + "max_source_mass": 0.31847020983695984, + "mixer_index": 31, + "normalized_entropy": 0.9288907675133354, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27174079418182373, + "median": 0.2585451304912567, + "p05": 0.13639721274375916, + "p95": 0.4698348045349121 + }, + { + "label": "completed_group_1", + "mean": 0.12178683280944824, + "median": 0.11714950203895569, + "p05": 0.08594125509262085, + "p95": 0.1723237782716751 + }, + { + "label": "completed_group_2", + "mean": 0.1529826819896698, + "median": 0.15223106741905212, + "p05": 0.10371473431587219, + "p95": 0.203575000166893 + }, + { + "label": "completed_group_3", + "mean": 0.1350194811820984, + "median": 0.13363957405090332, + "p05": 0.08964549005031586, + "p95": 0.18468207120895386 + }, + { + "label": "current_group_partial", + "mean": 0.31847020983695984, + "median": 0.3185216188430786, + "p05": 0.21444711089134216, + "p95": 0.4301546812057495 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.544264793395996, + "group": 5, + "latest_source_mass": 0.2861037850379944, + "layer": 17, + "max_source_mass": 0.2861037850379944, + "mixer_index": 32, + "normalized_entropy": 0.9595056643474137, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24016115069389343, + "median": 0.23868846893310547, + "p05": 0.18435582518577576, + "p95": 0.2983453571796417 + }, + { + "label": "completed_group_1", + "mean": 0.20730282366275787, + "median": 0.1998077780008316, + "p05": 0.16255050897598267, + "p95": 0.28107208013534546 + }, + { + "label": "completed_group_2", + "mean": 0.09692573547363281, + "median": 0.0967404842376709, + "p05": 0.08098706603050232, + "p95": 0.11221462488174438 + }, + { + "label": "completed_group_3", + "mean": 0.1695065200328827, + "median": 0.1696585714817047, + "p05": 0.136396586894989, + "p95": 0.20130634307861328 + }, + { + "label": "completed_group_4", + "mean": 0.2861037850379944, + "median": 0.28883200883865356, + "p05": 0.22401469945907593, + "p95": 0.3409370183944702 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.686682939529419, + "group": 5, + "latest_source_mass": 0.12281867861747742, + "layer": 17, + "max_source_mass": 0.2885604202747345, + "mixer_index": 33, + "normalized_entropy": 0.9413556721740635, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16755425930023193, + "median": 0.1327776312828064, + "p05": 0.0790192037820816, + "p95": 0.3667412996292114 + }, + { + "label": "completed_group_1", + "mean": 0.10962115228176117, + "median": 0.10525335371494293, + "p05": 0.07840172946453094, + "p95": 0.15405339002609253 + }, + { + "label": "completed_group_2", + "mean": 0.1783454418182373, + "median": 0.17896884679794312, + "p05": 0.10802385956048965, + "p95": 0.24048911035060883 + }, + { + "label": "completed_group_3", + "mean": 0.13310004770755768, + "median": 0.13033035397529602, + "p05": 0.0904831737279892, + "p95": 0.17934636771678925 + }, + { + "label": "completed_group_4", + "mean": 0.2885604202747345, + "median": 0.2831825315952301, + "p05": 0.16967125236988068, + "p95": 0.4189499616622925 + }, + { + "label": "current_group_partial", + "mean": 0.12281867861747742, + "median": 0.12062528729438782, + "p05": 0.07461707293987274, + "p95": 0.1782219111919403 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7070834636688232, + "group": 5, + "latest_source_mass": 0.2988377809524536, + "layer": 18, + "max_source_mass": 0.2988377809524536, + "mixer_index": 34, + "normalized_entropy": 0.9527414214834803, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.181177020072937, + "median": 0.18419909477233887, + "p05": 0.13624237477779388, + "p95": 0.22192169725894928 + }, + { + "label": "completed_group_1", + "mean": 0.14557164907455444, + "median": 0.144476979970932, + "p05": 0.1293315887451172, + "p95": 0.16486899554729462 + }, + { + "label": "completed_group_2", + "mean": 0.08318156749010086, + "median": 0.0820261612534523, + "p05": 0.07088497281074524, + "p95": 0.0990293100476265 + }, + { + "label": "completed_group_3", + "mean": 0.11424209177494049, + "median": 0.11267061531543732, + "p05": 0.0979941189289093, + "p95": 0.1353992223739624 + }, + { + "label": "completed_group_4", + "mean": 0.17698991298675537, + "median": 0.17597860097885132, + "p05": 0.14837387204170227, + "p95": 0.21050144731998444 + }, + { + "label": "current_group_partial", + "mean": 0.2988377809524536, + "median": 0.2989804744720459, + "p05": 0.2721370458602905, + "p95": 0.32572072744369507 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.7419235706329346, + "group": 5, + "latest_source_mass": 0.26016175746917725, + "layer": 18, + "max_source_mass": 0.26016175746917725, + "mixer_index": 35, + "normalized_entropy": 0.972186055410333, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1532188355922699, + "median": 0.1451343297958374, + "p05": 0.11514341086149216, + "p95": 0.22103679180145264 + }, + { + "label": "completed_group_1", + "mean": 0.11420352011919022, + "median": 0.11261284351348877, + "p05": 0.0928458571434021, + "p95": 0.14123988151550293 + }, + { + "label": "completed_group_2", + "mean": 0.16597424447536469, + "median": 0.1655328869819641, + "p05": 0.13871800899505615, + "p95": 0.19611573219299316 + }, + { + "label": "completed_group_3", + "mean": 0.1250365972518921, + "median": 0.12369821965694427, + "p05": 0.10375828295946121, + "p95": 0.14912742376327515 + }, + { + "label": "completed_group_4", + "mean": 0.18140508234500885, + "median": 0.1796584278345108, + "p05": 0.14511024951934814, + "p95": 0.22286373376846313 + }, + { + "label": "current_group_partial", + "mean": 0.26016175746917725, + "median": 0.26266711950302124, + "p05": 0.1889389455318451, + "p95": 0.3234381675720215 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7127439975738525, + "group": 5, + "latest_source_mass": 0.2907913327217102, + "layer": 19, + "max_source_mass": 0.2907913327217102, + "mixer_index": 36, + "normalized_entropy": 0.9559006256078307, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18731768429279327, + "median": 0.19042012095451355, + "p05": 0.13879702985286713, + "p95": 0.23792000114917755 + }, + { + "label": "completed_group_1", + "mean": 0.16715484857559204, + "median": 0.1667211651802063, + "p05": 0.14574480056762695, + "p95": 0.19026139378547668 + }, + { + "label": "completed_group_2", + "mean": 0.08496320247650146, + "median": 0.08452782034873962, + "p05": 0.07410907745361328, + "p95": 0.097386933863163 + }, + { + "label": "completed_group_3", + "mean": 0.11362075805664062, + "median": 0.11248823255300522, + "p05": 0.0977015346288681, + "p95": 0.13219426572322845 + }, + { + "label": "completed_group_4", + "mean": 0.15615218877792358, + "median": 0.15540224313735962, + "p05": 0.1289147436618805, + "p95": 0.18598109483718872 + }, + { + "label": "current_group_partial", + "mean": 0.2907913327217102, + "median": 0.2900415062904358, + "p05": 0.25822532176971436, + "p95": 0.32475030422210693 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6076834201812744, + "group": 5, + "latest_source_mass": 0.3738824725151062, + "layer": 19, + "max_source_mass": 0.3738824725151062, + "mixer_index": 37, + "normalized_entropy": 0.8972652009334232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15032559633255005, + "median": 0.12544110417366028, + "p05": 0.06976518034934998, + "p95": 0.35840943455696106 + }, + { + "label": "completed_group_1", + "mean": 0.08624927699565887, + "median": 0.08602266013622284, + "p05": 0.061998721212148666, + "p95": 0.11138717085123062 + }, + { + "label": "completed_group_2", + "mean": 0.15736976265907288, + "median": 0.15802210569381714, + "p05": 0.10121746361255646, + "p95": 0.20687000453472137 + }, + { + "label": "completed_group_3", + "mean": 0.08584291487932205, + "median": 0.0827617198228836, + "p05": 0.060561805963516235, + "p95": 0.11351694166660309 + }, + { + "label": "completed_group_4", + "mean": 0.14632999897003174, + "median": 0.1426895558834076, + "p05": 0.10106660425662994, + "p95": 0.20179018378257751 + }, + { + "label": "current_group_partial", + "mean": 0.3738824725151062, + "median": 0.3799295425415039, + "p05": 0.23631010949611664, + "p95": 0.4884527325630188 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.709290862083435, + "group": 5, + "latest_source_mass": 0.2901708781719208, + "layer": 20, + "max_source_mass": 0.2901708781719208, + "mixer_index": 38, + "normalized_entropy": 0.9539733939957076, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18991519510746002, + "median": 0.1924753338098526, + "p05": 0.14201581478118896, + "p95": 0.23709386587142944 + }, + { + "label": "completed_group_1", + "mean": 0.17091524600982666, + "median": 0.17006167769432068, + "p05": 0.15297070145606995, + "p95": 0.19153237342834473 + }, + { + "label": "completed_group_2", + "mean": 0.07822999358177185, + "median": 0.07778562605381012, + "p05": 0.06707640737295151, + "p95": 0.09089364111423492 + }, + { + "label": "completed_group_3", + "mean": 0.116765096783638, + "median": 0.11516137421131134, + "p05": 0.10162825882434845, + "p95": 0.13653048872947693 + }, + { + "label": "completed_group_4", + "mean": 0.1540035903453827, + "median": 0.15287795662879944, + "p05": 0.12593276798725128, + "p95": 0.18590238690376282 + }, + { + "label": "current_group_partial", + "mean": 0.2901708781719208, + "median": 0.29104921221733093, + "p05": 0.25643545389175415, + "p95": 0.3205650746822357 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6690887212753296, + "group": 5, + "latest_source_mass": 0.3342471718788147, + "layer": 20, + "max_source_mass": 0.3342471718788147, + "mixer_index": 39, + "normalized_entropy": 0.9315361520005944, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19029226899147034, + "median": 0.19644704461097717, + "p05": 0.09456793963909149, + "p95": 0.30468082427978516 + }, + { + "label": "completed_group_1", + "mean": 0.1100241169333458, + "median": 0.10808776319026947, + "p05": 0.09035106748342514, + "p95": 0.1359245330095291 + }, + { + "label": "completed_group_2", + "mean": 0.11471512168645859, + "median": 0.11494918912649155, + "p05": 0.08799737691879272, + "p95": 0.14024731516838074 + }, + { + "label": "completed_group_3", + "mean": 0.09999909996986389, + "median": 0.09682698547840118, + "p05": 0.07921088486909866, + "p95": 0.1264391839504242 + }, + { + "label": "completed_group_4", + "mean": 0.1507222205400467, + "median": 0.1476232260465622, + "p05": 0.11397356539964676, + "p95": 0.19991227984428406 + }, + { + "label": "current_group_partial", + "mean": 0.3342471718788147, + "median": 0.33802270889282227, + "p05": 0.2548266053199768, + "p95": 0.40130624175071716 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7458693981170654, + "group": 6, + "latest_source_mass": 0.2361394464969635, + "layer": 21, + "max_source_mass": 0.2361394464969635, + "mixer_index": 40, + "normalized_entropy": 0.9743882636597644, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20528709888458252, + "median": 0.2140858769416809, + "p05": 0.13168928027153015, + "p95": 0.27098023891448975 + }, + { + "label": "completed_group_1", + "mean": 0.1722964197397232, + "median": 0.17238952219486237, + "p05": 0.15053534507751465, + "p95": 0.19365043938159943 + }, + { + "label": "completed_group_2", + "mean": 0.10435444861650467, + "median": 0.10361101478338242, + "p05": 0.08888213336467743, + "p95": 0.12212536484003067 + }, + { + "label": "completed_group_3", + "mean": 0.12075909972190857, + "median": 0.1180175170302391, + "p05": 0.10358668118715286, + "p95": 0.14458929002285004 + }, + { + "label": "completed_group_4", + "mean": 0.16116347908973694, + "median": 0.15841388702392578, + "p05": 0.1337081342935562, + "p95": 0.19791515171527863 + }, + { + "label": "completed_group_5", + "mean": 0.2361394464969635, + "median": 0.23709020018577576, + "p05": 0.20446214079856873, + "p95": 0.26509594917297363 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4612125158309937, + "group": 6, + "latest_source_mass": 0.5543455481529236, + "layer": 21, + "max_source_mass": 0.5543455481529236, + "mixer_index": 41, + "normalized_entropy": 0.7509146897354808, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08934466540813446, + "median": 0.07911523431539536, + "p05": 0.04488275945186615, + "p95": 0.1737232506275177 + }, + { + "label": "completed_group_1", + "mean": 0.054379627108573914, + "median": 0.05344422906637192, + "p05": 0.04221750423312187, + "p95": 0.06924191862344742 + }, + { + "label": "completed_group_2", + "mean": 0.07567165046930313, + "median": 0.07438549399375916, + "p05": 0.05811355262994766, + "p95": 0.09632256627082825 + }, + { + "label": "completed_group_3", + "mean": 0.05758609622716904, + "median": 0.05612001568078995, + "p05": 0.044792283326387405, + "p95": 0.07347827404737473 + }, + { + "label": "completed_group_4", + "mean": 0.07884950935840607, + "median": 0.07757881283760071, + "p05": 0.05758974328637123, + "p95": 0.10452556610107422 + }, + { + "label": "completed_group_5", + "mean": 0.08982288837432861, + "median": 0.08912864327430725, + "p05": 0.0663854256272316, + "p95": 0.11604885756969452 + }, + { + "label": "current_group_partial", + "mean": 0.5543455481529236, + "median": 0.5596089363098145, + "p05": 0.45455610752105713, + "p95": 0.6367484331130981 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7241814136505127, + "group": 6, + "latest_source_mass": 0.39484071731567383, + "layer": 22, + "max_source_mass": 0.39484071731567383, + "mixer_index": 42, + "normalized_entropy": 0.886053970419732, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13360010087490082, + "median": 0.14072942733764648, + "p05": 0.07699814438819885, + "p95": 0.17506597936153412 + }, + { + "label": "completed_group_1", + "mean": 0.10609637200832367, + "median": 0.10586297512054443, + "p05": 0.08648421615362167, + "p95": 0.12602579593658447 + }, + { + "label": "completed_group_2", + "mean": 0.06363911926746368, + "median": 0.06324908882379532, + "p05": 0.054980404675006866, + "p95": 0.07316673547029495 + }, + { + "label": "completed_group_3", + "mean": 0.07426095008850098, + "median": 0.07362888753414154, + "p05": 0.06318790465593338, + "p95": 0.08665015548467636 + }, + { + "label": "completed_group_4", + "mean": 0.09383076429367065, + "median": 0.09294568747282028, + "p05": 0.07638302445411682, + "p95": 0.11445100605487823 + }, + { + "label": "completed_group_5", + "mean": 0.13373197615146637, + "median": 0.13327741622924805, + "p05": 0.11399447172880173, + "p95": 0.15641780197620392 + }, + { + "label": "current_group_partial", + "mean": 0.39484071731567383, + "median": 0.3879151940345764, + "p05": 0.3302172124385834, + "p95": 0.4852263331413269 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.3979300260543823, + "group": 6, + "latest_source_mass": 0.5830056071281433, + "layer": 22, + "max_source_mass": 0.5830056071281433, + "mixer_index": 43, + "normalized_entropy": 0.7183939231382495, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08959861099720001, + "median": 0.07544538378715515, + "p05": 0.04165133833885193, + "p95": 0.20099139213562012 + }, + { + "label": "completed_group_1", + "mean": 0.05414731800556183, + "median": 0.05336771905422211, + "p05": 0.04069708660244942, + "p95": 0.07046990841627121 + }, + { + "label": "completed_group_2", + "mean": 0.07668881118297577, + "median": 0.07607124000787735, + "p05": 0.0555989071726799, + "p95": 0.09980960935354233 + }, + { + "label": "completed_group_3", + "mean": 0.05603286623954773, + "median": 0.054573021829128265, + "p05": 0.04162490367889404, + "p95": 0.0730118378996849 + }, + { + "label": "completed_group_4", + "mean": 0.06979228556156158, + "median": 0.06884932518005371, + "p05": 0.04576495662331581, + "p95": 0.09806846082210541 + }, + { + "label": "completed_group_5", + "mean": 0.07073447108268738, + "median": 0.07023533433675766, + "p05": 0.04956750199198723, + "p95": 0.09379896521568298 + }, + { + "label": "current_group_partial", + "mean": 0.5830056071281433, + "median": 0.5862616896629333, + "p05": 0.4658474922180176, + "p95": 0.6849814057350159 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7459560632705688, + "group": 6, + "latest_source_mass": 0.37691548466682434, + "layer": 23, + "max_source_mass": 0.37691548466682434, + "mixer_index": 44, + "normalized_entropy": 0.897243926765161, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14318783581256866, + "median": 0.1493408977985382, + "p05": 0.08292607218027115, + "p95": 0.19464610517024994 + }, + { + "label": "completed_group_1", + "mean": 0.1094653457403183, + "median": 0.10858799517154694, + "p05": 0.09432064741849899, + "p95": 0.1279931515455246 + }, + { + "label": "completed_group_2", + "mean": 0.06349228322505951, + "median": 0.06325754523277283, + "p05": 0.05439673364162445, + "p95": 0.0737837627530098 + }, + { + "label": "completed_group_3", + "mean": 0.07859592139720917, + "median": 0.0775163471698761, + "p05": 0.06603655964136124, + "p95": 0.09357087314128876 + }, + { + "label": "completed_group_4", + "mean": 0.09567482024431229, + "median": 0.09468592703342438, + "p05": 0.07671141624450684, + "p95": 0.11774884164333344 + }, + { + "label": "completed_group_5", + "mean": 0.13266831636428833, + "median": 0.13199329376220703, + "p05": 0.11244359612464905, + "p95": 0.15504513680934906 + }, + { + "label": "current_group_partial", + "mean": 0.37691548466682434, + "median": 0.37180638313293457, + "p05": 0.31207239627838135, + "p95": 0.4595233201980591 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.4055393934249878, + "group": 6, + "latest_source_mass": 0.5772827863693237, + "layer": 23, + "max_source_mass": 0.5772827863693237, + "mixer_index": 45, + "normalized_entropy": 0.7223043644164862, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10154955089092255, + "median": 0.08766824007034302, + "p05": 0.04901552200317383, + "p95": 0.20287738740444183 + }, + { + "label": "completed_group_1", + "mean": 0.05619276314973831, + "median": 0.055477917194366455, + "p05": 0.041608311235904694, + "p95": 0.07371112704277039 + }, + { + "label": "completed_group_2", + "mean": 0.07791627198457718, + "median": 0.07736755907535553, + "p05": 0.055920060724020004, + "p95": 0.10158214718103409 + }, + { + "label": "completed_group_3", + "mean": 0.053915880620479584, + "median": 0.05215863138437271, + "p05": 0.03798118978738785, + "p95": 0.07156119495630264 + }, + { + "label": "completed_group_4", + "mean": 0.06398355215787888, + "median": 0.06229078769683838, + "p05": 0.040616974234580994, + "p95": 0.09270931780338287 + }, + { + "label": "completed_group_5", + "mean": 0.06915920227766037, + "median": 0.0684593990445137, + "p05": 0.046986356377601624, + "p95": 0.09408729523420334 + }, + { + "label": "current_group_partial", + "mean": 0.5772827863693237, + "median": 0.5824689269065857, + "p05": 0.4518115520477295, + "p95": 0.6926202178001404 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7853620052337646, + "group": 6, + "latest_source_mass": 0.3407441973686218, + "layer": 24, + "max_source_mass": 0.3407441973686218, + "mixer_index": 46, + "normalized_entropy": 0.9174945750195659, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15284113585948944, + "median": 0.1593906283378601, + "p05": 0.08678612112998962, + "p95": 0.211342453956604 + }, + { + "label": "completed_group_1", + "mean": 0.12206703424453735, + "median": 0.12153112143278122, + "p05": 0.1060115247964859, + "p95": 0.1399766057729721 + }, + { + "label": "completed_group_2", + "mean": 0.06508444249629974, + "median": 0.06482555717229843, + "p05": 0.055350616574287415, + "p95": 0.07557765394449234 + }, + { + "label": "completed_group_3", + "mean": 0.08347313106060028, + "median": 0.08190013468265533, + "p05": 0.06966532766819, + "p95": 0.10050469636917114 + }, + { + "label": "completed_group_4", + "mean": 0.09811970591545105, + "median": 0.09664031863212585, + "p05": 0.07688035070896149, + "p95": 0.12369741499423981 + }, + { + "label": "completed_group_5", + "mean": 0.13767032325267792, + "median": 0.1373325139284134, + "p05": 0.11642646044492722, + "p95": 0.1606694608926773 + }, + { + "label": "current_group_partial", + "mean": 0.3407441973686218, + "median": 0.33805108070373535, + "p05": 0.271301805973053, + "p95": 0.41781002283096313 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.509842872619629, + "group": 6, + "latest_source_mass": 0.5227814316749573, + "layer": 24, + "max_source_mass": 0.5227814316749573, + "mixer_index": 47, + "normalized_entropy": 0.7759057494780099, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11788781732320786, + "median": 0.10097194463014603, + "p05": 0.06265684217214584, + "p95": 0.2408325970172882 + }, + { + "label": "completed_group_1", + "mean": 0.06822571158409119, + "median": 0.0673636943101883, + "p05": 0.05209728330373764, + "p95": 0.08648915588855743 + }, + { + "label": "completed_group_2", + "mean": 0.08556114137172699, + "median": 0.08463940769433975, + "p05": 0.06234566867351532, + "p95": 0.1125786080956459 + }, + { + "label": "completed_group_3", + "mean": 0.06067090481519699, + "median": 0.058974385261535645, + "p05": 0.043267179280519485, + "p95": 0.07997901737689972 + }, + { + "label": "completed_group_4", + "mean": 0.07228855788707733, + "median": 0.06972275674343109, + "p05": 0.04689513146877289, + "p95": 0.106512151658535 + }, + { + "label": "completed_group_5", + "mean": 0.07258446514606476, + "median": 0.07082539796829224, + "p05": 0.050949547439813614, + "p95": 0.09970614314079285 + }, + { + "label": "current_group_partial", + "mean": 0.5227814316749573, + "median": 0.5290273427963257, + "p05": 0.38773614168167114, + "p95": 0.6392589807510376 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.839367389678955, + "group": 7, + "latest_source_mass": 0.29583215713500977, + "layer": 25, + "max_source_mass": 0.29583215713500977, + "mixer_index": 48, + "normalized_entropy": 0.9452478525649903, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15906387567520142, + "median": 0.15846745669841766, + "p05": 0.09519976377487183, + "p95": 0.24517810344696045 + }, + { + "label": "completed_group_1", + "mean": 0.13139380514621735, + "median": 0.1308426856994629, + "p05": 0.11478274315595627, + "p95": 0.15000829100608826 + }, + { + "label": "completed_group_2", + "mean": 0.07717745751142502, + "median": 0.07715500891208649, + "p05": 0.06616044044494629, + "p95": 0.08791667222976685 + }, + { + "label": "completed_group_3", + "mean": 0.09607303142547607, + "median": 0.09436552226543427, + "p05": 0.08152313530445099, + "p95": 0.11384639143943787 + }, + { + "label": "completed_group_4", + "mean": 0.1107850894331932, + "median": 0.1089654266834259, + "p05": 0.08814431726932526, + "p95": 0.139452263712883 + }, + { + "label": "completed_group_5", + "mean": 0.12967455387115479, + "median": 0.1299237608909607, + "p05": 0.10832332074642181, + "p95": 0.15138982236385345 + }, + { + "label": "completed_group_6", + "mean": 0.29583215713500977, + "median": 0.29647016525268555, + "p05": 0.2276897132396698, + "p95": 0.3604946434497833 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5841834545135498, + "group": 7, + "latest_source_mass": 0.40767741203308105, + "layer": 25, + "max_source_mass": 0.40767741203308105, + "mixer_index": 49, + "normalized_entropy": 0.7618312045616817, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07896879315376282, + "median": 0.06809917092323303, + "p05": 0.025932887569069862, + "p95": 0.19368338584899902 + }, + { + "label": "completed_group_1", + "mean": 0.048752427101135254, + "median": 0.04767637699842453, + "p05": 0.034443892538547516, + "p95": 0.0667153000831604 + }, + { + "label": "completed_group_2", + "mean": 0.055707164108753204, + "median": 0.0549345538020134, + "p05": 0.038696639239788055, + "p95": 0.0750277042388916 + }, + { + "label": "completed_group_3", + "mean": 0.04139826446771622, + "median": 0.03878612071275711, + "p05": 0.026673054322600365, + "p95": 0.05746251344680786 + }, + { + "label": "completed_group_4", + "mean": 0.04449018836021423, + "median": 0.04196460545063019, + "p05": 0.026162121444940567, + "p95": 0.07164917886257172 + }, + { + "label": "completed_group_5", + "mean": 0.04285799711942673, + "median": 0.04109475016593933, + "p05": 0.029393533244729042, + "p95": 0.06277710199356079 + }, + { + "label": "completed_group_6", + "mean": 0.2801477909088135, + "median": 0.2763082981109619, + "p05": 0.1455708146095276, + "p95": 0.4254714846611023 + }, + { + "label": "current_group_partial", + "mean": 0.40767741203308105, + "median": 0.4120897948741913, + "p05": 0.21399420499801636, + "p95": 0.5853031277656555 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8125314712524414, + "group": 7, + "latest_source_mass": 0.3700707256793976, + "layer": 26, + "max_source_mass": 0.3700707256793976, + "mixer_index": 50, + "normalized_entropy": 0.8716433883436914, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10889513790607452, + "median": 0.11333495378494263, + "p05": 0.04850132763385773, + "p95": 0.1620434820652008 + }, + { + "label": "completed_group_1", + "mean": 0.08656036853790283, + "median": 0.08640051633119583, + "p05": 0.0650845319032669, + "p95": 0.1084962785243988 + }, + { + "label": "completed_group_2", + "mean": 0.048123255372047424, + "median": 0.04800213500857353, + "p05": 0.03576180338859558, + "p95": 0.06107594817876816 + }, + { + "label": "completed_group_3", + "mean": 0.06561170518398285, + "median": 0.06379580497741699, + "p05": 0.050968848168849945, + "p95": 0.08332888036966324 + }, + { + "label": "completed_group_4", + "mean": 0.06917259097099304, + "median": 0.06724178791046143, + "p05": 0.050207968801259995, + "p95": 0.09373342245817184 + }, + { + "label": "completed_group_5", + "mean": 0.08286689221858978, + "median": 0.08183546364307404, + "p05": 0.06535022705793381, + "p95": 0.10340933501720428 + }, + { + "label": "completed_group_6", + "mean": 0.16869932413101196, + "median": 0.16509854793548584, + "p05": 0.11059704422950745, + "p95": 0.23654375970363617 + }, + { + "label": "current_group_partial", + "mean": 0.3700707256793976, + "median": 0.3694133758544922, + "p05": 0.23441238701343536, + "p95": 0.5025027990341187 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.50089693069458, + "group": 7, + "latest_source_mass": 0.49096518754959106, + "layer": 26, + "max_source_mass": 0.49096518754959106, + "mixer_index": 51, + "normalized_entropy": 0.7217788529328457, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07974998652935028, + "median": 0.07402504235506058, + "p05": 0.04579866677522659, + "p95": 0.1359037309885025 + }, + { + "label": "completed_group_1", + "mean": 0.04166398197412491, + "median": 0.04038450866937637, + "p05": 0.027992289513349533, + "p95": 0.05969971418380737 + }, + { + "label": "completed_group_2", + "mean": 0.04607272148132324, + "median": 0.044438350945711136, + "p05": 0.030038591474294662, + "p95": 0.06808309257030487 + }, + { + "label": "completed_group_3", + "mean": 0.03586764633655548, + "median": 0.03317485377192497, + "p05": 0.02203454077243805, + "p95": 0.05572832003235817 + }, + { + "label": "completed_group_4", + "mean": 0.04223340004682541, + "median": 0.03930673003196716, + "p05": 0.02465122565627098, + "p95": 0.06968094408512115 + }, + { + "label": "completed_group_5", + "mean": 0.04095568507909775, + "median": 0.03919431194663048, + "p05": 0.026239857077598572, + "p95": 0.06058088690042496 + }, + { + "label": "completed_group_6", + "mean": 0.22249138355255127, + "median": 0.2125401496887207, + "p05": 0.10647299885749817, + "p95": 0.3711881935596466 + }, + { + "label": "current_group_partial", + "mean": 0.49096518754959106, + "median": 0.4988659620285034, + "p05": 0.2878011167049408, + "p95": 0.6676807403564453 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8059929609298706, + "group": 7, + "latest_source_mass": 0.37139976024627686, + "layer": 27, + "max_source_mass": 0.37139976024627686, + "mixer_index": 52, + "normalized_entropy": 0.8684990295379666, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10711923241615295, + "median": 0.10972198843955994, + "p05": 0.04896187782287598, + "p95": 0.16201043128967285 + }, + { + "label": "completed_group_1", + "mean": 0.07651707530021667, + "median": 0.07666032761335373, + "p05": 0.05141128972172737, + "p95": 0.10152198374271393 + }, + { + "label": "completed_group_2", + "mean": 0.058284737169742584, + "median": 0.05812416970729828, + "p05": 0.039273619651794434, + "p95": 0.07745087146759033 + }, + { + "label": "completed_group_3", + "mean": 0.06609587371349335, + "median": 0.06383530795574188, + "p05": 0.04481930285692215, + "p95": 0.09336265176534653 + }, + { + "label": "completed_group_4", + "mean": 0.07139429450035095, + "median": 0.0694514587521553, + "p05": 0.0485839769244194, + "p95": 0.10034126043319702 + }, + { + "label": "completed_group_5", + "mean": 0.07493151724338531, + "median": 0.07319007813930511, + "p05": 0.051876723766326904, + "p95": 0.10402432829141617 + }, + { + "label": "completed_group_6", + "mean": 0.17425751686096191, + "median": 0.16937679052352905, + "p05": 0.10449051856994629, + "p95": 0.2624811828136444 + }, + { + "label": "current_group_partial", + "mean": 0.37139976024627686, + "median": 0.3732905089855194, + "p05": 0.23830299079418182, + "p95": 0.5014542937278748 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5622732639312744, + "group": 7, + "latest_source_mass": 0.4768889844417572, + "layer": 27, + "max_source_mass": 0.4768889844417572, + "mixer_index": 53, + "normalized_entropy": 0.7512946301290215, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08775827288627625, + "median": 0.08504067361354828, + "p05": 0.0428270623087883, + "p95": 0.144285649061203 + }, + { + "label": "completed_group_1", + "mean": 0.047765426337718964, + "median": 0.04646281152963638, + "p05": 0.031814731657505035, + "p95": 0.06807165592908859 + }, + { + "label": "completed_group_2", + "mean": 0.053928617388010025, + "median": 0.052262187004089355, + "p05": 0.035437777638435364, + "p95": 0.0784272700548172 + }, + { + "label": "completed_group_3", + "mean": 0.04293380305171013, + "median": 0.03996016085147858, + "p05": 0.02556057646870613, + "p95": 0.06623870134353638 + }, + { + "label": "completed_group_4", + "mean": 0.04174989461898804, + "median": 0.03903091698884964, + "p05": 0.025013400241732597, + "p95": 0.06869364529848099 + }, + { + "label": "completed_group_5", + "mean": 0.04734005779027939, + "median": 0.04609068110585213, + "p05": 0.029884062707424164, + "p95": 0.06911227852106094 + }, + { + "label": "completed_group_6", + "mean": 0.20163494348526, + "median": 0.19222964346408844, + "p05": 0.09963390231132507, + "p95": 0.3307305574417114 + }, + { + "label": "current_group_partial", + "mean": 0.4768889844417572, + "median": 0.48383551836013794, + "p05": 0.276212215423584, + "p95": 0.6487575173377991 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8757455348968506, + "group": 7, + "latest_source_mass": 0.3109493851661682, + "layer": 28, + "max_source_mass": 0.3109493851661682, + "mixer_index": 54, + "normalized_entropy": 0.9020429270551009, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11563669890165329, + "median": 0.12087491154670715, + "p05": 0.05733206868171692, + "p95": 0.1688953936100006 + }, + { + "label": "completed_group_1", + "mean": 0.08108697831630707, + "median": 0.08185897022485733, + "p05": 0.05175650864839554, + "p95": 0.10753051191568375 + }, + { + "label": "completed_group_2", + "mean": 0.07280858606100082, + "median": 0.07357072830200195, + "p05": 0.04870978742837906, + "p95": 0.0948689877986908 + }, + { + "label": "completed_group_3", + "mean": 0.07553227245807648, + "median": 0.07362619787454605, + "p05": 0.056317005306482315, + "p95": 0.09756095707416534 + }, + { + "label": "completed_group_4", + "mean": 0.07606014609336853, + "median": 0.07456053793430328, + "p05": 0.05528895929455757, + "p95": 0.1022825762629509 + }, + { + "label": "completed_group_5", + "mean": 0.06935927271842957, + "median": 0.06843917071819305, + "p05": 0.04978285729885101, + "p95": 0.09167985618114471 + }, + { + "label": "completed_group_6", + "mean": 0.19856664538383484, + "median": 0.19508783519268036, + "p05": 0.1281973123550415, + "p95": 0.27796900272369385 + }, + { + "label": "current_group_partial", + "mean": 0.3109493851661682, + "median": 0.3105033040046692, + "p05": 0.18172834813594818, + "p95": 0.43916669487953186 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6924852132797241, + "group": 7, + "latest_source_mass": 0.3315581679344177, + "layer": 28, + "max_source_mass": 0.3315581679344177, + "mixer_index": 55, + "normalized_entropy": 0.8139133413255193, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08638577908277512, + "median": 0.058608733117580414, + "p05": 0.026544082909822464, + "p95": 0.2740764319896698 + }, + { + "label": "completed_group_1", + "mean": 0.059912845492362976, + "median": 0.05901291221380234, + "p05": 0.03629481792449951, + "p95": 0.08707985281944275 + }, + { + "label": "completed_group_2", + "mean": 0.06609142571687698, + "median": 0.0647939145565033, + "p05": 0.038691695779561996, + "p95": 0.09703366458415985 + }, + { + "label": "completed_group_3", + "mean": 0.056745611131191254, + "median": 0.05418796092271805, + "p05": 0.032174739986658096, + "p95": 0.08370660990476608 + }, + { + "label": "completed_group_4", + "mean": 0.06277773529291153, + "median": 0.0600711889564991, + "p05": 0.03501531109213829, + "p95": 0.10094858705997467 + }, + { + "label": "completed_group_5", + "mean": 0.05352145433425903, + "median": 0.05179818347096443, + "p05": 0.032271772623062134, + "p95": 0.08084158599376678 + }, + { + "label": "completed_group_6", + "mean": 0.2830069661140442, + "median": 0.2759752869606018, + "p05": 0.12981083989143372, + "p95": 0.46204638481140137 + }, + { + "label": "current_group_partial", + "mean": 0.3315581679344177, + "median": 0.3204721510410309, + "p05": 0.13446375727653503, + "p95": 0.5667697787284851 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8980522155761719, + "group": 8, + "latest_source_mass": 0.20123262703418732, + "layer": 29, + "max_source_mass": 0.26602888107299805, + "mixer_index": 56, + "normalized_entropy": 0.9127701729200177, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11603894829750061, + "median": 0.09991352260112762, + "p05": 0.030035395175218582, + "p95": 0.2941743731498718 + }, + { + "label": "completed_group_1", + "mean": 0.08804173022508621, + "median": 0.08880250155925751, + "p05": 0.05664174258708954, + "p95": 0.11779069900512695 + }, + { + "label": "completed_group_2", + "mean": 0.07975024729967117, + "median": 0.07984748482704163, + "p05": 0.05428915470838547, + "p95": 0.10499341040849686 + }, + { + "label": "completed_group_3", + "mean": 0.09027000516653061, + "median": 0.08898856490850449, + "p05": 0.06189651042222977, + "p95": 0.11996553838253021 + }, + { + "label": "completed_group_4", + "mean": 0.08430293202400208, + "median": 0.0819794088602066, + "p05": 0.05464346706867218, + "p95": 0.12159879505634308 + }, + { + "label": "completed_group_5", + "mean": 0.07433462142944336, + "median": 0.07287181913852692, + "p05": 0.052520088851451874, + "p95": 0.1010315865278244 + }, + { + "label": "completed_group_6", + "mean": 0.26602888107299805, + "median": 0.2639757990837097, + "p05": 0.15474572777748108, + "p95": 0.3846370279788971 + }, + { + "label": "completed_group_7", + "mean": 0.20123262703418732, + "median": 0.18140269815921783, + "p05": 0.08419936150312424, + "p95": 0.3782624900341034 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7512003183364868, + "group": 8, + "latest_source_mass": 0.33482813835144043, + "layer": 29, + "max_source_mass": 0.33482813835144043, + "mixer_index": 57, + "normalized_entropy": 0.7970056117156376, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07744461297988892, + "median": 0.06970229744911194, + "p05": 0.014401955530047417, + "p95": 0.17144526541233063 + }, + { + "label": "completed_group_1", + "mean": 0.046086348593235016, + "median": 0.04394831880927086, + "p05": 0.028066720813512802, + "p95": 0.0718703418970108 + }, + { + "label": "completed_group_2", + "mean": 0.05025433003902435, + "median": 0.04861236736178398, + "p05": 0.029644077643752098, + "p95": 0.07761462777853012 + }, + { + "label": "completed_group_3", + "mean": 0.03748488053679466, + "median": 0.03434944897890091, + "p05": 0.020774798467755318, + "p95": 0.05938909202814102 + }, + { + "label": "completed_group_4", + "mean": 0.04005061462521553, + "median": 0.037498839199543, + "p05": 0.019541850313544273, + "p95": 0.06922487914562225 + }, + { + "label": "completed_group_5", + "mean": 0.041623204946517944, + "median": 0.039552345871925354, + "p05": 0.022081367671489716, + "p95": 0.06821337342262268 + }, + { + "label": "completed_group_6", + "mean": 0.198423832654953, + "median": 0.18318486213684082, + "p05": 0.07548617571592331, + "p95": 0.3721975088119507 + }, + { + "label": "completed_group_7", + "mean": 0.17380405962467194, + "median": 0.1514919549226761, + "p05": 0.05777408182621002, + "p95": 0.36326974630355835 + }, + { + "label": "current_group_partial", + "mean": 0.33482813835144043, + "median": 0.33602219820022583, + "p05": 0.0795319601893425, + "p95": 0.5979493856430054 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.0160694122314453, + "group": 8, + "latest_source_mass": 0.21711516380310059, + "layer": 30, + "max_source_mass": 0.21711516380310059, + "mixer_index": 58, + "normalized_entropy": 0.9175527313077866, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09385271370410919, + "median": 0.09211117774248123, + "p05": 0.032377734780311584, + "p95": 0.16496515274047852 + }, + { + "label": "completed_group_1", + "mean": 0.06208505108952522, + "median": 0.06140740588307381, + "p05": 0.0407925546169281, + "p95": 0.0863042101264 + }, + { + "label": "completed_group_2", + "mean": 0.06560768187046051, + "median": 0.06516212224960327, + "p05": 0.041897669434547424, + "p95": 0.09085705131292343 + }, + { + "label": "completed_group_3", + "mean": 0.07029052078723907, + "median": 0.06780634075403214, + "p05": 0.04846132919192314, + "p95": 0.09917552024126053 + }, + { + "label": "completed_group_4", + "mean": 0.06935097277164459, + "median": 0.06708167493343353, + "p05": 0.04668639227747917, + "p95": 0.09815650433301926 + }, + { + "label": "completed_group_5", + "mean": 0.056832700967788696, + "median": 0.05595892667770386, + "p05": 0.038822539150714874, + "p95": 0.0771557092666626 + }, + { + "label": "completed_group_6", + "mean": 0.19425815343856812, + "median": 0.19145764410495758, + "p05": 0.12249615788459778, + "p95": 0.27496030926704407 + }, + { + "label": "completed_group_7", + "mean": 0.17060703039169312, + "median": 0.16317546367645264, + "p05": 0.09239593893289566, + "p95": 0.2726203501224518 + }, + { + "label": "current_group_partial", + "mean": 0.21711516380310059, + "median": 0.21198657155036926, + "p05": 0.11019522696733475, + "p95": 0.3391069769859314 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7837426662445068, + "group": 8, + "latest_source_mass": 0.29026859998703003, + "layer": 30, + "max_source_mass": 0.29026859998703003, + "mixer_index": 59, + "normalized_entropy": 0.8118162725118464, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08836327493190765, + "median": 0.07904411852359772, + "p05": 0.02631576545536518, + "p95": 0.17534467577934265 + }, + { + "label": "completed_group_1", + "mean": 0.04953558370471001, + "median": 0.04658038914203644, + "p05": 0.025181813165545464, + "p95": 0.08318150043487549 + }, + { + "label": "completed_group_2", + "mean": 0.04813014343380928, + "median": 0.045740917325019836, + "p05": 0.026714935898780823, + "p95": 0.07843323051929474 + }, + { + "label": "completed_group_3", + "mean": 0.0373263955116272, + "median": 0.03401809185743332, + "p05": 0.01990434154868126, + "p95": 0.06290259957313538 + }, + { + "label": "completed_group_4", + "mean": 0.037015400826931, + "median": 0.03405017405748367, + "p05": 0.018646355718374252, + "p95": 0.06544196605682373 + }, + { + "label": "completed_group_5", + "mean": 0.03390160948038101, + "median": 0.03198609501123428, + "p05": 0.018189728260040283, + "p95": 0.05570102855563164 + }, + { + "label": "completed_group_6", + "mean": 0.1840541511774063, + "median": 0.17327937483787537, + "p05": 0.07966600358486176, + "p95": 0.32982927560806274 + }, + { + "label": "completed_group_7", + "mean": 0.23140482604503632, + "median": 0.22406929731369019, + "p05": 0.08831030130386353, + "p95": 0.40350067615509033 + }, + { + "label": "current_group_partial", + "mean": 0.29026859998703003, + "median": 0.2745037376880646, + "p05": 0.10280418395996094, + "p95": 0.5399085879325867 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9973840713500977, + "group": 8, + "latest_source_mass": 0.19101998209953308, + "layer": 31, + "max_source_mass": 0.20831164717674255, + "mixer_index": 60, + "normalized_entropy": 0.9090486661912383, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10261696577072144, + "median": 0.10389091819524765, + "p05": 0.01371697522699833, + "p95": 0.21163305640220642 + }, + { + "label": "completed_group_1", + "mean": 0.06039813905954361, + "median": 0.06085003539919853, + "p05": 0.026107093319296837, + "p95": 0.09348798543214798 + }, + { + "label": "completed_group_2", + "mean": 0.06671924144029617, + "median": 0.06656201928853989, + "p05": 0.034684643149375916, + "p95": 0.09819839894771576 + }, + { + "label": "completed_group_3", + "mean": 0.08377155661582947, + "median": 0.0804532840847969, + "p05": 0.05448959022760391, + "p95": 0.12039553374052048 + }, + { + "label": "completed_group_4", + "mean": 0.07002593576908112, + "median": 0.06737451255321503, + "p05": 0.044841229915618896, + "p95": 0.10655929893255234 + }, + { + "label": "completed_group_5", + "mean": 0.05768897384405136, + "median": 0.056077782064676285, + "p05": 0.039495617151260376, + "p95": 0.08242568373680115 + }, + { + "label": "completed_group_6", + "mean": 0.20831164717674255, + "median": 0.2021285891532898, + "p05": 0.11652229726314545, + "p95": 0.31706833839416504 + }, + { + "label": "completed_group_7", + "mean": 0.1594475358724594, + "median": 0.1546279788017273, + "p05": 0.06593266129493713, + "p95": 0.2735573351383209 + }, + { + "label": "current_group_partial", + "mean": 0.19101998209953308, + "median": 0.1713225245475769, + "p05": 0.06408411264419556, + "p95": 0.3850628137588501 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.969987154006958, + "group": 8, + "latest_source_mass": 0.16864418983459473, + "layer": 31, + "max_source_mass": 0.24503237009048462, + "mixer_index": 61, + "normalized_entropy": 0.8965797917640489, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08120635151863098, + "median": 0.061164144426584244, + "p05": 0.0259944386780262, + "p95": 0.23622018098831177 + }, + { + "label": "completed_group_1", + "mean": 0.07894632965326309, + "median": 0.07658770680427551, + "p05": 0.0496862456202507, + "p95": 0.11596276611089706 + }, + { + "label": "completed_group_2", + "mean": 0.08615730702877045, + "median": 0.08467075228691101, + "p05": 0.050409119576215744, + "p95": 0.12811316549777985 + }, + { + "label": "completed_group_3", + "mean": 0.07609779387712479, + "median": 0.0722871869802475, + "p05": 0.04099677503108978, + "p95": 0.11950768530368805 + }, + { + "label": "completed_group_4", + "mean": 0.058728985488414764, + "median": 0.056080836802721024, + "p05": 0.03323862701654434, + "p95": 0.09344407916069031 + }, + { + "label": "completed_group_5", + "mean": 0.05076971277594566, + "median": 0.04946017265319824, + "p05": 0.03047419898211956, + "p95": 0.07563400268554688 + }, + { + "label": "completed_group_6", + "mean": 0.24503237009048462, + "median": 0.24110376834869385, + "p05": 0.10613486170768738, + "p95": 0.3991503119468689 + }, + { + "label": "completed_group_7", + "mean": 0.1544169783592224, + "median": 0.1400495171546936, + "p05": 0.06199025735259056, + "p95": 0.2995939254760742 + }, + { + "label": "current_group_partial", + "mean": 0.16864418983459473, + "median": 0.15222924947738647, + "p05": 0.0543956533074379, + "p95": 0.3329208791255951 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.071648359298706, + "group": 8, + "latest_source_mass": 0.11520501971244812, + "layer": 32, + "max_source_mass": 0.20955611765384674, + "mixer_index": 62, + "normalized_entropy": 0.9428478002054053, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10569609701633453, + "median": 0.08689188957214355, + "p05": 0.03173588588833809, + "p95": 0.24571751058101654 + }, + { + "label": "completed_group_1", + "mean": 0.09922641515731812, + "median": 0.09795533865690231, + "p05": 0.06407676637172699, + "p95": 0.14120742678642273 + }, + { + "label": "completed_group_2", + "mean": 0.08180510997772217, + "median": 0.080217644572258, + "p05": 0.054607540369033813, + "p95": 0.11442367732524872 + }, + { + "label": "completed_group_3", + "mean": 0.10646799206733704, + "median": 0.10510502755641937, + "p05": 0.07415887713432312, + "p95": 0.14216329157352448 + }, + { + "label": "completed_group_4", + "mean": 0.0826798751950264, + "median": 0.08022409677505493, + "p05": 0.05491747707128525, + "p95": 0.11860857903957367 + }, + { + "label": "completed_group_5", + "mean": 0.06615597754716873, + "median": 0.06485182791948318, + "p05": 0.04477996751666069, + "p95": 0.09238749742507935 + }, + { + "label": "completed_group_6", + "mean": 0.20955611765384674, + "median": 0.206287682056427, + "p05": 0.129981130361557, + "p95": 0.2996031939983368 + }, + { + "label": "completed_group_7", + "mean": 0.13320739567279816, + "median": 0.12465103715658188, + "p05": 0.06303662061691284, + "p95": 0.23065781593322754 + }, + { + "label": "current_group_partial", + "mean": 0.11520501971244812, + "median": 0.10386130958795547, + "p05": 0.03912060335278511, + "p95": 0.22735844552516937 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9781701564788818, + "group": 8, + "latest_source_mass": 0.16112206876277924, + "layer": 32, + "max_source_mass": 0.22251944243907928, + "mixer_index": 63, + "normalized_entropy": 0.9003040366848136, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09446848928928375, + "median": 0.074150450527668, + "p05": 0.042592622339725494, + "p95": 0.19443637132644653 + }, + { + "label": "completed_group_1", + "mean": 0.07056106626987457, + "median": 0.06868506222963333, + "p05": 0.04818308353424072, + "p95": 0.0982733890414238 + }, + { + "label": "completed_group_2", + "mean": 0.08033448457717896, + "median": 0.08047192543745041, + "p05": 0.048296794295310974, + "p95": 0.11253741383552551 + }, + { + "label": "completed_group_3", + "mean": 0.0658978745341301, + "median": 0.06256154924631119, + "p05": 0.04223886504769325, + "p95": 0.0967884361743927 + }, + { + "label": "completed_group_4", + "mean": 0.058234501630067825, + "median": 0.0563921295106411, + "p05": 0.03529864549636841, + "p95": 0.08846980333328247 + }, + { + "label": "completed_group_5", + "mean": 0.04316519945859909, + "median": 0.042268507182598114, + "p05": 0.029443470761179924, + "p95": 0.05996086448431015 + }, + { + "label": "completed_group_6", + "mean": 0.2036968618631363, + "median": 0.19645483791828156, + "p05": 0.09398645162582397, + "p95": 0.3395216464996338 + }, + { + "label": "completed_group_7", + "mean": 0.22251944243907928, + "median": 0.2130982130765915, + "p05": 0.11867167800664902, + "p95": 0.35465875267982483 + }, + { + "label": "current_group_partial", + "mean": 0.16112206876277924, + "median": 0.14023703336715698, + "p05": 0.06008770316839218, + "p95": 0.3247341811656952 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.6036605834960938, + "group": null, + "latest_source_mass": 0.31639498472213745, + "layer": null, + "max_source_mass": 0.3412485122680664, + "mixer_index": null, + "normalized_entropy": 0.7298573846467136, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09759747982025146, + "median": 0.09915056824684143, + "p05": 0.025461740791797638, + "p95": 0.16398483514785767 + }, + { + "label": "completed_group_1", + "mean": 0.04215940088033676, + "median": 0.04050712287425995, + "p05": 0.025720715522766113, + "p95": 0.06398092210292816 + }, + { + "label": "completed_group_2", + "mean": 0.041145481169223785, + "median": 0.03881259262561798, + "p05": 0.023510465398430824, + "p95": 0.0662468671798706 + }, + { + "label": "completed_group_3", + "mean": 0.029554028064012527, + "median": 0.027420252561569214, + "p05": 0.01627083122730255, + "p95": 0.04924973472952843 + }, + { + "label": "completed_group_4", + "mean": 0.0289996936917305, + "median": 0.027226522564888, + "p05": 0.01560397818684578, + "p95": 0.048351090401411057 + }, + { + "label": "completed_group_5", + "mean": 0.03647579252719879, + "median": 0.0347352996468544, + "p05": 0.020744025707244873, + "p95": 0.058257561177015305 + }, + { + "label": "completed_group_6", + "mean": 0.06642461568117142, + "median": 0.057809896767139435, + "p05": 0.022837381809949875, + "p95": 0.14078694581985474 + }, + { + "label": "completed_group_7", + "mean": 0.3412485122680664, + "median": 0.3336990475654602, + "p05": 0.0819513276219368, + "p95": 0.6423859596252441 + }, + { + "label": "completed_group_8", + "mean": 0.31639498472213745, + "median": 0.30713334679603577, + "p05": 0.05841828137636185, + "p95": 0.6156357526779175 + } + ], + "sources": 9 + } + ], + "interventions": { + "detached_learned": { + "statistics": { + "first_quartile_mean": 5.772373378931661e-05, + "first_to_last_ratio": 0.7709541968266947, + "imbalance_abs_log_ratio": 0.26012631467157654, + "last_quartile_mean": 7.487310404030723e-05, + "mean": 7.085559036568156e-05, + "normalized": [ + 0.767456541889667, + 0.8910969874796977, + 0.7949526654342541, + 0.7325741840266811, + 0.943544133913626, + 0.868390294625501, + 0.8054913948148298, + 0.7138323441642012, + 0.5978064063949543, + 0.5556378314836203, + 0.5261854153191788, + 0.505768953649729, + 0.715108334510866, + 0.6608161569684796, + 0.612256437426044, + 0.5785464585899268, + 0.739960161627241, + 0.6892568994732284, + 0.697060043562065, + 0.8113764458982461, + 3.2405349047340315, + 2.5249733605503017, + 1.9894958890200054, + 1.5842775624032772, + 2.3150907133984013, + 1.7369124395330813, + 1.1174777605817472, + 0.8878712764898554, + 0.8457898829566253, + 0.5666580094993431, + 0.5093321459106411, + 0.474467963670652 + ], + "peak_layer": 21, + "peak_normalized": 3.2405349047340315, + "population_cv": 0.6532808754860918, + "rest_mean": 5.3392646400930564e-05, + "spike_contrast": 3.093225357948515, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016515548777533694, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.437858635559678e-05, + 6.313920312095433e-05, + 5.632684042211622e-05, + 5.1906976295867935e-05, + 6.685537664452568e-05, + 6.153030699351802e-05, + 5.707356831408106e-05, + 5.057901216787286e-05, + 4.2357925849501044e-05, + 3.9370046579279006e-05, + 3.728317824425176e-05, + 3.583655779948458e-05, + 5.0669423217186704e-05, + 4.682251892518252e-05, + 4.338179132901132e-05, + 4.099325087736361e-05, + 5.2430314099183306e-05, + 4.8837704525794834e-05, + 4.9390600906917825e-05, + 5.749055708292872e-05, + 0.00022961001377552748, + 0.00017890847811941057, + 0.00014096690574660897, + 0.00011225492198718712, + 0.0001640371192479506, + 0.00012306995631661266, + 7.917954644653946e-05, + 6.291064346441999e-05, + 5.9928941482212394e-05, + 4.015088779851794e-05, + 3.608902989071794e-05, + 3.36187076754868e-05 + ] + }, + "learned": { + "statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "rest_mean": 5.7697108584559625e-05, + "spike_contrast": 3.04626897714722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017576091195223854, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ] + }, + "uniform_value_backward": { + "statistics": { + "first_quartile_mean": 3.269887292844942e-05, + "first_to_last_ratio": 2.3568639793204684, + "imbalance_abs_log_ratio": 0.8573319130483905, + "last_quartile_mean": 1.3873890566173941e-05, + "mean": 2.12624314031018e-05, + "normalized": [ + 1.6816001107908207, + 1.968609461712434, + 1.7075260440141944, + 1.4905139409933708, + 1.4972595154646264, + 1.4034185225085787, + 1.3238781474315933, + 1.2301608589864579, + 1.121553773570522, + 1.0499501635664128, + 1.001131281922829, + 0.9633193641679035, + 0.9560899219831485, + 0.9203740514079343, + 0.8930538331784411, + 0.8703809179546808, + 0.8841528402061924, + 0.8558056863273277, + 0.8550540488218812, + 0.8483642525948266, + 0.8350365031100208, + 0.8178976965352276, + 0.8078428125234317, + 0.7969682791900239, + 0.7797478584358322, + 0.7532962238158083, + 0.7097589943815636, + 0.6713244476982704, + 0.629948240254275, + 0.5812766747709892, + 0.5573943041371163, + 0.5373112275432649 + ], + "peak_layer": 2, + "peak_normalized": 1.968609461712434, + "population_cv": 0.35388972104860866, + "rest_mean": 2.2020403102268808e-05, + "spike_contrast": 0.7797034481094922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.716938422759995e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.5754907003138214e-05, + 4.185742363915779e-05, + 3.630615537986159e-05, + 3.169195042573847e-05, + 3.183537774020806e-05, + 2.9840090064681135e-05, + 2.8148868295829743e-05, + 2.6156210878980346e-05, + 2.3846960175433196e-05, + 2.2324493329506367e-05, + 2.128648520738352e-05, + 2.048251189989969e-05, + 2.0328796381363645e-05, + 1.9569390133256093e-05, + 1.8988495867233723e-05, + 1.8506414562580176e-05, + 1.8799239114741795e-05, + 1.819650969991926e-05, + 1.8180528059019707e-05, + 1.8038286725641228e-05, + 1.775490636646282e-05, + 1.739049366733525e-05, + 1.7176702385768294e-05, + 1.6945483366725966e-05, + 1.6579335351707414e-05, + 1.6016909285099246e-05, + 1.5091201930772513e-05, + 1.4273990018409677e-05, + 1.3394231245911215e-05, + 1.2359355423541274e-05, + 1.1851558156195097e-05, + 1.1424543117755093e-05 + ] + } + }, + "positions": { + "attention_branch_output": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6127158914969186e-05, + "first_to_last_ratio": 0.6403133863364645, + "imbalance_abs_log_ratio": 0.4457975563250414, + "last_quartile_mean": 2.518635290016391e-05, + "mean": 2.1593947650444534e-05, + "normalized": [ + 0.6929919556166857, + 0.8032314127318161, + 0.7123629256452019, + 0.6640523991876173, + 0.8725736234454132, + 0.8071370183700546, + 0.7516577913014878, + 0.6706888120378312, + 0.5882487760705537, + 0.5190639183458972, + 0.4863138588077732, + 0.4652918617004374, + 0.6603036411218667, + 0.6189718629251246, + 0.5802646281311779, + 0.5410038780854491, + 0.6697462103535458, + 0.6259079034957141, + 0.6246747715872017, + 0.6676260304525075, + 3.1377243999857436, + 2.6752853614525707, + 2.136977079899559, + 1.6970064982433113, + 2.2713553531497497, + 1.9850638882393974, + 1.4820693911808296, + 0.9615863897898643, + 1.0163024380148542, + 0.7191699586662397, + 0.4720839007000451, + 0.42326206126447913 + ], + "peak_layer": 21, + "peak_normalized": 3.1377243999857436, + "population_cv": 0.6918392109604528, + "rest_mean": 1.6060819520875897e-05, + "spike_contrast": 3.2048700555541787, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.147283955011517e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4964432011765894e-05, + 1.7344937077723444e-05, + 1.53827277245e-05, + 1.4339512745209504e-05, + 1.8842309145838954e-05, + 1.7429274521418847e-05, + 1.623125899641309e-05, + 1.4482819096883759e-05, + 1.2702613275905605e-05, + 1.1208639079995919e-05, + 1.0501436008780729e-05, + 1.0047488103737123e-05, + 1.4258562259783503e-05, + 1.3366046005103271e-05, + 1.2530204003269318e-05, + 1.1682409422064666e-05, + 1.446246460545808e-05, + 1.3515822502085939e-05, + 1.3489194316207431e-05, + 1.4416681551665533e-05, + 6.775585643481463e-05, + 5.7769972045207396e-05, + 4.61457711935509e-05, + 3.664506948553026e-05, + 4.9047528591472656e-05, + 4.2865365685429424e-05, + 3.2003728847485036e-05, + 2.076444616250228e-05, + 2.1945981643511914e-05, + 1.5529718439211138e-05, + 1.019415503833443e-05, + 9.13989879336441e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.422064188882359e-05, + "first_to_last_ratio": 0.6376814356469364, + "imbalance_abs_log_ratio": 0.4499164375054873, + "last_quartile_mean": 0.00010070959933727863, + "mean": 8.629182207187114e-05, + "normalized": [ + 0.6940773401510585, + 0.8062528427202595, + 0.7180374505283312, + 0.6647119372719499, + 0.8588689882759897, + 0.7941431696191276, + 0.7473948010342778, + 0.6703237246625169, + 0.591781846497542, + 0.522694550559361, + 0.4893812082436792, + 0.4670959071660659, + 0.6619658672285874, + 0.6164719403371215, + 0.5787012511245688, + 0.5393739243504363, + 0.6707710740427385, + 0.62668399370843, + 0.6317390712162502, + 0.6712190136492653, + 3.138964807406113, + 2.6795820619127526, + 2.136435916928567, + 1.6866743661161174, + 2.264472697470994, + 1.978410075097681, + 1.4778942897678682, + 0.9605324338748193, + 1.022040338716437, + 0.7239267286003249, + 0.47869752189559756, + 0.4306788598251707 + ], + "peak_layer": 21, + "peak_normalized": 3.138964807406113, + "population_cv": 0.6905024564046187, + "rest_mean": 6.421987658271703e-05, + "spike_contrast": 3.1996375366532934, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020548032771330326, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.989319834043272e-05, + 6.957302684895694e-05, + 6.196075992193073e-05, + 5.7359204220119864e-05, + 7.411336991935968e-05, + 6.852806109236553e-05, + 6.449405918829143e-05, + 5.784345557913184e-05, + 5.1065933803329244e-05, + 4.5104265154805034e-05, + 4.222959614708088e-05, + 4.03065569116734e-05, + 5.712224083254114e-05, + 5.319648698787205e-05, + 4.993718539481051e-05, + 4.654355871025473e-05, + 5.788205817225389e-05, + 5.4077703680377454e-05, + 5.4513915529241785e-05, + 5.792071169707924e-05, + 0.0002708669926505536, + 0.00023122601851355284, + 0.00018435694801155478, + 0.00014554620429407805, + 0.00019540547509677708, + 0.0001707206101855263, + 0.00012753019109368324, + 8.288609387818724e-05, + 8.81937230587937e-05, + 6.246895645745099e-05, + 4.1307681385660544e-05, + 3.716406354214996e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05695150885730982, + "first_to_last_ratio": 0.6376814663645546, + "imbalance_abs_log_ratio": 0.4499163893346989, + "last_quartile_mean": 0.08931027771905065, + "mean": 0.07652445149142295, + "normalized": [ + 0.6940773558515339, + 0.8062528453975966, + 0.7180374361894372, + 0.6647119756359035, + 0.8588690718004696, + 0.7941431432641803, + 0.7473947840325739, + 0.6703237318272384, + 0.5917818305157142, + 0.5226945671206367, + 0.4893812028028422, + 0.46709594515141906, + 0.6619658291829611, + 0.6164719318106706, + 0.5787012336125282, + 0.5393739103061749, + 0.6707710619535627, + 0.6266839981127393, + 0.6317390378687008, + 0.6712190249334776, + 3.138964907679676, + 2.679582133407049, + 2.136436026253706, + 1.6866743790728473, + 2.2644726279739604, + 1.9784099533155926, + 1.4778942270811004, + 0.9605324250805215, + 1.0220403417906543, + 0.7239267230206866, + 0.4786975145327867, + 0.43067882342105884 + ], + "peak_layer": 21, + "peak_normalized": 3.138964907679676, + "population_cv": 0.6905024682030625, + "rest_mean": 0.05695082868138949, + "spike_contrast": 3.1996376327558256, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18222201466560364, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05311388894915581, + 0.061698056757450104, + 0.054947420954704285, + 0.05086671933531761, + 0.06572448462247849, + 0.06077136844396591, + 0.057193975895643234, + 0.05129615589976311, + 0.04528577998280525, + 0.039998915046453476, + 0.03744962811470032, + 0.035744260996580124, + 0.05065657198429108, + 0.047175176441669464, + 0.044284794479608536, + 0.04127529263496399, + 0.051330387592315674, + 0.04795664921402931, + 0.0483434833586216, + 0.05136466771364212, + 0.24020756781101227, + 0.20505355298519135, + 0.16348959505558014, + 0.12907183170318604, + 0.1732875257730484, + 0.1513967365026474, + 0.11309504508972168, + 0.07350421696901321, + 0.07821107655763626, + 0.05539809539914131, + 0.03663206472992897, + 0.03295746073126793 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.592033183645981e-06, + "first_to_last_ratio": 0.3776007228014791, + "imbalance_abs_log_ratio": 0.9739179305137912, + "last_quartile_mean": 6.864481520096888e-06, + "mean": 4.875440396290287e-06, + "normalized": [ + 0.58913988046903, + 0.5446521181917551, + 0.5278862817384906, + 0.53898932284961, + 0.5219441801529001, + 0.509889287242028, + 0.5048315077449682, + 0.5158761132703409, + 0.4368296481828547, + 0.4047199700448834, + 0.3956716882892422, + 0.40126275663148436, + 0.5110508169296636, + 0.5085656955747676, + 0.5112234187675316, + 0.511163584085558, + 0.5662253891905324, + 0.5872486754624854, + 0.609842074156892, + 0.6494088419115644, + 3.025689231381945, + 2.8686186674565652, + 2.4420116557657345, + 2.053486551003887, + 2.698068277749679, + 2.429082514259166, + 1.8298867229318267, + 1.2191127574146867, + 1.182073363720965, + 0.8592876866491284, + 0.5581944369380665, + 0.48806688384176716 + ], + "peak_layer": 21, + "peak_normalized": 3.025689231381945, + "population_cv": 0.8235639638153561, + "rest_mean": 3.4149978226330994e-06, + "spike_contrast": 3.736995148125518, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2761830294039101e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.87231637230434e-06, + 2.6554189389571548e-06, + 2.573678102635313e-06, + 2.6278103177901357e-06, + 2.5447077405260643e-06, + 2.485934828655445e-06, + 2.461275926179951e-06, + 2.5151232421194436e-06, + 2.1297369130479638e-06, + 1.9731880911422195e-06, + 1.92907373275375e-06, + 1.9563326532079373e-06, + 2.491597797416034e-06, + 2.4794817363726906e-06, + 2.4924393073888496e-06, + 2.492147586963256e-06, + 2.7605981358647114e-06, + 2.863095915017766e-06, + 2.973248683701968e-06, + 3.166154101563734e-06, + 1.4751567505300045e-05, + 1.3985779332870152e-05, + 1.1905882274731994e-05, + 1.0011651284003165e-05, + 1.3154271073290147e-05, + 1.1842847015941516e-05, + 8.92150364961708e-06, + 5.943711585132405e-06, + 5.763128228863934e-06, + 4.18940589952399e-06, + 2.7214437068323605e-06, + 2.3795410015736707e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.8627718115312746e-05, + "first_to_last_ratio": 0.778294860341054, + "imbalance_abs_log_ratio": 0.2506498287376175, + "last_quartile_mean": 6.247981400520075e-05, + "mean": 6.0048676687074476e-05, + "normalized": [ + 0.768607661628769, + 0.8690595287737466, + 0.7879569251847977, + 0.7385864958726717, + 0.9274786983184008, + 0.8617431831538573, + 0.8054883103443636, + 0.7195191288766225, + 0.6496482220749353, + 0.5799654886485536, + 0.5429742744436054, + 0.5176928240267351, + 0.720980532018643, + 0.6651070950014618, + 0.6225467122437304, + 0.5784265987882409, + 0.7443177261701239, + 0.697888823545442, + 0.701818835999622, + 0.7379727210931422, + 3.1748914800668615, + 2.647277084571918, + 2.0581907003495648, + 1.5579720703470805, + 2.1379505239352077, + 1.7650981274587072, + 1.310550617537982, + 0.8840683850126465, + 0.8342765345805514, + 0.5872947992080164, + 0.42062138312406294, + 0.3840285075999366 + ], + "peak_layer": 21, + "peak_normalized": 3.1748914800668615, + "population_cv": 0.6535432926484233, + "rest_mean": 4.5422861014096134e-05, + "spike_contrast": 3.0607513093024403, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013902808132115752, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.615387297235429e-05, + 5.218587466515601e-05, + 4.7315770643763244e-05, + 4.435114169609733e-05, + 5.569386848947033e-05, + 5.174653779249638e-05, + 4.83685071230866e-05, + 4.3206171540077776e-05, + 3.901051604771055e-05, + 3.482616011751816e-05, + 3.2604886655462906e-05, + 3.1086769013199955e-05, + 4.329392686486244e-05, + 3.993880091002211e-05, + 3.738310624612495e-05, + 3.473375181783922e-05, + 4.4695294491248205e-05, + 4.1907300328603014e-05, + 4.214329237584025e-05, + 4.431428533280268e-05, + 0.00019064803200308233, + 0.0001589654857525602, + 0.00012359162792563438, + 9.355416113976389e-05, + 0.00012838109978474677, + 0.00010599180677672848, + 7.869683031458408e-05, + 5.3087136620888487e-05, + 5.009720189264044e-05, + 3.52662755176425e-05, + 2.525775744288694e-05, + 2.3060403691488318e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.645518927442026e-05, + "first_to_last_ratio": 0.9865830246987611, + "imbalance_abs_log_ratio": 0.013507796190281324, + "last_quartile_mean": 3.6950959383830195e-05, + "mean": 4.1335310356771515e-05, + "normalized": [ + 0.8560051014711698, + 0.9467981965967722, + 0.8689980494059457, + 0.8159268191754907, + 1.0103083717536356, + 0.9426554996877177, + 0.8489560918930947, + 0.765857482001633, + 0.7201445334320749, + 0.6498760503731635, + 0.6109444686941845, + 0.5791405787907885, + 0.7989198470081217, + 0.7291371869136885, + 0.6790968935040115, + 0.6275849613644364, + 0.8454614229708973, + 0.7893993011096712, + 0.7856135784658201, + 0.8181223517247423, + 3.2763150867214206, + 2.615409622894913, + 1.9276349807221327, + 1.3402369957405367, + 1.940332562376331, + 1.4667183642001942, + 1.0945246384976166, + 0.8103162680270223, + 0.6598082696260246, + 0.4694457579306494, + 0.3666062775483438, + 0.3437043893777548 + ], + "peak_layer": 21, + "peak_normalized": 3.2763150867214206, + "population_cv": 0.6207682663855968, + "rest_mean": 3.199670040716969e-05, + "spike_contrast": 2.867914594845514, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.176380408462136e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5383236536290497e-05, + 3.913619730155915e-05, + 3.592030407162383e-05, + 3.37265882990323e-05, + 4.176141010248102e-05, + 3.896495763910934e-05, + 3.509186353767291e-05, + 3.1656956707593054e-05, + 2.9767397791147232e-05, + 2.6862828235607594e-05, + 2.5253579224226996e-05, + 2.3938955564517528e-05, + 3.3023599826265126e-05, + 3.0139111913740635e-05, + 2.8070680855307728e-05, + 2.594141915324144e-05, + 3.4947410313179716e-05, + 3.263006510678679e-05, + 3.2473581086378545e-05, + 3.381734131835401e-05, + 0.0001354275009362027, + 0.00010810876847244799, + 7.967939018271863e-05, + 5.539911217056215e-05, + 8.020424866117537e-05, + 6.062725879019126e-05, + 4.5242515625432134e-05, + 3.349467442603782e-05, + 2.7273379600956105e-05, + 1.9404686099733226e-05, + 1.5153784261201508e-05, + 1.4207127605914138e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012982768839719938, + "first_to_last_ratio": 0.6037915882942751, + "imbalance_abs_log_ratio": 0.5045261930845532, + "last_quartile_mean": 0.00021502069739653962, + "mean": 0.0001790727353636612, + "normalized": [ + 0.6739765257680471, + 0.7636067305383979, + 0.68849644503889, + 0.6447045606228033, + 0.8477445590156324, + 0.7818709864142286, + 0.7391626168129651, + 0.6604355769831363, + 0.5800901925673246, + 0.5130947085748045, + 0.4841531977734674, + 0.46136480942786895, + 0.6373637447050293, + 0.5885397553357715, + 0.5617314689244236, + 0.5244758601010534, + 0.6356987553893274, + 0.6101287890369049, + 0.6118132813754859, + 0.6643470301154188, + 3.09815074914609, + 2.693324715468958, + 2.18351992306606, + 1.7462447599227975, + 2.367437166894702, + 2.0445672235354038, + 1.5627333414952298, + 1.026078331181794, + 1.036030398706582, + 0.6947075889876938, + 0.46018158540857673, + 0.4142246216651318 + ], + "peak_layer": 21, + "peak_normalized": 3.09815074914609, + "population_cv": 0.7095514290759102, + "rest_mean": 0.00013205833400004648, + "spike_contrast": 3.278479211520481, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004329505027271807, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012069082004018128, + 0.00013674114597961307, + 0.00012329094170127064, + 0.00011544900917215273, + 0.00015180793707259, + 0.00014001177623867989, + 0.0001323638716712594, + 0.00011826600530184805, + 0.00010387833754066378, + 9.188127296511084e-05, + 8.669863746035844e-05, + 8.261785842478275e-05, + 0.00011413446918595582, + 0.00010539142385823652, + 0.00010059079068014398, + 9.391932690050453e-05, + 0.00011383631499484181, + 0.00010925743117695674, + 0.00010955907782772556, + 0.00011896643991349265, + 0.0005547943292185664, + 0.0004823010240215808, + 0.0003910088853444904, + 0.0003127048257738352, + 0.00042394344927743077, + 0.0003661262453533709, + 0.0002798429341055453, + 0.0001837426534621045, + 0.00018552479741629213, + 0.0001244031882379204, + 8.240597526310012e-05, + 7.417633605655283e-05 + ] + } + }, + "mlp_branch_output": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5986858215910615e-05, + "first_to_last_ratio": 0.8170320074925627, + "imbalance_abs_log_ratio": 0.20207700803204537, + "last_quartile_mean": 1.9566991340980167e-05, + "mean": 1.9039267499465495e-05, + "normalized": [ + 0.7853354674160987, + 0.9097466615865485, + 0.8087937381555731, + 0.7523818308832718, + 0.9858416226009721, + 0.9045115156777315, + 0.8288509697382657, + 0.741963942115696, + 0.6090758738841542, + 0.5628736284922787, + 0.5329013290216021, + 0.5131548852635935, + 0.7265824288035061, + 0.6757039396775449, + 0.6225875966900762, + 0.5898833165121294, + 0.7532717273459274, + 0.6927780667296713, + 0.6897910451069404, + 0.809449516707258, + 3.207633782072864, + 2.5002059817245676, + 1.9824571594317655, + 1.5924827370882133, + 2.273874867822024, + 1.6940739554643818, + 1.0968135124011316, + 0.8719494928225496, + 0.815651324531277, + 0.538892521487639, + 0.48196570767333513, + 0.44851985507141146 + ], + "peak_layer": 21, + "peak_normalized": 3.207633782072864, + "population_cv": 0.6428607369119902, + "rest_mean": 1.4415789741582961e-05, + "spike_contrast": 3.0526282764165082, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.4006047392031175e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4952212040952872e-05, + 1.7320910046692006e-05, + 1.539884033263661e-05, + 1.4324798939924221e-05, + 1.8769702364807017e-05, + 1.7221236703335308e-05, + 1.578071533003822e-05, + 1.4126449968898669e-05, + 1.1596358490351122e-05, + 1.0716701581259258e-05, + 1.0146050954062957e-05, + 9.770093129191082e-06, + 1.3833597222401295e-05, + 1.2864908057963476e-05, + 1.18536117952317e-05, + 1.1230946256546304e-05, + 1.434174191672355e-05, + 1.318998693022877e-05, + 1.3133116226526909e-05, + 1.541132587590255e-05, + 6.107099761720747e-05, + 4.760209048981778e-05, + 3.77445321646519e-05, + 3.0319704819703475e-05, + 4.3292911868775263e-05, + 3.225392720196396e-05, + 2.088252585963346e-05, + 1.660127963987179e-05, + 1.5529403754044324e-05, + 1.0260118870064616e-05, + 9.176274033961818e-06, + 8.539489499526098e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.365280069076107e-05, + "first_to_last_ratio": 0.8123684077728273, + "imbalance_abs_log_ratio": 0.2078013375593398, + "last_quartile_mean": 7.83545988269907e-05, + "mean": 7.614439766712167e-05, + "normalized": [ + 0.7854660713099145, + 0.9130496523819581, + 0.8147298517264512, + 0.752236196131495, + 0.9689924576782674, + 0.8892579571020611, + 0.8242433765322099, + 0.7396130336387524, + 0.6125904066287632, + 0.5670744382794471, + 0.5359292329408729, + 0.5145351492524995, + 0.7290151059417195, + 0.673146260009013, + 0.621425875046763, + 0.5883337897938153, + 0.7535116607120764, + 0.6934400787203782, + 0.6978162466203766, + 0.8168476312813977, + 3.2190719364475022, + 2.5076839554965473, + 1.973830744692153, + 1.575947332799018, + 2.2647184639164535, + 1.6880790923847124, + 1.0924366554795215, + 0.8732736568641918, + 0.8206255787733402, + 0.5459821164389519, + 0.48994786526283923, + 0.4571481297165359 + ], + "peak_layer": 21, + "peak_normalized": 3.2190719364475022, + "population_cv": 0.642032487569562, + "rest_mean": 5.76970003883751e-05, + "spike_contrast": 3.0462648281411457, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001757603429723531, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.9808840887853876e-05, + 6.952361582079902e-05, + 6.203711382113397e-05, + 5.727857205783948e-05, + 7.378334703389555e-05, + 6.771201151423156e-05, + 6.276151543715969e-05, + 5.6317388953175396e-05, + 4.664532752940431e-05, + 4.3179541535209864e-05, + 4.0808008634485304e-05, + 3.917896901839413e-05, + 5.551041613216512e-05, + 5.125631651026197e-05, + 4.731809895019978e-05, + 4.479832205106504e-05, + 5.7375691540073603e-05, + 5.280157711240463e-05, + 5.3134797781240195e-05, + 6.219837086973712e-05, + 0.00024511429364793, + 0.00019094608433078974, + 0.0001502961531514302, + 0.00011999956041108817, + 0.00017244562332052737, + 0.00012853776570409536, + 8.318293112097308e-05, + 6.649489660048857e-05, + 6.248604040592909e-05, + 4.157347939326428e-05, + 3.7306785088730976e-05, + 3.480926898191683e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05644794087857008, + "first_to_last_ratio": 0.8123684042388034, + "imbalance_abs_log_ratio": 0.2078013419096122, + "last_quartile_mean": 0.06948564294725657, + "mean": 0.06752561370376498, + "normalized": [ + 0.7854660588804813, + 0.9130496786513765, + 0.814729886022376, + 0.7522361645698925, + 0.968992465622299, + 0.8892579649660906, + 0.8242433717205022, + 0.7396130482968516, + 0.6125904375263521, + 0.5670744516239514, + 0.535929266538159, + 0.5145351229333333, + 0.7290151143815181, + 0.673146253269631, + 0.6214258465577596, + 0.5883338184574164, + 0.7535116614810096, + 0.6934400601144122, + 0.6978162502950979, + 0.8168476412095322, + 3.21907174067581, + 2.5076838612014067, + 1.9738308079311102, + 1.5759473804424433, + 2.264718470873259, + 1.6880792267982687, + 1.0924366234670762, + 0.8732736495232089, + 0.8206255836175469, + 0.5459820799959341, + 0.4899478787351448, + 0.45714813362074885 + ], + "peak_layer": 21, + "peak_normalized": 3.21907174067581, + "population_cv": 0.6420324679566953, + "rest_mean": 0.05116627765474496, + "spike_contrast": 3.0462647570380494, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15586602836847305, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.053039077669382095, + 0.061654239892959595, + 0.05501513555645943, + 0.05079520866274834, + 0.06543181091547012, + 0.0600476898252964, + 0.05565753951668739, + 0.049942824989557266, + 0.041365545243024826, + 0.0382920503616333, + 0.03618895262479782, + 0.034744299948215485, + 0.049227192997932434, + 0.045454613864421844, + 0.0419621616601944, + 0.039727602154016495, + 0.050881337374448776, + 0.04682496562600136, + 0.04712047055363655, + 0.055158138275146484, + 0.21736979484558105, + 0.16933289170265198, + 0.13328413665294647, + 0.10641681402921677, + 0.152926504611969, + 0.11398858577013016, + 0.07376745343208313, + 0.05896833911538124, + 0.055413246154785156, + 0.03686777502298355, + 0.03308403119444847, + 0.03086920827627182 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.57929028180115e-06, + "first_to_last_ratio": 0.4759084167214945, + "imbalance_abs_log_ratio": 0.7425298451079264, + "last_quartile_mean": 5.41971982670475e-06, + "mean": 4.213722270662856e-06, + "normalized": [ + 0.6771451422985035, + 0.6338625145334242, + 0.6120949335724296, + 0.6258725064817227, + 0.6050121598754559, + 0.5852465596314514, + 0.5785033580390577, + 0.5791973952882404, + 0.4696864977471604, + 0.4522091645384124, + 0.4457992216966758, + 0.45590579514105095, + 0.5773211960070957, + 0.5667516162949074, + 0.5644132471304436, + 0.568160519465189, + 0.6503432281671621, + 0.663170344529487, + 0.6837490184222774, + 0.7583004470800568, + 3.021070651777523, + 2.6274098317695387, + 2.3058345630897463, + 2.003282670865861, + 2.819487095674051, + 2.140274896548892, + 1.4825743986898414, + 1.122684410298403, + 0.9855482185618173, + 0.6429507763662341, + 0.5708772581826982, + 0.5252603622351902 + ], + "peak_layer": 21, + "peak_normalized": 3.021070651777523, + "population_cv": 0.7577432386283482, + "rest_mean": 3.000000956658537e-06, + "spike_contrast": 3.5892713108597136, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0767817366286181e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8533015665743733e-06, + 2.670920594027848e-06, + 2.579198053354048e-06, + 2.6372529191576177e-06, + 2.549353212089045e-06, + 2.4660664621478645e-06, + 2.4376524834224256e-06, + 2.4405769636359764e-06, + 1.9791284557868494e-06, + 1.9054838276133523e-06, + 1.8784741087074508e-06, + 1.921060402310104e-06, + 2.4326711809408152e-06, + 2.3881339075160213e-06, + 2.378280669290689e-06, + 2.394070634181844e-06, + 2.7403657441027462e-06, + 2.7944156499870587e-06, + 2.881128466469818e-06, + 3.1952674817148363e-06, + 1.27299526866409e-05, + 1.1071175322285853e-05, + 9.71614645095542e-06, + 8.441276804660447e-06, + 1.1880535566888284e-05, + 9.018523996928707e-06, + 6.247156761673978e-06, + 4.730680302600376e-06, + 4.152826477366034e-06, + 2.7092160053143743e-06, + 2.4055182166193845e-06, + 2.2133012862468604e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.826902750210138e-05, + "first_to_last_ratio": 0.9723604213825404, + "imbalance_abs_log_ratio": 0.028028739360968564, + "last_quartile_mean": 4.964108620697516e-05, + "mean": 5.395586174472555e-05, + "normalized": [ + 0.8546738942457572, + 0.9667007478721396, + 0.8780654782701063, + 0.82151592085777, + 1.0287569123437954, + 0.9492731152891766, + 0.8750422046820956, + 0.7827885676521403, + 0.6672237173045179, + 0.6202571530123847, + 0.5854234638750982, + 0.5620770546607131, + 0.7815896821357725, + 0.715521927114241, + 0.659703864497871, + 0.6215694475292887, + 0.8222045332968153, + 0.7616919988241317, + 0.764120987673305, + 0.8753289635877315, + 3.236716069528407, + 2.479479992593944, + 1.8829977352397056, + 1.4470254883420661, + 2.056662650357765, + 1.5065673712263334, + 1.0009483781315074, + 0.8155586445002443, + 0.6595748803016609, + 0.47597993512775083, + 0.43187920683965, + 0.4130800130861142 + ], + "peak_layer": 21, + "peak_normalized": 3.236716069528407, + "population_cv": 0.610766278714797, + "rest_mean": 4.176007456003895e-05, + "spike_contrast": 2.8690828214345374, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001198131125420332, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.6114666474750265e-05, + 5.2159171900711954e-05, + 4.737677954835817e-05, + 4.432559944689274e-05, + 5.550746573135257e-05, + 5.121884896652773e-05, + 4.721365621662699e-05, + 4.223603173159063e-05, + 3.600063064368442e-05, + 3.3466509194113314e-05, + 3.158702747896314e-05, + 3.032735185115598e-05, + 4.217134483042173e-05, + 3.860660217469558e-05, + 3.5594890505308285e-05, + 3.353731517563574e-05, + 4.4362754124449566e-05, + 4.109774818061851e-05, + 4.122880636714399e-05, + 4.722912854049355e-05, + 0.00017463980475440621, + 0.00013378247967921197, + 0.00010159876546822488, + 7.80755071900785e-05, + 0.00011096900561824441, + 8.128814079100266e-05, + 5.400703230407089e-05, + 4.400416946737096e-05, + 3.558793105185032e-05, + 2.568190757301636e-05, + 2.3302414774661884e-05, + 2.22880880755838e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.626883517426904e-05, + "first_to_last_ratio": 1.1972713068748915, + "imbalance_abs_log_ratio": 0.1800450565949769, + "last_quartile_mean": 3.0292912697404972e-05, + "mean": 3.794855416572318e-05, + "normalized": [ + 0.934241229967689, + 1.0326674049103126, + 0.9483113974161322, + 0.8884949951731808, + 1.1009339220259078, + 1.0148817561286225, + 0.9027362835592571, + 0.8236285600155873, + 0.7333418938739371, + 0.6823440677494514, + 0.6463725772705814, + 0.6166567793546124, + 0.850307320786337, + 0.7701976529963521, + 0.7042915982263429, + 0.6605966097637871, + 0.9142336932467707, + 0.8480352948917127, + 0.839474071242197, + 0.9443144532969522, + 3.2998505290664992, + 2.4485943259044243, + 1.7642227386757172, + 1.2451697951023746, + 1.763789999228952, + 1.2716404306318911, + 0.9094502638769789, + 0.7804639985266583, + 0.5161011792987255, + 0.40746526774101827, + 0.37633786588868356, + 0.36085204416235356 + ], + "peak_layer": 21, + "peak_normalized": 3.2998505290664992, + "population_cv": 0.5857907121700514, + "rest_mean": 3.01878957947742e-05, + "spike_contrast": 2.6453022731935993, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.985610936884768e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.545310391928069e-05, + 3.918823495041579e-05, + 3.5987046430818737e-05, + 3.3717100450303406e-05, + 4.177885057288222e-05, + 3.851329529425129e-05, + 3.425753675401211e-05, + 3.12555130221881e-05, + 2.7829264581669122e-05, + 2.5893970814649947e-05, + 2.452890475979075e-05, + 2.3401233192998916e-05, + 3.2267933420371264e-05, + 2.9227887353044935e-05, + 2.672684786375612e-05, + 2.5068686227314174e-05, + 3.469384682830423e-05, + 3.218171332264319e-05, + 3.185682726325467e-05, + 3.583536818041466e-05, + 0.00012522455654107034, + 9.292061440646648e-05, + 6.694970215903595e-05, + 4.72523934149649e-05, + 6.693328032270074e-05, + 4.825691576115787e-05, + 3.4512322599766776e-05, + 2.961748032248579e-05, + 1.9585293557611294e-05, + 1.546271778352093e-05, + 1.4281477888289373e-05, + 1.3693813343707006e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012846768913732376, + "first_to_last_ratio": 0.7741677488258334, + "imbalance_abs_log_ratio": 0.2559666991257704, + "last_quartile_mean": 0.00016594296175753698, + "mean": 0.0001574651118971815, + "normalized": [ + 0.7618335990450072, + 0.8698560675479529, + 0.7822046486498666, + 0.7351395196601278, + 0.9594102821252986, + 0.8815927774487453, + 0.812858049389013, + 0.7238937569203876, + 0.5978223149190788, + 0.5606129559633132, + 0.5319350341866579, + 0.513003926829316, + 0.6981154037320007, + 0.6408130676433959, + 0.6075107239537738, + 0.5706379352682298, + 0.7181374534396903, + 0.6783796513052409, + 0.6819318446184696, + 0.7908903240994326, + 3.2180960304949577, + 2.5452212715171636, + 2.028532306848687, + 1.6608547003669625, + 2.321647122642583, + 1.7738587692821628, + 1.1712480907534195, + 0.9285924776215382, + 0.7896984661161683, + 0.5235677697382035, + 0.4733567899608167, + 0.4487468679123384 + ], + "peak_layer": 21, + "peak_normalized": 3.2180960304949577, + "population_cv": 0.6626251267593958, + "rest_mean": 0.00011795681536939271, + "spike_contrast": 3.1436073616095452, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037080991314724087, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011996221292065457, + 0.00013697198301088065, + 0.00012316994252614677, + 0.00011575882672332227, + 0.0001510736474301666, + 0.0001388201053487137, + 0.0001279967837035656, + 0.00011398801143513992, + 9.413615771336481e-05, + 8.82769818417728e-05, + 8.376120968023315e-05, + 8.078022074187174e-05, + 0.00010992882016580552, + 0.00010090570140164346, + 9.566174412611872e-05, + 8.985556632978842e-05, + 0.0001130815944634378, + 0.00010682112770155072, + 0.00010738047421909869, + 0.00012453763338271528, + 0.0005067378515377641, + 0.0004007835523225367, + 0.0003194230666849762, + 0.0002615266712382436, + 0.0003655784239526838, + 0.0002793208695948124, + 0.00018443071166984737, + 0.0001462209183955565, + 0.00012434995733201504, + 8.244365744758397e-05, + 7.453717989847064e-05, + 7.066197576932609e-05 + ] + } + }, + "post_attention_state": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.6127134472299076e-05, + "first_to_last_ratio": 0.6403082943802734, + "imbalance_abs_log_ratio": 0.4458055086442437, + "last_quartile_mean": 2.5186515017594502e-05, + "mean": 2.159394537670778e-05, + "normalized": [ + 0.6929830574423667, + 0.8032565154251953, + 0.7123500282965459, + 0.6640666207712503, + 0.8725494553308608, + 0.8071350816916083, + 0.7516540798235798, + 0.6706926732818747, + 0.5882490907185235, + 0.5190611932100045, + 0.48631129869562834, + 0.46529523801872363, + 0.6602888008616835, + 0.6189890701429912, + 0.5802627939183058, + 0.5409897412703016, + 0.6697392892795451, + 0.6259125602673482, + 0.6246778698614331, + 0.6676379780381497, + 3.137658015393984, + 2.675256160517513, + 2.137000554072058, + 1.6970284098395438, + 2.2713857488305376, + 1.9850977916906365, + 1.4820653354312663, + 0.9615821107638199, + 1.0163133272453195, + 0.7191715506408339, + 0.4720786856528992, + 0.423259873575668 + ], + "peak_layer": 21, + "peak_normalized": 3.137658015393984, + "population_cv": 0.691835904675886, + "rest_mean": 1.6060833668571258e-05, + "spike_contrast": 3.20486156963133, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.147274860064499e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4964238289394416e-05, + 1.7345477317576297e-05, + 1.5382447600131854e-05, + 1.43398183354293e-05, + 1.8841785276890732e-05, + 1.7429230865673162e-05, + 1.623117714188993e-05, + 1.4482900951406918e-05, + 1.2702618732873816e-05, + 1.1208579053345602e-05, + 1.0501379620109219e-05, + 1.0047559953818563e-05, + 1.4258240298659075e-05, + 1.3366416169446893e-05, + 1.2530163076007739e-05, + 1.1682102922350168e-05, + 1.4462313629337586e-05, + 1.3515921637008432e-05, + 1.3489259799825959e-05, + 1.4416938029171433e-05, + 6.775441579520702e-05, + 5.7769335398916155e-05, + 4.614627323462628e-05, + 3.664553878479637e-05, + 4.9048179789679125e-05, + 4.286609328119084e-05, + 3.200363789801486e-05, + 2.0764351575053297e-05, + 2.1946214474155568e-05, + 1.55297511810204e-05, + 1.0194041351496708e-05, + 9.139850590145215e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.422088154067751e-05, + "first_to_last_ratio": 0.637682021407296, + "imbalance_abs_log_ratio": 0.44991551892756126, + "last_quartile_mean": 0.00010070988264487823, + "mean": 8.629204910448607e-05, + "normalized": [ + 0.6940783808537224, + 0.8062729813904731, + 0.7180303336815479, + 0.6647219507645815, + 0.8588589713696609, + 0.794136864353616, + 0.7473889560340541, + 0.6703283692089702, + 0.5917748510321142, + 0.5226947352403994, + 0.4893764215021349, + 0.4670989784552037, + 0.661952700545094, + 0.6164859593679979, + 0.5786990118719374, + 0.539368710966703, + 0.6707742418503443, + 0.6266861813765591, + 0.6317438172781369, + 0.6712283354793382, + 3.1389231589965445, + 2.679577878793134, + 2.1364533990925567, + 1.686690164787602, + 2.264487313238182, + 1.9784244316759128, + 1.477878596963198, + 0.9605191983702851, + 1.022047261980904, + 0.7239260887312458, + 0.4786954192739982, + 0.4306763354738471 + ], + "peak_layer": 21, + "peak_normalized": 3.1389231589965445, + "population_cv": 0.690500620274875, + "rest_mean": 6.42200389443638e-05, + "spike_contrast": 3.1996384204494492, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002054809039691463, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.98934457229916e-05, + 6.957494770176709e-05, + 6.196030881255865e-05, + 5.736021921620704e-05, + 7.411270053125918e-05, + 6.852769729448482e-05, + 6.449372449424118e-05, + 5.784400855191052e-05, + 5.106546450406313e-05, + 4.51043997600209e-05, + 4.2229294194839895e-05, + 4.030692798551172e-05, + 5.7121254940284416e-05, + 5.319783667800948e-05, + 4.993712354917079e-05, + 4.654323129216209e-05, + 5.788248381577432e-05, + 5.40780347364489e-05, + 5.451446850202046e-05, + 5.7921668485505506e-05, + 0.00027086411137133837, + 0.00023122626589611173, + 0.00018435894162394106, + 0.00014554795052390546, + 0.00019540725043043494, + 0.0001707222982076928, + 0.00012752917245961726, + 8.288516983157024e-05, + 8.819455251796171e-05, + 6.24690655968152e-05, + 4.13076086260844e-05, + 3.716394348884933e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.056951718870550394, + "first_to_last_ratio": 0.6376820191187264, + "imbalance_abs_log_ratio": 0.4499155225164498, + "last_quartile_mean": 0.08931052964180708, + "mean": 0.07652465114369988, + "normalized": [ + 0.6940783685018588, + 0.8062729890693022, + 0.7180303052956253, + 0.6647219248274491, + 0.8588589447083164, + 0.7941368847913186, + 0.7473889396113633, + 0.6703284088385635, + 0.5917748342982165, + 0.5226947125226558, + 0.4893764209845987, + 0.46709901042220126, + 0.6619527107838585, + 0.616485998695461, + 0.5786990422522198, + 0.5393687059717984, + 0.6707742773702546, + 0.6266862088911475, + 0.6317438155472213, + 0.671228324296238, + 3.138923030946758, + 2.6795778685275287, + 2.136453429700619, + 1.6866902298133055, + 2.2644871659665227, + 1.9784244587489672, + 1.4778786878450325, + 0.9605192092561265, + 1.0220472167545944, + 0.7239261000048334, + 0.4786954380387232, + 0.4306763367173204 + ], + "peak_layer": 21, + "peak_normalized": 3.138923030946758, + "population_cv": 0.6905006031140464, + "rest_mean": 0.056950972587973984, + "spike_contrast": 3.199638339154521, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18222251534461975, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05311410501599312, + 0.0616997592151165, + 0.05494701862335205, + 0.05086761340498924, + 0.06572388112545013, + 0.060771048069000244, + 0.05719367787241936, + 0.051296647638082504, + 0.045285362750291824, + 0.03999903053045273, + 0.03744935989379883, + 0.03574458882212639, + 0.05065570026636124, + 0.04717637598514557, + 0.04428474232554436, + 0.04127500206232071, + 0.05133076757192612, + 0.04795694351196289, + 0.048343975096940994, + 0.05136551335453987, + 0.24020498991012573, + 0.20505376160144806, + 0.163491353392601, + 0.1290733814239502, + 0.17328909039497375, + 0.15139824151992798, + 0.11309415102005005, + 0.07350339740514755, + 0.07821180671453476, + 0.05539819225668907, + 0.036632001399993896, + 0.03295735642313957 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.591977136034984e-06, + "first_to_last_ratio": 0.377584641115206, + "imbalance_abs_log_ratio": 0.9739605205507381, + "last_quartile_mean": 6.864625447633443e-06, + "mean": 4.875532418679995e-06, + "normalized": [ + 0.5891470886653372, + 0.5445974877366685, + 0.5278757119769764, + 0.5390247128297342, + 0.5219320902964267, + 0.5098223481982903, + 0.5047854170324074, + 0.5158515929518226, + 0.43682532071303765, + 0.4047462819890774, + 0.3956760190748421, + 0.40124851416781243, + 0.5110306315304836, + 0.5086020328656453, + 0.5111981468235955, + 0.5112038363741968, + 0.5662160078662817, + 0.5872378713533353, + 0.609816433193417, + 0.6494206487580901, + 3.0257058078799917, + 2.8686033250815077, + 2.442085511360172, + 2.0535509509100836, + 2.6979608311079244, + 2.429139638501957, + 1.8298842703872822, + 1.2190711864897739, + 1.1821681083908706, + 0.8593060718331675, + 0.5582216762792698, + 0.48804442738052095 + ], + "peak_layer": 21, + "peak_normalized": 3.0257058078799917, + "population_cv": 0.8235699242655958, + "rest_mean": 3.415056493462611e-06, + "spike_contrast": 3.7370106290435205, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2762102414853871e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.872405730158789e-06, + 2.6552027065918082e-06, + 2.573675146777532e-06, + 2.6280324618710438e-06, + 2.544696826589643e-06, + 2.4856553864083253e-06, + 2.4610976652184036e-06, + 2.515051164664328e-06, + 2.129756012436701e-06, + 1.973353619177942e-06, + 1.929131258293637e-06, + 1.956300138772349e-06, + 2.491546410965384e-06, + 2.4797056994430022e-06, + 2.492363137207576e-06, + 2.49239087679598e-06, + 2.7606045023276238e-06, + 2.8630972792598186e-06, + 2.9731797894783085e-06, + 3.1662714263802627e-06, + 1.4751926755707245e-05, + 1.398596850776812e-05, + 1.1906467079825234e-05, + 1.0012154234573245e-05, + 1.315399549639551e-05, + 1.1843349057016894e-05, + 8.921660082705785e-06, + 5.943621090409579e-06, + 5.7636989367892966e-06, + 4.189574610791169e-06, + 2.7216278795094695e-06, + 2.3794764274498448e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.862782361669815e-05, + "first_to_last_ratio": 0.7782948608369358, + "imbalance_abs_log_ratio": 0.25064982810047876, + "last_quartile_mean": 6.247994951991132e-05, + "mean": 6.004880788168521e-05, + "normalized": [ + 0.7686076787185943, + 0.8690658694356951, + 0.7879558094938062, + 0.7385895471539009, + 0.9274752179545845, + 0.8617380288980451, + 0.805485823509466, + 0.7195218583140428, + 0.6496432282856196, + 0.579963373368429, + 0.542973633406911, + 0.5176967820022496, + 0.7209744736270646, + 0.6651136389238139, + 0.6225458367735092, + 0.578423456947426, + 0.7443181598307745, + 0.6978871776308329, + 0.7018192413444355, + 0.7379772277240845, + 3.17488720924795, + 2.647274451154299, + 2.058197351015695, + 1.557976178868823, + 2.137956515672529, + 1.7650930593990968, + 1.3105561147977765, + 0.8840594863751997, + 0.8342799220515003, + 0.5872955759322932, + 0.4206210094039151, + 0.3840270627376372 + ], + "peak_layer": 21, + "peak_normalized": 3.17488720924795, + "population_cv": 0.6535433957622422, + "rest_mean": 4.5422938354830774e-05, + "spike_contrast": 3.060755388404179, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00013902850332669914, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.615397483576089e-05, + 5.218636943027377e-05, + 4.7315807023551315e-05, + 4.4351421820465475e-05, + 5.569378117797896e-05, + 5.17463413416408e-05, + 4.8368463467340916e-05, + 4.320642983657308e-05, + 3.9010301406960934e-05, + 3.482610918581486e-05, + 3.260491939727217e-05, + 3.108707460341975e-05, + 4.329365765443072e-05, + 3.9939281123224646e-05, + 3.738313534995541e-05, + 3.47336390404962e-05, + 4.4695418182527646e-05, + 4.19072930526454e-05, + 4.2143408791162074e-05, + 4.43146527686622e-05, + 0.00019064819207414985, + 0.00015896567492745817, + 0.0001235922973137349, + 9.355461224913597e-05, + 0.00012838174006901681, + 0.00010599173401715234, + 7.869733235565946e-05, + 5.308671825332567e-05, + 5.009751475881785e-05, + 3.526639920892194e-05, + 2.5257790184696205e-05, + 2.3060367311700247e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.645344395408756e-05, + "first_to_last_ratio": 0.9865245275167812, + "imbalance_abs_log_ratio": 0.01356709065898008, + "last_quartile_mean": 3.695138127568498e-05, + "mean": 4.1334421752026174e-05, + "normalized": [ + 0.8559009893021382, + 0.9469134291395908, + 0.8691128415923758, + 0.8158459610499553, + 1.0100395593992384, + 0.9426509450849102, + 0.8490005706596633, + 0.7658551995239993, + 0.720151565776352, + 0.6499584076998435, + 0.6111014164691825, + 0.5791973878203037, + 0.798669989787535, + 0.7292203680582068, + 0.6791147051708346, + 0.6276623067938342, + 0.8453470506184937, + 0.7894265691211152, + 0.7857081832788244, + 0.8180697050541997, + 3.276793902438668, + 2.6146895715088023, + 1.9277214837041228, + 1.3401559674726915, + 1.9403857172500238, + 1.4668533992566692, + 1.0945373428822949, + 0.8102509556533821, + 0.6598620161196013, + 0.46946557551492035, + 0.3666267887349235, + 0.3437101280633033 + ], + "peak_layer": 21, + "peak_normalized": 3.276793902438668, + "population_cv": 0.6207652283001258, + "rest_mean": 3.1996292111418574e-05, + "spike_contrast": 2.867842357851226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.176032181130722e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5378172469791025e-05, + 3.9140119042713195e-05, + 3.5924276744481176e-05, + 3.372252103872597e-05, + 4.174940113443881e-05, + 3.8963931729085743e-05, + 3.509294765535742e-05, + 3.165618181810714e-05, + 2.976704854518175e-05, + 2.6865654945140705e-05, + 2.5259523681597784e-05, + 2.3940789105836302e-05, + 3.301256219856441e-05, + 3.0141902243485674e-05, + 2.8070813641534187e-05, + 2.5944058506865986e-05, + 3.4941931517096236e-05, + 3.263049075030722e-05, + 3.2476793421665207e-05, + 3.381443821126595e-05, + 0.0001354443811578676, + 0.00010807668149936944, + 7.968125282786787e-05, + 5.53945719730109e-05, + 8.020472159842029e-05, + 6.06315370532684e-05, + 4.524206815403886e-05, + 3.349125472595915e-05, + 2.7275014872429892e-05, + 1.940508809639141e-05, + 1.5154306311160326e-05, + 1.4207059393811505e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012982848875253694, + "first_to_last_ratio": 0.6036924070795879, + "imbalance_abs_log_ratio": 0.5046904705661434, + "last_quartile_mean": 0.00021505734912352636, + "mean": 0.00017907720302900998, + "normalized": [ + 0.6739415088564852, + 0.7635979187076084, + 0.6884132846665824, + 0.6448880523181134, + 0.847636297931654, + 0.7820437425942102, + 0.7390348805248977, + 0.6603333703627584, + 0.580097091853396, + 0.5130765445550399, + 0.4840585582545864, + 0.4614791311662735, + 0.637447225260648, + 0.5884103323685723, + 0.5619029319457929, + 0.5245290839622635, + 0.6356966288449003, + 0.6100272686595811, + 0.6118819192326126, + 0.6643605628507145, + 3.097964891544288, + 2.6928767346712488, + 2.1827809090098653, + 1.7461631642015993, + 2.367937988760414, + 2.0449314566347367, + 1.563314534665903, + 1.0260809297257236, + 1.0359670091642164, + 0.6947146354097079, + 0.460142110411202, + 0.41426930088440395 + ], + "peak_layer": 21, + "peak_normalized": 3.097964891544288, + "population_cv": 0.7095182139952283, + "rest_mean": 0.00013206795362445214, + "spike_contrast": 3.278063587209748, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004329271498136222, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00012068756041117013, + 0.00013674297952093184, + 0.0001232791255461052, + 0.0001154847486759536, + 0.00015179233741946518, + 0.0001400462060701102, + 0.00013234429934527725, + 0.00011825065303128213, + 0.00010388216469436884, + 9.188031253870577e-05, + 8.668385271448642e-05, + 8.264039206551388e-05, + 0.00011415226617828012, + 0.00010537087655393407, + 0.00010062400542665273, + 9.393120126333088e-05, + 0.00011383877426851541, + 0.00010924197704298422, + 0.00010957410268019885, + 0.00011897183139808476, + 0.0005547748878598213, + 0.0004822328337468207, + 0.00039088630001060665, + 0.0003126980154775083, + 0.0004240437119733542, + 0.00036620060564018786, + 0.0002799539943225682, + 0.00018374770297668874, + 0.00018551807443145663, + 0.0001244075538124889, + 8.240096212830395e-05, + 7.418618770316243e-05 + ] + } + }, + "post_mlp_state": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.5986864127626177e-05, + "first_to_last_ratio": 0.8170274391577745, + "imbalance_abs_log_ratio": 0.20208259942567244, + "last_quartile_mean": 1.956710798367567e-05, + "mean": 1.9039307829871177e-05, + "normalized": [ + 0.7853167979867789, + 0.9097665173048055, + 0.808777694116954, + 0.752402497635975, + 0.9858540561935626, + 0.9044938358034963, + 0.8288447236919866, + 0.7419578801196516, + 0.6090850929460484, + 0.5628647453138417, + 0.5328979550345672, + 0.5131403750850531, + 0.7265824660927316, + 0.6757007408890954, + 0.6225826951824261, + 0.5898791530514104, + 0.7532772493405897, + 0.6927950381940383, + 0.6897848547806317, + 0.8094505726944553, + 3.2076147584918284, + 2.5002121502474965, + 1.982484105644761, + 1.5924612114390058, + 2.273889158865966, + 1.6940625327896233, + 1.0968336406315982, + 0.8719471681044353, + 0.815653609383569, + 0.5388945327422838, + 0.4819664542042268, + 0.448525735997107 + ], + "peak_layer": 21, + "peak_normalized": 3.2076147584918284, + "population_cv": 0.6428608089466793, + "rest_mean": 1.4415815443229534e-05, + "spike_contrast": 3.052631111367605, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.400616671773605e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.4951888260839041e-05, + 1.7321324776276015e-05, + 1.5398567484226078e-05, + 1.4325222764455248e-05, + 1.8769978851196356e-05, + 1.7220936570083722e-05, + 1.5780629837536253e-05, + 1.4126364476396702e-05, + 1.1596558579185512e-05, + 1.0716555152612273e-05, + 1.0146008207811974e-05, + 9.769837561179884e-06, + 1.3833627235726453e-05, + 1.286487440665951e-05, + 1.1853543583129067e-05, + 1.1230890777369495e-05, + 1.4341877431434114e-05, + 1.3190337995183654e-05, + 1.3133026186551433e-05, + 1.541137862659525e-05, + 6.107076478656381e-05, + 4.760230876854621e-05, + 3.7745125155197456e-05, + 3.03193592117168e-05, + 4.329327566665597e-05, + 3.225377804483287e-05, + 2.0882953322143294e-05, + 1.6601270544924773e-05, + 1.5529480151599273e-05, + 1.0260178896714933e-05, + 9.176307685265783e-06, + 8.539619557268452e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 6.365272747643758e-05, + "first_to_last_ratio": 0.8123633197064085, + "imbalance_abs_log_ratio": 0.2078076008289402, + "last_quartile_mean": 7.835499945940683e-05, + "mean": 7.614457786075945e-05, + "normalized": [ + 0.7854618236676242, + 0.9130540849409994, + 0.8147131127464502, + 0.7522410092502885, + 0.9690072688462938, + 0.8892440039434748, + 0.8242354060534288, + 0.7396083689563772, + 0.6126026690255093, + 0.5670644008562938, + 0.5359283946772616, + 0.5145269561429746, + 0.7290174418180176, + 0.6731434248219464, + 0.6214223500409742, + 0.5883307730946398, + 0.7535191463387267, + 0.6934458909688136, + 0.6978155985835179, + 0.8168473226649812, + 3.219066994145898, + 2.5077095541323673, + 1.973834673592298, + 1.5759251613516239, + 2.2647260999477155, + 1.6880724220748888, + 1.092454232265147, + 0.8732751258075867, + 0.8206258345435784, + 0.5459870832104624, + 0.48994656248557816, + 0.45714680900426247 + ], + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "rest_mean": 5.7697108584559625e-05, + "spike_contrast": 3.04626897714722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017576091195223854, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.980865898891352e-05, + 6.95241178618744e-05, + 6.203598604770377e-05, + 5.727907409891486e-05, + 7.378464943030849e-05, + 6.77111092954874e-05, + 6.276105705183e-05, + 5.631716703646816e-05, + 4.664637162932195e-05, + 4.3178879423066974e-05, + 4.080804137629457e-05, + 3.917843787348829e-05, + 5.551072536036372e-05, + 5.1256221922812983e-05, + 4.731794251711108e-05, + 4.47981983597856e-05, + 5.737639730796218e-05, + 5.2802144637098536e-05, + 5.3134874178795144e-05, + 6.219849456101656e-05, + 0.0002451144973747432, + 0.00019094848539680243, + 0.00015029680798761547, + 0.00011999815615126863, + 0.0001724466128507629, + 0.00012853756197728217, + 8.318446634802967e-05, + 6.64951658109203e-05, + 6.248620775295421e-05, + 4.157395596848801e-05, + 3.7306774174794555e-05, + 3.4809250792022794e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.05644787335768342, + "first_to_last_ratio": 0.8123632868395458, + "imbalance_abs_log_ratio": 0.2078076412872703, + "last_quartile_mean": 0.06948599754832685, + "mean": 0.06752577290171757, + "normalized": [ + 0.7854618348329956, + 0.9130540359301327, + 0.8147131249195937, + 0.7522410113175043, + 0.9690071730110875, + 0.88924395208314, + 0.8242353599678512, + 0.7396083806676593, + 0.6126026750590019, + 0.5670643980854334, + 0.5359283892166958, + 0.5145269586503932, + 0.7290174229589911, + 0.6731434525628229, + 0.6214223402600404, + 0.5883307763560162, + 0.7535191533057549, + 0.6934459281737582, + 0.6978155981634587, + 0.8168473704719704, + 3.2190670201887848, + 2.507709505453073, + 1.9738347607250095, + 1.5759251284181852, + 2.264726151347608, + 1.68807248858185, + 1.0924542395938426, + 0.8732750663142743, + 0.820625855657368, + 0.5459870819951741, + 0.48994655813463567, + 0.45714680759589343 + ], + "peak_layer": 21, + "peak_normalized": 3.2190670201887848, + "population_cv": 0.6420341436220799, + "rest_mean": 0.05116637251167386, + "spike_contrast": 3.0462690113979045, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15586653500795364, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.05303891748189926, + 0.06165467947721481, + 0.05501413345336914, + 0.05079565569758415, + 0.06543295830488205, + 0.060046885162591934, + 0.05565712973475456, + 0.04994262754917145, + 0.041366469115018845, + 0.038291461765766144, + 0.03618897870182991, + 0.03474383056163788, + 0.04922746494412422, + 0.04545453190803528, + 0.041962023824453354, + 0.03972749039530754, + 0.05088196322321892, + 0.04682547226548195, + 0.04712053760886192, + 0.05515825003385544, + 0.21736998856067657, + 0.1693350225687027, + 0.13328471779823303, + 0.10641556233167648, + 0.15292738378047943, + 0.11398839950561523, + 0.07376881688833237, + 0.058968573808670044, + 0.055413395166397095, + 0.036868199706077576, + 0.033084020018577576, + 0.030869191512465477 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.579253447265728e-06, + "first_to_last_ratio": 0.47590496457778414, + "imbalance_abs_log_ratio": 0.7425370989326324, + "last_quartile_mean": 5.419681741614113e-06, + "mean": 4.2137082871818166e-06, + "normalized": [ + 0.6771144196047112, + 0.6338426021784607, + 0.6121020910961615, + 0.6258737201152189, + 0.6050362914422536, + 0.5852204423685657, + 0.5784631886764509, + 0.5792281322851218, + 0.4697078599262646, + 0.4522117174548025, + 0.44580914592548354, + 0.4558904724294877, + 0.5773213311917524, + 0.5667240346830847, + 0.5644321177223985, + 0.5681434108598962, + 0.6503548834184826, + 0.6631947770191623, + 0.6837887360559592, + 0.7582966501774209, + 3.020973835679826, + 2.627473806530212, + 2.3058644468737186, + 2.00331262981993, + 2.819406877957528, + 2.1402761714678284, + 1.4825745701937771, + 1.1227288221975413, + 0.9855094000056028, + 0.6429466506282437, + 0.5709200007532207, + 0.5252567632614316 + ], + "peak_layer": 21, + "peak_normalized": 3.020973835679826, + "population_cv": 0.7577358346351384, + "rest_mean": 2.999999306094078e-06, + "spike_contrast": 3.589246425218318, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0767736785055605e-05, + "top_five_layers": [ + 21, + 25, + 22, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8531626412586775e-06, + 2.670827825568267e-06, + 2.5792196538532153e-06, + 2.6372492811788106e-06, + 2.5494464352959767e-06, + 2.465948227836634e-06, + 2.43747513195558e-06, + 2.440698381178663e-06, + 1.979211901925737e-06, + 1.9054882614000235e-06, + 1.8785096926876577e-06, + 1.920989461723366e-06, + 2.4326636776095256e-06, + 2.3880097614892293e-06, + 2.3783522919984534e-06, + 2.393990598648088e-06, + 2.7404057618696243e-06, + 2.7945093279413413e-06, + 2.8812862638005754e-06, + 3.1952408789948095e-06, + 1.2729502486763522e-05, + 1.1071408152929507e-05, + 9.716240128909703e-06, + 8.441375030088238e-06, + 1.1880158126587048e-05, + 9.018499440571759e-06, + 6.247136752790539e-06, + 4.7308517423516605e-06, + 4.152649125899188e-06, + 2.709189629968023e-06, + 2.405690338491695e-06, + 2.2132787762529915e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 4.826905797017389e-05, + "first_to_last_ratio": 0.9723593605402814, + "imbalance_abs_log_ratio": 0.028029830358518852, + "last_quartile_mean": 4.964117169947713e-05, + "mean": 5.395593586854375e-05, + "normalized": [ + 0.8546740011831107, + 0.9667023865336807, + 0.8780594173977533, + 0.8215201862718362, + 1.028762848378348, + 0.9492707323927138, + 0.8750383055629682, + 0.7827836490446655, + 0.6672270484578459, + 0.6202534690629704, + 0.5854228619045188, + 0.5620710233383177, + 0.781592991024926, + 0.7155185842665527, + 0.6597031604820425, + 0.621567177701999, + 0.8222052916650392, + 0.7616961441483728, + 0.7641199379358804, + 0.8753280307745216, + 3.236716477578425, + 2.479496544122304, + 1.8830004075567146, + 1.4470156791415152, + 2.05665793704697, + 1.5065644924277526, + 1.0009572516438536, + 0.8155596816984735, + 0.6595753226879756, + 0.4759799891964506, + 0.4318790517932509, + 0.413079917578251 + ], + "peak_layer": 21, + "peak_normalized": 3.236716477578425, + "population_cv": 0.6107674157639074, + "rest_mean": 4.176012171902349e-05, + "spike_contrast": 2.869084843241085, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011981333227595314, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 4.61147355963476e-05, + 5.2159331971779466e-05, + 4.737651761388406e-05, + 4.4325890485197306e-05, + 5.550786227104254e-05, + 5.121879075886682e-05, + 4.72135106974747e-05, + 4.2235824366798624e-05, + 3.6000859836349264e-05, + 3.3466356399003416e-05, + 3.158703839289956e-05, + 3.0327068088809028e-05, + 4.217158129904419e-05, + 3.860647484543733e-05, + 3.559490141924471e-05, + 3.353723877808079e-05, + 4.4362855987856165e-05, + 4.1098028304986656e-05, + 4.122880636714399e-05, + 4.7229143092408776e-05, + 0.00017464006668888032, + 0.00013378355652093887, + 0.00010159904923057184, + 7.807508518453687e-05, + 0.00011096890375483781, + 8.128809713525698e-05, + 5.400758527684957e-05, + 4.4004285882692784e-05, + 3.558800381142646e-05, + 2.5681945771793835e-05, + 2.330243842152413e-05, + 2.228811354143545e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 3.626825309765991e-05, + "first_to_last_ratio": 1.1971952602722011, + "imbalance_abs_log_ratio": 0.17998153797771219, + "last_quartile_mean": 3.0294350722215313e-05, + "mean": 3.794959516767449e-05, + "normalized": [ + 0.9342317076920968, + 1.032527492584725, + 0.9483990781630639, + 0.8885231558783968, + 1.1008949985426397, + 1.0148066560537, + 0.9027782413985963, + 0.8234017777861798, + 0.7333936750261243, + 0.6823073758552121, + 0.6464481216407498, + 0.616777331910669, + 0.8501710687237772, + 0.7702178906455867, + 0.7042577074377577, + 0.660638930723233, + 0.914060026408416, + 0.847914539167618, + 0.8394293783449699, + 0.9442746494437388, + 3.300220154812763, + 2.4489522165880904, + 1.7640184700183157, + 1.245126339879113, + 1.7643693302152461, + 1.271277790934411, + 0.9095623054467832, + 0.780451313056054, + 0.5159478762516835, + 0.40747033934370974, + 0.37633772798986104, + 0.36081233203671814 + ], + "peak_layer": 21, + "peak_normalized": 3.300220154812763, + "population_cv": 0.5858778602410362, + "rest_mean": 3.018723526582579e-05, + "spike_contrast": 2.6456990159702416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.986633863765746e-05, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.5453715099720284e-05, + 3.9184000343084335e-05, + 3.599136107368395e-05, + 3.371909406268969e-05, + 4.1778519516810775e-05, + 3.85115017706994e-05, + 3.4260068787261844e-05, + 3.124776412732899e-05, + 2.783199306577444e-05, + 2.5893288693623617e-05, + 2.453244451317005e-05, + 2.3406450054608285e-05, + 3.226364788133651e-05, + 2.9229457140900195e-05, + 2.6726294890977442e-05, + 2.5070979972952046e-05, + 3.468820796115324e-05, + 3.217801349819638e-05, + 3.185600508004427e-05, + 3.5834840673487633e-05, + 0.00012524201883934438, + 9.293674520449713e-05, + 6.694378680549562e-05, + 4.725204053102061e-05, + 6.695710180792958e-05, + 4.8244477511616424e-05, + 3.451752127148211e-05, + 2.9617811378557235e-05, + 1.95800130313728e-05, + 1.546333442092873e-05, + 1.4281864423537627e-05, + 1.3692681932298001e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00012846494701079791, + "first_to_last_ratio": 0.7741545804954827, + "imbalance_abs_log_ratio": 0.25598370893064326, + "last_quartile_mean": 0.00016594224234722788, + "mean": 0.000157460202672155, + "normalized": [ + 0.7615886502877881, + 0.869859991005712, + 0.7823534284439785, + 0.735201115807512, + 0.9596786286759813, + 0.8813992955944876, + 0.8128285893567467, + 0.7239431731448671, + 0.5977864740547744, + 0.5606030792342883, + 0.5318673348078621, + 0.5130567490014816, + 0.6987134321316841, + 0.6407269063222457, + 0.6075335461477536, + 0.5704889146486948, + 0.7180428901779269, + 0.6784140170824181, + 0.6819219612414157, + 0.7908928021790416, + 3.2176296651190497, + 2.545498211708922, + 2.0288816729118655, + 1.6601468184271124, + 2.321862751456626, + 1.7738386620365922, + 1.1711907122839438, + 0.9284857615000071, + 0.7896970255450443, + 0.5236030848875767, + 0.47341239614817127, + 0.4488522586284289 + ], + "peak_layer": 21, + "peak_normalized": 3.2176296651190497, + "population_cv": 0.6625973254291921, + "rest_mean": 0.0001179550758765631, + "spike_contrast": 3.143466990402101, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037078788736835124, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00011991990322712809, + 0.00013696833048015833, + 0.00012318952940404415, + 0.00011576491669984534, + 0.0001511111913714558, + 0.00013878531171940267, + 0.00012798815441783518, + 0.00011399223876651376, + 9.412757935933769e-05, + 8.82726744748652e-05, + 8.374793833354488e-05, + 8.078601968009025e-05, + 0.000110019558633212, + 0.00010088898852700368, + 9.566235530655831e-05, + 8.982930012280121e-05, + 0.0001130631790147163, + 0.00010682320862542838, + 0.00010737557022366673, + 0.00012453414092306048, + 0.0005066486191935837, + 0.00040081466431729496, + 0.00031946811941452324, + 0.0002614070544950664, + 0.0003656009794212878, + 0.00027930899523198605, + 0.00018441592692397535, + 0.00014619955618400127, + 0.00012434585369192064, + 8.24466478661634e-05, + 7.454361184500158e-05, + 7.067636761348695e-05 + ] + } + }, + "pre_attention_input": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0311521975836513e-05, + "first_to_last_ratio": 1.1418129560476864, + "imbalance_abs_log_ratio": 0.13261731152138012, + "last_quartile_mean": 9.030832870848826e-06, + "mean": 1.3945096057454975e-05, + "normalized": [ + 0.7145337044030123, + 0.932335477800909, + 0.645487324949067, + 0.558162738870377, + 0.34661223918607953, + 0.7511338800577496, + 0.9447212820009926, + 1.0225104899919193, + 0.6315142693979814, + 0.5614582240786927, + 0.4574638139025647, + 0.40615350248375953, + 0.4194348381092698, + 0.7040945119904685, + 0.6483274463519548, + 0.5709286433652233, + 0.8217341836062412, + 1.1893729541204203, + 1.2408864574521172, + 1.5700693567578727, + 2.5502861886942485, + 4.114342551839228, + 2.785136777749787, + 2.232505646034574, + 1.1806687356339707, + 1.2053474712098855, + 0.7898562388784587, + 0.5945521897391881, + 0.46830824879649285, + 0.2867747481020556, + 0.32588927285831853, + 0.32939659158711965 + ], + "peak_layer": 22, + "peak_normalized": 4.114342551839228, + "population_cv": 0.8262244130468728, + "rest_mean": 9.884005235276331e-06, + "spike_contrast": 3.629600110811422, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.587498649721965e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.964241144189145e-06, + 1.3001507795706857e-05, + 9.001382750284392e-06, + 7.783633009239566e-06, + 4.833540970139438e-06, + 1.0474634109414183e-05, + 1.3174229025025852e-05, + 1.425900700269267e-05, + 8.80652714840835e-06, + 7.82958886702545e-06, + 6.3793768276809715e-06, + 5.663849606207805e-06, + 5.849059107276844e-06, + 9.818665603233967e-06, + 9.040988516062498e-06, + 7.961654773680493e-06, + 1.1459162124083377e-05, + 1.658592009334825e-05, + 1.730428084556479e-05, + 2.189476799685508e-05, + 3.556398587534204e-05, + 5.7374902098672464e-05, + 3.883899989887141e-05, + 3.113250568276271e-05, + 1.6464538930449635e-05, + 1.68086862686323e-05, + 1.1014621122740209e-05, + 8.291087397083174e-06, + 6.530603513965616e-06, + 3.999101409135619e-06, + 4.544557214103406e-06, + 4.593467110680649e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 3.944479362871789e-05, + "first_to_last_ratio": 1.1135282861542686, + "imbalance_abs_log_ratio": 0.1075336103151947, + "last_quartile_mean": 3.54232524841791e-05, + "mean": 5.533300526394669e-05, + "normalized": [ + 0.5280207224829799, + 0.8999098868607942, + 0.6284748654319019, + 0.5778568458898549, + 0.34988006202074273, + 0.7764713506345448, + 0.9190909808587969, + 1.023190827540606, + 0.6226417920568781, + 0.5519145939388094, + 0.4568960180345979, + 0.4213932325876586, + 0.4074494455716704, + 0.7465319443901276, + 0.6474109861274324, + 0.5764712279753077, + 0.7979703549327781, + 1.2369697451324837, + 1.2324489817728508, + 1.5808985864286207, + 2.5860289117498976, + 4.232714993086349, + 2.835646318943954, + 2.2422528664695807, + 1.1646676435019565, + 1.2175941057769888, + 0.783401544111113, + 0.5289102464485725, + 0.46735876466607296, + 0.2966943922188728, + 0.3266155865457417, + 0.3362221758114643 + ], + "peak_layer": 22, + "peak_normalized": 4.232714993086349, + "population_cv": 0.8496980089222349, + "rest_mean": 3.881239232820614e-05, + "spike_contrast": 3.7241794809927478, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014454431511694564, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.921697341662366e-05, + 4.9794718506745994e-05, + 3.477540303720161e-05, + 3.197455589543097e-05, + 1.9359915313543752e-05, + 4.296449333196506e-05, + 5.085606608190574e-05, + 5.661622344632633e-05, + 3.4452641557436436e-05, + 3.053909313166514e-05, + 2.5281429770984687e-05, + 2.3316953956964426e-05, + 2.2545402316609398e-05, + 4.1307856008643284e-05, + 3.582319550332613e-05, + 3.189788549207151e-05, + 4.415409784996882e-05, + 6.844525341875851e-05, + 6.819510599598289e-05, + 8.747586980462074e-05, + 0.0001430927513865754, + 0.00023420884099323303, + 0.00015690483269281685, + 0.00012407058966346085, + 6.444456084864214e-05, + 6.737314106430858e-05, + 4.3347961764084175e-05, + 2.9266193450894207e-05, + 2.5860364985419437e-05, + 1.641699236643035e-05, + 1.807262196962256e-05, + 1.8604183424031362e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03498003748245537, + "first_to_last_ratio": 1.1135283344223057, + "imbalance_abs_log_ratio": 0.10753365366212776, + "last_quartile_mean": 0.03141369321383536, + "mean": 0.04906986153218895, + "normalized": [ + 0.528020735774383, + 0.8999099163028887, + 0.6284749107820657, + 0.5778568640801447, + 0.34988006993508275, + 0.776471348923419, + 0.919091002023713, + 1.0231908018037408, + 0.622641782085556, + 0.5519145889381298, + 0.45689603549337854, + 0.4213932430966296, + 0.4074494429214576, + 0.7465319861987265, + 0.6474110467766695, + 0.5764712070514831, + 0.7979703658517325, + 1.236969773835328, + 1.2324490791670337, + 1.5808986244806464, + 2.5860289675304746, + 4.232714870782151, + 2.835646138092529, + 2.242252864087746, + 1.1646675839359508, + 1.2175940341247224, + 0.7834015320190452, + 0.528910268066999, + 0.467358763362216, + 0.2966943974186558, + 0.32661557776013733, + 0.33622217729716364 + ], + "peak_layer": 22, + "peak_normalized": 4.232714870782151, + "population_cv": 0.8496979833458974, + "rest_mean": 0.034419217595347655, + "spike_contrast": 3.72417933196884, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12818333879113197, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0259099043905735, + 0.0441584549844265, + 0.03083917684853077, + 0.02835535630583763, + 0.017168566584587097, + 0.03810134157538414, + 0.04509966820478439, + 0.05020783096551895, + 0.030552946031093597, + 0.027082372456789017, + 0.022419825196266174, + 0.020677708089351654, + 0.019993487745523453, + 0.036632221192121506, + 0.03176837041974068, + 0.028287362307310104, + 0.039156295359134674, + 0.06069793552160263, + 0.06047610566020012, + 0.0775744765996933, + 0.12689608335494995, + 0.2076987326145172, + 0.13914476335048676, + 0.11002703756093979, + 0.05715007707476616, + 0.05974717065691948, + 0.038441404700279236, + 0.02595355361700058, + 0.022933229804039, + 0.014558752998709679, + 0.01602698117494583, + 0.016498375684022903 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.750052800593039e-06, + "first_to_last_ratio": 0.4308304317433478, + "imbalance_abs_log_ratio": 0.8420406961220296, + "last_quartile_mean": 6.383144267374519e-06, + "mean": 4.5530042669383874e-06, + "normalized": [ + 2.0777537806616246, + 0.8353410859824805, + 0.1910970826520623, + 0.2941663520688202, + 0.18051642612757607, + 0.3343816911052641, + 0.3564217354179324, + 0.562389007550559, + 0.34578362377189115, + 0.35956396375636174, + 0.3051237753595574, + 0.33780495264365074, + 0.3038063276744268, + 0.5221079484446093, + 0.49376666786286805, + 0.5929821493588614, + 0.3686987288124583, + 0.7392110518834059, + 0.5965749796632107, + 0.9039444986896679, + 1.6537371809131796, + 3.0803086468030596, + 2.8630898340929525, + 2.485723197420275, + 1.724168620325731, + 2.41211172564755, + 1.9299601312238395, + 1.6334426636441552, + 1.3339260437132254, + 0.9558102182238872, + 0.6013349904117006, + 0.6249509180931564 + ], + "peak_layer": 22, + "peak_normalized": 3.0803086468030596, + "population_cv": 0.8333981127885269, + "rest_mean": 3.4051366684353625e-06, + "spike_contrast": 3.1574325337710922, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0751489298854722e-05, + "top_five_layers": [ + 22, + 23, + 24, + 26, + 1 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.460021828999743e-06, + 3.80331152882718e-06, + 8.700658327143174e-07, + 1.3393406561590382e-06, + 8.218920584113221e-07, + 1.5224412663883413e-06, + 1.622789682187431e-06, + 2.56055955105694e-06, + 1.5743543144708383e-06, + 1.6370962612199946e-06, + 1.389229851156415e-06, + 1.5380273907794617e-06, + 1.383231506224547e-06, + 2.3771597170707537e-06, + 2.248121745651588e-06, + 2.699850256249192e-06, + 1.678686885497882e-06, + 3.3656310733931605e-06, + 2.71620842795528e-06, + 4.1156631596095394e-06, + 7.529472441092366e-06, + 1.4024658412381541e-05, + 1.3035660231253132e-05, + 1.1317508324282244e-05, + 7.850147085264325e-06, + 1.0982354979205411e-05, + 8.787116712483112e-06, + 7.437071417371044e-06, + 6.073370968806557e-06, + 4.35180800195667e-06, + 2.737880777203827e-06, + 2.8454041967052035e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6044932951663213e-05, + "first_to_last_ratio": 1.1246393261284713, + "imbalance_abs_log_ratio": 0.11746238525662386, + "last_quartile_mean": 2.315847609679622e-05, + "mean": 3.690820258839267e-05, + "normalized": [ + 0.45689793146127605, + 0.8738188272728166, + 0.6345310201461108, + 0.5870234156151962, + 0.3959041381169584, + 0.8145991894129532, + 0.9605505534893847, + 0.9220179377165436, + 0.6534489876582346, + 0.574567881594872, + 0.47330901138441, + 0.4295574998000546, + 0.4505301728221963, + 0.8093943395842734, + 0.6650197233282829, + 0.5723463492360998, + 0.7892367771238881, + 1.2091363576952083, + 1.2417470885367035, + 1.5367390457984618, + 2.460873737770247, + 4.227774989574034, + 3.004456821926061, + 2.2368262162922274, + 1.1669129504749511, + 1.3012136399895533, + 0.7651380597325796, + 0.47754123850698527, + 0.4891606669175266, + 0.2974921422063077, + 0.24654864385331124, + 0.27568464496229084 + ], + "peak_layer": 22, + "peak_normalized": 4.227774989574034, + "population_cv": 0.8549059646786937, + "rest_mean": 2.5840054991864168e-05, + "spike_contrast": 3.7413310319999478, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.667619960964657e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.6863281416590326e-05, + 3.2251082302536815e-05, + 2.341939944017213e-05, + 2.166597914765589e-05, + 1.4612110135203693e-05, + 3.006539191119373e-05, + 3.545219442457892e-05, + 3.4030024835374206e-05, + 2.4117627617670223e-05, + 2.120626777468715e-05, + 1.7468984879087657e-05, + 1.5854195225983858e-05, + 1.6628258890705183e-05, + 2.9873290259274654e-05, + 2.4544682673877105e-05, + 2.1124275008332916e-05, + 2.9129310860298574e-05, + 4.462704964680597e-05, + 4.583065310725942e-05, + 5.671827602782287e-05, + 9.082642645807937e-05, + 0.00015603957581333816, + 0.00011088910105172545, + 8.255723514594138e-05, + 4.3068659579148516e-05, + 4.8025456635514274e-05, + 2.823987051669974e-05, + 1.7625188775127754e-05, + 1.805404099286534e-05, + 1.0979900253005326e-05, + 9.099667295231484e-06, + 1.0175024726777337e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.700510824775847e-05, + "first_to_last_ratio": 1.1234226056354075, + "imbalance_abs_log_ratio": 0.1163799234305708, + "last_quartile_mean": 1.5136875617827172e-05, + "mean": 2.416565462226572e-05, + "normalized": [ + 0.4144047676027832, + 0.875113510868426, + 0.648392503192907, + 0.5978987553197436, + 0.4472461776170949, + 0.8581546509904864, + 0.9915956817879598, + 0.7967069167230741, + 0.6805807808463835, + 0.6196150111411964, + 0.5091007553108428, + 0.44255818216184833, + 0.49857092482915655, + 0.8652430602969526, + 0.6897944860164179, + 0.5800177420993363, + 0.7901724317041942, + 1.1871172558602836, + 1.243251110659502, + 1.5081298165112624, + 2.2807123105787017, + 4.096637658310102, + 3.1477300512572848, + 2.2202178168572866, + 1.1479124533540228, + 1.3987972507734237, + 0.7581679674914413, + 0.4356681137302007, + 0.511763265149204, + 0.3118761594955322, + 0.21315449183792956, + 0.2336979396250189 + ], + "peak_layer": 22, + "peak_normalized": 4.096637658310102, + "population_cv": 0.8395518179977153, + "rest_mean": 1.710104003938829e-05, + "spike_contrast": 3.6439054716132255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.231457336980384e-05, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.001436248770915e-05, + 2.114769085892476e-05, + 1.5668829291826114e-05, + 1.4448614820139483e-05, + 1.0807996659423225e-05, + 2.0737868908327073e-05, + 2.396255877101794e-05, + 1.9252944184700027e-05, + 1.644668009248562e-05, + 1.497340235800948e-05, + 1.2302753020776436e-05, + 1.0694708180380985e-05, + 1.2048292774125002e-05, + 2.090916495944839e-05, + 1.6669335309416056e-05, + 1.4016508430358954e-05, + 1.9095034076599404e-05, + 2.868746560125146e-05, + 3.0043976948945783e-05, + 3.644494427135214e-05, + 5.511490599019453e-05, + 9.899793076328933e-05, + 7.606695726281032e-05, + 5.365301694837399e-05, + 2.7740055884351023e-05, + 3.3802851248765364e-05, + 1.8321625248063356e-05, + 1.0528205166338012e-05, + 1.236709431395866e-05, + 7.536691555287689e-06, + 5.1510178309399635e-06, + 5.647463694913313e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.642002219654387e-05, + "first_to_last_ratio": 1.1203608181269622, + "imbalance_abs_log_ratio": 0.11365079246685211, + "last_quartile_mean": 6.821018814662239e-05, + "mean": 0.00011010971388714097, + "normalized": [ + 0.5163109854936148, + 0.8673817094155928, + 0.5924673517780332, + 0.5654643004921971, + 0.34399113467095715, + 0.7609201839652954, + 0.9076117604661024, + 0.9981345295043799, + 0.6327009050241252, + 0.5203303165972624, + 0.4437849422865529, + 0.41423566265400885, + 0.39884159677379066, + 0.7686876572920235, + 0.6339609024838357, + 0.5703063832410542, + 0.781268072457197, + 1.1797447140196398, + 1.2760717444836593, + 1.5599326259342914, + 2.6126342040159805, + 4.419910912775975, + 2.9287458133972417, + 2.3507635425478965, + 1.1736586910474214, + 1.2201215929679614, + 0.7838637940961916, + 0.4869646019277815, + 0.46617791476107356, + 0.26920984270675297, + 0.25672408594104645, + 0.2990775247810641 + ], + "peak_layer": 22, + "peak_normalized": 4.419910912775975, + "population_cv": 0.8912016880930539, + "rest_mean": 7.550380838370916e-05, + "spike_contrast": 3.9333327677514873, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002969816036056727, + "top_five_layers": [ + 22, + 23, + 21, + 24, + 20 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 5.685085488948971e-05, + 9.550715185469016e-05, + 6.523641059175134e-05, + 6.226311234058812e-05, + 3.787676541833207e-05, + 8.378470374736935e-05, + 9.993687126552686e-05, + 0.00010990430746460333, + 6.966651562834159e-05, + 5.729342228733003e-05, + 4.886503302259371e-05, + 4.561137029668316e-05, + 4.3916334107052535e-05, + 8.463997801300138e-05, + 6.980525358812883e-05, + 6.279627268668264e-05, + 8.602520392742008e-05, + 0.00012990135292056948, + 0.0001405078946845606, + 0.00017176373512484133, + 0.0002876764046959579, + 0.0004866751260124147, + 0.0003224833635613322, + 0.0002588419010862708, + 0.00012923122267238796, + 0.0001343472395092249, + 8.631101809442043e-05, + 5.3619532991433516e-05, + 5.13307168148458e-05, + 2.9642618756042793e-05, + 2.8267815650906414e-05, + 3.293134068371728e-05 + ] + } + }, + "pre_mlp_input": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 4.6955231027823174e-06, + "first_to_last_ratio": 0.14308275980953641, + "imbalance_abs_log_ratio": 1.9443320762007668, + "last_quartile_mean": 3.2816833481774665e-05, + "mean": 1.7864957968072304e-05, + "normalized": [ + 0.5367890577532618, + 0.2021050510263675, + 0.1411976282823933, + 0.2660218188543664, + 0.19890690834720978, + 0.16151559172232732, + 0.2167464101659875, + 0.3793916565295996, + 0.424775345287454, + 0.3093373886676644, + 0.2783613720705816, + 0.2825637413001494, + 0.3733172205433837, + 0.3886765425479794, + 0.44060863402002287, + 0.44323494933287805, + 0.3667341230574315, + 0.45728447370296305, + 0.42556973592368313, + 0.5658685710706646, + 2.6227066493164064, + 3.0085127149589037, + 2.58733070947387, + 2.226933740479959, + 2.8743975419651484, + 2.84666799381699, + 2.39504866979417, + 1.5908588860103314, + 1.8185955715005, + 1.4035945808786974, + 0.9127606447015644, + 0.8535860768970909 + ], + "peak_layer": 22, + "peak_normalized": 3.0085127149589037, + "population_cv": 0.9632431009960319, + "rest_mean": 1.2359982755933069e-05, + "spike_contrast": 3.850476578600324, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.759182411362417e-05, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 9.589713954483159e-06, + 3.6105982417211635e-06, + 2.5224896944564534e-06, + 4.7524686124233995e-06, + 3.5534635571821127e-06, + 2.8854692573077045e-06, + 3.872165507345926e-06, + 6.777815997338621e-06, + 7.588593689433765e-06, + 5.526299446501071e-06, + 4.972914211975876e-06, + 5.047989361628424e-06, + 6.669296453765128e-06, + 6.9436900957953185e-06, + 7.871454727137461e-06, + 7.918373739812523e-06, + 6.5516896938788705e-06, + 8.1693679021555e-06, + 7.6027854447602294e-06, + 1.0109218237630557e-05, + 4.685454405262135e-05, + 5.3746953199151903e-05, + 4.622255437425338e-05, + 3.97840776713565e-05, + 5.135099127073772e-05, + 5.085560405859724e-05, + 4.278744381736033e-05, + 2.8420627131708898e-05, + 3.248913344577886e-05, + 2.507515819161199e-05, + 1.6306430552504025e-05, + 1.5249279385898262e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.8878105947806034e-05, + "first_to_last_ratio": 0.1437388462831867, + "imbalance_abs_log_ratio": 1.939757193388504, + "last_quartile_mean": 0.00013133614493199275, + "mean": 7.144442986373178e-05, + "normalized": [ + 0.543944422065902, + 0.20498362677674017, + 0.14117605251679072, + 0.2675823635170904, + 0.19429426405696296, + 0.15941273479469137, + 0.21634544427316113, + 0.386139638870723, + 0.42865458135412043, + 0.3129364237141443, + 0.2788876084512782, + 0.2815826978190512, + 0.3736543391719335, + 0.3909795972451435, + 0.4389936626557141, + 0.44232523153890874, + 0.3688366584716598, + 0.45947150441297413, + 0.42734480687548726, + 0.5800824758005463, + 2.6187902433353907, + 2.987231504916195, + 2.580721746390286, + 2.2092452217225382, + 2.879951595578818, + 2.8406783220158585, + 2.397028447242086, + 1.5906421267626043, + 1.8247593500794728, + 1.4018234858691145, + 0.9198949147470885, + 0.8516049069575247 + ], + "peak_layer": 22, + "peak_normalized": 2.987231504916195, + "population_cv": 0.9601858610516129, + "rest_mean": 4.954554700547036e-05, + "spike_contrast": 3.828767846228416, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00018969839729834347, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 3.886179911205545e-05, + 1.4644938346464187e-05, + 1.008624258247437e-05, + 1.911726940306835e-05, + 1.3881242921343073e-05, + 1.1389151950425003e-05, + 1.5456676919711754e-05, + 2.7587526346906088e-05, + 3.062498217332177e-05, + 2.2357564375852235e-05, + 1.9924966181861237e-05, + 2.0117515305173583e-05, + 2.6695521228248253e-05, + 2.7933314413530752e-05, + 3.13636519422289e-05, + 3.160167398164049e-05, + 2.6351324777351692e-05, + 3.282667967141606e-05, + 3.0531406082445756e-05, + 4.144366175751202e-05, + 0.00018709797586780041, + 0.00021342105173971504, + 0.0001843781938077882, + 0.00015783826529514045, + 0.0002057564997812733, + 0.0002029506431426853, + 0.00017125433078035712, + 0.00011364251986378804, + 0.00013036889140494168, + 0.00010015247971750796, + 6.572136771865189e-05, + 6.08424270467367e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.016741293016821146, + "first_to_last_ratio": 0.14373884545887533, + "imbalance_abs_log_ratio": 1.9397571991232891, + "last_quartile_mean": 0.11647020652890205, + "mean": 0.06335763493552804, + "normalized": [ + 0.5439444372437541, + 0.20498362168576487, + 0.14117604516877305, + 0.2675823445899417, + 0.1942942529879486, + 0.15941273039509152, + 0.2163454315588509, + 0.38613966530889043, + 0.4286545697273512, + 0.3129364189494029, + 0.278887605667477, + 0.2815826915705289, + 0.3736543321474427, + 0.3909795777927573, + 0.43899363576159356, + 0.44232520797796643, + 0.3688366746353626, + 0.45947150248290297, + 0.4273447896704886, + 0.580082454376481, + 2.6187903327724635, + 2.987231533325295, + 2.5807218070831297, + 2.2092452282911403, + 2.8799516329199686, + 2.84067822247012, + 2.397028505583422, + 1.5906421518499794, + 1.8247592808395534, + 1.4018234450909468, + 0.9198949415397243, + 0.8516049285354858 + ], + "peak_layer": 22, + "peak_normalized": 2.987231533325295, + "population_cv": 0.9601858709388058, + "rest_mean": 0.04393748607900408, + "spike_contrast": 3.8287679558695995, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.16822643876075744, + "top_five_layers": [ + 22, + 25, + 26, + 21, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.03446303308010101, + 0.01298727747052908, + 0.00894458033144474, + 0.01695338450372219, + 0.012310024350881577, + 0.010100013576447964, + 0.013707134872674942, + 0.024464895948767662, + 0.02715853974223137, + 0.01982691138982773, + 0.017669659107923508, + 0.017840413376688957, + 0.023673854768276215, + 0.024771541357040405, + 0.02781359851360321, + 0.02802467904984951, + 0.02336861938238144, + 0.029111027717590332, + 0.02707555517554283, + 0.03675265237689018, + 0.16592036187648773, + 0.18926392495632172, + 0.16350843012332916, + 0.13997255265712738, + 0.18246692419052124, + 0.1799786537885666, + 0.15187005698680878, + 0.10077932476997375, + 0.11561243236064911, + 0.08881621807813644, + 0.05828236788511276, + 0.053955674171447754 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0825735472508313e-06, + "first_to_last_ratio": 0.12187576812955331, + "imbalance_abs_log_ratio": 2.104753047082524, + "last_quartile_mean": 8.882598763193528e-06, + "mean": 4.656443639561303e-06, + "normalized": [ + 0.5462983207096154, + 0.1799290732737773, + 0.13161762512632208, + 0.2712494331106157, + 0.1613911681124062, + 0.13237661267085615, + 0.15721616286614848, + 0.2798363906909718, + 0.34662434040104295, + 0.2682122377700395, + 0.24445710198520895, + 0.24681111820039023, + 0.3364691861798734, + 0.3619231630019147, + 0.42744951901080175, + 0.43112099033231693, + 0.36249737820633765, + 0.4368387762566429, + 0.42578881412518377, + 0.38711372693052853, + 2.644134376023103, + 3.0274176414789977, + 2.680305221816438, + 2.2521785297578925, + 2.965193895729998, + 2.901136489590974, + 2.603841896063781, + 1.6862657755052308, + 1.872923926339135, + 1.3894795988927409, + 0.9615945171774488, + 0.8803069926632656 + ], + "peak_layer": 22, + "peak_normalized": 3.0274176414789977, + "population_cv": 1.0082602587575777, + "rest_mean": 3.1785867888713795e-06, + "spike_contrast": 3.9756254815914156, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2636870633286889e-05, + "top_five_layers": [ + 22, + 25, + 26, + 23, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.543807340771309e-06, + 8.3782958881784e-07, + 6.128700533736264e-07, + 1.2630576975425356e-06, + 7.515088782383827e-07, + 6.164042360978783e-07, + 7.32068201614311e-07, + 1.3030423815507675e-06, + 1.6140367051775684e-06, + 1.2489151686168043e-06, + 1.138300717684615e-06, + 1.14926206151722e-06, + 1.5667498018956394e-06, + 1.6852748103701742e-06, + 1.990394594031386e-06, + 2.0074905933142873e-06, + 1.687948611106549e-06, + 2.034115141213988e-06, + 1.9826616153295618e-06, + 1.8025732515525306e-06, + 1.2312262697378173e-05, + 1.409699962096056e-05, + 1.2480690202210099e-05, + 1.0487142390047666e-05, + 1.3807258255837951e-05, + 1.3508978554455098e-05, + 1.2124643035349436e-05, + 7.85200154496124e-06, + 8.721164704184048e-06, + 6.470033440564293e-06, + 4.477610673347954e-06, + 4.099099896848202e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.2557812397062662e-05, + "first_to_last_ratio": 0.15959566659574662, + "imbalance_abs_log_ratio": 1.8351117459907644, + "last_quartile_mean": 7.868517150200205e-05, + "mean": 4.5438396199415365e-05, + "normalized": [ + 0.5876869810338485, + 0.2186593308079107, + 0.115659623677568, + 0.24728992953724943, + 0.2138823134903866, + 0.17309343907534852, + 0.22828222046505, + 0.4264066618299725, + 0.47941738273017553, + 0.34947398487973863, + 0.31642185290041636, + 0.3326608895247323, + 0.42753132054545495, + 0.43329608717407087, + 0.4807098955780001, + 0.491094994316893, + 0.41296023678331367, + 0.5280992844682807, + 0.4827771475203192, + 0.6138123378227869, + 2.7117823469683264, + 3.061642080099734, + 2.646250140140893, + 2.167597408443836, + 2.9446942269972274, + 2.694165709502317, + 2.239316799454017, + 1.5139318638986488, + 1.6066586018929343, + 1.2634426578175135, + 0.8290147862338136, + 0.7622874643892231 + ], + "peak_layer": 22, + "peak_normalized": 3.061642080099734, + "population_cv": 0.9412918554701797, + "rest_mean": 3.107991987808082e-05, + "spike_contrast": 3.9567080229621103, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012297416833462194, + "top_five_layers": [ + 22, + 25, + 21, + 26, + 23 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.6703553885454312e-05, + 9.935529305948876e-06, + 5.255387804936618e-06, + 1.1236457794439048e-05, + 9.718469300423749e-06, + 7.86508826422505e-06, + 1.0372777978773229e-05, + 1.9375234842300415e-05, + 2.178395698138047e-05, + 1.587953738635406e-05, + 1.4377701518242247e-05, + 1.511557729827473e-05, + 1.9426337530603632e-05, + 1.968827928067185e-05, + 2.1842686692252755e-05, + 2.231456892332062e-05, + 1.876425085356459e-05, + 2.3995984520297498e-05, + 2.1936619305051863e-05, + 2.789064819808118e-05, + 0.00012321904068812728, + 0.0001391161058563739, + 0.00012024136231048033, + 9.849214984569699e-05, + 0.00013380218297243118, + 0.00012241856893524528, + 0.00010175096394959837, + 6.879063585074618e-05, + 7.300399011000991e-05, + 5.740880806115456e-05, + 3.766910231206566e-05, + 3.463711982476525e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.367274347165221e-06, + "first_to_last_ratio": 0.1920014554991945, + "imbalance_abs_log_ratio": 1.6502523262581177, + "last_quartile_mean": 4.3579223529377487e-05, + "mean": 2.787530080183842e-05, + "normalized": [ + 0.6522901960458961, + 0.23732362933486428, + 0.10383521258256681, + 0.24073339013803968, + 0.24132146388193534, + 0.19483084771733855, + 0.2475207738927628, + 0.483488789493259, + 0.550030749638162, + 0.4005903921146771, + 0.3626794403805512, + 0.398664404925254, + 0.5057551655640742, + 0.49633936232785464, + 0.5444163811542189, + 0.5645330449101175, + 0.481087325236049, + 0.6303807295780811, + 0.5534160889368265, + 0.6675682370643486, + 2.937265214456223, + 3.231070824381384, + 2.7157464748293116, + 2.0522050942602807, + 2.8791759008165254, + 2.406039643209466, + 1.9404796965168836, + 1.3618287054639213, + 1.3825321378732942, + 1.123237294980051, + 0.7355691904186271, + 0.6780441978771551 + ], + "peak_layer": 22, + "peak_normalized": 3.231070824381384, + "population_cv": 0.9137826743997257, + "rest_mean": 1.8774052764288002e-05, + "spike_contrast": 4.1025792976954865, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.702204020461067e-05, + "top_five_layers": [ + 22, + 21, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.8182785424869508e-05, + 6.615467555093346e-06, + 2.8944377845618874e-06, + 6.710515663144179e-06, + 6.726908395648934e-06, + 5.430968485597987e-06, + 6.899716026964597e-06, + 1.3477395441441331e-05, + 1.5332272596424446e-05, + 1.1166577678523026e-05, + 1.0109798495250288e-05, + 1.111289020627737e-05, + 1.409807737218216e-05, + 1.3835609024681617e-05, + 1.5175770386122167e-05, + 1.5736528439447284e-05, + 1.3410453902906738e-05, + 1.7572052456671372e-05, + 1.5426639947691e-05, + 1.86086654139217e-05, + 8.187715138774365e-05, + 9.006707114167511e-05, + 7.570224988739938e-05, + 5.720583430957049e-05, + 8.025789429666474e-05, + 6.706907879561186e-05, + 5.409145524026826e-05, + 3.796138480538502e-05, + 3.8538499211426824e-05, + 3.131057746941224e-05, + 2.0504212443483993e-05, + 1.890068597276695e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.742885974133969e-05, + "first_to_last_ratio": 0.13285814939480156, + "imbalance_abs_log_ratio": 2.018473265828943, + "last_quartile_mean": 0.00028172046586405486, + "mean": 0.00014909372896454443, + "normalized": [ + 0.5351570886371335, + 0.1998816464903299, + 0.1134003859562838, + 0.2373819769757024, + 0.18971959520637977, + 0.1492636791238371, + 0.21004413698536548, + 0.3734913450927867, + 0.42927632587153325, + 0.3024206473119055, + 0.27462175768495817, + 0.27240928064339276, + 0.3512855646324889, + 0.3655094888356237, + 0.41973324501001746, + 0.42402062842094873, + 0.360798486332806, + 0.4559662563090299, + 0.41985007515873884, + 0.5597866310716989, + 2.4844541141001115, + 2.96085466615825, + 2.5443412288142064, + 2.249909733543605, + 2.9984626521607822, + 2.895026684700318, + 2.601768175835175, + 1.7136369023458127, + 1.8313353225818492, + 1.363588302533558, + 0.8854239763588411, + 0.82717999911653 + ], + "peak_layer": 25, + "peak_normalized": 2.9984626521607822, + "population_cv": 0.9800748961570518, + "rest_mean": 0.00010360345199673989, + "spike_contrast": 3.810116525877056, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003947412245906889, + "top_five_layers": [ + 25, + 22, + 26, + 27, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 7.978856592671946e-05, + 2.980110002681613e-05, + 1.6907286408240907e-05, + 3.53921641362831e-05, + 2.8286001906963065e-05, + 2.2254278519540094e-05, + 3.131626363028772e-05, + 5.568521737586707e-05, + 6.400240818038583e-05, + 4.508902202360332e-05, + 4.0944381908047944e-05, + 4.061451545567252e-05, + 5.2374474762473255e-05, + 5.449517266242765e-05, + 6.257959466893226e-05, + 6.321881664916873e-05, + 5.379279173212126e-05, + 6.79817094351165e-05, + 6.259701331146061e-05, + 8.34606762509793e-05, + 0.0003704165283124894, + 0.00044144486309960485, + 0.0003793453215621412, + 0.0003354474320076406, + 0.0004470519779715687, + 0.0004316303238738328, + 0.00038790731923654675, + 0.00025549251586198807, + 0.00027304061222821474, + 0.00020330246479716152, + 0.00013201116234995425, + 0.00012332735059317201 + ] + } + } + }, + "seed": 2026073001 + }, + { + "final_mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.2200722098350525, + "group": 1, + "latest_source_mass": 0.06159520894289017, + "layer": 1, + "max_source_mass": 0.9384047985076904, + "mixer_index": 1, + "normalized_entropy": 0.3174970857665056, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9384047985076904, + "median": 0.949082612991333, + "p05": 0.8598991632461548, + "p95": 0.9761404991149902 + }, + { + "label": "current_group_partial", + "mean": 0.06159520894289017, + "median": 0.05091731622815132, + "p05": 0.02385946735739708, + "p95": 0.14010077714920044 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5651201605796814, + "group": 1, + "latest_source_mass": 0.3442652225494385, + "layer": 2, + "max_source_mass": 0.6557347774505615, + "mixer_index": 2, + "normalized_entropy": 0.815296053174681, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6557347774505615, + "median": 0.6636689305305481, + "p05": 0.3472050428390503, + "p95": 0.9190996289253235 + }, + { + "label": "current_group_partial", + "mean": 0.3442652225494385, + "median": 0.3363310396671295, + "p05": 0.08090042322874069, + "p95": 0.6527949571609497 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4018177092075348, + "group": 1, + "latest_source_mass": 0.14357808232307434, + "layer": 2, + "max_source_mass": 0.856421947479248, + "mixer_index": 3, + "normalized_entropy": 0.5797004164150741, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.856421947479248, + "median": 0.8660546541213989, + "p05": 0.7540444135665894, + "p95": 0.9155591726303101 + }, + { + "label": "current_group_partial", + "mean": 0.14357808232307434, + "median": 0.13394534587860107, + "p05": 0.08444082736968994, + "p95": 0.24595558643341064 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.687358021736145, + "group": 1, + "latest_source_mass": 0.5312775373458862, + "layer": 3, + "max_source_mass": 0.5312775373458862, + "mixer_index": 4, + "normalized_entropy": 0.9916480092739848, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.46872249245643616, + "median": 0.4674533009529114, + "p05": 0.3992259204387665, + "p95": 0.5431661605834961 + }, + { + "label": "current_group_partial", + "mean": 0.5312775373458862, + "median": 0.5325466990470886, + "p05": 0.45683377981185913, + "p95": 0.6007740497589111 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.2678314447402954, + "group": 1, + "latest_source_mass": 0.07946884632110596, + "layer": 3, + "max_source_mass": 0.920531153678894, + "mixer_index": 5, + "normalized_entropy": 0.38639909712095066, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.920531153678894, + "median": 0.9314625263214111, + "p05": 0.8444089889526367, + "p95": 0.9650533199310303 + }, + { + "label": "current_group_partial", + "mean": 0.07946884632110596, + "median": 0.06853749603033066, + "p05": 0.03494669869542122, + "p95": 0.1555909812450409 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.657238245010376, + "group": 1, + "latest_source_mass": 0.3777551054954529, + "layer": 4, + "max_source_mass": 0.6222448348999023, + "mixer_index": 6, + "normalized_entropy": 0.948194356759035, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.6222448348999023, + "median": 0.6208515167236328, + "p05": 0.5314525961875916, + "p95": 0.7065955400466919 + }, + { + "label": "current_group_partial", + "mean": 0.3777551054954529, + "median": 0.3791484832763672, + "p05": 0.2934044897556305, + "p95": 0.4685474634170532 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.5044373869895935, + "group": 1, + "latest_source_mass": 0.2207653522491455, + "layer": 4, + "max_source_mass": 0.7792346477508545, + "mixer_index": 7, + "normalized_entropy": 0.7277493166488734, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7792346477508545, + "median": 0.7866228222846985, + "p05": 0.6320486664772034, + "p95": 0.9385759830474854 + }, + { + "label": "current_group_partial", + "mean": 0.2207653522491455, + "median": 0.21337716281414032, + "p05": 0.06142403930425644, + "p95": 0.367951363325119 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.650418758392334, + "group": 2, + "latest_source_mass": 0.4602207541465759, + "layer": 5, + "max_source_mass": 0.5397792458534241, + "mixer_index": 8, + "normalized_entropy": 0.9383559172337771, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5397792458534241, + "median": 0.5213949680328369, + "p05": 0.34510529041290283, + "p95": 0.7807042598724365 + }, + { + "label": "completed_group_1", + "mean": 0.4602207541465759, + "median": 0.4786049723625183, + "p05": 0.2192957103252411, + "p95": 0.6548947095870972 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0032464265823364, + "group": 2, + "latest_source_mass": 0.37668097019195557, + "layer": 5, + "max_source_mass": 0.4337552785873413, + "mixer_index": 9, + "normalized_entropy": 0.913194251448444, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4337552785873413, + "median": 0.42766237258911133, + "p05": 0.2500072717666626, + "p95": 0.6592159271240234 + }, + { + "label": "completed_group_1", + "mean": 0.1895637810230255, + "median": 0.18978442251682281, + "p05": 0.1284591555595398, + "p95": 0.25696656107902527 + }, + { + "label": "current_group_partial", + "mean": 0.37668097019195557, + "median": 0.3803442120552063, + "p05": 0.1759825050830841, + "p95": 0.568291425704956 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0385558605194092, + "group": 2, + "latest_source_mass": 0.4949595034122467, + "layer": 6, + "max_source_mass": 0.4949595034122467, + "mixer_index": 10, + "normalized_entropy": 0.9453342832879565, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.24478834867477417, + "median": 0.23865866661071777, + "p05": 0.19890648126602173, + "p95": 0.32154381275177 + }, + { + "label": "completed_group_1", + "mean": 0.26025211811065674, + "median": 0.26050686836242676, + "p05": 0.23401768505573273, + "p95": 0.2860541343688965 + }, + { + "label": "current_group_partial", + "mean": 0.4949595034122467, + "median": 0.5017017722129822, + "p05": 0.42678141593933105, + "p95": 0.5358227491378784 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.031773328781128, + "group": 2, + "latest_source_mass": 0.33116668462753296, + "layer": 6, + "max_source_mass": 0.40370824933052063, + "mixer_index": 11, + "normalized_entropy": 0.9391605568439314, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40370824933052063, + "median": 0.38690465688705444, + "p05": 0.18056227266788483, + "p95": 0.7100913524627686 + }, + { + "label": "completed_group_1", + "mean": 0.2651250660419464, + "median": 0.2730395793914795, + "p05": 0.12547528743743896, + "p95": 0.35344114899635315 + }, + { + "label": "current_group_partial", + "mean": 0.33116668462753296, + "median": 0.33928292989730835, + "p05": 0.1502782255411148, + "p95": 0.47667479515075684 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0838522911071777, + "group": 2, + "latest_source_mass": 0.38127952814102173, + "layer": 7, + "max_source_mass": 0.38127952814102173, + "mixer_index": 12, + "normalized_entropy": 0.9865648712351232, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35612422227859497, + "median": 0.36401283740997314, + "p05": 0.2946957051753998, + "p95": 0.39828816056251526 + }, + { + "label": "completed_group_1", + "mean": 0.2625962495803833, + "median": 0.2590169608592987, + "p05": 0.24180665612220764, + "p95": 0.2929879426956177 + }, + { + "label": "current_group_partial", + "mean": 0.38127952814102173, + "median": 0.37846946716308594, + "p05": 0.3489896059036255, + "p95": 0.41648316383361816 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.96053147315979, + "group": 2, + "latest_source_mass": 0.3146248161792755, + "layer": 7, + "max_source_mass": 0.5271762013435364, + "mixer_index": 13, + "normalized_entropy": 0.874313425279704, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5271762013435364, + "median": 0.5114221572875977, + "p05": 0.3548204302787781, + "p95": 0.7728348970413208 + }, + { + "label": "completed_group_1", + "mean": 0.1581989824771881, + "median": 0.1618250608444214, + "p05": 0.07541970163583755, + "p95": 0.21691158413887024 + }, + { + "label": "current_group_partial", + "mean": 0.3146248161792755, + "median": 0.32365772128105164, + "p05": 0.14578065276145935, + "p95": 0.4483411908149719 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0704245567321777, + "group": 2, + "latest_source_mass": 0.38260507583618164, + "layer": 8, + "max_source_mass": 0.38260507583618164, + "mixer_index": 14, + "normalized_entropy": 0.9743424206822726, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3788670599460602, + "median": 0.39538437128067017, + "p05": 0.27489906549453735, + "p95": 0.43731769919395447 + }, + { + "label": "completed_group_1", + "mean": 0.23852786421775818, + "median": 0.2294245958328247, + "p05": 0.20573674142360687, + "p95": 0.29735344648361206 + }, + { + "label": "current_group_partial", + "mean": 0.38260507583618164, + "median": 0.37669745087623596, + "p05": 0.34335845708847046, + "p95": 0.435106486082077 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9674104452133179, + "group": 2, + "latest_source_mass": 0.3281455934047699, + "layer": 8, + "max_source_mass": 0.48829177021980286, + "mixer_index": 15, + "normalized_entropy": 0.8805749354816949, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.48829177021980286, + "median": 0.44956350326538086, + "p05": 0.2823859453201294, + "p95": 0.8209846615791321 + }, + { + "label": "completed_group_1", + "mean": 0.18356263637542725, + "median": 0.18210050463676453, + "p05": 0.05149933695793152, + "p95": 0.2906108498573303 + }, + { + "label": "current_group_partial", + "mean": 0.3281455934047699, + "median": 0.34843379259109497, + "p05": 0.12376503646373749, + "p95": 0.46937939524650574 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0832805633544922, + "group": 3, + "latest_source_mass": 0.31498202681541443, + "layer": 9, + "max_source_mass": 0.40388280153274536, + "mixer_index": 16, + "normalized_entropy": 0.9860444622076776, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.40388280153274536, + "median": 0.403384804725647, + "p05": 0.3489384353160858, + "p95": 0.4601682126522064 + }, + { + "label": "completed_group_1", + "mean": 0.2811351716518402, + "median": 0.28186845779418945, + "p05": 0.25417307019233704, + "p95": 0.30605652928352356 + }, + { + "label": "completed_group_2", + "mean": 0.31498202681541443, + "median": 0.3153840899467468, + "p05": 0.2613564431667328, + "p95": 0.3658811151981354 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2917464971542358, + "group": 3, + "latest_source_mass": 0.33769795298576355, + "layer": 9, + "max_source_mass": 0.33769795298576355, + "mixer_index": 17, + "normalized_entropy": 0.9317981327650527, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3349480926990509, + "median": 0.3352771997451782, + "p05": 0.18474003672599792, + "p95": 0.48752525448799133 + }, + { + "label": "completed_group_1", + "mean": 0.1244347020983696, + "median": 0.11896269023418427, + "p05": 0.07986846566200256, + "p95": 0.1843445897102356 + }, + { + "label": "completed_group_2", + "mean": 0.20291924476623535, + "median": 0.20060715079307556, + "p05": 0.1253921389579773, + "p95": 0.2836313247680664 + }, + { + "label": "current_group_partial", + "mean": 0.33769795298576355, + "median": 0.33908897638320923, + "p05": 0.26979348063468933, + "p95": 0.4016563892364502 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3533791303634644, + "group": 3, + "latest_source_mass": 0.28473395109176636, + "layer": 10, + "max_source_mass": 0.32115638256073, + "mixer_index": 18, + "normalized_entropy": 0.976256679908994, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32115638256073, + "median": 0.3355404734611511, + "p05": 0.22348493337631226, + "p95": 0.3863494396209717 + }, + { + "label": "completed_group_1", + "mean": 0.20602071285247803, + "median": 0.2052205502986908, + "p05": 0.1802767962217331, + "p95": 0.23218519985675812 + }, + { + "label": "completed_group_2", + "mean": 0.18808895349502563, + "median": 0.18706050515174866, + "p05": 0.16633892059326172, + "p95": 0.2130376398563385 + }, + { + "label": "current_group_partial", + "mean": 0.28473395109176636, + "median": 0.275795578956604, + "p05": 0.22748424112796783, + "p95": 0.3660258948802948 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.308367371559143, + "group": 3, + "latest_source_mass": 0.29719752073287964, + "layer": 10, + "max_source_mass": 0.3113970160484314, + "mixer_index": 19, + "normalized_entropy": 0.9437875593046517, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3113970160484314, + "median": 0.33913683891296387, + "p05": 0.0915825292468071, + "p95": 0.47554588317871094 + }, + { + "label": "completed_group_1", + "mean": 0.16179001331329346, + "median": 0.16501447558403015, + "p05": 0.10682816803455353, + "p95": 0.20702867209911346 + }, + { + "label": "completed_group_2", + "mean": 0.2296154499053955, + "median": 0.2261994481086731, + "p05": 0.14510810375213623, + "p95": 0.32595163583755493 + }, + { + "label": "current_group_partial", + "mean": 0.29719752073287964, + "median": 0.2761377692222595, + "p05": 0.194716677069664, + "p95": 0.4515363574028015 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3437637090682983, + "group": 3, + "latest_source_mass": 0.3098427951335907, + "layer": 11, + "max_source_mass": 0.3217787742614746, + "mixer_index": 20, + "normalized_entropy": 0.969320619599697, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3217787742614746, + "median": 0.31754863262176514, + "p05": 0.25967735052108765, + "p95": 0.40468457341194153 + }, + { + "label": "completed_group_1", + "mean": 0.1920534074306488, + "median": 0.1915990114212036, + "p05": 0.16391108930110931, + "p95": 0.22057634592056274 + }, + { + "label": "completed_group_2", + "mean": 0.1763250231742859, + "median": 0.1779440939426422, + "p05": 0.14714840054512024, + "p95": 0.1996835470199585 + }, + { + "label": "current_group_partial", + "mean": 0.3098427951335907, + "median": 0.31089526414871216, + "p05": 0.2459525167942047, + "p95": 0.36900240182876587 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.2280659675598145, + "group": 3, + "latest_source_mass": 0.29383569955825806, + "layer": 11, + "max_source_mass": 0.42534634470939636, + "mixer_index": 21, + "normalized_entropy": 0.8858623406415255, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.42534634470939636, + "median": 0.4574331045150757, + "p05": 0.17354995012283325, + "p95": 0.6023476123809814 + }, + { + "label": "completed_group_1", + "mean": 0.14848899841308594, + "median": 0.14150214195251465, + "p05": 0.07509882748126984, + "p95": 0.23531809449195862 + }, + { + "label": "completed_group_2", + "mean": 0.13232898712158203, + "median": 0.13064970076084137, + "p05": 0.07000484317541122, + "p95": 0.19124652445316315 + }, + { + "label": "current_group_partial", + "mean": 0.29383569955825806, + "median": 0.27858495712280273, + "p05": 0.187642902135849, + "p95": 0.4577033817768097 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3462648391723633, + "group": 3, + "latest_source_mass": 0.3101534843444824, + "layer": 12, + "max_source_mass": 0.3221249580383301, + "mixer_index": 22, + "normalized_entropy": 0.9711248035985732, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3221249580383301, + "median": 0.3120178282260895, + "p05": 0.2763444483280182, + "p95": 0.3882659375667572 + }, + { + "label": "completed_group_1", + "mean": 0.18557845056056976, + "median": 0.1851869523525238, + "p05": 0.16533124446868896, + "p95": 0.20681798458099365 + }, + { + "label": "completed_group_2", + "mean": 0.18214309215545654, + "median": 0.18278372287750244, + "p05": 0.15564095973968506, + "p95": 0.2072875201702118 + }, + { + "label": "current_group_partial", + "mean": 0.3101534843444824, + "median": 0.31112557649612427, + "p05": 0.2608441412448883, + "p95": 0.36039745807647705 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.147221326828003, + "group": 3, + "latest_source_mass": 0.2585829496383667, + "layer": 12, + "max_source_mass": 0.5053804516792297, + "mixer_index": 23, + "normalized_entropy": 0.8275452595084083, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5053804516792297, + "median": 0.48097139596939087, + "p05": 0.30125147104263306, + "p95": 0.7477225065231323 + }, + { + "label": "completed_group_1", + "mean": 0.12089012563228607, + "median": 0.11902710795402527, + "p05": 0.05662901699542999, + "p95": 0.1844676434993744 + }, + { + "label": "completed_group_2", + "mean": 0.11514648050069809, + "median": 0.11994394659996033, + "p05": 0.05982710421085358, + "p95": 0.16130289435386658 + }, + { + "label": "current_group_partial", + "mean": 0.2585829496383667, + "median": 0.25007814168930054, + "p05": 0.12433226406574249, + "p95": 0.42273372411727905 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3729398250579834, + "group": 4, + "latest_source_mass": 0.22271886467933655, + "layer": 13, + "max_source_mass": 0.30785465240478516, + "mixer_index": 24, + "normalized_entropy": 0.9903667385250569, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30785465240478516, + "median": 0.3129041790962219, + "p05": 0.25217515230178833, + "p95": 0.3537779450416565 + }, + { + "label": "completed_group_1", + "mean": 0.23079943656921387, + "median": 0.22906282544136047, + "p05": 0.19741898775100708, + "p95": 0.26931890845298767 + }, + { + "label": "completed_group_2", + "mean": 0.23862707614898682, + "median": 0.23815912008285522, + "p05": 0.21215641498565674, + "p95": 0.2663886845111847 + }, + { + "label": "completed_group_3", + "mean": 0.22271886467933655, + "median": 0.22075486183166504, + "p05": 0.186451256275177, + "p95": 0.2638319730758667 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5232064723968506, + "group": 4, + "latest_source_mass": 0.24461600184440613, + "layer": 13, + "max_source_mass": 0.3136816620826721, + "mixer_index": 25, + "normalized_entropy": 0.9464213938474744, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.3136816620826721, + "median": 0.326117604970932, + "p05": 0.20238417387008667, + "p95": 0.4362232983112335 + }, + { + "label": "completed_group_1", + "mean": 0.11285123229026794, + "median": 0.11373943835496902, + "p05": 0.08906605839729309, + "p95": 0.13395145535469055 + }, + { + "label": "completed_group_2", + "mean": 0.13196846842765808, + "median": 0.1334267258644104, + "p05": 0.10183953493833542, + "p95": 0.16085752844810486 + }, + { + "label": "completed_group_3", + "mean": 0.19688265025615692, + "median": 0.1940557211637497, + "p05": 0.13830327987670898, + "p95": 0.2671412229537964 + }, + { + "label": "current_group_partial", + "mean": 0.24461600184440613, + "median": 0.2419893592596054, + "p05": 0.18994617462158203, + "p95": 0.3077915906906128 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4876129627227783, + "group": 4, + "latest_source_mass": 0.396677166223526, + "layer": 14, + "max_source_mass": 0.396677166223526, + "mixer_index": 26, + "normalized_entropy": 0.9243059028433878, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2035508155822754, + "median": 0.20008960366249084, + "p05": 0.16096048057079315, + "p95": 0.255715548992157 + }, + { + "label": "completed_group_1", + "mean": 0.15372203290462494, + "median": 0.15584711730480194, + "p05": 0.12048184871673584, + "p95": 0.18010932207107544 + }, + { + "label": "completed_group_2", + "mean": 0.13167491555213928, + "median": 0.13113251328468323, + "p05": 0.11609883606433868, + "p95": 0.14900363981723785 + }, + { + "label": "completed_group_3", + "mean": 0.11437506973743439, + "median": 0.11201842874288559, + "p05": 0.09515875577926636, + "p95": 0.1399364471435547 + }, + { + "label": "current_group_partial", + "mean": 0.396677166223526, + "median": 0.39533013105392456, + "p05": 0.3409242630004883, + "p95": 0.4550648033618927 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4587602615356445, + "group": 4, + "latest_source_mass": 0.33461427688598633, + "layer": 14, + "max_source_mass": 0.33461427688598633, + "mixer_index": 27, + "normalized_entropy": 0.906378711639412, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28519001603126526, + "median": 0.28675830364227295, + "p05": 0.1500462144613266, + "p95": 0.45491817593574524 + }, + { + "label": "completed_group_1", + "mean": 0.10532153397798538, + "median": 0.09640030562877655, + "p05": 0.06345808506011963, + "p95": 0.1692449301481247 + }, + { + "label": "completed_group_2", + "mean": 0.09675674885511398, + "median": 0.09718051552772522, + "p05": 0.058796193450689316, + "p95": 0.13363884389400482 + }, + { + "label": "completed_group_3", + "mean": 0.17811740934848785, + "median": 0.1759425848722458, + "p05": 0.1284305453300476, + "p95": 0.23726706206798553 + }, + { + "label": "current_group_partial", + "mean": 0.33461427688598633, + "median": 0.33546411991119385, + "p05": 0.23162710666656494, + "p95": 0.4484652876853943 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.4565696716308594, + "group": 4, + "latest_source_mass": 0.4221859574317932, + "layer": 15, + "max_source_mass": 0.4221859574317932, + "mixer_index": 28, + "normalized_entropy": 0.9050176216042753, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20232918858528137, + "median": 0.1998670995235443, + "p05": 0.15170496702194214, + "p95": 0.26285088062286377 + }, + { + "label": "completed_group_1", + "mean": 0.14928826689720154, + "median": 0.14749380946159363, + "p05": 0.12863489985466003, + "p95": 0.17874351143836975 + }, + { + "label": "completed_group_2", + "mean": 0.12029382586479187, + "median": 0.11950792372226715, + "p05": 0.10211217403411865, + "p95": 0.14088989794254303 + }, + { + "label": "completed_group_3", + "mean": 0.1059027910232544, + "median": 0.10411787033081055, + "p05": 0.08704904466867447, + "p95": 0.13138693571090698 + }, + { + "label": "current_group_partial", + "mean": 0.4221859574317932, + "median": 0.4195709228515625, + "p05": 0.35523390769958496, + "p95": 0.4969034492969513 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.4788098335266113, + "group": 4, + "latest_source_mass": 0.34753501415252686, + "layer": 15, + "max_source_mass": 0.34753501415252686, + "mixer_index": 29, + "normalized_entropy": 0.9188362111403675, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2697942852973938, + "median": 0.25651901960372925, + "p05": 0.146310955286026, + "p95": 0.42084938287734985 + }, + { + "label": "completed_group_1", + "mean": 0.11948063224554062, + "median": 0.11549928039312363, + "p05": 0.0849025622010231, + "p95": 0.16021209955215454 + }, + { + "label": "completed_group_2", + "mean": 0.11090938746929169, + "median": 0.11184529960155487, + "p05": 0.08139486610889435, + "p95": 0.13886672258377075 + }, + { + "label": "completed_group_3", + "mean": 0.15228068828582764, + "median": 0.15164361894130707, + "p05": 0.11261995136737823, + "p95": 0.19522227346897125 + }, + { + "label": "current_group_partial", + "mean": 0.34753501415252686, + "median": 0.3453425168991089, + "p05": 0.25332537293434143, + "p95": 0.44237542152404785 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5018762350082397, + "group": 4, + "latest_source_mass": 0.31088152527809143, + "layer": 16, + "max_source_mass": 0.31088152527809143, + "mixer_index": 30, + "normalized_entropy": 0.9331681721954809, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.208971306681633, + "median": 0.20881521701812744, + "p05": 0.16508029401302338, + "p95": 0.2516583502292633 + }, + { + "label": "completed_group_1", + "mean": 0.27331358194351196, + "median": 0.26948875188827515, + "p05": 0.23055149614810944, + "p95": 0.33120542764663696 + }, + { + "label": "completed_group_2", + "mean": 0.12766750156879425, + "median": 0.1261082887649536, + "p05": 0.10924770683050156, + "p95": 0.1494429111480713 + }, + { + "label": "completed_group_3", + "mean": 0.07916608452796936, + "median": 0.07811431586742401, + "p05": 0.0561223030090332, + "p95": 0.10760308802127838 + }, + { + "label": "current_group_partial", + "mean": 0.31088152527809143, + "median": 0.3123162090778351, + "p05": 0.2707694172859192, + "p95": 0.3467172384262085 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5627843141555786, + "group": 4, + "latest_source_mass": 0.22652018070220947, + "layer": 16, + "max_source_mass": 0.27452588081359863, + "mixer_index": 31, + "normalized_entropy": 0.9710124895666443, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.27452588081359863, + "median": 0.2678954601287842, + "p05": 0.19414913654327393, + "p95": 0.3916982412338257 + }, + { + "label": "completed_group_1", + "mean": 0.21448183059692383, + "median": 0.21617943048477173, + "p05": 0.17793960869312286, + "p95": 0.24279263615608215 + }, + { + "label": "completed_group_2", + "mean": 0.143874391913414, + "median": 0.1446641981601715, + "p05": 0.11340779811143875, + "p95": 0.17275942862033844 + }, + { + "label": "completed_group_3", + "mean": 0.14059773087501526, + "median": 0.13659776747226715, + "p05": 0.11039599776268005, + "p95": 0.18301524221897125 + }, + { + "label": "current_group_partial", + "mean": 0.22652018070220947, + "median": 0.22071748971939087, + "p05": 0.16668984293937683, + "p95": 0.293569415807724 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.580754280090332, + "group": 5, + "latest_source_mass": 0.24107450246810913, + "layer": 17, + "max_source_mass": 0.2518634498119354, + "mixer_index": 32, + "normalized_entropy": 0.9821778571747528, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2518634498119354, + "median": 0.24593469500541687, + "p05": 0.21242231130599976, + "p95": 0.3016672730445862 + }, + { + "label": "completed_group_1", + "mean": 0.18758200109004974, + "median": 0.1801876276731491, + "p05": 0.15723729133605957, + "p95": 0.24892441928386688 + }, + { + "label": "completed_group_2", + "mean": 0.13903990387916565, + "median": 0.13839681446552277, + "p05": 0.12513971328735352, + "p95": 0.15491701662540436 + }, + { + "label": "completed_group_3", + "mean": 0.18044015765190125, + "median": 0.1819951832294464, + "p05": 0.14810611307621002, + "p95": 0.20801132917404175 + }, + { + "label": "completed_group_4", + "mean": 0.24107450246810913, + "median": 0.24384509027004242, + "p05": 0.1871698796749115, + "p95": 0.2886185050010681 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.7564777135849, + "group": 5, + "latest_source_mass": 0.2205006182193756, + "layer": 17, + "max_source_mass": 0.2205006182193756, + "mixer_index": 33, + "normalized_entropy": 0.9803088772521708, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.15440437197685242, + "median": 0.15525281429290771, + "p05": 0.11778812110424042, + "p95": 0.18865147233009338 + }, + { + "label": "completed_group_1", + "mean": 0.19411899149417877, + "median": 0.2002902328968048, + "p05": 0.14482440054416656, + "p95": 0.2256385236978531 + }, + { + "label": "completed_group_2", + "mean": 0.17806550860404968, + "median": 0.1781877875328064, + "p05": 0.15101563930511475, + "p95": 0.20418882369995117 + }, + { + "label": "completed_group_3", + "mean": 0.14696575701236725, + "median": 0.14620032906532288, + "p05": 0.10836094617843628, + "p95": 0.1874053031206131 + }, + { + "label": "completed_group_4", + "mean": 0.10594475269317627, + "median": 0.10578415542840958, + "p05": 0.08951808512210846, + "p95": 0.1216079369187355 + }, + { + "label": "current_group_partial", + "mean": 0.2205006182193756, + "median": 0.21638520061969757, + "p05": 0.16007332503795624, + "p95": 0.2944811284542084 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5899518728256226, + "group": 5, + "latest_source_mass": 0.4157395362854004, + "layer": 18, + "max_source_mass": 0.4157395362854004, + "mixer_index": 34, + "normalized_entropy": 0.8873690359290373, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1636897772550583, + "median": 0.16218292713165283, + "p05": 0.1178273856639862, + "p95": 0.21766462922096252 + }, + { + "label": "completed_group_1", + "mean": 0.11789833009243011, + "median": 0.11790023744106293, + "p05": 0.1032494530081749, + "p95": 0.13267967104911804 + }, + { + "label": "completed_group_2", + "mean": 0.07030131667852402, + "median": 0.0698043555021286, + "p05": 0.06258689612150192, + "p95": 0.07938572019338608 + }, + { + "label": "completed_group_3", + "mean": 0.09238425642251968, + "median": 0.09013497084379196, + "p05": 0.0768275037407875, + "p95": 0.1164383590221405 + }, + { + "label": "completed_group_4", + "mean": 0.1399867981672287, + "median": 0.1400289535522461, + "p05": 0.12679682672023773, + "p95": 0.15317322313785553 + }, + { + "label": "current_group_partial", + "mean": 0.4157395362854004, + "median": 0.4171099066734314, + "p05": 0.367803692817688, + "p95": 0.4605288803577423 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.2894535064697266, + "group": 5, + "latest_source_mass": 0.5977407693862915, + "layer": 18, + "max_source_mass": 0.5977407693862915, + "mixer_index": 35, + "normalized_entropy": 0.7196577044045218, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1212274506688118, + "median": 0.10892187058925629, + "p05": 0.07249757647514343, + "p95": 0.2145335078239441 + }, + { + "label": "completed_group_1", + "mean": 0.06944876164197922, + "median": 0.0678764283657074, + "p05": 0.05350388586521149, + "p95": 0.08999023586511612 + }, + { + "label": "completed_group_2", + "mean": 0.05726535990834236, + "median": 0.0559837631881237, + "p05": 0.0409373864531517, + "p95": 0.07825973629951477 + }, + { + "label": "completed_group_3", + "mean": 0.09096579253673553, + "median": 0.08796361088752747, + "p05": 0.06407731771469116, + "p95": 0.12661314010620117 + }, + { + "label": "completed_group_4", + "mean": 0.06335186958312988, + "median": 0.06283852458000183, + "p05": 0.05032140761613846, + "p95": 0.0781375840306282 + }, + { + "label": "current_group_partial", + "mean": 0.5977407693862915, + "median": 0.6085352301597595, + "p05": 0.4878399074077606, + "p95": 0.6737192869186401 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6581614017486572, + "group": 5, + "latest_source_mass": 0.35773903131484985, + "layer": 19, + "max_source_mass": 0.35773903131484985, + "mixer_index": 36, + "normalized_entropy": 0.9254374988530375, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18132120370864868, + "median": 0.1806161105632782, + "p05": 0.1410265862941742, + "p95": 0.22331592440605164 + }, + { + "label": "completed_group_1", + "mean": 0.12711825966835022, + "median": 0.12722507119178772, + "p05": 0.11291128396987915, + "p95": 0.14178316295146942 + }, + { + "label": "completed_group_2", + "mean": 0.08560527861118317, + "median": 0.08513904362916946, + "p05": 0.07705061137676239, + "p95": 0.09564155340194702 + }, + { + "label": "completed_group_3", + "mean": 0.09748885035514832, + "median": 0.0953599065542221, + "p05": 0.08346174657344818, + "p95": 0.1194010004401207 + }, + { + "label": "completed_group_4", + "mean": 0.15072739124298096, + "median": 0.150638610124588, + "p05": 0.13762864470481873, + "p95": 0.1634974181652069 + }, + { + "label": "current_group_partial", + "mean": 0.35773903131484985, + "median": 0.35794761776924133, + "p05": 0.32225728034973145, + "p95": 0.39409786462783813 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3699336051940918, + "group": 5, + "latest_source_mass": 0.5385624766349792, + "layer": 19, + "max_source_mass": 0.5385624766349792, + "mixer_index": 37, + "normalized_entropy": 0.7645745027284836, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13604649901390076, + "median": 0.11998516321182251, + "p05": 0.06740576773881912, + "p95": 0.26897257566452026 + }, + { + "label": "completed_group_1", + "mean": 0.10151474922895432, + "median": 0.0991508960723877, + "p05": 0.07438431680202484, + "p95": 0.13600951433181763 + }, + { + "label": "completed_group_2", + "mean": 0.07715044170618057, + "median": 0.0767412930727005, + "p05": 0.054624155163764954, + "p95": 0.09943743050098419 + }, + { + "label": "completed_group_3", + "mean": 0.11300846189260483, + "median": 0.11003986746072769, + "p05": 0.08064281940460205, + "p95": 0.1516837179660797 + }, + { + "label": "completed_group_4", + "mean": 0.033717282116413116, + "median": 0.03286907076835632, + "p05": 0.024589763954281807, + "p95": 0.04592139273881912 + }, + { + "label": "current_group_partial", + "mean": 0.5385624766349792, + "median": 0.5415897369384766, + "p05": 0.4144754409790039, + "p95": 0.6506978273391724 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6926933526992798, + "group": 5, + "latest_source_mass": 0.3198680281639099, + "layer": 20, + "max_source_mass": 0.3198680281639099, + "mixer_index": 38, + "normalized_entropy": 0.9447101476341264, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19425281882286072, + "median": 0.19403350353240967, + "p05": 0.139950692653656, + "p95": 0.2494640350341797 + }, + { + "label": "completed_group_1", + "mean": 0.14001017808914185, + "median": 0.14058321714401245, + "p05": 0.12015820294618607, + "p95": 0.158815398812294 + }, + { + "label": "completed_group_2", + "mean": 0.09434760361909866, + "median": 0.09424005448818207, + "p05": 0.08333250135183334, + "p95": 0.10548526048660278 + }, + { + "label": "completed_group_3", + "mean": 0.10331346094608307, + "median": 0.10153411328792572, + "p05": 0.08690286427736282, + "p95": 0.12658309936523438 + }, + { + "label": "completed_group_4", + "mean": 0.14820793271064758, + "median": 0.14894580841064453, + "p05": 0.13278263807296753, + "p95": 0.16199614107608795 + }, + { + "label": "current_group_partial", + "mean": 0.3198680281639099, + "median": 0.3201481103897095, + "p05": 0.2800636887550354, + "p95": 0.3604968786239624 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.4148362874984741, + "group": 5, + "latest_source_mass": 0.4996800422668457, + "layer": 20, + "max_source_mass": 0.4996800422668457, + "mixer_index": 39, + "normalized_entropy": 0.789635166883214, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.16652248799800873, + "median": 0.14292314648628235, + "p05": 0.08186839520931244, + "p95": 0.350705623626709 + }, + { + "label": "completed_group_1", + "mean": 0.10329679399728775, + "median": 0.10057851672172546, + "p05": 0.07478320598602295, + "p95": 0.13858266174793243 + }, + { + "label": "completed_group_2", + "mean": 0.08063912391662598, + "median": 0.0807541012763977, + "p05": 0.056308042258024216, + "p95": 0.10469165444374084 + }, + { + "label": "completed_group_3", + "mean": 0.1165856122970581, + "median": 0.11373114585876465, + "p05": 0.08339665830135345, + "p95": 0.15619920194149017 + }, + { + "label": "completed_group_4", + "mean": 0.03327595070004463, + "median": 0.032142095267772675, + "p05": 0.022759035229682922, + "p95": 0.04896518588066101 + }, + { + "label": "current_group_partial", + "mean": 0.4996800422668457, + "median": 0.5001581907272339, + "p05": 0.3463003933429718, + "p95": 0.6266770362854004 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.740808129310608, + "group": 6, + "latest_source_mass": 0.24231041967868805, + "layer": 21, + "max_source_mass": 0.24231041967868805, + "mixer_index": 40, + "normalized_entropy": 0.9715635157550481, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21459883451461792, + "median": 0.21599377691745758, + "p05": 0.1570315659046173, + "p95": 0.27208006381988525 + }, + { + "label": "completed_group_1", + "mean": 0.15640269219875336, + "median": 0.1569681316614151, + "p05": 0.13785089552402496, + "p95": 0.1749057024717331 + }, + { + "label": "completed_group_2", + "mean": 0.10570268332958221, + "median": 0.10564640164375305, + "p05": 0.09223169088363647, + "p95": 0.11907710134983063 + }, + { + "label": "completed_group_3", + "mean": 0.11097013205289841, + "median": 0.10862545669078827, + "p05": 0.09454305469989777, + "p95": 0.13635265827178955 + }, + { + "label": "completed_group_4", + "mean": 0.17001524567604065, + "median": 0.17010903358459473, + "p05": 0.15088173747062683, + "p95": 0.18767674267292023 + }, + { + "label": "completed_group_5", + "mean": 0.24231041967868805, + "median": 0.24274763464927673, + "p05": 0.2087050974369049, + "p95": 0.27555277943611145 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.3179612159729004, + "group": 6, + "latest_source_mass": 0.603651762008667, + "layer": 21, + "max_source_mass": 0.603651762008667, + "mixer_index": 41, + "normalized_entropy": 0.6772980841960945, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08068332821130753, + "median": 0.06917878985404968, + "p05": 0.039396293461322784, + "p95": 0.17595769464969635 + }, + { + "label": "completed_group_1", + "mean": 0.049232251942157745, + "median": 0.04861311614513397, + "p05": 0.03742668777704239, + "p95": 0.06363335251808167 + }, + { + "label": "completed_group_2", + "mean": 0.045073024928569794, + "median": 0.04482679069042206, + "p05": 0.03333880379796028, + "p95": 0.057495348155498505 + }, + { + "label": "completed_group_3", + "mean": 0.06111945956945419, + "median": 0.059528060257434845, + "p05": 0.042710162699222565, + "p95": 0.08362975716590881 + }, + { + "label": "completed_group_4", + "mean": 0.028317855671048164, + "median": 0.027924545109272003, + "p05": 0.022549759596586227, + "p95": 0.03542562201619148 + }, + { + "label": "completed_group_5", + "mean": 0.13192236423492432, + "median": 0.12939827144145966, + "p05": 0.10056710243225098, + "p95": 0.17195004224777222 + }, + { + "label": "current_group_partial", + "mean": 0.603651762008667, + "median": 0.6094026565551758, + "p05": 0.5065152645111084, + "p95": 0.6787958145141602 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.5319744348526, + "group": 6, + "latest_source_mass": 0.5132461190223694, + "layer": 22, + "max_source_mass": 0.5132461190223694, + "mixer_index": 42, + "normalized_entropy": 0.7872791226235868, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12406684458255768, + "median": 0.1289612352848053, + "p05": 0.06980825215578079, + "p95": 0.16972985863685608 + }, + { + "label": "completed_group_1", + "mean": 0.07462044060230255, + "median": 0.07392798364162445, + "p05": 0.06016036123037338, + "p95": 0.0917147696018219 + }, + { + "label": "completed_group_2", + "mean": 0.05378449708223343, + "median": 0.053309448063373566, + "p05": 0.04370884597301483, + "p95": 0.06546851992607117 + }, + { + "label": "completed_group_3", + "mean": 0.05575074627995491, + "median": 0.053552836179733276, + "p05": 0.0437602773308754, + "p95": 0.07444392144680023 + }, + { + "label": "completed_group_4", + "mean": 0.07357081025838852, + "median": 0.07329098880290985, + "p05": 0.06283499300479889, + "p95": 0.08542977273464203 + }, + { + "label": "completed_group_5", + "mean": 0.10496053099632263, + "median": 0.1038457602262497, + "p05": 0.08522139489650726, + "p95": 0.1291121542453766 + }, + { + "label": "current_group_partial", + "mean": 0.5132461190223694, + "median": 0.5090781450271606, + "p05": 0.4398295283317566, + "p95": 0.59932541847229 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.2461028099060059, + "group": 6, + "latest_source_mass": 0.6375080347061157, + "layer": 22, + "max_source_mass": 0.6375080347061157, + "mixer_index": 43, + "normalized_entropy": 0.640370168432985, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08556011319160461, + "median": 0.07947102189064026, + "p05": 0.043556369841098785, + "p95": 0.16172736883163452 + }, + { + "label": "completed_group_1", + "mean": 0.0446893647313118, + "median": 0.04398339241743088, + "p05": 0.03128381073474884, + "p95": 0.05990619957447052 + }, + { + "label": "completed_group_2", + "mean": 0.04084867238998413, + "median": 0.04042873531579971, + "p05": 0.02942952886223793, + "p95": 0.05392487719655037 + }, + { + "label": "completed_group_3", + "mean": 0.05386577919125557, + "median": 0.052273523062467575, + "p05": 0.033789411187171936, + "p95": 0.076903335750103 + }, + { + "label": "completed_group_4", + "mean": 0.025119207799434662, + "median": 0.024625632911920547, + "p05": 0.019138969480991364, + "p95": 0.03265690058469772 + }, + { + "label": "completed_group_5", + "mean": 0.11240880191326141, + "median": 0.11003861576318741, + "p05": 0.07852104306221008, + "p95": 0.155371755361557 + }, + { + "label": "current_group_partial", + "mean": 0.6375080347061157, + "median": 0.6392797231674194, + "p05": 0.5317062735557556, + "p95": 0.7386638522148132 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.6323832273483276, + "group": 6, + "latest_source_mass": 0.4532663822174072, + "layer": 23, + "max_source_mass": 0.4532663822174072, + "mixer_index": 44, + "normalized_entropy": 0.8388790346464895, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1418324112892151, + "median": 0.14852997660636902, + "p05": 0.07787702977657318, + "p95": 0.1923113763332367 + }, + { + "label": "completed_group_1", + "mean": 0.08991365134716034, + "median": 0.08918912708759308, + "p05": 0.0752246081829071, + "p95": 0.10654127597808838 + }, + { + "label": "completed_group_2", + "mean": 0.06188817322254181, + "median": 0.06161077320575714, + "p05": 0.04994015768170357, + "p95": 0.07481888681650162 + }, + { + "label": "completed_group_3", + "mean": 0.05739111453294754, + "median": 0.0548475980758667, + "p05": 0.04428064078092575, + "p95": 0.07816634327173233 + }, + { + "label": "completed_group_4", + "mean": 0.09848874807357788, + "median": 0.09768162667751312, + "p05": 0.08320866525173187, + "p95": 0.11606857925653458 + }, + { + "label": "completed_group_5", + "mean": 0.0972195416688919, + "median": 0.09613433480262756, + "p05": 0.07767169177532196, + "p95": 0.12004032731056213 + }, + { + "label": "current_group_partial", + "mean": 0.4532663822174072, + "median": 0.45108917355537415, + "p05": 0.37653055787086487, + "p95": 0.5389904975891113 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.301480770111084, + "group": 6, + "latest_source_mass": 0.6134276390075684, + "layer": 23, + "max_source_mass": 0.6134276390075684, + "mixer_index": 45, + "normalized_entropy": 0.6688288103861927, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09513165056705475, + "median": 0.08380547165870667, + "p05": 0.055099427700042725, + "p95": 0.1882638931274414 + }, + { + "label": "completed_group_1", + "mean": 0.050291214138269424, + "median": 0.04947858303785324, + "p05": 0.03757611662149429, + "p95": 0.06516233086585999 + }, + { + "label": "completed_group_2", + "mean": 0.045042067766189575, + "median": 0.0443364642560482, + "p05": 0.031873252242803574, + "p95": 0.06115761026740074 + }, + { + "label": "completed_group_3", + "mean": 0.05902485549449921, + "median": 0.05624900758266449, + "p05": 0.038259170949459076, + "p95": 0.08772744238376617 + }, + { + "label": "completed_group_4", + "mean": 0.02615261636674404, + "median": 0.02565642073750496, + "p05": 0.01966264471411705, + "p95": 0.03415588289499283 + }, + { + "label": "completed_group_5", + "mean": 0.1109299436211586, + "median": 0.10745325684547424, + "p05": 0.07642066478729248, + "p95": 0.15984633564949036 + }, + { + "label": "current_group_partial", + "mean": 0.6134276390075684, + "median": 0.6200529336929321, + "p05": 0.49638131260871887, + "p95": 0.7104787826538086 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.629331111907959, + "group": 6, + "latest_source_mass": 0.45539987087249756, + "layer": 24, + "max_source_mass": 0.45539987087249756, + "mixer_index": 46, + "normalized_entropy": 0.837310557580963, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14790503680706024, + "median": 0.15420395135879517, + "p05": 0.08257082104682922, + "p95": 0.20396333932876587 + }, + { + "label": "completed_group_1", + "mean": 0.08658000826835632, + "median": 0.08578156679868698, + "p05": 0.07299856841564178, + "p95": 0.10278668999671936 + }, + { + "label": "completed_group_2", + "mean": 0.06608415395021439, + "median": 0.06551258265972137, + "p05": 0.054121918976306915, + "p95": 0.07912646979093552 + }, + { + "label": "completed_group_3", + "mean": 0.06427294760942459, + "median": 0.06133077293634415, + "p05": 0.04914242774248123, + "p95": 0.09066928923130035 + }, + { + "label": "completed_group_4", + "mean": 0.0819322019815445, + "median": 0.08114765584468842, + "p05": 0.06886082887649536, + "p95": 0.09821749478578568 + }, + { + "label": "completed_group_5", + "mean": 0.0978257954120636, + "median": 0.09643188118934631, + "p05": 0.07740232348442078, + "p95": 0.12209523469209671 + }, + { + "label": "current_group_partial", + "mean": 0.45539987087249756, + "median": 0.45274412631988525, + "p05": 0.3780696392059326, + "p95": 0.5462272763252258 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.338958501815796, + "group": 6, + "latest_source_mass": 0.5946891903877258, + "layer": 24, + "max_source_mass": 0.5946891903877258, + "mixer_index": 47, + "normalized_entropy": 0.6880885545850224, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10803166776895523, + "median": 0.09507323801517487, + "p05": 0.04510519653558731, + "p95": 0.2335781455039978 + }, + { + "label": "completed_group_1", + "mean": 0.0559723861515522, + "median": 0.05525296553969383, + "p05": 0.042058296501636505, + "p95": 0.07207933813333511 + }, + { + "label": "completed_group_2", + "mean": 0.047874726355075836, + "median": 0.04723185673356056, + "p05": 0.03294869884848595, + "p95": 0.06497423350811005 + }, + { + "label": "completed_group_3", + "mean": 0.060293328016996384, + "median": 0.05662515014410019, + "p05": 0.040346790105104446, + "p95": 0.08978720009326935 + }, + { + "label": "completed_group_4", + "mean": 0.029215168207883835, + "median": 0.028496116399765015, + "p05": 0.021991699934005737, + "p95": 0.039112430065870285 + }, + { + "label": "completed_group_5", + "mean": 0.10392352938652039, + "median": 0.10063231736421585, + "p05": 0.06900570541620255, + "p95": 0.1502695530653 + }, + { + "label": "current_group_partial", + "mean": 0.5946891903877258, + "median": 0.600748598575592, + "p05": 0.46449366211891174, + "p95": 0.7016042470932007 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7611865997314453, + "group": 7, + "latest_source_mass": 0.3679775297641754, + "layer": 25, + "max_source_mass": 0.3679775297641754, + "mixer_index": 48, + "normalized_entropy": 0.9050708742058073, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1573912650346756, + "median": 0.15898014605045319, + "p05": 0.09474922716617584, + "p95": 0.2298184633255005 + }, + { + "label": "completed_group_1", + "mean": 0.10317619889974594, + "median": 0.10270150750875473, + "p05": 0.08812867850065231, + "p95": 0.11994096636772156 + }, + { + "label": "completed_group_2", + "mean": 0.08533225208520889, + "median": 0.08495655655860901, + "p05": 0.07036984711885452, + "p95": 0.1013721376657486 + }, + { + "label": "completed_group_3", + "mean": 0.08218386024236679, + "median": 0.07942138612270355, + "p05": 0.06530377268791199, + "p95": 0.1088358536362648 + }, + { + "label": "completed_group_4", + "mean": 0.09052872657775879, + "median": 0.08931052684783936, + "p05": 0.07639768719673157, + "p95": 0.10804671049118042 + }, + { + "label": "completed_group_5", + "mean": 0.11341015249490738, + "median": 0.1123356968164444, + "p05": 0.09177497029304504, + "p95": 0.13803066313266754 + }, + { + "label": "completed_group_6", + "mean": 0.3679775297641754, + "median": 0.3671194612979889, + "p05": 0.2961871325969696, + "p95": 0.4462817311286926 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.476489782333374, + "group": 7, + "latest_source_mass": 0.3396714925765991, + "layer": 25, + "max_source_mass": 0.3836256265640259, + "mixer_index": 49, + "normalized_entropy": 0.7100414956318613, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09070298075675964, + "median": 0.0818760022521019, + "p05": 0.04576021432876587, + "p95": 0.1613437533378601 + }, + { + "label": "completed_group_1", + "mean": 0.03713828697800636, + "median": 0.035560622811317444, + "p05": 0.02337636426091194, + "p95": 0.054472144693136215 + }, + { + "label": "completed_group_2", + "mean": 0.030013620853424072, + "median": 0.028701743111014366, + "p05": 0.01836019568145275, + "p95": 0.04582716524600983 + }, + { + "label": "completed_group_3", + "mean": 0.03533655032515526, + "median": 0.031870774924755096, + "p05": 0.018502909690141678, + "p95": 0.061826568096876144 + }, + { + "label": "completed_group_4", + "mean": 0.020730752497911453, + "median": 0.020491186529397964, + "p05": 0.014567261561751366, + "p95": 0.027856238186359406 + }, + { + "label": "completed_group_5", + "mean": 0.06278066337108612, + "median": 0.059403590857982635, + "p05": 0.03608490154147148, + "p95": 0.10122650861740112 + }, + { + "label": "completed_group_6", + "mean": 0.3836256265640259, + "median": 0.3773127794265747, + "p05": 0.1874689906835556, + "p95": 0.5895293951034546 + }, + { + "label": "current_group_partial", + "mean": 0.3396714925765991, + "median": 0.34821513295173645, + "p05": 0.09212851524353027, + "p95": 0.587702751159668 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7213938236236572, + "group": 7, + "latest_source_mass": 0.37924307584762573, + "layer": 26, + "max_source_mass": 0.37924307584762573, + "mixer_index": 50, + "normalized_entropy": 0.8278154442529139, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10584281384944916, + "median": 0.10982948541641235, + "p05": 0.03868259862065315, + "p95": 0.16580678522586823 + }, + { + "label": "completed_group_1", + "mean": 0.06887699663639069, + "median": 0.06925956159830093, + "p05": 0.04082746431231499, + "p95": 0.09546782076358795 + }, + { + "label": "completed_group_2", + "mean": 0.05207539349794388, + "median": 0.051871635019779205, + "p05": 0.035383015871047974, + "p95": 0.07048650085926056 + }, + { + "label": "completed_group_3", + "mean": 0.050678279250860214, + "median": 0.04831986129283905, + "p05": 0.03378130868077278, + "p95": 0.07452918589115143 + }, + { + "label": "completed_group_4", + "mean": 0.04768219590187073, + "median": 0.04721469432115555, + "p05": 0.03680850565433502, + "p95": 0.060343313962221146 + }, + { + "label": "completed_group_5", + "mean": 0.06809806823730469, + "median": 0.06714263558387756, + "p05": 0.04759174585342407, + "p95": 0.09223967790603638 + }, + { + "label": "completed_group_6", + "mean": 0.2275032103061676, + "median": 0.22425106167793274, + "p05": 0.14320722222328186, + "p95": 0.3203037679195404 + }, + { + "label": "current_group_partial", + "mean": 0.37924307584762573, + "median": 0.3716055750846863, + "p05": 0.20950253307819366, + "p95": 0.5737906098365784 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.5405172109603882, + "group": 7, + "latest_source_mass": 0.4017985463142395, + "layer": 26, + "max_source_mass": 0.4017985463142395, + "mixer_index": 51, + "normalized_entropy": 0.7408321802188831, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.074287548661232, + "median": 0.06164507195353508, + "p05": 0.028203874826431274, + "p95": 0.19368377327919006 + }, + { + "label": "completed_group_1", + "mean": 0.04491765797138214, + "median": 0.04365704953670502, + "p05": 0.02776508405804634, + "p95": 0.06526201963424683 + }, + { + "label": "completed_group_2", + "mean": 0.035157643258571625, + "median": 0.03418130800127983, + "p05": 0.022541508078575134, + "p95": 0.050758615136146545 + }, + { + "label": "completed_group_3", + "mean": 0.046252522617578506, + "median": 0.04353209584951401, + "p05": 0.026215802878141403, + "p95": 0.07329559326171875 + }, + { + "label": "completed_group_4", + "mean": 0.023427214473485947, + "median": 0.023055266588926315, + "p05": 0.016644639894366264, + "p95": 0.03150089830160141 + }, + { + "label": "completed_group_5", + "mean": 0.07749318331480026, + "median": 0.07363636791706085, + "p05": 0.04229677468538284, + "p95": 0.12579958140850067 + }, + { + "label": "completed_group_6", + "mean": 0.29666566848754883, + "median": 0.2920765280723572, + "p05": 0.14379237592220306, + "p95": 0.45965293049812317 + }, + { + "label": "current_group_partial", + "mean": 0.4017985463142395, + "median": 0.4004032611846924, + "p05": 0.18456703424453735, + "p95": 0.6275562047958374 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8063910007476807, + "group": 7, + "latest_source_mass": 0.3437207341194153, + "layer": 27, + "max_source_mass": 0.3437207341194153, + "mixer_index": 52, + "normalized_entropy": 0.868690446228377, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11433833092451096, + "median": 0.11714513599872589, + "p05": 0.06115472689270973, + "p95": 0.16258974373340607 + }, + { + "label": "completed_group_1", + "mean": 0.07374332845211029, + "median": 0.0731307864189148, + "p05": 0.05447638779878616, + "p95": 0.09509895741939545 + }, + { + "label": "completed_group_2", + "mean": 0.05932536721229553, + "median": 0.05929289758205414, + "p05": 0.04452420026063919, + "p95": 0.07498160749673843 + }, + { + "label": "completed_group_3", + "mean": 0.05389238893985748, + "median": 0.051720209419727325, + "p05": 0.03926049917936325, + "p95": 0.07536239922046661 + }, + { + "label": "completed_group_4", + "mean": 0.06126017868518829, + "median": 0.06108957529067993, + "p05": 0.0506296306848526, + "p95": 0.07238088548183441 + }, + { + "label": "completed_group_5", + "mean": 0.07349519431591034, + "median": 0.07249070703983307, + "p05": 0.05706139653921127, + "p95": 0.09332700818777084 + }, + { + "label": "completed_group_6", + "mean": 0.22022445499897003, + "median": 0.21959853172302246, + "p05": 0.15368635952472687, + "p95": 0.2904376685619354 + }, + { + "label": "current_group_partial", + "mean": 0.3437207341194153, + "median": 0.33986932039260864, + "p05": 0.22015926241874695, + "p95": 0.4802429676055908 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.4859801530838013, + "group": 7, + "latest_source_mass": 0.4424910843372345, + "layer": 27, + "max_source_mass": 0.4424910843372345, + "mixer_index": 53, + "normalized_entropy": 0.7146053992378076, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08387120068073273, + "median": 0.07644708454608917, + "p05": 0.0386628657579422, + "p95": 0.15876422822475433 + }, + { + "label": "completed_group_1", + "mean": 0.041406162083148956, + "median": 0.0394824743270874, + "p05": 0.025755178183317184, + "p95": 0.06136181950569153 + }, + { + "label": "completed_group_2", + "mean": 0.032444946467876434, + "median": 0.0307333804666996, + "p05": 0.018246563151478767, + "p95": 0.052875272929668427 + }, + { + "label": "completed_group_3", + "mean": 0.03686721250414848, + "median": 0.03247762471437454, + "p05": 0.017809681594371796, + "p95": 0.06780916452407837 + }, + { + "label": "completed_group_4", + "mean": 0.025052767246961594, + "median": 0.024561133235692978, + "p05": 0.017057543620467186, + "p95": 0.034822143614292145 + }, + { + "label": "completed_group_5", + "mean": 0.06012478470802307, + "median": 0.056938059628009796, + "p05": 0.03254516050219536, + "p95": 0.09905251860618591 + }, + { + "label": "completed_group_6", + "mean": 0.27774184942245483, + "median": 0.2747848629951477, + "p05": 0.11654777824878693, + "p95": 0.4562506675720215 + }, + { + "label": "current_group_partial", + "mean": 0.4424910843372345, + "median": 0.4387660026550293, + "p05": 0.21194897592067719, + "p95": 0.6782112717628479 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.7777256965637207, + "group": 7, + "latest_source_mass": 0.3409850299358368, + "layer": 28, + "max_source_mass": 0.3409850299358368, + "mixer_index": 54, + "normalized_entropy": 0.8549053488311195, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10586117208003998, + "median": 0.10652337223291397, + "p05": 0.05095779523253441, + "p95": 0.1629783660173416 + }, + { + "label": "completed_group_1", + "mean": 0.07087038457393646, + "median": 0.06906798481941223, + "p05": 0.04985840618610382, + "p95": 0.09773125499486923 + }, + { + "label": "completed_group_2", + "mean": 0.05850498005747795, + "median": 0.05767729878425598, + "p05": 0.04179685190320015, + "p95": 0.07779045403003693 + }, + { + "label": "completed_group_3", + "mean": 0.06360965222120285, + "median": 0.06068425625562668, + "p05": 0.04235687851905823, + "p95": 0.09371945261955261 + }, + { + "label": "completed_group_4", + "mean": 0.04672512412071228, + "median": 0.04617916792631149, + "p05": 0.0356140062212944, + "p95": 0.05914818495512009 + }, + { + "label": "completed_group_5", + "mean": 0.07485505938529968, + "median": 0.07274073362350464, + "p05": 0.052147433161735535, + "p95": 0.10333258658647537 + }, + { + "label": "completed_group_6", + "mean": 0.2385886013507843, + "median": 0.2345724105834961, + "p05": 0.15389031171798706, + "p95": 0.33735349774360657 + }, + { + "label": "current_group_partial", + "mean": 0.3409850299358368, + "median": 0.33667099475860596, + "p05": 0.18655306100845337, + "p95": 0.5089476704597473 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6318418979644775, + "group": 7, + "latest_source_mass": 0.3292362689971924, + "layer": 28, + "max_source_mass": 0.3292362689971924, + "mixer_index": 55, + "normalized_entropy": 0.7847500712360619, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09244748950004578, + "median": 0.06782519817352295, + "p05": 0.025312144309282303, + "p95": 0.24087309837341309 + }, + { + "label": "completed_group_1", + "mean": 0.05762210115790367, + "median": 0.056058838963508606, + "p05": 0.03864594176411629, + "p95": 0.08028930425643921 + }, + { + "label": "completed_group_2", + "mean": 0.045099519193172455, + "median": 0.04426075518131256, + "p05": 0.03003283590078354, + "p95": 0.06244165822863579 + }, + { + "label": "completed_group_3", + "mean": 0.056236449629068375, + "median": 0.052902162075042725, + "p05": 0.03282861411571503, + "p95": 0.08620619773864746 + }, + { + "label": "completed_group_4", + "mean": 0.029373466968536377, + "median": 0.02887408435344696, + "p05": 0.020401213318109512, + "p95": 0.04018516466021538 + }, + { + "label": "completed_group_5", + "mean": 0.07237439602613449, + "median": 0.06993190199136734, + "p05": 0.04555661231279373, + "p95": 0.10748817771673203 + }, + { + "label": "completed_group_6", + "mean": 0.31761035323143005, + "median": 0.3173876404762268, + "p05": 0.16091161966323853, + "p95": 0.4724525213241577 + }, + { + "label": "current_group_partial", + "mean": 0.3292362689971924, + "median": 0.3128150701522827, + "p05": 0.12605780363082886, + "p95": 0.5847846865653992 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.8334553241729736, + "group": 8, + "latest_source_mass": 0.270294725894928, + "layer": 29, + "max_source_mass": 0.27155569195747375, + "mixer_index": 56, + "normalized_entropy": 0.8817056346252721, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1157165914773941, + "median": 0.113271564245224, + "p05": 0.06316040456295013, + "p95": 0.17676270008087158 + }, + { + "label": "completed_group_1", + "mean": 0.07161639630794525, + "median": 0.07118864357471466, + "p05": 0.04294680058956146, + "p95": 0.102086141705513 + }, + { + "label": "completed_group_2", + "mean": 0.06577154994010925, + "median": 0.06490545719861984, + "p05": 0.04486069828271866, + "p95": 0.09022088348865509 + }, + { + "label": "completed_group_3", + "mean": 0.07425849884748459, + "median": 0.07179903984069824, + "p05": 0.045271825045347214, + "p95": 0.11117851734161377 + }, + { + "label": "completed_group_4", + "mean": 0.04881400242447853, + "median": 0.04833249747753143, + "p05": 0.0363147035241127, + "p95": 0.0629924014210701 + }, + { + "label": "completed_group_5", + "mean": 0.08197253197431564, + "median": 0.08029600977897644, + "p05": 0.05680408701300621, + "p95": 0.11223416775465012 + }, + { + "label": "completed_group_6", + "mean": 0.27155569195747375, + "median": 0.2687028646469116, + "p05": 0.18294592201709747, + "p95": 0.3740781545639038 + }, + { + "label": "completed_group_7", + "mean": 0.270294725894928, + "median": 0.2569413185119629, + "p05": 0.13936233520507812, + "p95": 0.43686625361442566 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.6227548122406006, + "group": 8, + "latest_source_mass": 0.45173269510269165, + "layer": 29, + "max_source_mass": 0.45173269510269165, + "mixer_index": 57, + "normalized_entropy": 0.7385475426494315, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0605507455766201, + "median": 0.05293343961238861, + "p05": 0.02206447720527649, + "p95": 0.1254502236843109 + }, + { + "label": "completed_group_1", + "mean": 0.040063515305519104, + "median": 0.03803131729364395, + "p05": 0.022060411050915718, + "p95": 0.06445677578449249 + }, + { + "label": "completed_group_2", + "mean": 0.032619886100292206, + "median": 0.030926726758480072, + "p05": 0.020269516855478287, + "p95": 0.050711989402770996 + }, + { + "label": "completed_group_3", + "mean": 0.03868011012673378, + "median": 0.0344192236661911, + "p05": 0.020861119031906128, + "p95": 0.06798171997070312 + }, + { + "label": "completed_group_4", + "mean": 0.020290488377213478, + "median": 0.01974661275744438, + "p05": 0.014590941369533539, + "p95": 0.027847949415445328 + }, + { + "label": "completed_group_5", + "mean": 0.04760073497891426, + "median": 0.044880062341690063, + "p05": 0.027415042743086815, + "p95": 0.07675685733556747 + }, + { + "label": "completed_group_6", + "mean": 0.19551873207092285, + "median": 0.18908299505710602, + "p05": 0.10506269335746765, + "p95": 0.30520933866500854 + }, + { + "label": "completed_group_7", + "mean": 0.11294309794902802, + "median": 0.09412655979394913, + "p05": 0.03902517631649971, + "p95": 0.24146248400211334 + }, + { + "label": "current_group_partial", + "mean": 0.45173269510269165, + "median": 0.4618796110153198, + "p05": 0.24265648424625397, + "p95": 0.6204620599746704 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9787883758544922, + "group": 8, + "latest_source_mass": 0.1984800100326538, + "layer": 30, + "max_source_mass": 0.24502354860305786, + "mixer_index": 58, + "normalized_entropy": 0.9005854004479842, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0788881778717041, + "median": 0.055554650723934174, + "p05": 0.03367501497268677, + "p95": 0.21530622243881226 + }, + { + "label": "completed_group_1", + "mean": 0.08108963817358017, + "median": 0.07848536968231201, + "p05": 0.05139587074518204, + "p95": 0.12043403089046478 + }, + { + "label": "completed_group_2", + "mean": 0.057054273784160614, + "median": 0.056182146072387695, + "p05": 0.04127398133277893, + "p95": 0.07570968568325043 + }, + { + "label": "completed_group_3", + "mean": 0.08211587369441986, + "median": 0.0802752673625946, + "p05": 0.055154237896203995, + "p95": 0.113825723528862 + }, + { + "label": "completed_group_4", + "mean": 0.026040805503726006, + "median": 0.025504354387521744, + "p05": 0.01917124353349209, + "p95": 0.03502596169710159 + }, + { + "label": "completed_group_5", + "mean": 0.10537257045507431, + "median": 0.10299044847488403, + "p05": 0.07248030602931976, + "p95": 0.14682361483573914 + }, + { + "label": "completed_group_6", + "mean": 0.24502354860305786, + "median": 0.24406442046165466, + "p05": 0.1495005488395691, + "p95": 0.3413054645061493 + }, + { + "label": "completed_group_7", + "mean": 0.12593510746955872, + "median": 0.11572076380252838, + "p05": 0.051278628408908844, + "p95": 0.2299422323703766 + }, + { + "label": "current_group_partial", + "mean": 0.1984800100326538, + "median": 0.19106052815914154, + "p05": 0.11614266037940979, + "p95": 0.3049595355987549 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.7590525150299072, + "group": 8, + "latest_source_mass": 0.3258843719959259, + "layer": 30, + "max_source_mass": 0.3258843719959259, + "mixer_index": 59, + "normalized_entropy": 0.800579300438408, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07131639868021011, + "median": 0.052984386682510376, + "p05": 0.03260878100991249, + "p95": 0.19837424159049988 + }, + { + "label": "completed_group_1", + "mean": 0.044687941670417786, + "median": 0.04335416108369827, + "p05": 0.027018578723073006, + "p95": 0.06556333601474762 + }, + { + "label": "completed_group_2", + "mean": 0.033290669322013855, + "median": 0.03222261369228363, + "p05": 0.01970716379582882, + "p95": 0.05042369291186333 + }, + { + "label": "completed_group_3", + "mean": 0.043597716838121414, + "median": 0.03997292369604111, + "p05": 0.024005703628063202, + "p95": 0.07244924455881119 + }, + { + "label": "completed_group_4", + "mean": 0.02658817172050476, + "median": 0.02605675905942917, + "p05": 0.017710059881210327, + "p95": 0.03680335357785225 + }, + { + "label": "completed_group_5", + "mean": 0.06922741234302521, + "median": 0.06634850800037384, + "p05": 0.03838749974966049, + "p95": 0.10990548133850098 + }, + { + "label": "completed_group_6", + "mean": 0.23554196953773499, + "median": 0.2292930632829666, + "p05": 0.10460641980171204, + "p95": 0.3925905227661133 + }, + { + "label": "completed_group_7", + "mean": 0.14986534416675568, + "median": 0.12543796002864838, + "p05": 0.04750998318195343, + "p95": 0.3270293176174164 + }, + { + "label": "current_group_partial", + "mean": 0.3258843719959259, + "median": 0.31322309374809265, + "p05": 0.1389845609664917, + "p95": 0.5583901405334473 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.9736928939819336, + "group": 8, + "latest_source_mass": 0.20976582169532776, + "layer": 31, + "max_source_mass": 0.22560220956802368, + "mixer_index": 60, + "normalized_entropy": 0.8982663467084998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09849467873573303, + "median": 0.09779177606105804, + "p05": 0.017307927832007408, + "p95": 0.19556622207164764 + }, + { + "label": "completed_group_1", + "mean": 0.068942591547966, + "median": 0.0661388635635376, + "p05": 0.044743385165929794, + "p95": 0.10368303209543228 + }, + { + "label": "completed_group_2", + "mean": 0.0599900484085083, + "median": 0.05786054581403732, + "p05": 0.040265996009111404, + "p95": 0.08643331378698349 + }, + { + "label": "completed_group_3", + "mean": 0.06335455179214478, + "median": 0.05964011698961258, + "p05": 0.04099462926387787, + "p95": 0.09641734510660172 + }, + { + "label": "completed_group_4", + "mean": 0.04671281576156616, + "median": 0.04506188631057739, + "p05": 0.03403949737548828, + "p95": 0.06450185924768448 + }, + { + "label": "completed_group_5", + "mean": 0.0713595524430275, + "median": 0.06962447613477707, + "p05": 0.048271458595991135, + "p95": 0.10105480253696442 + }, + { + "label": "completed_group_6", + "mean": 0.22560220956802368, + "median": 0.22400957345962524, + "p05": 0.13957585394382477, + "p95": 0.31992682814598083 + }, + { + "label": "completed_group_7", + "mean": 0.1557777225971222, + "median": 0.1399117112159729, + "p05": 0.06281191110610962, + "p95": 0.2967374622821808 + }, + { + "label": "current_group_partial", + "mean": 0.20976582169532776, + "median": 0.2105630338191986, + "p05": 0.08139262348413467, + "p95": 0.34620192646980286 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.809161901473999, + "group": 8, + "latest_source_mass": 0.288146048784256, + "layer": 31, + "max_source_mass": 0.288146048784256, + "mixer_index": 61, + "normalized_entropy": 0.8233850650202157, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08854076266288757, + "median": 0.08406365662813187, + "p05": 0.020090684294700623, + "p95": 0.1807844340801239 + }, + { + "label": "completed_group_1", + "mean": 0.05356752499938011, + "median": 0.05036265030503273, + "p05": 0.031541675329208374, + "p95": 0.08521421998739243 + }, + { + "label": "completed_group_2", + "mean": 0.04243364930152893, + "median": 0.03939494490623474, + "p05": 0.023794323205947876, + "p95": 0.0721740797162056 + }, + { + "label": "completed_group_3", + "mean": 0.04702243581414223, + "median": 0.043501369655132294, + "p05": 0.02430840954184532, + "p95": 0.0788305327296257 + }, + { + "label": "completed_group_4", + "mean": 0.02648795023560524, + "median": 0.025087328627705574, + "p05": 0.01698113977909088, + "p95": 0.040197182446718216 + }, + { + "label": "completed_group_5", + "mean": 0.05820769816637039, + "median": 0.05441046878695488, + "p05": 0.02834305912256241, + "p95": 0.10125020891427994 + }, + { + "label": "completed_group_6", + "mean": 0.18489578366279602, + "median": 0.17386877536773682, + "p05": 0.07876831293106079, + "p95": 0.32456493377685547 + }, + { + "label": "completed_group_7", + "mean": 0.21069815754890442, + "median": 0.19004324078559875, + "p05": 0.07121473550796509, + "p95": 0.42417171597480774 + }, + { + "label": "current_group_partial", + "mean": 0.288146048784256, + "median": 0.28367581963539124, + "p05": 0.0962119847536087, + "p95": 0.5029257535934448 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 1.8544633388519287, + "group": 8, + "latest_source_mass": 0.12106549739837646, + "layer": 32, + "max_source_mass": 0.3525030016899109, + "mixer_index": 62, + "normalized_entropy": 0.8440026376822011, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07977832853794098, + "median": 0.053830958902835846, + "p05": 0.02079753391444683, + "p95": 0.2620829939842224 + }, + { + "label": "completed_group_1", + "mean": 0.05858254432678223, + "median": 0.056158941239118576, + "p05": 0.03545547276735306, + "p95": 0.09179478138685226 + }, + { + "label": "completed_group_2", + "mean": 0.053744811564683914, + "median": 0.05185101926326752, + "p05": 0.03345105051994324, + "p95": 0.08065605163574219 + }, + { + "label": "completed_group_3", + "mean": 0.07626347988843918, + "median": 0.07124648243188858, + "p05": 0.043040163815021515, + "p95": 0.12173676490783691 + }, + { + "label": "completed_group_4", + "mean": 0.031090255826711655, + "median": 0.02958102896809578, + "p05": 0.01873353309929371, + "p95": 0.04807420074939728 + }, + { + "label": "completed_group_5", + "mean": 0.0994817242026329, + "median": 0.0973888486623764, + "p05": 0.0575672909617424, + "p95": 0.14856761693954468 + }, + { + "label": "completed_group_6", + "mean": 0.3525030016899109, + "median": 0.3572760224342346, + "p05": 0.18325307965278625, + "p95": 0.5024369955062866 + }, + { + "label": "completed_group_7", + "mean": 0.1274903416633606, + "median": 0.1073581799864769, + "p05": 0.039848171174526215, + "p95": 0.2741084098815918 + }, + { + "label": "current_group_partial", + "mean": 0.12106549739837646, + "median": 0.10882791876792908, + "p05": 0.0503288172185421, + "p95": 0.22616314888000488 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9456019401550293, + "group": 8, + "latest_source_mass": 0.18013209104537964, + "layer": 32, + "max_source_mass": 0.2524128556251526, + "mixer_index": 63, + "normalized_entropy": 0.885481602665194, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08823961019515991, + "median": 0.07105386257171631, + "p05": 0.04482517018914223, + "p95": 0.1774362027645111 + }, + { + "label": "completed_group_1", + "mean": 0.06737187504768372, + "median": 0.06628458946943283, + "p05": 0.04582075774669647, + "p95": 0.09290395677089691 + }, + { + "label": "completed_group_2", + "mean": 0.05197303742170334, + "median": 0.051264435052871704, + "p05": 0.035227708518505096, + "p95": 0.07133325934410095 + }, + { + "label": "completed_group_3", + "mean": 0.0631948709487915, + "median": 0.06016676500439644, + "p05": 0.03969947248697281, + "p95": 0.09776238352060318 + }, + { + "label": "completed_group_4", + "mean": 0.02739548496901989, + "median": 0.026907969266176224, + "p05": 0.020028207451105118, + "p95": 0.036658842116594315 + }, + { + "label": "completed_group_5", + "mean": 0.08291903138160706, + "median": 0.08066994696855545, + "p05": 0.05307728424668312, + "p95": 0.12080682814121246 + }, + { + "label": "completed_group_6", + "mean": 0.2524128556251526, + "median": 0.25180506706237793, + "p05": 0.14340050518512726, + "p95": 0.36010295152664185 + }, + { + "label": "completed_group_7", + "mean": 0.186361163854599, + "median": 0.17697101831436157, + "p05": 0.08843781799077988, + "p95": 0.3156980276107788 + }, + { + "label": "current_group_partial", + "mean": 0.18013209104537964, + "median": 0.16958200931549072, + "p05": 0.08391770720481873, + "p95": 0.3105517029762268 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.596132755279541, + "group": null, + "latest_source_mass": 0.29075440764427185, + "layer": null, + "max_source_mass": 0.3670029938220978, + "mixer_index": null, + "normalized_entropy": 0.7264313223797062, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09668220579624176, + "median": 0.1013428270816803, + "p05": 0.02658359333872795, + "p95": 0.15407970547676086 + }, + { + "label": "completed_group_1", + "mean": 0.037084318697452545, + "median": 0.035886794328689575, + "p05": 0.02348068356513977, + "p95": 0.053765345364809036 + }, + { + "label": "completed_group_2", + "mean": 0.03457995876669884, + "median": 0.03355112299323082, + "p05": 0.019709739834070206, + "p95": 0.0531856045126915 + }, + { + "label": "completed_group_3", + "mean": 0.026012206450104713, + "median": 0.023950476199388504, + "p05": 0.013590577989816666, + "p95": 0.04525115340948105 + }, + { + "label": "completed_group_4", + "mean": 0.0258483849465847, + "median": 0.02512776106595993, + "p05": 0.016705092042684555, + "p95": 0.03687155246734619 + }, + { + "label": "completed_group_5", + "mean": 0.037477150559425354, + "median": 0.03506678715348244, + "p05": 0.02071693353354931, + "p95": 0.061963990330696106 + }, + { + "label": "completed_group_6", + "mean": 0.0845583900809288, + "median": 0.07240844517946243, + "p05": 0.03393660485744476, + "p95": 0.1794772744178772 + }, + { + "label": "completed_group_7", + "mean": 0.3670029938220978, + "median": 0.3608853220939636, + "p05": 0.1103469580411911, + "p95": 0.6465575098991394 + }, + { + "label": "completed_group_8", + "mean": 0.29075440764427185, + "median": 0.28802984952926636, + "p05": 0.039541684091091156, + "p95": 0.5658512711524963 + } + ], + "sources": 9 + } + ], + "interventions": { + "detached_learned": { + "statistics": { + "first_quartile_mean": 4.0602767057862366e-05, + "first_to_last_ratio": 0.6220527400735582, + "imbalance_abs_log_ratio": 0.47473039872247574, + "last_quartile_mean": 6.52722260383598e-05, + "mean": 6.584079096683126e-05, + "normalized": [ + 0.6772375898692375, + 0.7419156446131505, + 0.6265276809050203, + 0.5983548462080928, + 0.6909826192767372, + 0.5826528795658069, + 0.521809962087311, + 0.49396705008933894, + 0.6208428547650034, + 0.583552196572707, + 0.5417821373953783, + 0.5073870191655783, + 0.43782941267698894, + 0.4086714236070873, + 0.40807611508949443, + 0.4979465665227966, + 1.2379953934660572, + 1.244289507430791, + 1.1312943811704865, + 1.027048864560996, + 3.6326329119144347, + 2.8432062989929086, + 2.2290691465672023, + 1.784011106006968, + 1.8683393699616757, + 1.5302817088353375, + 1.1091498325862283, + 0.8793222326330687, + 0.8937044532253541, + 0.6693159640810682, + 0.5098515212827301, + 0.47095130887496306 + ], + "peak_layer": 21, + "peak_normalized": 3.6326329119144347, + "population_cv": 0.7406534740263053, + "rest_mean": 4.7899763524549565e-05, + "spike_contrast": 3.3971428496041334, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016272233915515244, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.458985858946107e-05, + 4.884831287199631e-05, + 4.1251078073401004e-05, + 3.93961563531775e-05, + 4.549484219751321e-05, + 3.83623264497146e-05, + 3.435638063820079e-05, + 3.252318128943443e-05, + 4.087678462383337e-05, + 3.8421538192778826e-05, + 3.567136445781216e-05, + 3.340676266816445e-05, + 2.882703483919613e-05, + 2.6907249775831588e-05, + 2.686805419216398e-05, + 3.278519579907879e-05, + 8.15105959190987e-05, + 8.192500536097214e-05, + 7.448531687259674e-05, + 6.762170960428193e-05, + 0.00023917542421258986, + 0.00018719895160757005, + 0.00014676367572974414, + 0.00011746070231311023, + 0.00012301294191274792, + 0.0001007549581117928, + 7.302730227820575e-05, + 5.789527131128125e-05, + 5.8842208090936765e-05, + 4.4068292481824756e-05, + 3.356902743689716e-05, + 3.100780668319203e-05 + ] + }, + "learned": { + "statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "rest_mean": 5.15456125598306e-05, + "spike_contrast": 3.3328477735441013, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179368005599828, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ] + }, + "uniform_value_backward": { + "statistics": { + "first_quartile_mean": 3.093152440669655e-05, + "first_to_last_ratio": 2.2774685844866087, + "imbalance_abs_log_ratio": 0.8230645561980515, + "last_quartile_mean": 1.3581537245954678e-05, + "mean": 2.01113496416383e-05, + "normalized": [ + 1.8305028390484899, + 2.063895445405066, + 1.6144389462797186, + 1.4292254312672816, + 1.5742789705777873, + 1.395222470726611, + 1.2383859090556808, + 1.15815685579556, + 1.0870603073810894, + 1.0303295548521039, + 0.9686496018996943, + 0.9150613022126579, + 0.8858369611012855, + 0.866010221809998, + 0.8620422691094124, + 0.8950413704487017, + 0.898565866832108, + 0.8961986259205167, + 0.8726168436106015, + 0.8416436403028231, + 0.8273619591175169, + 0.8203659676564082, + 0.8150936944275661, + 0.8114785712381606, + 0.790107287070579, + 0.7656015997474095, + 0.7288914121037795, + 0.6928588452491169, + 0.6522528807444096, + 0.6198966689630461, + 0.586304513764539, + 0.566623166280281 + ], + "peak_layer": 2, + "peak_normalized": 2.063895445405066, + "population_cv": 0.3585699713300157, + "rest_mean": 2.0808239578737238e-05, + "spike_contrast": 0.7856572354159773, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.634814398130402e-05, + "top_five_layers": [ + 2, + 1, + 3, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.6813882616115734e-05, + 4.1507722926326096e-05, + 3.246854612370953e-05, + 2.874365236493759e-05, + 3.166087481076829e-05, + 2.805980693665333e-05, + 2.4905612008296885e-05, + 2.329209746676497e-05, + 2.186224992328789e-05, + 2.072131792374421e-05, + 1.9480850824038498e-05, + 1.8403117792331614e-05, + 1.7815376850194298e-05, + 1.7416634364053607e-05, + 1.7336833479930647e-05, + 1.800048994482495e-05, + 1.8071372323902324e-05, + 1.8023763914243318e-05, + 1.7549502445035614e-05, + 1.6926589523791336e-05, + 1.6639365640003234e-05, + 1.6498666809638962e-05, + 1.6392634279327467e-05, + 1.631992927286774e-05, + 1.5890123904682696e-05, + 1.539728145871777e-05, + 1.4658990039606579e-05, + 1.3934326489106752e-05, + 1.3117685739416629e-05, + 1.246695865120273e-05, + 1.1791375072789378e-05, + 1.1395556612114888e-05 + ] + } + }, + "positions": { + "attention_branch_output": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1261602139711613e-05, + "first_to_last_ratio": 0.5245701598350286, + "imbalance_abs_log_ratio": 0.6451760949151356, + "last_quartile_mean": 2.146824772353284e-05, + "mean": 2.0086123484475138e-05, + "normalized": [ + 0.6085748623432731, + 0.672343922700359, + 0.5609816766389839, + 0.5375859907162287, + 0.641556317168069, + 0.5340725535941322, + 0.4789883309018991, + 0.4512226054976404, + 0.5900913956594765, + 0.540676107431406, + 0.5086908977229898, + 0.473500107930304, + 0.40487760145311497, + 0.38162857664898325, + 0.3654861410934707, + 0.44670015436893523, + 1.1143309981822054, + 1.1164864049647665, + 1.036008527266956, + 0.9253333108915559, + 3.7714016683827163, + 3.048770143020733, + 2.3679155285839633, + 1.872296940712075, + 1.7902480367185682, + 1.5773203624246581, + 1.2887757729919698, + 0.9509073151516803, + 1.1740622366025057, + 0.7631942144418731, + 0.5687018744547615, + 0.43726942333974633 + ], + "peak_layer": 21, + "peak_normalized": 3.7714016683827163, + "population_cv": 0.7926339194277571, + "rest_mean": 1.4245798663776239e-05, + "spike_contrast": 3.6237966529399817, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.1623877516249196e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2223909834574442e-05, + 1.350478305539582e-05, + 1.1267947229498532e-05, + 1.0798018593050074e-05, + 1.2886379408882931e-05, + 1.0727447261160705e-05, + 9.621018762118183e-06, + 9.063312973012216e-06, + 1.1852648640342522e-05, + 1.0860087058972567e-05, + 1.0217628187092487e-05, + 9.510781637800392e-06, + 8.132421498885378e-06, + 7.665438715775963e-06, + 7.341199761867756e-06, + 8.97247446118854e-06, + 2.238259003206622e-05, + 2.2425883798860013e-05, + 2.0809395209653303e-05, + 1.8586359146866016e-05, + 7.575283962069079e-05, + 6.123797356849536e-05, + 4.756224370794371e-05, + 3.760718755074777e-05, + 3.595914313336834e-05, + 3.168225157423876e-05, + 2.5886509320116602e-05, + 1.9100041754427366e-05, + 2.3582359062856995e-05, + 1.532961323391646e-05, + 1.1423016076150816e-05, + 8.78304763318738e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.418448133947095e-05, + "first_to_last_ratio": 0.515514370401758, + "imbalance_abs_log_ratio": 0.662590099285622, + "last_quartile_mean": 8.570950467401417e-05, + "mean": 8.019432914352365e-05, + "normalized": [ + 0.5977618675800617, + 0.6560769396425044, + 0.5549086183167843, + 0.5332641078022274, + 0.6187504975834317, + 0.5235180355140238, + 0.4739614950807707, + 0.44949966520899015, + 0.5901273331445138, + 0.5434806557289369, + 0.5105162863240122, + 0.47519844997761185, + 0.40525254510695746, + 0.3803726356397213, + 0.3632907544081655, + 0.4413534187685878, + 1.1120243122062796, + 1.1101422281940985, + 1.048076642833664, + 0.9240879167666287, + 3.7925106998539326, + 3.0713745688935163, + 2.3834694147362985, + 1.8907998104142179, + 1.7924928574135153, + 1.575834720390749, + 1.2856009256033192, + 0.946613406637061, + 1.179458790809137, + 0.7622280566154953, + 0.5700461300235112, + 0.43790621278127495 + ], + "peak_layer": 21, + "peak_normalized": 3.7925106998539326, + "population_cv": 0.799556889668146, + "rest_mean": 5.663903492084636e-05, + "spike_contrast": 3.6616605179769537, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00020739291794598104, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.793711195816286e-05, + 5.261365004116669e-05, + 4.450052438187413e-05, + 4.27647573815193e-05, + 4.9620281060924754e-05, + 4.1983177652582526e-05, + 3.800902413786389e-05, + 3.604732410167344e-05, + 4.7324865590780973e-05, + 4.358406658866443e-05, + 4.09405110985972e-05, + 3.810822090599686e-05, + 3.249895598855801e-05, + 3.0503728339681402e-05, + 2.913385833380744e-05, + 3.539404133334756e-05, + 8.917804370867088e-05, + 8.902711124392226e-05, + 8.404980326304212e-05, + 7.410661055473611e-05, + 0.0003041378513444215, + 0.0002463068231008947, + 0.00019114073074888438, + 0.0001516314223408699, + 0.00014374776219483465, + 0.00012637300824280828, + 0.00010309790377505124, + 7.591302710352466e-05, + 9.458590648137033e-05, + 6.11263676546514e-05, + 4.5714466978097335e-05, + 3.511759496177547e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.039183240849524736, + "first_to_last_ratio": 0.5155143831205776, + "imbalance_abs_log_ratio": 0.6625900746135269, + "last_quartile_mean": 0.07600804581306875, + "mean": 0.07111713383346796, + "normalized": [ + 0.5977618468462375, + 0.6560769852418055, + 0.5549086482011255, + 0.5332641116607529, + 0.618750502626057, + 0.5235180409966333, + 0.4739615115435027, + 0.44949963686515554, + 0.590127313076766, + 0.543480629182993, + 0.5105162957833653, + 0.47519841853167755, + 0.40525256719553, + 0.38037265430860584, + 0.36329073465232276, + 0.44135337886041204, + 1.1120242741359478, + 1.1101421724087162, + 1.048076711910138, + 0.9240879569048468, + 3.79251093493639, + 3.0713746221735128, + 2.383469321905541, + 1.8907997296698473, + 1.7924929415619593, + 1.5758345890827283, + 1.2856009880693582, + 0.9466133568642321, + 1.179458727081905, + 0.7622280464822554, + 0.5700461310047829, + 0.4379062202348958 + ], + "peak_layer": 21, + "peak_normalized": 3.79251093493639, + "population_cv": 0.7995569122050559, + "rest_mean": 0.05022806260320875, + "spike_contrast": 3.6616606125102327, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18391811847686768, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04251110926270485, + 0.046658314764499664, + 0.03946351259946823, + 0.03792421519756317, + 0.04400376230478287, + 0.03723110258579254, + 0.03370678424835205, + 0.031967125833034515, + 0.04196816310286522, + 0.03865078464150429, + 0.03630645573139191, + 0.03379474952816963, + 0.028820401057600975, + 0.027051012963056564, + 0.025836195796728134, + 0.03138778731226921, + 0.07908397912979126, + 0.07895012944936752, + 0.07453621178865433, + 0.06571848690509796, + 0.26971250772476196, + 0.2184273600578308, + 0.1695055067539215, + 0.13446825742721558, + 0.12747696042060852, + 0.11206883937120438, + 0.0914282575249672, + 0.06732042878866196, + 0.08387972414493561, + 0.05420747399330139, + 0.040540046989917755, + 0.03114263527095318 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8925995988183786e-06, + "first_to_last_ratio": 0.3261648413799994, + "imbalance_abs_log_ratio": 1.1203523770075996, + "last_quartile_mean": 5.802586173331292e-06, + "mean": 4.670730803013612e-06, + "normalized": [ + 0.5310348802393875, + 0.46146348274830745, + 0.44476201867725484, + 0.43677383492791344, + 0.3556033019654118, + 0.3423979278935454, + 0.3333438108043655, + 0.33625410358276536, + 0.4223966230117983, + 0.41208725398123947, + 0.41437801392793316, + 0.4155995059181246, + 0.382562022554051, + 0.3815658727614762, + 0.38949800087449105, + 0.4096468013886889, + 0.8179581009736591, + 0.8423033340072443, + 0.8553008421936446, + 0.8530642636826707, + 3.896213232744094, + 3.3402464703409684, + 2.714333567686501, + 2.2725775720601913, + 2.07067894660348, + 1.8582090508366296, + 1.5332685227055798, + 1.1680391073619418, + 1.334393737375711, + 0.8510160788430678, + 0.6469677209502473, + 0.4760619963776144 + ], + "peak_layer": 21, + "peak_normalized": 3.896213232744094, + "population_cv": 0.9180343092487918, + "rest_mean": 3.0629528535226e-06, + "spike_contrast": 4.359431035612757, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3352731730265078e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4803209726087516e-06, + 2.15537170333846e-06, + 2.0773636606463697e-06, + 2.040053004748188e-06, + 1.6609272961431998e-06, + 1.599248548700416e-06, + 1.5569592051178915e-06, + 1.570552399243752e-06, + 1.9729009181901347e-06, + 1.9247486306994688e-06, + 1.935448153744801e-06, + 1.9411534140090225e-06, + 1.786844222806394e-06, + 1.7821914752857992e-06, + 1.819240310396708e-06, + 1.9133499336021487e-06, + 3.820462097792188e-06, + 3.934172127628699e-06, + 3.99487998947734e-06, + 3.984433533332776e-06, + 1.8198163161287084e-05, + 1.5601392078679055e-05, + 1.2677921404247172e-05, + 1.0614598068059422e-05, + 9.671583939052653e-06, + 8.679194252181333e-06, + 7.161484518292127e-06, + 5.455596237879945e-06, + 6.2325939325091895e-06, + 3.974867013312178e-06, + 3.021812062797835e-06, + 2.2235574306250783e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.4072450716848834e-05, + "first_to_last_ratio": 0.634285224932807, + "imbalance_abs_log_ratio": 0.4552565441138088, + "last_quartile_mean": 5.371786915020493e-05, + "mean": 5.588273802459298e-05, + "normalized": [ + 0.6676093152090673, + 0.7197771199409104, + 0.6218041460469672, + 0.5988669281463308, + 0.6666598935307795, + 0.5782912860000531, + 0.5252896450390306, + 0.4994083671227573, + 0.6402001675671648, + 0.5894906078705415, + 0.5499130564119773, + 0.5108819562402718, + 0.4646654187577452, + 0.4356758329586985, + 0.4154501696208735, + 0.4880908883442812, + 1.1821250357964774, + 1.1664741605550812, + 1.078174429062179, + 0.962548943315326, + 3.8682876465312988, + 3.0459843127744204, + 2.2760577554709505, + 1.7581888525052323, + 1.7091478115698895, + 1.4520832989438006, + 1.1543347914386743, + 0.8578423927708055, + 0.9984708567512235, + 0.639080834302184, + 0.48627901888802055, + 0.39284506051698587 + ], + "peak_layer": 21, + "peak_normalized": 3.8682876465312988, + "population_cv": 0.7787255777921933, + "rest_mean": 4.0033428242033415e-05, + "spike_contrast": 3.533772076553717, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014146901085041464, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.7307836464606225e-05, + 4.022311622975394e-05, + 3.4748118196148425e-05, + 3.346632365719415e-05, + 3.72547801816836e-05, + 3.231650043744594e-05, + 2.935462362074759e-05, + 2.7908306947210804e-05, + 3.57761382474564e-05, + 3.294234920758754e-05, + 3.073064726777375e-05, + 2.8549482522066683e-05, + 2.596677586552687e-05, + 2.434675843687728e-05, + 2.321649299119599e-05, + 2.727585524553433e-05, + 6.606038368772715e-05, + 6.518576992675662e-05, + 6.025133916409686e-05, + 5.3789870435139164e-05, + 0.0002161705051548779, + 0.00017021794337779284, + 0.00012719233927782625, + 9.825240704230964e-05, + 9.551185939926654e-05, + 8.114639058476314e-05, + 6.450738874264061e-05, + 4.7938581701600924e-05, + 5.579728531301953e-05, + 3.571358683984727e-05, + 2.7174603019375354e-05, + 2.1953257601126097e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.604143833195849e-05, + "first_to_last_ratio": 0.864500772440069, + "imbalance_abs_log_ratio": 0.14560308034841146, + "last_quartile_mean": 3.012309434780036e-05, + "mean": 3.750934104118642e-05, + "normalized": [ + 0.7613938466965254, + 0.8106407342403823, + 0.7210553143827715, + 0.6926649057839166, + 0.7441900072557481, + 0.6549135446116822, + 0.6010420947141472, + 0.5682228952686117, + 0.7146883031764985, + 0.6605877661848203, + 0.6100617443729247, + 0.5720177685677394, + 0.5632312795246776, + 0.5253333685644455, + 0.49762299559665957, + 0.5644048416919647, + 1.293995056666398, + 1.2838583891993025, + 1.1578542801495904, + 1.0246076437213654, + 3.9830246051038323, + 3.000750743357827, + 2.06181294387121, + 1.5073651467427196, + 1.5060057544008438, + 1.2137998314477398, + 0.951522556430117, + 0.7372102220186553, + 0.7702443482214394, + 0.5110189167704164, + 0.405035438834493, + 0.3298227124305357 + ], + "peak_layer": 21, + "peak_normalized": 3.9830246051038323, + "population_cv": 0.7527493617080463, + "rest_mean": 2.770278890104102e-05, + "spike_contrast": 3.2655456792139823, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.046472259797156e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.855938146240078e-05, + 3.0406599762500264e-05, + 2.7046309696743265e-05, + 2.5981404178310186e-05, + 2.791407678159885e-05, + 2.4565375497331843e-05, + 2.2544692910742015e-05, + 2.1313666366040707e-05, + 2.6807487301994115e-05, + 2.4778211809461936e-05, + 2.2883014025865123e-05, + 2.1456009562825784e-05, + 2.1126434148754925e-05, + 1.9704908481799066e-05, + 1.866551065177191e-05, + 2.1170453692320734e-05, + 4.853690188610926e-05, + 4.815668216906488e-05, + 4.343035107012838e-05, + 3.8432357541751117e-05, + 0.0001494006282882765, + 0.00011255618301220238, + 7.733724487479776e-05, + 5.654027336277068e-05, + 5.648928345181048e-05, + 4.5528831833507866e-05, + 3.5690984077518806e-05, + 2.7652269636746496e-05, + 2.889135794248432e-05, + 1.9167982827639207e-05, + 1.5192612409009598e-05, + 1.2371432603686117e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.932713899412192e-05, + "first_to_last_ratio": 0.48428070567546516, + "imbalance_abs_log_ratio": 0.7250905699794873, + "last_quartile_mean": 0.00018445322712068446, + "mean": 0.00016839920272104791, + "normalized": [ + 0.580113378070693, + 0.6316011322538936, + 0.5371709395526841, + 0.5193916381660382, + 0.5958096458866964, + 0.49825289372245124, + 0.4493748616089108, + 0.43187515015476213, + 0.5789448986804199, + 0.5298739922879939, + 0.5012758869564349, + 0.4661125381937263, + 0.3655735366227801, + 0.35379219756409647, + 0.33290305767004397, + 0.4094948207474536, + 1.0315615517644154, + 1.032222526387711, + 0.9835380166521113, + 0.8863325773824939, + 3.853305598042274, + 3.162866144945307, + 2.502991043300759, + 2.002956870927857, + 1.8436650111759416, + 1.637369378572084, + 1.333674654700857, + 1.0052835546379626, + 1.1909970302670063, + 0.7575881320943505, + 0.5664246618886806, + 0.4276626791211106 + ], + "peak_layer": 21, + "peak_normalized": 3.853305598042274, + "population_cv": 0.8344872340072493, + "rest_mean": 0.00011622174093241079, + "spike_contrast": 3.873264096443705, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004501574963796884, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.769063035491854e-05, + 0.00010636112710926682, + 9.045915794558823e-05, + 8.746513776713982e-05, + 0.00010033386934082955, + 8.390539005631581e-05, + 7.567436841782182e-05, + 7.27274309610948e-05, + 9.749385935720056e-05, + 8.923035784391686e-05, + 8.441445970674977e-05, + 7.84929798101075e-05, + 6.156229210318998e-05, + 5.9578323998721316e-05, + 5.606060949503444e-05, + 6.895860133226961e-05, + 0.00017371414287481457, + 0.00017382545047439635, + 0.0001656270178500563, + 0.00014925769937690347, + 0.0006488935905508697, + 0.000532624137122184, + 0.00042150169610977173, + 0.000337296340148896, + 0.00031047171796672046, + 0.0002757316979113966, + 0.0002245897485408932, + 0.0001692889491096139, + 0.00020056295034009963, + 0.00012757723743561655, + 9.538546146359295e-05, + 7.201805419754237e-05 + ] + } + }, + "mlp_branch_output": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1220664760003274e-05, + "first_to_last_ratio": 0.6555362798323586, + "imbalance_abs_log_ratio": 0.42230163050239594, + "last_quartile_mean": 1.711677157345548e-05, + "mean": 1.7638031224009865e-05, + "normalized": [ + 0.6920470508787427, + 0.765871148622978, + 0.6390872031764239, + 0.6123477040772916, + 0.7313980706371319, + 0.6060462235938255, + 0.5358729377549785, + 0.5066345241410947, + 0.6361374094466793, + 0.5935131324046057, + 0.553200745756426, + 0.5203044559127653, + 0.44270176965120756, + 0.41555808291178226, + 0.42011495887552464, + 0.5249791307747981, + 1.2555818719274776, + 1.2709584825941431, + 1.1358536396128, + 1.0637844334884017, + 3.6056946953031805, + 2.7844654381945673, + 2.1779734918269082, + 1.746298720031033, + 1.816337422755615, + 1.4912504148810115, + 1.091129247215256, + 0.8818465230617194, + 0.8715920116565027, + 0.6503144256419822, + 0.49863208841232076, + 0.4624725447808252 + ], + "peak_layer": 21, + "peak_normalized": 3.6056946953031805, + "population_cv": 0.7234178171751035, + "rest_mean": 1.2979781601003466e-05, + "spike_contrast": 3.2968643466956427, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2792579188244416e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2206347491883207e-05, + 1.3508459232980385e-05, + 1.12722400444909e-05, + 1.0800607924466021e-05, + 1.2900422007078305e-05, + 1.0689462214941159e-05, + 9.451743608224206e-06, + 8.936035555962007e-06, + 1.1220211490581278e-05, + 1.0468403161212336e-05, + 9.757372026797384e-06, + 9.177146239380818e-06, + 7.808387636032421e-06, + 7.3296264417876955e-06, + 7.410000762320124e-06, + 9.259598300559446e-06, + 2.2145992261357605e-05, + 2.2417205400415696e-05, + 2.0034221961395815e-05, + 1.8763063053484075e-05, + 6.359735562000424e-05, + 4.9112488341052085e-05, + 3.84151644539088e-05, + 3.080127135035582e-05, + 3.2036616175901145e-05, + 2.6302721380488947e-05, + 1.9245371731813066e-05, + 1.5554036508547142e-05, + 1.5373167116194963e-05, + 1.1470266144897323e-05, + 8.794888344709761e-06, + 8.157105185091496e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.3983808154735016e-05, + "first_to_last_ratio": 0.6452579224313538, + "imbalance_abs_log_ratio": 0.43810516237842645, + "last_quartile_mean": 6.816469294790295e-05, + "mean": 7.033422900803998e-05, + "normalized": [ + 0.6809924600708206, + 0.7475303658027369, + 0.6323190441913445, + 0.6077859654839745, + 0.7056348319250383, + 0.5942536325242065, + 0.530225745030374, + 0.5040918553425093, + 0.6368908391102093, + 0.5978963063844708, + 0.5559498242106791, + 0.5233025181432952, + 0.44344812436490716, + 0.41411157823288425, + 0.41744328645242573, + 0.5192458958427938, + 1.2557243607052013, + 1.2676253654309775, + 1.1574430802993265, + 1.060137110625566, + 3.6189237818679025, + 2.8123811167698722, + 2.2018107363871295, + 1.7616009064115667, + 1.8179427908533603, + 1.4871270732826793, + 1.0868453970932423, + 0.8727044844737096, + 0.8717769668640218, + 0.6523346900704022, + 0.4999162674775265, + 0.4645835982748448 + ], + "peak_layer": 21, + "peak_normalized": 3.6189237818679025, + "population_cv": 0.7299683580686596, + "rest_mean": 5.154545740677147e-05, + "spike_contrast": 3.3328561680843984, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179359565488994, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789707963936962e-05, + 5.2576971938833594e-05, + 4.4473672460298985e-05, + 4.274815728422254e-05, + 4.963028186466545e-05, + 4.1796371078817174e-05, + 3.729301897692494e-05, + 3.545491199474782e-05, + 4.479522613110021e-05, + 4.205257573630661e-05, + 3.910230225301348e-05, + 3.680607915157452e-05, + 3.1189581932267174e-05, + 2.9126218578312546e-05, + 2.9360551707213745e-05, + 3.6520759749691933e-05, + 8.832040475681424e-05, + 8.915745274862275e-05, + 8.140786667354405e-05, + 7.456392631866038e-05, + 0.0002545342140365392, + 0.00019780665752477944, + 0.00015486266056541353, + 0.00012390084157232195, + 0.00012786360457539558, + 0.00010459593613632023, + 7.644243305549026e-05, + 6.138099706731737e-05, + 6.13157608313486e-05, + 4.588145748130046e-05, + 3.516122524160892e-05, + 3.267612919444218e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.039005281403660774, + "first_to_last_ratio": 0.6452579411239904, + "imbalance_abs_log_ratio": 0.4381051334091792, + "last_quartile_mean": 0.06044913036748767, + "mean": 0.06237309757852927, + "normalized": [ + 0.6809924890770891, + 0.7475303831256814, + 0.6323190938885936, + 0.6077859577612714, + 0.7056348041762752, + 0.5942536183897815, + 0.5302257656275956, + 0.5040918555375298, + 0.6368908140206065, + 0.5978963019204037, + 0.5559498364917699, + 0.5233025167864885, + 0.4434481328217215, + 0.4141115698767486, + 0.41744329043307676, + 0.5192458729418684, + 1.2557243339288315, + 1.2676253196693095, + 1.1574430718922224, + 1.0601371340408086, + 3.6189238450886454, + 2.8123810331522527, + 2.2018108921566237, + 1.761600919246031, + 1.8179427635119274, + 1.487127014661369, + 1.086845368525537, + 0.8727044924490052, + 0.8717770087213936, + 0.6523346812884329, + 0.4999162376710536, + 0.46458358112005493 + ], + "peak_layer": 21, + "peak_normalized": 3.6189238450886454, + "population_cv": 0.7299683644727474, + "rest_mean": 0.045711026768441555, + "spike_contrast": 3.3328562214267015, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15234827995300293, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.042475610971450806, + 0.046625785529613495, + 0.03943970054388046, + 0.03790949285030365, + 0.04401262849569321, + 0.03706543892621994, + 0.03307182341814041, + 0.03144177049398422, + 0.03972485288977623, + 0.03729264438152313, + 0.034676313400268555, + 0.0326399989426136, + 0.027659233659505844, + 0.02582942135632038, + 0.02603723108768463, + 0.03238697350025177, + 0.07832341641187668, + 0.0790657177567482, + 0.07219330966472626, + 0.06612403690814972, + 0.2257234901189804, + 0.17541691660881042, + 0.13733376562595367, + 0.10987650603055954, + 0.1133907213807106, + 0.0927567183971405, + 0.06778991222381592, + 0.05443328246474266, + 0.054375432431697845, + 0.04068813472986221, + 0.031181324273347855, + 0.028977517038583755 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8962810770517535e-06, + "first_to_last_ratio": 0.40446687944391657, + "imbalance_abs_log_ratio": 0.9051854260734966, + "last_quartile_mean": 4.688346990633363e-06, + "mean": 3.997706922831412e-06, + "normalized": [ + 0.617154002791989, + 0.5425132643975126, + 0.522616779881192, + 0.5130533608023556, + 0.4179497664431836, + 0.4022508178223506, + 0.3871440054888278, + 0.39205556299135513, + 0.47572519958981047, + 0.4709737480952342, + 0.46924028063152856, + 0.46856760789403773, + 0.43552024907631937, + 0.4334760110167188, + 0.4573112739567068, + 0.48343767377708236, + 0.9432261829651593, + 0.9556187861127372, + 0.9329616531367625, + 0.91445072544449, + 3.5932868350685228, + 3.016174008278517, + 2.572613821305946, + 2.200605947741084, + 2.1897185116963183, + 1.8429631378409383, + 1.3739246533430545, + 1.1483111158298067, + 1.0092131774108768, + 0.7452469424242577, + 0.5556201735322511, + 0.5170747232130729 + ], + "peak_layer": 21, + "peak_normalized": 3.5932868350685228, + "population_cv": 0.8307963422495312, + "rest_mean": 2.7284499108443724e-06, + "spike_contrast": 3.9772380443675277, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0851694787561427e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.467200829414651e-06, + 2.168809032809804e-06, + 2.089268718918902e-06, + 2.0510369722614996e-06, + 1.670840674705687e-06, + 1.6080808791230083e-06, + 1.5476882708753692e-06, + 1.5673232383051072e-06, + 1.9018099237655406e-06, + 1.8828150132321753e-06, + 1.8758851183520164e-06, + 1.8731959698925493e-06, + 1.7410823147656629e-06, + 1.7329100501228822e-06, + 1.8281964457855793e-06, + 1.932642135216156e-06, + 3.7707418414356653e-06, + 3.82028383683064e-06, + 3.7297072594810743e-06, + 3.6557059956976445e-06, + 1.4364907656272408e-05, + 1.2057779713359196e-05, + 1.0284556083206553e-05, + 8.797377631708514e-06, + 8.753852853260469e-06, + 7.367626494669821e-06, + 5.492548098118277e-06, + 4.590611297317082e-06, + 4.034538505948149e-06, + 2.9792788609483978e-06, + 2.2212066141946707e-06, + 2.067113200610038e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.395081762391783e-05, + "first_to_last_ratio": 0.7815758779298154, + "imbalance_abs_log_ratio": 0.246443041153289, + "last_quartile_mean": 4.343892715041875e-05, + "mean": 4.974787287892468e-05, + "normalized": [ + 0.7500101371289282, + 0.8083785365909619, + 0.6989343871323576, + 0.6730384758360606, + 0.7487834825409285, + 0.6472056011592257, + 0.5806038659736669, + 0.5527069076510954, + 0.6849789424977496, + 0.6414689031465389, + 0.5923280921669694, + 0.5570919747617912, + 0.5025384608304704, + 0.4685796759451795, + 0.4679593283316108, + 0.5622406113180932, + 1.3163192795863237, + 1.3145042344644593, + 1.1718309380215874, + 1.0894177246512402, + 3.678688405943946, + 2.7746227673728385, + 2.0966738705116796, + 1.635642610362089, + 1.6803690663352613, + 1.3632931153614756, + 0.986286743821586, + 0.8095332329033809, + 0.728391794299929, + 0.5544865952259644, + 0.4452050831707052, + 0.41788715495590656 + ], + "peak_layer": 21, + "peak_normalized": 3.678688405943946, + "population_cv": 0.7109227971967927, + "rest_mean": 3.7097179099671645e-05, + "spike_contrast": 3.182495870364873, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011806161928689108, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.731140895979479e-05, + 4.021511267637834e-05, + 3.4770499041769654e-05, + 3.3482232538517565e-05, + 3.725038550328463e-05, + 3.219710197299719e-05, + 2.88838073174702e-05, + 2.7495992981130257e-05, + 3.4076245356118307e-05, + 3.1911713449517265e-05, + 2.9467062631738372e-05, + 2.7714140742318705e-05, + 2.5000219466164708e-05, + 2.331084215256851e-05, + 2.3279981178347953e-05, + 2.79702744592214e-05, + 6.548408418893814e-05, + 6.539378955494612e-05, + 5.8296096540289e-05, + 5.419621447799727e-05, + 0.0001830069231800735, + 0.00013803158071823418, + 0.00010430506517877802, + 8.136974065564573e-05, + 8.359478670172393e-05, + 6.782093259971589e-05, + 4.9065667553804815e-05, + 4.027255636174232e-05, + 3.623594238888472e-05, + 2.7584528652369045e-05, + 2.214800588262733e-05, + 2.078899706248194e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.6004547407865175e-05, + "first_to_last_ratio": 1.0323157315408855, + "imbalance_abs_log_ratio": 0.031804561684426046, + "last_quartile_mean": 2.519049803595408e-05, + "mean": 3.413751699099521e-05, + "normalized": [ + 0.838119937847341, + 0.8931761838723624, + 0.7929921852930517, + 0.7615904360235652, + 0.8189280439687685, + 0.7189183912075784, + 0.6527804223286999, + 0.6175632597625974, + 0.7513453279894857, + 0.7058548167769552, + 0.6474515253469071, + 0.6094592164179198, + 0.5931149886633756, + 0.5496093129413688, + 0.5439605510615998, + 0.633537428927029, + 1.4126123740339225, + 1.4259512145086812, + 1.2377137858897012, + 1.131469831457994, + 3.7563109168199906, + 2.718723537642896, + 1.8748185902822487, + 1.4106982999267925, + 1.4327179849324012, + 1.1438362357528318, + 0.8490271550305547, + 0.7287425009809699, + 0.5778680951931764, + 0.464037753169555, + 0.3681285307530028, + 0.3389411651966756 + ], + "peak_layer": 21, + "peak_normalized": 3.7563109168199906, + "population_cv": 0.6884877712720907, + "rest_mean": 2.6307041547692347e-05, + "spike_contrast": 2.9050048917999454, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.642208438483067e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8611333618755452e-05, + 3.0490817152895033e-05, + 2.7070784199167974e-05, + 2.599880644993391e-05, + 2.7956170015386306e-05, + 2.454208879498765e-05, + 2.2284302758635022e-05, + 2.1082076273160055e-05, + 2.5649063900345936e-05, + 2.4096130800899118e-05, + 2.2102387447375804e-05, + 2.0805424355785362e-05, + 2.0247473003109917e-05, + 1.8762297258945182e-05, + 1.856946255429648e-05, + 2.1627394744427875e-05, + 4.822307892027311e-05, + 4.867843381362036e-05, + 4.225247539579868e-05, + 3.862557059619576e-05, + 0.00012823112774640322, + 9.281047096010298e-05, + 6.400165148079395e-05, + 4.8157737182918936e-05, + 4.890943455393426e-05, + 3.9047728932928294e-05, + 2.8983678930671886e-05, + 2.4877459509298205e-05, + 1.9726981918211095e-05, + 1.5841096683288924e-05, + 1.2566993973450735e-05, + 1.1570609785849229e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.908438576327171e-05, + "first_to_last_ratio": 0.6062325309661494, + "imbalance_abs_log_ratio": 0.5004916520467982, + "last_quartile_mean": 0.00014694755100208567, + "mean": 0.00014725192045261792, + "normalized": [ + 0.6637009362863341, + 0.7248732277041845, + 0.6138698495621964, + 0.598513900406796, + 0.680481175716416, + 0.565076258862256, + 0.5041321503581428, + 0.4891880965619703, + 0.6294201840754665, + 0.5850404355839129, + 0.5479525097707834, + 0.5179728424254814, + 0.4076252060436077, + 0.3853916272098706, + 0.3845475777225467, + 0.4800015848780767, + 1.1784027781257396, + 1.1715870365157128, + 1.1011549065529929, + 1.0246566094283616, + 3.66881165389385, + 2.9055737038215397, + 2.326629675317946, + 1.8619320585590522, + 1.8930003135554039, + 1.5415479300660813, + 1.1509068852552828, + 0.9338288403787282, + 0.8717742548708616, + 0.6455051030286945, + 0.49036587220767053, + 0.45653481525404077 + ], + "peak_layer": 21, + "peak_normalized": 3.66881165389385, + "population_cv": 0.7616800575204435, + "rest_mean": 0.00010549810718253462, + "spike_contrast": 3.5329781933070796, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00037272251211106777, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.77312374743633e-05, + 0.00010673897486412898, + 9.039351425599307e-05, + 8.81323212524876e-05, + 0.0001002021599560976, + 8.320856431964785e-05, + 7.423442730214447e-05, + 7.203388668131083e-05, + 9.268333087675273e-05, + 8.614832768216729e-05, + 8.068705938057974e-05, + 7.627249578945339e-05, + 6.002359441481531e-05, + 5.674965723301284e-05, + 5.662536932504736e-05, + 7.068115519359708e-05, + 0.00017352207214571536, + 0.00017251844110433012, + 0.00016214717470575124, + 0.0001508826535427943, + 0.0005402395618148148, + 0.0004278513079043478, + 0.00034260068787261844, + 0.0002741730713751167, + 0.00027874793158844113, + 0.00022699589317198843, + 0.00016947324911598116, + 0.0001375080901198089, + 0.00012837043323088437, + 9.505186608294025e-05, + 7.220731640700251e-05, + 6.722562829963863e-05 + ] + } + }, + "post_attention_state": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1261542454121809e-05, + "first_to_last_ratio": 0.5245675018823146, + "imbalance_abs_log_ratio": 0.6451811618435493, + "last_quartile_mean": 2.146824272131198e-05, + "mean": 2.008613361681455e-05, + "normalized": [ + 0.6085515532483275, + 0.6723602464807182, + 0.5609760959264884, + 0.53758893439475, + 0.6415342139885134, + 0.5340779441503937, + 0.4789823840329274, + 0.45122885288223713, + 0.5900862077803115, + 0.5406841208808664, + 0.5086899166409938, + 0.47350367257308246, + 0.40488038567700324, + 0.38162938029260524, + 0.36548145139289145, + 0.4467012874253964, + 1.1143303455046147, + 1.1164970711323288, + 1.0360116270365431, + 0.9253438018068647, + 3.771407372914525, + 3.0487664316584073, + 2.367876661366877, + 1.872337110236809, + 1.790253835037814, + 1.5773117786405777, + 1.2887586410545064, + 0.9509019452609131, + 1.1740685268722015, + 0.7631964556770826, + 0.5687161676492125, + 0.4372655803832159 + ], + "peak_layer": 21, + "peak_normalized": 3.771407372914525, + "population_cv": 0.7926334965261557, + "rest_mean": 1.42457990848386e-05, + "spike_contrast": 3.6238009382307355, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 5.1623940089484675e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.2223447811265942e-05, + 1.3505117749446072e-05, + 1.1267840818618424e-05, + 1.07980831671739e-05, + 1.2885941941931378e-05, + 1.0727560947998427e-05, + 9.62090416578576e-06, + 9.06344303075457e-06, + 1.1852550414914731e-05, + 1.0860253496502992e-05, + 1.0217613635177258e-05, + 9.51085803535534e-06, + 8.132481525535695e-06, + 7.665458724659402e-06, + 7.34110926714493e-06, + 8.972501746029593e-06, + 2.2382588213076815e-05, + 2.2426109353546053e-05, + 2.0809467969229445e-05, + 1.8586579244583845e-05, + 7.575299241580069e-05, + 6.123792991274968e-05, + 4.756148700835183e-05, + 3.760801337193698e-05, + 3.5959277738584206e-05, + 3.168209514115006e-05, + 2.5886178264045157e-05, + 1.9099943528999574e-05, + 2.3582497306051664e-05, + 1.5329665984609164e-05, + 1.1423308933444787e-05, + 8.782974873611238e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.4184340367792174e-05, + "first_to_last_ratio": 0.5155127283755759, + "imbalance_abs_log_ratio": 0.6625932845097232, + "last_quartile_mean": 8.570950421926682e-05, + "mean": 8.019429117211985e-05, + "normalized": [ + 0.5977553912963086, + 0.6560764337279869, + 0.5549115575710053, + 0.5332632715493767, + 0.618731238431461, + 0.5235220032899371, + 0.4739610390296197, + 0.44950831585208795, + 0.5901295178770004, + 0.5434863568103182, + 0.5105118554999238, + 0.4751981306058326, + 0.4052536442822589, + 0.3803760366271764, + 0.36329074496540736, + 0.44135367311072854, + 1.112029465926781, + 1.1101484697729092, + 1.04807895367264, + 0.9240971550402413, + 3.7925041485004933, + 3.0713698535863756, + 2.383447135178006, + 1.890810504439103, + 1.7925006015588865, + 1.5758247605004587, + 1.2855904653729862, + 0.9466104071448787, + 1.1794713255182339, + 0.7622315023149855, + 0.5700532953493974, + 0.43790274559719394 + ], + "peak_layer": 21, + "peak_normalized": 3.7925041485004933, + "population_cv": 0.7995548542275265, + "rest_mean": 5.6639052976000435e-05, + "spike_contrast": 3.6616533387138865, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002073925774311647, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.79365698993206e-05, + 5.2613584557548165e-05, + 4.450073902262375e-05, + 4.276467007002793e-05, + 4.96187130920589e-05, + 4.198347596684471e-05, + 3.800896956818178e-05, + 3.604800076573156e-05, + 4.732501838589087e-05, + 4.358450314612128e-05, + 4.094013638678007e-05, + 3.8108177250251174e-05, + 3.249902874813415e-05, + 3.0503986636176705e-05, + 2.913384378189221e-05, + 3.5394044971326366e-05, + 8.917841478250921e-05, + 8.902756962925196e-05, + 8.40499487821944e-05, + 7.410731632262468e-05, + 0.000304137181956321, + 0.0002463063283357769, + 0.00019113885355181992, + 0.00015163220814429224, + 0.00014374831516761333, + 0.0001263721496798098, + 0.0001030970161082223, + 7.591275061713532e-05, + 9.45868669077754e-05, + 6.112661503721029e-05, + 4.571501995087601e-05, + 3.511730028549209e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03918311558663845, + "first_to_last_ratio": 0.5155127429948628, + "imbalance_abs_log_ratio": 0.6625932561509924, + "last_quartile_mean": 0.07600804464891553, + "mean": 0.07111709873424843, + "normalized": [ + 0.5977553845258899, + 0.656076418540609, + 0.5549115935781082, + 0.5332632748164047, + 0.6187312693383394, + 0.5235220185307409, + 0.4739610644907502, + 0.4495083446741889, + 0.5901295424808332, + 0.5434863451914381, + 0.5105118857031089, + 0.4751981292368421, + 0.4052536577066065, + 0.38037603736972314, + 0.36329075680382195, + 0.4413536490689804, + 1.1120294326243692, + 1.1101484823828944, + 1.0480789054186883, + 0.9240971084727143, + 3.792504006439516, + 3.0713698521253847, + 2.383447240422923, + 1.8908105107644724, + 1.7925005311890823, + 1.575824785560249, + 1.2855905174797924, + 0.9466103668129862, + 1.1794713571647604, + 0.7622314608572955, + 0.5700533268339548, + 0.4379027433945313 + ], + "peak_layer": 21, + "peak_normalized": 3.792504006439516, + "population_cv": 0.7995548405481961, + "rest_mean": 0.05022807840119909, + "spike_contrast": 3.6616532900913086, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.18391780853271483, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04251062870025635, + 0.04665825143456459, + 0.03946370258927345, + 0.037924136966466904, + 0.04400237277150154, + 0.03723136708140373, + 0.03370673581957817, + 0.031967729330062866, + 0.04196830093860626, + 0.03865117207169533, + 0.036306124180555344, + 0.03379471227526665, + 0.02882046438753605, + 0.02705124020576477, + 0.02583618462085724, + 0.03138779103755951, + 0.07908430695533752, + 0.07895053923130035, + 0.07453633099794388, + 0.0657191053032875, + 0.2697118818759918, + 0.218426913022995, + 0.169503852725029, + 0.1344689577817917, + 0.12747743725776672, + 0.11206808686256409, + 0.09142746776342392, + 0.06732018291950226, + 0.08388058096170425, + 0.0542076900601387, + 0.04054053872823715, + 0.031142372637987137 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.8926682940900719e-06, + "first_to_last_ratio": 0.32617819149698246, + "imbalance_abs_log_ratio": 1.1203114472626277, + "last_quartile_mean": 5.802559286394171e-06, + "mean": 4.670736920786567e-06, + "normalized": [ + 0.5310214304016295, + 0.4614624888755425, + 0.44477058805262604, + 0.4368101139545488, + 0.3555922725307191, + 0.3424096008550773, + 0.333381953461308, + 0.3362983274867149, + 0.4224036639064773, + 0.41210428787587466, + 0.41438837559751834, + 0.41562174402282875, + 0.3825706733995329, + 0.3815673445421813, + 0.3895113889811354, + 0.40964670295389133, + 0.8179200324465078, + 0.8423754461799106, + 0.8552890122092341, + 0.8530595439765862, + 3.8961508812147314, + 3.3402234019552477, + 2.714242582301369, + 2.272602051203693, + 2.0707052479706927, + 1.8581861711397907, + 1.5332291278146746, + 1.1680063245911945, + 1.3343957866516452, + 0.8510283999847308, + 0.6469666788242595, + 0.4760583546381262 + ], + "peak_layer": 21, + "peak_normalized": 3.8961508812147314, + "population_cv": 0.9180173062016742, + "rest_mean": 3.0629785972754084e-06, + "spike_contrast": 4.359361792349546, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.3352631867746823e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4802614007057855e-06, + 2.1553698843490565e-06, + 2.077406406897353e-06, + 2.0402251266204985e-06, + 1.6608779560556286e-06, + 1.5993051647456014e-06, + 1.5571393987556803e-06, + 1.5707610145909712e-06, + 1.972936388483504e-06, + 1.9248307125963038e-06, + 1.9354990854481002e-06, + 1.94125982488913e-06, + 1.7868869690573774e-06, + 1.7822006839196547e-06, + 1.8193052255810471e-06, + 1.9133519799652277e-06, + 3.820289293798851e-06, + 3.9345140976365656e-06, + 3.994829967268743e-06, + 3.9844167076807935e-06, + 1.8197895769844763e-05, + 1.5601304767187685e-05, + 1.2677513041126076e-05, + 1.0614726306812372e-05, + 9.671719453763217e-06, + 8.679098755237646e-06, + 7.161309895309387e-06, + 5.45545026398031e-06, + 6.232611667655874e-06, + 3.9749297684466e-06, + 3.0218111533031333e-06, + 2.2235433334572008e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.407248482290015e-05, + "first_to_last_ratio": 0.6342850678385603, + "imbalance_abs_log_ratio": 0.4552567917851402, + "last_quartile_mean": 5.371793622543919e-05, + "mean": 5.588281288737562e-05, + "normalized": [ + 0.667605556447693, + 0.7197799315075553, + 0.6218050056576038, + 0.5988650842787754, + 0.6666530112346837, + 0.5782950032081196, + 0.5252885832899008, + 0.4994128735568847, + 0.6402020441359789, + 0.5894915107696465, + 0.5499104969232754, + 0.5108819879445103, + 0.46466609827668737, + 0.43567603051211307, + 0.41544997111847654, + 0.48809065762969883, + 1.1821280091855806, + 1.1664813213195084, + 1.07816907868966, + 0.9625539685607898, + 3.8682944428485135, + 3.0459807530568472, + 2.2760455923564815, + 1.7581896219738866, + 1.7091546359484804, + 1.4520780986698394, + 1.1543285578371303, + 0.8578398764689898, + 0.9984751177727861, + 0.6390807593663731, + 0.4862828593592872, + 0.39284346009424276 + ], + "peak_layer": 21, + "peak_normalized": 3.8682944428485135, + "population_cv": 0.7787255378412773, + "rest_mean": 4.0033484630704925e-05, + "spike_contrast": 3.5337714610257334, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014146918547339738, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.7307676393538713e-05, + 4.022332723252475e-05, + 3.474821278359741e-05, + 3.346626544953324e-05, + 3.725444548763335e-05, + 3.231675145798363e-05, + 2.935460361186415e-05, + 2.7908596166525967e-05, + 3.57762910425663e-05, + 3.2942443795036525e-05, + 3.073054540436715e-05, + 2.854952253983356e-05, + 2.5966848625103012e-05, + 2.4346802092622966e-05, + 2.321651300007943e-05, + 2.7275878892396577e-05, + 6.606063834624365e-05, + 6.518625741591677e-05, + 6.0251120885368437e-05, + 5.379022331908345e-05, + 0.0002161711745429784, + 0.0001702179724816233, + 0.00012719182996079326, + 9.825258166529238e-05, + 9.551236871629953e-05, + 8.114620868582278e-05, + 6.45071268081665e-05, + 4.7938505304045975e-05, + 5.579759817919694e-05, + 3.571363049559295e-05, + 2.7174854039913043e-05, + 2.195319757447578e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.60414058175229e-05, + "first_to_last_ratio": 0.8644500280195812, + "imbalance_abs_log_ratio": 0.14566178001816363, + "last_quartile_mean": 3.0124825002530997e-05, + "mean": 3.7509687871306596e-05, + "normalized": [ + 0.7612877820794312, + 0.810711701779243, + 0.7209397300058344, + 0.6925594766630951, + 0.7442562064075515, + 0.6550198977679047, + 0.601099191291831, + 0.5681910666111114, + 0.7147935701981128, + 0.6606027044500686, + 0.6100202180357266, + 0.5720600519271453, + 0.5632479423839041, + 0.5253282201512559, + 0.49767906018209906, + 0.5644858935407635, + 1.2937489635148631, + 1.2840324435565709, + 1.1578011905188617, + 1.0245733409201465, + 3.982499346289561, + 3.0010314180971562, + 2.0616055293573603, + 1.5074555677980233, + 1.505937419140994, + 1.2140551303841993, + 0.9515353865179874, + 0.7371695567558888, + 0.7704610253685644, + 0.5109788396566676, + 0.4050543403397244, + 0.3297777883083533 + ], + "peak_layer": 21, + "peak_normalized": 3.982499346289561, + "population_cv": 0.7526960469489694, + "rest_mean": 2.7703642310236183e-05, + "spike_contrast": 3.2653588610499065, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.046233390108682e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.855566708603874e-05, + 3.04095428873552e-05, + 2.70422242465429e-05, + 2.5977689801948145e-05, + 2.7916817998629995e-05, + 2.4569591914769262e-05, + 2.2547043045051396e-05, + 2.1312669559847564e-05, + 2.6811683710548095e-05, + 2.4779001250863075e-05, + 2.28816679737065e-05, + 2.1457793991430663e-05, + 2.1127254512975924e-05, + 1.9704897567862645e-05, + 1.8667786207515746e-05, + 2.1173689674469642e-05, + 4.852811980526894e-05, + 4.8163656174438074e-05, + 4.342876127338968e-05, + 3.84314262191765e-05, + 0.00014938230742700398, + 0.00011256775178480893, + 7.733017991995439e-05, + 5.654418782796711e-05, + 5.64872425456997e-05, + 4.5538828999269754e-05, + 3.569179534679279e-05, + 2.765099998214282e-05, + 2.8899752578581683e-05, + 1.9166656784364022e-05, + 1.5193461877061054e-05, + 1.2369861906336155e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.932642231229693e-05, + "first_to_last_ratio": 0.48429290534855984, + "imbalance_abs_log_ratio": 0.7250653789708609, + "last_quartile_mean": 0.00018444710076437332, + "mean": 0.00016840073044477322, + "normalized": [ + 0.5799699418435614, + 0.631638262959337, + 0.5372322150726369, + 0.519448365501922, + 0.5960027732717111, + 0.4981229459623488, + 0.4492943531121762, + 0.43180823754645586, + 0.5789679033812432, + 0.5298804189104165, + 0.5013705840728442, + 0.4662175781515477, + 0.3655512526294271, + 0.35395025516272205, + 0.3328677409454649, + 0.4094748170873465, + 1.0315739694159025, + 1.0322187789196813, + 0.9834616923499129, + 0.8864297869456468, + 3.853800867660393, + 3.163217666231461, + 2.5024524541124884, + 2.002752567834613, + 1.843634113893829, + 1.6377082968935632, + 1.3336197815157738, + 1.0052459186517901, + 1.1908891844448808, + 0.7572668908929535, + 0.5664028889185376, + 0.4275274957074122 + ], + "peak_layer": 21, + "peak_normalized": 3.853800867660393, + "population_cv": 0.8345380635674979, + "rest_mean": 0.00011622233998625436, + "spike_contrast": 3.873300425494927, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00045016403892077505, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.766736184246838e-05, + 0.00010636834485922009, + 9.047029743669555e-05, + 8.747548417886719e-05, + 0.00010036730236606672, + 8.388426795136184e-05, + 7.566149724880233e-05, + 7.271682261489332e-05, + 9.749861783348024e-05, + 8.923224959289655e-05, + 8.443117258138955e-05, + 7.851138070691377e-05, + 6.155909795779735e-05, + 5.9605481510516256e-05, + 5.605517071671784e-05, + 6.895585829624906e-05, + 0.00017371780995745212, + 0.0001738263963488862, + 0.00016561566735617816, + 0.00014927542360965163, + 0.000648982881102711, + 0.0005326881655491889, + 0.0004214148211758584, + 0.0003372649953234941, + 0.000310469331452623, + 0.00027579127345234156, + 0.00022458254534285516, + 0.00016928414697758853, + 0.00020054660853929818, + 0.00012752429756801575, + 9.538266021991149e-05, + 7.199594256235287e-05 + ] + } + }, + "post_mlp_state": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.1220713531656656e-05, + "first_to_last_ratio": 0.6555413366591524, + "imbalance_abs_log_ratio": 0.4222939165017446, + "last_quartile_mean": 1.7116713934228756e-05, + "mean": 1.7638015336274293e-05, + "normalized": [ + 0.6920608231884311, + 0.7659177824262653, + 0.6390754033749174, + 0.6123424804403663, + 0.7313911494807722, + 0.6060429021656846, + 0.5358717188243214, + 0.5066293084101811, + 0.6361394778273108, + 0.5935071698997143, + 0.5532047504443424, + 0.5203003869133469, + 0.4427001574083234, + 0.41555518288954896, + 0.42010731902783316, + 0.5249828006548773, + 1.2555950689969066, + 1.2709301325611273, + 1.1358461030518159, + 1.0637808540376092, + 3.605678554956403, + 2.784492491030135, + 2.1780228929741425, + 1.7462695606216458, + 1.8163019324419567, + 1.491254439501435, + 1.0911345614825427, + 0.8818559802285882, + 0.8715835151550508, + 0.6503123816361532, + 0.4986363533333601, + 0.46247636461489006 + ], + "peak_layer": 21, + "peak_normalized": 3.605678554956403, + "population_cv": 0.7234183193184713, + "rest_mean": 1.2979772741851371e-05, + "spike_contrast": 3.2968624487684486, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.2792525346158075e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.220657941303216e-05, + 1.3509269592759665e-05, + 1.1272021765762474e-05, + 1.0800506061059423e-05, + 1.2900288311357144e-05, + 1.0689394002838526e-05, + 9.451713594899047e-06, + 8.935935511544812e-06, + 1.122023786592763e-05, + 1.0468288564879913e-05, + 9.757433872437105e-06, + 9.177066203847062e-06, + 7.808352165739052e-06, + 7.329568688874133e-06, + 7.409859335893998e-06, + 9.259654689230956e-06, + 2.214620508311782e-05, + 2.2416685169446282e-05, + 2.003407098527532e-05, + 1.876298301795032e-05, + 6.359701364999637e-05, + 4.911292126053013e-05, + 3.841600118903443e-05, + 3.080072929151356e-05, + 3.203596133971587e-05, + 2.630276867421344e-05, + 1.9245448129368015e-05, + 1.555418930365704e-05, + 1.5373003407148644e-05, + 1.1470219760667533e-05, + 8.794955647317693e-06, + 8.157165211741813e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.398412329464918e-05, + "first_to_last_ratio": 0.6452634668607965, + "imbalance_abs_log_ratio": 0.43809656983670076, + "last_quartile_mean": 6.816459563196986e-05, + "mean": 7.03343731061068e-05, + "normalized": [ + 0.6809932372926891, + 0.7475632825133391, + 0.6323153694166137, + 0.6077851857944574, + 0.7056447138179579, + 0.5942446047107566, + 0.530223262177042, + 0.504089839823597, + 0.6368974480554273, + 0.5978888745536934, + 0.5559539610577222, + 0.5232972563751404, + 0.4434509399790644, + 0.41411238498840724, + 0.4174418363858031, + 0.5192489699581264, + 1.2557404086895352, + 1.2676084925356499, + 1.1574347089891701, + 1.060138766241709, + 3.6189242296270336, + 2.81239769967395, + 2.2018329150221976, + 1.7615672973682381, + 1.8179181698124194, + 1.4871187506667403, + 1.086854135909831, + 0.872703317202576, + 0.8717669049561169, + 0.6523365087620864, + 0.4999226915339381, + 0.46458383610897114 + ], + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "rest_mean": 5.15456125598306e-05, + "spike_contrast": 3.3328477735441013, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00017179368005599828, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.789723243447952e-05, + 5.257939483271912e-05, + 4.447350511327386e-05, + 4.274819002603181e-05, + 4.96310785820242e-05, + 4.17958217440173e-05, + 3.729292075149715e-05, + 3.545484287315048e-05, + 4.479578274185769e-05, + 4.205213917884976e-05, + 3.91026733268518e-05, + 3.680578447529115e-05, + 3.1189843866741285e-05, + 2.9126334993634373e-05, + 2.9360509870457463e-05, + 3.65210507879965e-05, + 8.83217144291848e-05, + 8.915644866647199e-05, + 8.140744466800243e-05, + 7.45641955290921e-05, + 0.0002545347670093179, + 0.0001978082291316241, + 0.000154864537762478, + 0.00012389873154461384, + 0.00012786213483195752, + 0.00010459556506248191, + 7.644320430699736e-05, + 6.138104072306305e-05, + 6.131517875473946e-05, + 4.588167939800769e-05, + 3.516174911055714e-05, + 3.2676212867954746e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.039005559869110584, + "first_to_last_ratio": 0.6452634722813873, + "imbalance_abs_log_ratio": 0.43809656143611553, + "last_quartile_mean": 0.06044904375448823, + "mean": 0.06237322464585304, + "normalized": [ + 0.6809932518812878, + 0.7475633220322179, + 0.6323153569646397, + 0.6077851973803072, + 0.7056447145510248, + 0.5942445836918125, + 0.530223252027616, + 0.5040898132583496, + 0.6368974600710677, + 0.5978889321237267, + 0.5559539597752757, + 0.523297269904143, + 0.4434509324241475, + 0.4141123687040429, + 0.4174418427557242, + 0.5192489959347639, + 1.2557404102027183, + 1.2676085225122937, + 1.157434741358947, + 1.0601387967701486, + 3.61892435638739, + 2.812397521737949, + 2.201832924858681, + 1.7615672286963322, + 1.8179181559541986, + 1.487118729202359, + 1.086854143939758, + 0.8727033118250799, + 0.8717668711180101, + 0.6523365178130685, + 0.4999226849610752, + 0.46458382918184327 + ], + "peak_layer": 21, + "peak_normalized": 3.61892435638739, + "population_cv": 0.7299683256836244, + "rest_mean": 0.045711164397222025, + "spike_contrast": 3.3328477188762906, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.15234834998846053, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04247574508190155, + 0.04662793502211571, + 0.039439547806978226, + 0.03790952265262604, + 0.044013336300849915, + 0.03706495091319084, + 0.03307173401117325, + 0.03144170716404915, + 0.039725348353385925, + 0.03729226067662239, + 0.03467664122581482, + 0.03263973817229271, + 0.02765946462750435, + 0.02582952380180359, + 0.026037193834781647, + 0.03238723427057266, + 0.0783245787024498, + 0.07906483113765717, + 0.07219293713569641, + 0.06612427532672882, + 0.2257239818572998, + 0.17541830241680145, + 0.1373354196548462, + 0.10987462848424911, + 0.11338941752910614, + 0.09275639057159424, + 0.06779059767723083, + 0.054433319717645645, + 0.05437491089105606, + 0.04068833217024803, + 0.031181789934635162, + 0.028977591544389725 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.896312141980161e-06, + "first_to_last_ratio": 0.40447227453095713, + "imbalance_abs_log_ratio": 0.905172087401449, + "last_quartile_mean": 4.688361258331497e-06, + "mean": 3.997668517996544e-06, + "normalized": [ + 0.6172058310645722, + 0.5425329228673286, + 0.5226213455478163, + 0.5130242205418598, + 0.4179738021592733, + 0.4022397236307717, + 0.38715722309433015, + 0.3920811131204843, + 0.47573503087335345, + 0.47091681751525505, + 0.4692356598433531, + 0.46858709630953144, + 0.435518404119361, + 0.43348810961922396, + 0.457344361488258, + 0.4834240606909693, + 0.943243320831244, + 0.9556851843891272, + 0.9328847891755226, + 0.9144352241016374, + 3.593133434897055, + 3.0161686306352355, + 2.572714523011844, + 2.200458051372808, + 2.189693136620607, + 1.8429744726585007, + 1.3739587829516955, + 1.1483183935808434, + 1.0092761091983693, + 0.7452413615169265, + 0.5556410385724699, + 0.5170878240003723 + ], + "peak_layer": 21, + "peak_normalized": 3.593133434897055, + "population_cv": 0.8307777929041875, + "rest_mean": 2.728457953135485e-06, + "spike_contrast": 3.9771203202072667, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0851405568246264e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.4673843199707335e-06, + 2.1688667857233668e-06, + 2.0892668999294983e-06, + 2.050900775429909e-06, + 1.670920710239443e-06, + 1.6080210798463668e-06, + 1.5477262422791682e-06, + 1.567410322422802e-06, + 1.901830955830519e-06, + 1.8825693359758588e-06, + 1.8758486248771078e-06, + 1.8732558828560286e-06, + 1.741058213156066e-06, + 1.7329417687506066e-06, + 1.8283111558048404e-06, + 1.9325691482663387e-06, + 3.770774128497578e-06, + 3.820512574748136e-06, + 3.7293641526048305e-06, + 3.6556089071382303e-06, + 1.4364156413648743e-05, + 1.2057642379659228e-05, + 1.0284859854436945e-05, + 8.796701877145097e-06, + 8.753667316341307e-06, + 7.3676010288181715e-06, + 5.49263177163084e-06, + 4.590596290654503e-06, + 4.0347513277083635e-06, + 2.9792279292450985e-06, + 2.2212686872080667e-06, + 2.0671457150456263e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 3.3950954957617796e-05, + "first_to_last_ratio": 0.7815786467251918, + "imbalance_abs_log_ratio": 0.24643949857917652, + "last_quartile_mean": 4.343894897829159e-05, + "mean": 4.974799179535694e-05, + "normalized": [ + 0.7500094412446059, + 0.8083859646655934, + 0.6989360803110847, + 0.6730394265060721, + 0.7487882742003255, + 0.6472039809659893, + 0.5806034653418889, + 0.5527037948336986, + 0.6849833016487895, + 0.641463932771085, + 0.5923270784846439, + 0.5570934219720602, + 0.5025383930607131, + 0.4685801281186291, + 0.467958282860767, + 0.5622422290415673, + 1.3163199357516149, + 1.3145043099431595, + 1.1718295263393368, + 1.089421848322846, + 3.6786851702332073, + 2.7746307606104894, + 2.0966820217393844, + 1.6356296326692725, + 1.680361539469081, + 1.3632870777077486, + 0.9862895783238333, + 0.8095320290956769, + 0.7283886637343489, + 0.5544846482049088, + 0.4452099057793211, + 0.417886156048257 + ], + "peak_layer": 21, + "peak_normalized": 3.6786851702332073, + "population_cv": 0.710922698103497, + "rest_mean": 3.709728177151798e-05, + "spike_contrast": 3.18249263251226, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011806182592408732, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.7311463529476896e-05, + 4.021557833766565e-05, + 3.477066638879478e-05, + 3.348235986777581e-05, + 3.725071292137727e-05, + 3.219709833501838e-05, + 2.8883856430184096e-05, + 2.7495903850649484e-05, + 3.407654367038049e-05, + 3.191154246451333e-05, + 2.946708264062181e-05, + 2.7714278985513374e-05, + 2.5000275854836218e-05, + 2.3310920369112864e-05, + 2.327998481632676e-05, + 2.7970421797363088e-05, + 6.548427336383611e-05, + 6.539394962601364e-05, + 5.8296165661886334e-05, + 5.4196549172047526e-05, + 0.00018300719966646284, + 0.0001380323083139956, + 0.0001043057200149633, + 8.136928954627365e-05, + 8.359461207874119e-05, + 6.782079435652122e-05, + 4.906592585030012e-05, + 4.027259274153039e-05, + 3.623587326728739e-05, + 2.7584497729549184e-05, + 2.21482987399213e-05, + 2.078899706248194e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 2.600453626655508e-05, + "first_to_last_ratio": 1.032180910721196, + "imbalance_abs_log_ratio": 0.031673952782361334, + "last_quartile_mean": 2.5193777560161834e-05, + "mean": 3.414015603198095e-05, + "normalized": [ + 0.8380570158080917, + 0.8932999084125006, + 0.7929620023886231, + 0.7615443521416788, + 0.8187638282400099, + 0.7186815599217726, + 0.6527123798513835, + 0.6175741300582618, + 0.7513411682592698, + 0.705872928051971, + 0.6475562024934578, + 0.6094685286755085, + 0.5930459106503145, + 0.5494388493370179, + 0.5439464215496852, + 0.6333820560847124, + 1.4124654565316987, + 1.425994860990475, + 1.2380543674563398, + 1.1313103339756791, + 3.756835524951107, + 2.7186060870583466, + 1.87468091251249, + 1.410793634954463, + 1.4329939421834126, + 1.1438971076949418, + 0.8490265267268121, + 0.7287288463621843, + 0.5779352074283672, + 0.4639524923457935, + 0.36814597511060676, + 0.33893148179302335 + ], + "peak_layer": 21, + "peak_normalized": 3.756835524951107, + "population_cv": 0.6885594503102112, + "rest_mean": 2.6308265020176163e-05, + "spike_contrast": 2.9052606637917697, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.643236749572679e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 18 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 2.8611397283384576e-05, + 3.0497398256557062e-05, + 2.7071846488979645e-05, + 2.599924300739076e-05, + 2.795272484945599e-05, + 2.4535900593036786e-05, + 2.228370249213185e-05, + 2.1084077161503956e-05, + 2.5650904717622325e-05, + 2.4098611902445555e-05, + 2.21076697926037e-05, + 2.0807350665563717e-05, + 2.024667992372997e-05, + 1.8757928046397865e-05, + 1.8570415704743937e-05, + 2.1623762222588994e-05, + 4.82217910757754e-05, + 4.86836870550178e-05, + 4.226736928103492e-05, + 3.8623111322522163e-05, + 0.00012825895100831985, + 9.281363600166515e-05, + 6.400189886335284e-05, + 4.816471482627094e-05, + 4.89226367790252e-05, + 3.905282574123703e-05, + 2.898589809774421e-05, + 2.4878916519810446e-05, + 1.9730798157979734e-05, + 1.5839410480111837e-05, + 1.256856103282189e-05, + 1.1571173672564328e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 8.909050666261464e-05, + "first_to_last_ratio": 0.6062699294291527, + "imbalance_abs_log_ratio": 0.5004299639855584, + "last_quartile_mean": 0.00014694858236907749, + "mean": 0.00014725223013556388, + "normalized": [ + 0.6636255714147841, + 0.7248917149077715, + 0.6138660879695628, + 0.5985975801010156, + 0.6806345015153643, + 0.5652332367621573, + 0.5042378190281714, + 0.4890714447869022, + 0.6294477660992072, + 0.5850399957826026, + 0.5478904329704705, + 0.518038310415621, + 0.40778029155663986, + 0.3853440981019431, + 0.3845833582250337, + 0.48007518680371525, + 1.178544482689773, + 1.1716070054101007, + 1.1012637666749874, + 1.0246953672401915, + 3.668461219741745, + 2.9054304267727518, + 2.326469432391579, + 1.8616676451976302, + 1.8927854440285927, + 1.5414183922061209, + 1.1508159193509495, + 0.9341585266360799, + 0.8717798331906053, + 0.6455697098701952, + 0.49047863567768574, + 0.45649679648005004 + ], + "peak_layer": 21, + "peak_normalized": 3.668461219741745, + "population_cv": 0.7615887971484168, + "rest_mean": 0.00010550450948627528, + "spike_contrast": 3.5324549012779802, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003726899216417223, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 9.772034536581486e-05, + 0.00010674192162696272, + 9.039315045811236e-05, + 8.814482862362638e-05, + 0.00010022494825534523, + 8.323185465997085e-05, + 7.42501433705911e-05, + 7.201686094049364e-05, + 9.268758731195703e-05, + 8.614844409748912e-05, + 8.067808812484145e-05, + 7.628229650435969e-05, + 6.004655733704567e-05, + 5.674277781508863e-05, + 5.663075717166066e-05, + 7.06921418895945e-05, + 0.00017354330339003354, + 0.00017252174438908696, + 0.00016216354561038315, + 0.00015088867803569883, + 0.0005401890957728028, + 0.0004278311098460108, + 0.00034257781226187944, + 0.00027413471252657473, + 0.0002787168778013438, + 0.00022697729582432657, + 0.00016946021059993654, + 0.0001375569263473153, + 0.0001283715246245265, + 9.506157948635519e-05, + 7.222407293738797e-05, + 6.722017133142799e-05 + ] + } + }, + "pre_attention_input": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 7.792817086738069e-06, + "first_to_last_ratio": 0.9644943347481165, + "imbalance_abs_log_ratio": 0.03615132042434886, + "last_quartile_mean": 8.079691923512655e-06, + "mean": 1.231628780118399e-05, + "normalized": [ + 0.899719870687867, + 0.727426048108731, + 0.7370807482678542, + 0.3965794568165141, + 0.5003083522454715, + 0.7486231449014935, + 0.6212780315806493, + 0.4307803771911971, + 0.40860273125956736, + 0.5996948368722662, + 0.5787702645812915, + 0.5364690416901848, + 0.2710899543731052, + 0.3344315013927567, + 0.25145386433912675, + 1.0500832430459097, + 0.7256359007107253, + 1.3787305863467312, + 2.2756402165685268, + 2.51490055711116, + 3.4405522362788585, + 3.2818395195720442, + 2.448485364965397, + 1.5936896131932947, + 0.9036087632058596, + 1.0199413906977872, + 1.0227020812544427, + 0.5747747031450406, + 0.6830971927098096, + 0.5175337025733492, + 0.3003192317874535, + 0.22615747252553492 + ], + "peak_layer": 21, + "peak_normalized": 3.4405522362788585, + "population_cv": 0.8497149496404188, + "rest_mean": 9.274540818498483e-06, + "spike_contrast": 3.098991321528187, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.8741721507685725e-05, + "top_five_layers": [ + 21, + 22, + 20, + 23, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1081208867835812e-05, + 8.959188562585041e-06, + 9.078098628378939e-06, + 4.884386726189405e-06, + 6.161941655591363e-06, + 9.220258107234258e-06, + 7.651839041500352e-06, + 5.3056151045893785e-06, + 5.032468834542669e-06, + 7.386014203802915e-06, + 7.128301149350591e-06, + 6.6073071138816886e-06, + 3.338821898069e-06, + 4.118954620935256e-06, + 3.0969781619205605e-06, + 1.293312743655406e-05, + 8.937140592024662e-06, + 1.6980842701741494e-05, + 2.8027439839206636e-05, + 3.0974239052738994e-05, + 4.23748315370176e-05, + 4.0420080040348694e-05, + 3.0156250431900844e-05, + 1.9628339941846207e-05, + 1.112910558731528e-05, + 1.2561891708173789e-05, + 1.259589316759957e-05, + 7.079090664774412e-06, + 8.413221621594857e-06, + 6.374094027705723e-06, + 3.6988180909247603e-06, + 2.785420520012849e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.7120979211758822e-05, + "first_to_last_ratio": 0.8594772224120714, + "imbalance_abs_log_ratio": 0.15143095548529256, + "last_quartile_mean": 3.1555204145661264e-05, + "mean": 4.800072150601409e-05, + "normalized": [ + 0.509760479963835, + 0.6173326836717695, + 0.7324351629463104, + 0.39842116793036353, + 0.4759789060227602, + 0.7458666061827044, + 0.6096831903000134, + 0.4306170616938511, + 0.4047810550951731, + 0.5961537125874838, + 0.6210341956779442, + 0.5605157741769637, + 0.27837672031341976, + 0.3443893546178484, + 0.25310408037089555, + 1.0527443019909122, + 0.7465088513707779, + 1.4090569908205048, + 2.369608868078584, + 2.622524932193837, + 3.51885122958336, + 3.2980052855840127, + 2.541370727154349, + 1.6037570225598534, + 0.918310576031736, + 1.0270328185033981, + 1.0005888996255676, + 0.5708475165844191, + 0.6850494580580762, + 0.5307877571911549, + 0.29761830732641337, + 0.22888630579170677 + ], + "peak_layer": 21, + "peak_normalized": 3.51885122958336, + "population_cv": 0.8786708090610466, + "rest_mean": 3.5768902374979075e-05, + "spike_contrast": 3.1885950432011234, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011405254481360317, + "top_five_layers": [ + 21, + 22, + 20, + 23, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.446887083351612e-05, + 2.96324142254889e-05, + 3.515741627779789e-05, + 1.912450352392625e-05, + 2.2847330910735764e-05, + 3.580213524401188e-05, + 2.9265233024489135e-05, + 2.0669929654104635e-05, + 1.9429782696533948e-05, + 2.8615808332688175e-05, + 2.9810089472448453e-05, + 2.6905161575996317e-05, + 1.3362283425522037e-05, + 1.653093750064727e-05, + 1.2149178473919164e-05, + 5.0532486056908965e-05, + 3.5832963476423174e-05, + 6.76357522024773e-05, + 0.00011374293535482138, + 0.00012588308891281486, + 0.00016890739789232612, + 0.00015830663323868066, + 0.00012198762851767242, + 7.698149420320988e-05, + 4.407957021612674e-05, + 4.929831629851833e-05, + 4.802898911293596e-05, + 2.7401092665968463e-05, + 3.288286825409159e-05, + 2.5478195311734453e-05, + 1.4285893485066481e-05, + 1.0986707820848096e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.024051155894994736, + "first_to_last_ratio": 0.8594772374722103, + "imbalance_abs_log_ratio": 0.15143093796285173, + "last_quartile_mean": 0.027983470470644534, + "mean": 0.042567519849399105, + "normalized": [ + 0.5097604616222596, + 0.6173327141338325, + 0.7324351822855276, + 0.3984211759638306, + 0.4759788964629159, + 0.7458666120674253, + 0.6096832157472697, + 0.430617059477717, + 0.40478105545710685, + 0.5961537255711045, + 0.6210341986548017, + 0.5605157515644588, + 0.278376722109868, + 0.3443893736332492, + 0.25310409496395464, + 1.0527443624204256, + 0.7465088835832933, + 1.4090569785936171, + 2.3696089052435614, + 2.6225247338771562, + 3.5188511717225395, + 3.298005336484889, + 2.541370809226851, + 1.6037569634689308, + 0.9183106068252992, + 1.0270328482800881, + 1.000588834809077, + 0.5708474927211314, + 0.6850494767792821, + 0.5307877431850209, + 0.29761830682761936, + 0.2288863062358959 + ], + "peak_layer": 21, + "peak_normalized": 3.5188511717225395, + "population_cv": 0.8786708002138178, + "rest_mean": 0.0317202202492842, + "spike_contrast": 3.1885950631853515, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.10114293769001961, + "top_five_layers": [ + 21, + 22, + 20, + 23, + 19 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.021699238568544388, + 0.02627832256257534, + 0.031177949160337448, + 0.01695980131626129, + 0.020261241123080254, + 0.03174969181418419, + 0.02595270238816738, + 0.018330300226807594, + 0.017230525612831116, + 0.025376785546541214, + 0.026435885578393936, + 0.023859765380620956, + 0.011849806644022465, + 0.014659801498055458, + 0.010774013586342335, + 0.04481271654367447, + 0.0317770317196846, + 0.059980060905218124, + 0.10086837410926819, + 0.11163437366485596, + 0.1497887670993805, + 0.1403879076242447, + 0.10817985236644745, + 0.06826795637607574, + 0.03909020498394966, + 0.04371824115514755, + 0.04259258508682251, + 0.024299561977386475, + 0.02916085720062256, + 0.02259431779384613, + 0.012668873183429241, + 0.00974312238395214 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.670534108517586e-06, + "first_to_last_ratio": 0.516077911622693, + "imbalance_abs_log_ratio": 0.6614975333817992, + "last_quartile_mean": 5.174672367047606e-06, + "mean": 3.915999473136367e-06, + "normalized": [ + 2.91792462644838, + 1.0330499050224828, + 0.21324233730395303, + 0.31813747476101534, + 0.1980657666006576, + 0.22572388219114395, + 0.26853595304608496, + 0.28095737396172366, + 0.2790646446283966, + 0.3792913632687372, + 0.3694120454168363, + 0.4225628265552778, + 0.22944152357264785, + 0.35050475469892045, + 0.38580405805880497, + 0.330583746531144, + 0.275347554842962, + 0.6571160489647494, + 0.9522439259464477, + 1.1118382618025555, + 1.9569752952109616, + 3.466111019148444, + 2.5633468283111025, + 2.243374425678547, + 1.2633072391816056, + 1.9652520232619801, + 1.881284353579538, + 1.3019726140072538, + 1.7404895149568589, + 1.1347055768273686, + 0.7207866671170219, + 0.5635463690963973 + ], + "peak_layer": 22, + "peak_normalized": 3.466111019148444, + "population_cv": 0.881830620778404, + "rest_mean": 2.974257467013996e-06, + "spike_contrast": 3.026438163483582, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.001406306197169e-06, + "top_five_layers": [ + 22, + 1, + 23, + 24, + 26 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1426591299823485e-05, + 4.045422883791616e-06, + 8.350568805326475e-07, + 1.2458261835490703e-06, + 7.756254376545257e-07, + 8.83934603734815e-07, + 1.051586650646641e-06, + 1.100228928407887e-06, + 1.0928170013357885e-06, + 1.4853047787255491e-06, + 1.4466173752225586e-06, + 1.6547558061574819e-06, + 8.984928854260943e-07, + 1.372576434732764e-06, + 1.5108084880921524e-06, + 1.294565777243406e-06, + 1.078260879694426e-06, + 2.5732661015354097e-06, + 3.7289867123035947e-06, + 4.353958047431661e-06, + 7.663514224987011e-06, + 1.3573288924817462e-05, + 1.0038064829132054e-05, + 8.785053069004789e-06, + 4.947110483044526e-06, + 7.695925887674093e-06, + 7.3671085374371614e-06, + 5.0985240704903845e-06, + 6.8157560235704295e-06, + 4.4435064410208724e-06, + 2.8226002086739754e-06, + 2.2068472844694043e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.836107992403413e-05, + "first_to_last_ratio": 0.9160653111165196, + "imbalance_abs_log_ratio": 0.08766761650421023, + "last_quartile_mean": 2.004341797601228e-05, + "mean": 3.121470177802621e-05, + "normalized": [ + 0.5128089259228045, + 0.6437171011700566, + 0.7320107521642394, + 0.42714084499329735, + 0.4777939631770697, + 0.7761341517210342, + 0.6673269525799402, + 0.4688191183265314, + 0.43572863736759326, + 0.6442483222440443, + 0.6589268299586913, + 0.5254485027294052, + 0.30608252008544473, + 0.35836487745274936, + 0.2721971888139584, + 1.0581434296571979, + 0.7638096596387699, + 1.3818036797996578, + 2.4488923222411234, + 2.174595772916911, + 3.2150744119330064, + 3.5033815383059985, + 2.7401154537196812, + 1.6705176716332228, + 0.9292103758537513, + 1.0277197763891293, + 1.0216310132104693, + 0.5115354754917107, + 0.6598856618533152, + 0.50319316016592, + 0.28185438701748666, + 0.20188752146578876 + ], + "peak_layer": 22, + "peak_normalized": 3.5033815383059985, + "population_cv": 0.8621259030079774, + "rest_mean": 2.3054601317400942e-05, + "spike_contrast": 3.2652589922942665, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.527924426540267e-05, + "top_five_layers": [ + 22, + 21, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.6007177691790275e-05, + 2.0093437342438847e-05, + 2.2849497327115387e-05, + 1.3333074093679897e-05, + 1.4914196071913466e-05, + 2.422679608571343e-05, + 2.0830411813221872e-05, + 1.463404896639986e-05, + 1.3601139471575152e-05, + 2.011001924984157e-05, + 2.0568204490700737e-05, + 1.6401718312408775e-05, + 9.554274583933875e-06, + 1.118625277740648e-05, + 8.496554073644802e-06, + 3.3029631595127285e-05, + 2.3842090740799904e-05, + 4.313258978072554e-05, + 7.644144352525473e-05, + 6.787935853935778e-05, + 0.00010035758896265179, + 0.00010935700993286446, + 8.553188672522083e-05, + 5.2144710934953764e-05, + 2.9005024771322496e-05, + 3.207996633136645e-05, + 3.188990740454756e-05, + 1.5967427316354588e-05, + 2.059813414234668e-05, + 1.570702443132177e-05, + 8.798000635579228e-06, + 6.301858775259461e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.2228491186760948e-05, + "first_to_last_ratio": 0.9571529484026068, + "imbalance_abs_log_ratio": 0.04379207960676997, + "last_quartile_mean": 1.2775900870565238e-05, + "mean": 2.0228415721135207e-05, + "normalized": [ + 0.4968956893086432, + 0.680742973654979, + 0.7082959423523417, + 0.4553628765634101, + 0.48416833058916253, + 0.8025982815331242, + 0.71544927601163, + 0.49265031392139635, + 0.46201741019996967, + 0.6900348438458223, + 0.6974262922446428, + 0.46524054678094645, + 0.3363013254017149, + 0.39144273721666156, + 0.29629939799755733, + 1.053294975127128, + 0.7915013073840591, + 1.3752427386344863, + 2.531667923112097, + 1.9941217404782703, + 2.7172330416452364, + 3.752050564767614, + 2.871442549447273, + 1.6858638659668104, + 0.9381697357727052, + 1.0373218643217643, + 1.0308781989118676, + 0.4725668463395891, + 0.6400815997091446, + 0.4765607085370807, + 0.2663935850069364, + 0.19068251721593543 + ], + "peak_layer": 22, + "peak_normalized": 3.752050564767614, + "population_cv": 0.8541640127467388, + "rest_mean": 1.5010413655410823e-05, + "spike_contrast": 3.224802992594281, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.840562687604688e-05, + "top_five_layers": [ + 22, + 23, + 21, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.0051412573375273e-05, + 1.3770351870334707e-05, + 1.4327704775496386e-05, + 9.211269571096636e-06, + 9.793958270165604e-06, + 1.623529169592075e-05, + 1.4472405382548459e-05, + 9.965535355149768e-06, + 9.34588024392724e-06, + 1.3958311683381908e-05, + 1.410782897437457e-05, + 9.411079190613236e-06, + 6.8028430177946575e-06, + 7.918266419437714e-06, + 5.993667400616687e-06, + 2.1306488633854315e-05, + 1.601081748958677e-05, + 2.7818981834570877e-05, + 5.121163121657446e-05, + 4.033792356494814e-05, + 5.496531957760453e-05, + 7.589803863083944e-05, + 5.808473360957578e-05, + 3.4102355130016804e-05, + 1.8977687432197854e-05, + 2.0983377908123657e-05, + 2.0853032765444368e-05, + 9.55927862378303e-06, + 1.2947836694365833e-05, + 9.640068128646817e-06, + 5.38872018296388e-06, + 3.857205228996463e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 5.4042374358687084e-05, + "first_to_last_ratio": 0.8965146957898991, + "imbalance_abs_log_ratio": 0.10924059366762352, + "last_quartile_mean": 6.028052257534e-05, + "mean": 9.390137262244025e-05, + "normalized": [ + 0.5389360174242739, + 0.5974650105374263, + 0.7267651782696123, + 0.4014182444828728, + 0.4819177901244493, + 0.7732899466309664, + 0.6299309978744704, + 0.45445865774623606, + 0.41201557220312385, + 0.5831302663155696, + 0.6244048764918407, + 0.5807706511889962, + 0.27596931886992426, + 0.3314258726431575, + 0.23879880407819865, + 1.0945434973487749, + 0.734405519454758, + 1.357347632155138, + 2.3148429937772614, + 2.252423783537609, + 3.7724800608232885, + 3.3168691995820305, + 2.6904907774938076, + 1.6802536301987592, + 0.9186543261041, + 0.9881029074846317, + 1.0077452263535627, + 0.5222119037959454, + 0.6630699433389028, + 0.5491195746896886, + 0.2929408036924596, + 0.19380101528816393 + ], + "peak_layer": 21, + "peak_normalized": 3.7724800608232885, + "population_cv": 0.8932088546517, + "rest_mean": 6.823935170056454e-05, + "spike_contrast": 3.406777465012843, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002324762856005691, + "top_five_layers": [ + 21, + 22, + 23, + 19, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.060683179181069e-05, + 5.6102784583345056e-05, + 6.824424781370908e-05, + 3.769372415263206e-05, + 4.525274198385887e-05, + 7.26129874237813e-05, + 5.915138535783626e-05, + 4.267429176252335e-05, + 3.868882777169347e-05, + 5.475673242472112e-05, + 5.863247497472912e-05, + 5.45351613254752e-05, + 2.5913897843565792e-05, + 3.1121344363782555e-05, + 2.2423535483540036e-05, + 0.00010277913679601625, + 6.896168633829802e-05, + 0.0001274568057851866, + 0.00021736693452112377, + 0.0002115056850016117, + 0.00035424105590209365, + 0.0003114585706498474, + 0.000252640777034685, + 0.0001577781222295016, + 8.626290218671784e-05, + 9.2784219305031e-05, + 9.462866000831127e-05, + 4.903641456621699e-05, + 6.226317782420665e-05, + 5.156308179721236e-05, + 2.7507543563842773e-05, + 1.819818135118112e-05 + ] + } + }, + "pre_mlp_input": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 3.6929306190813804e-06, + "first_to_last_ratio": 0.13688476606561775, + "imbalance_abs_log_ratio": 1.9886158307055912, + "last_quartile_mean": 2.6978390110343753e-05, + "mean": 1.631737328722238e-05, + "normalized": [ + 0.4314194517233057, + 0.19806398425123214, + 0.34668833324353665, + 0.22046602853167355, + 0.10956900273172894, + 0.12688527812920286, + 0.16715726937306416, + 0.21030216864948645, + 0.2743958044688482, + 0.24925629504156072, + 0.32761471427281874, + 0.40984521150785025, + 0.28786819126149943, + 0.29273992694234957, + 0.30660983581768325, + 0.24937431986069702, + 0.3349491137477184, + 0.5177030633789016, + 0.6154108922798558, + 0.7890850685582481, + 3.8103077986911464, + 3.428556172833401, + 2.788926643404586, + 2.2799755314310137, + 2.2723621920706982, + 2.1349169811597655, + 2.003931559830236, + 1.5323918909280079, + 1.9997634861702531, + 1.35547405261909, + 1.065904631827636, + 0.8620851052629044 + ], + "peak_layer": 21, + "peak_normalized": 3.8103077986911464, + "population_cv": 1.0256784776243844, + "rest_mean": 1.0527649945086321e-05, + "spike_contrast": 4.519705687684409, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.7581879334757106e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 7.039632237137994e-06, + 3.23188396578189e-06, + 5.657042947859736e-06, + 3.597426484702737e-06, + 1.78787831828231e-06, + 2.070434447887237e-06, + 2.727567562033073e-06, + 3.4315789889660664e-06, + 4.47741876996588e-06, + 4.067208010383183e-06, + 5.345811587176286e-06, + 6.687597306154203e-06, + 4.697252734331414e-06, + 4.776746663992526e-06, + 5.003067144571105e-06, + 4.069133865414187e-06, + 5.465489721245831e-06, + 8.447554137092084e-06, + 1.004188925435301e-05, + 1.28757956190384e-05, + 6.217421469045803e-05, + 5.594503090833314e-05, + 4.5507957111112773e-05, + 3.7203211832093075e-05, + 3.707898213178851e-05, + 3.48362373188138e-05, + 3.269889930379577e-05, + 2.5004610506584868e-05, + 3.263088728999719e-05, + 2.2117776097729802e-05, + 1.7392763766110875e-05, + 1.406696446792921e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 1.4786839983571554e-05, + "first_to_last_ratio": 0.1368590004352419, + "imbalance_abs_log_ratio": 1.9888040770410698, + "last_quartile_mean": 0.00010804433713929029, + "mean": 6.53327875852483e-05, + "normalized": [ + 0.4303813973935663, + 0.1954661274427069, + 0.34854762874244455, + 0.22114030543306437, + 0.10883074349044422, + 0.12627252893821267, + 0.16836982659974983, + 0.21164003420329566, + 0.27785076214706256, + 0.25390299121217014, + 0.3323483997673342, + 0.40930824768453605, + 0.28788144954928885, + 0.2920379121787492, + 0.30710829102986115, + 0.24768611608853355, + 0.33445308089382414, + 0.5081445956986791, + 0.6162106751311494, + 0.7807238268363051, + 3.828274701557073, + 3.4515650116091967, + 2.7664213382241543, + 2.2654046605337452, + 2.2812415827715613, + 2.1356021257971247, + 2.0106696974531495, + 1.537539473929636, + 2.0088419319348305, + 1.3422522168191928, + 1.0598743605751482, + 0.8540079583342102 + ], + "peak_layer": 21, + "peak_normalized": 3.828274701557073, + "population_cv": 1.0279395854576847, + "rest_mean": 4.2120514451569225e-05, + "spike_contrast": 4.52698797699541, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001906790625071153, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.8118016416556202e-05, + 1.2770346984325442e-05, + 2.2771588191972114e-05, + 1.4447712601395324e-05, + 7.110215847205836e-06, + 8.249736310972366e-06, + 1.1000070117006544e-05, + 1.38270333991386e-05, + 1.8152764823753387e-05, + 1.6588190192123875e-05, + 2.1713247406296432e-05, + 2.6741248802863993e-05, + 1.8808097593137063e-05, + 1.9079650883213617e-05, + 2.0064240743522532e-05, + 1.6182024410227314e-05, + 2.1850752091268077e-05, + 3.3198502933373675e-05, + 4.0258761146105826e-05, + 5.1006863941438496e-05, + 0.00025011185789480805, + 0.0002255003637401387, + 0.000180738017661497, + 0.0001480052014812827, + 0.00014903987175785005, + 0.00013952484005130827, + 0.00013136265624780208, + 0.00010045173985417932, + 0.0001312432432314381, + 8.769307896727696e-05, + 6.924454646650702e-05, + 5.579472053796053e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.013113117485772818, + "first_to_last_ratio": 0.136858997118483, + "imbalance_abs_log_ratio": 1.9888041012759314, + "last_quartile_mean": 0.09581480035558343, + "mean": 0.05793777071812656, + "normalized": [ + 0.4303813960583082, + 0.19546612151423692, + 0.3485475910344393, + 0.22114030042116126, + 0.10883074113033038, + 0.12627253310783348, + 0.1683698287227952, + 0.21164002670866952, + 0.27785076861558566, + 0.25390299449217174, + 0.3323483845259541, + 0.40930823057023097, + 0.28788143760030005, + 0.2920379258991919, + 0.3071082822929024, + 0.2476861047748367, + 0.3344530552067231, + 0.5081445507864877, + 0.6162106424834214, + 0.7807238372353658, + 3.828274894940391, + 3.4515650124215562, + 2.7664213738142998, + 2.265404688647299, + 2.281241575194701, + 2.135602067657764, + 2.0106696442072307, + 1.5375394436971075, + 2.00884203416751, + 1.3422522638779362, + 1.0598743603945742, + 0.8540078877986846 + ], + "peak_layer": 21, + "peak_normalized": 3.828274894940391, + "population_cv": 1.0279396080658147, + "rest_mean": 0.03735289376022087, + "spike_contrast": 4.526988119750361, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.16909610629081726, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.024935338646173477, + 0.011324871331453323, + 0.020194070413708687, + 0.012812376022338867, + 0.006305410526692867, + 0.007315949071198702, + 0.009754972532391548, + 0.012261951342225075, + 0.016098054125905037, + 0.014710573479533195, + 0.01925552450120449, + 0.023714406415820122, + 0.01667920872569084, + 0.016920026391744614, + 0.017793169245123863, + 0.01435038074851036, + 0.019377464428544044, + 0.029440762475132942, + 0.035701870918273926, + 0.04523339867591858, + 0.22180171310901642, + 0.19997598230838776, + 0.1602802872657776, + 0.1312524974346161, + 0.13217005133628845, + 0.12373202294111252, + 0.11649371683597565, + 0.08908160775899887, + 0.11638782918453217, + 0.07776710391044617, + 0.061406757682561874, + 0.04947931319475174 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 9.471203732402955e-07, + "first_to_last_ratio": 0.13038144854942058, + "imbalance_abs_log_ratio": 2.037290905335615, + "last_quartile_mean": 7.264226496772608e-06, + "mean": 4.254466777098287e-06, + "normalized": [ + 0.436364407529321, + 0.1911843551159761, + 0.3504684074329156, + 0.22599177156694852, + 0.09379418393340402, + 0.12911388991485367, + 0.1538375030619483, + 0.20018851917359348, + 0.21440564422614256, + 0.20760264501895292, + 0.28595251377841346, + 0.3787305694072829, + 0.27369502895368186, + 0.29556005516396594, + 0.2777983154447535, + 0.10238349267586677, + 0.302592474036744, + 0.3867848818041926, + 0.5416819159884301, + 0.6344790509535327, + 4.041575473617487, + 3.5020644333651707, + 2.77570587266537, + 2.338562686179214, + 2.3623514652944007, + 2.22406080211689, + 2.0998230053367104, + 1.589088734701869, + 1.995381756812838, + 1.3883763621564047, + 1.1160049955561004, + 0.8843947870166255 + ], + "peak_layer": 21, + "peak_normalized": 4.041575473617487, + "population_cv": 1.0776419744012766, + "rest_mean": 2.6755459261780573e-06, + "spike_contrast": 4.776834232976263, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2780639372067526e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.8564978745416738e-06, + 8.133874871418811e-07, + 1.4910561958458857e-06, + 9.614744840291678e-07, + 3.9904423942971334e-07, + 5.493107551046705e-07, + 6.54496545848815e-07, + 8.516954039805569e-07, + 9.121816901824786e-07, + 8.832385560708644e-07, + 1.2165754696980002e-06, + 1.611296625014802e-06, + 1.1644264077403932e-06, + 1.25745043533243e-06, + 1.1818837037935737e-06, + 4.35587168112761e-07, + 1.2873696277893032e-06, + 1.6455634295198251e-06, + 2.304567715327721e-06, + 2.699370043046656e-06, + 1.7194748579640873e-05, + 1.4899416783009656e-05, + 1.1809148418251425e-05, + 9.949337254511192e-06, + 1.0050545824924484e-05, + 9.462192792852875e-06, + 8.933627213991713e-06, + 6.760725227650255e-06, + 8.489285391988233e-06, + 5.9068011069030035e-06, + 4.748006176669151e-06, + 3.762628239201149e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 9.853123231096106e-06, + "first_to_last_ratio": 0.15080465707059237, + "imbalance_abs_log_ratio": 1.89176994145415, + "last_quartile_mean": 6.533699570354656e-05, + "mean": 4.1971269041596315e-05, + "normalized": [ + 0.4681313077158978, + 0.20820887504839763, + 0.36906491110676237, + 0.20482622912946075, + 0.11627096124793933, + 0.12407290917887392, + 0.1787682927302214, + 0.20872662343689463, + 0.3073254147861055, + 0.27783033377805905, + 0.35982154325450194, + 0.4577819740053479, + 0.34235028174668086, + 0.3396230936294121, + 0.3348757051705037, + 0.17757659165663509, + 0.3534791092433193, + 0.5341192315280864, + 0.6428744767315707, + 0.8246883036040644, + 4.08439765980215, + 3.623760084464356, + 2.7887442899646078, + 2.2190205767814533, + 2.346653344716544, + 2.0729802100068144, + 1.8735227194091606, + 1.4202102880569494, + 1.8401836496959596, + 1.1967241409050977, + 0.93540866538392, + 0.7679782020842529 + ], + "peak_layer": 21, + "peak_normalized": 4.08439765980215, + "population_cv": 1.0467961731297342, + "rest_mean": 2.6329080793840586e-05, + "spike_contrast": 4.8022597738792445, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00012643908557947726, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.9648065062938258e-05, + 8.738790711504407e-06, + 1.549012267787475e-05, + 8.59681676956825e-06, + 4.880039796262281e-06, + 5.207497451920062e-06, + 7.50313211028697e-06, + 8.760521268413868e-06, + 1.2898837667307816e-05, + 1.166089168691542e-05, + 1.5102166798897088e-05, + 1.9213690393371508e-05, + 1.4368875781656243e-05, + 1.4254412235459313e-05, + 1.4055158317205496e-05, + 7.453114903910318e-06, + 1.483596679463517e-05, + 2.2417661966755986e-05, + 2.6982257622876205e-05, + 3.461321466602385e-05, + 0.0001714273530524224, + 0.00015209380944725126, + 0.00011704713688232005, + 9.313510963693261e-05, + 9.849201887845993e-05, + 8.700561011210084e-05, + 7.863412611186504e-05, + 5.960802809568122e-05, + 7.723484304733574e-05, + 5.0228030886501074e-05, + 3.926028875866905e-05, + 3.2233019737759605e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 6.539388465398588e-06, + "first_to_last_ratio": 0.18539523353460505, + "imbalance_abs_log_ratio": 1.6852653352714542, + "last_quartile_mean": 3.52726892742794e-05, + "mean": 2.5340588074129755e-05, + "normalized": [ + 0.5326820585081189, + 0.22955400672130744, + 0.4075699321599326, + 0.20399783458406082, + 0.13335091090399903, + 0.12913807236762478, + 0.20439277726135907, + 0.22379324161336578, + 0.3549539057327596, + 0.3176654165435907, + 0.4067751656526541, + 0.5369739555670685, + 0.4337606182550087, + 0.42213995757068684, + 0.3963582115760412, + 0.12453444551735621, + 0.40318005313905786, + 0.6070024892440241, + 0.7294977435014097, + 0.8929234728836388, + 4.460419429548373, + 3.910252651443441, + 2.738748156484065, + 2.0647804157793166, + 2.3165049051088284, + 1.8671237368915825, + 1.5983578514794328, + 1.197615503415084, + 1.590133109210288, + 1.0414283798969246, + 0.8387268861490448, + 0.6856647052905551 + ], + "peak_layer": 21, + "peak_normalized": 4.460419429548373, + "population_cv": 1.0654839953495887, + "rest_mean": 1.5494638142222862e-05, + "spike_contrast": 5.066831311954996, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.850871770642698e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.3498476619133726e-05, + 5.817033525090665e-06, + 1.0328061762265861e-05, + 5.169425094209146e-06, + 3.3791905025282176e-06, + 3.2724346965551376e-06, + 5.179433173907455e-06, + 5.6710523494984955e-06, + 8.994740710477345e-06, + 8.049828466027975e-06, + 1.0307921911589801e-05, + 1.3607235814561136e-05, + 1.0991749149980024e-05, + 1.0697274774429388e-05, + 1.0043950169347227e-05, + 3.1557760848954786e-06, + 1.021681964630261e-05, + 1.5381800039904192e-05, + 1.848590181907639e-05, + 2.2627205908065662e-05, + 0.00011302965140203014, + 9.908810170600191e-05, + 6.940148887224495e-05, + 5.2322749979794025e-05, + 5.870159657206386e-05, + 4.731401349999942e-05, + 4.0503327909391373e-05, + 3.034828114323318e-05, + 4.029490810353309e-05, + 2.639040758367628e-05, + 2.125383252860047e-05, + 1.7375146853737533e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 2.8892148407066998e-05, + "first_to_last_ratio": 0.12323172113154097, + "imbalance_abs_log_ratio": 2.0936887843071057, + "last_quartile_mean": 0.00023445382521458669, + "mean": 0.00013867109626630736, + "normalized": [ + 0.40760164765243684, + 0.18549021085330977, + 0.32629733370345226, + 0.19134578357607018, + 0.0997358093357467, + 0.1135317976584037, + 0.15336948879180487, + 0.18942939953458193, + 0.2684956409479822, + 0.25415697053702413, + 0.3183782664979506, + 0.3953176437860155, + 0.26214994057141733, + 0.2686491657455234, + 0.2734202925021933, + 0.2068336337941873, + 0.30327711407815156, + 0.45518103395508264, + 0.5758332130814036, + 0.7564185139065018, + 3.831959637972952, + 3.474608896989398, + 2.829513018719538, + 2.3332553915396175, + 2.3723457642549124, + 2.19075436705495, + 2.160872753795178, + 1.6201515958627097, + 2.023759263900917, + 1.3235642487571164, + 1.0060145229556705, + 0.8282876376878003 + ], + "peak_layer": 21, + "peak_normalized": 3.831959637972952, + "population_cv": 1.0570572109274479, + "rest_mean": 8.812454328748302e-05, + "spike_contrast": 4.670917624039755, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0004116224823519588, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 5.652256731991656e-05, + 2.572213088569697e-05, + 4.524800897343084e-05, + 2.6534129574429244e-05, + 1.3830474017595407e-05, + 1.5743578842375427e-05, + 2.126791514456272e-05, + 2.626838249852881e-05, + 3.7232584872981533e-05, + 3.5244225728092715e-05, + 4.4149863242637366e-05, + 5.481913103722036e-05, + 3.635261964518577e-05, + 3.7253874324960634e-05, + 3.791549170273356e-05, + 2.8681846742983907e-05, + 4.2055769881699234e-05, + 6.312045297818258e-05, + 7.985142292454839e-05, + 0.00010489338455954567, + 0.0005313820438459516, + 0.0004818278248421848, + 0.00039237167220562696, + 0.000323555083014071, + 0.00032897578785195947, + 0.0003037943097297102, + 0.00029965059366077185, + 0.00022466819791588932, + 0.00028063691570423543, + 0.00018354010535404086, + 0.00013950513675808907, + 0.00011485955474199727 + ] + } + } + }, + "seed": 2026073002 + }, + { + "final_mixers": [ + { + "branch": "attention", + "entropy_mean": -0.0, + "group": 1, + "latest_source_mass": 1.0, + "layer": 1, + "max_source_mass": 1.0, + "mixer_index": 0, + "normalized_entropy": 1.0, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 1.0, + "median": 1.0, + "p05": 1.0, + "p95": 1.0 + } + ], + "sources": 1 + }, + { + "branch": "mlp", + "entropy_mean": 0.21598131954669952, + "group": 1, + "latest_source_mass": 0.061274416744709015, + "layer": 1, + "max_source_mass": 0.9387255907058716, + "mixer_index": 1, + "normalized_entropy": 0.31159517863467795, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.9387255907058716, + "median": 0.9528161883354187, + "p05": 0.8532108068466187, + "p95": 0.9815421104431152 + }, + { + "label": "current_group_partial", + "mean": 0.061274416744709015, + "median": 0.047183819115161896, + "p05": 0.018457919359207153, + "p95": 0.14678917825222015 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.5377621054649353, + "group": 1, + "latest_source_mass": 0.2917928099632263, + "layer": 2, + "max_source_mass": 0.7082071900367737, + "mixer_index": 2, + "normalized_entropy": 0.7758267227322699, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.7082071900367737, + "median": 0.7268136143684387, + "p05": 0.38468095660209656, + "p95": 0.923936128616333 + }, + { + "label": "current_group_partial", + "mean": 0.2917928099632263, + "median": 0.27318641543388367, + "p05": 0.07606390863656998, + "p95": 0.6153190732002258 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.43774232268333435, + "group": 1, + "latest_source_mass": 0.170285165309906, + "layer": 2, + "max_source_mass": 0.829714834690094, + "mixer_index": 3, + "normalized_entropy": 0.6315286781224629, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.829714834690094, + "median": 0.85542893409729, + "p05": 0.687734842300415, + "p95": 0.9260365962982178 + }, + { + "label": "current_group_partial", + "mean": 0.170285165309906, + "median": 0.14457105100154877, + "p05": 0.07396337389945984, + "p95": 0.31226515769958496 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6749181151390076, + "group": 1, + "latest_source_mass": 0.45927006006240845, + "layer": 3, + "max_source_mass": 0.5407299995422363, + "mixer_index": 4, + "normalized_entropy": 0.9737010177171727, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5407299995422363, + "median": 0.5310744643211365, + "p05": 0.4232047200202942, + "p95": 0.6916534900665283 + }, + { + "label": "current_group_partial", + "mean": 0.45927006006240845, + "median": 0.4689255356788635, + "p05": 0.3083464205265045, + "p95": 0.576795220375061 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.6663151383399963, + "group": 1, + "latest_source_mass": 0.49885618686676025, + "layer": 3, + "max_source_mass": 0.5011438131332397, + "mixer_index": 5, + "normalized_entropy": 0.9612895457523564, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5011438131332397, + "median": 0.4804786741733551, + "p05": 0.3554112911224365, + "p95": 0.7207092642784119 + }, + { + "label": "current_group_partial", + "mean": 0.49885618686676025, + "median": 0.5195213556289673, + "p05": 0.27929070591926575, + "p95": 0.6445887684822083 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6865031719207764, + "group": 1, + "latest_source_mass": 0.4691343307495117, + "layer": 4, + "max_source_mass": 0.5308656692504883, + "mixer_index": 6, + "normalized_entropy": 0.9904147216846476, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5308656692504883, + "median": 0.5278033018112183, + "p05": 0.45786383748054504, + "p95": 0.6093498468399048 + }, + { + "label": "current_group_partial", + "mean": 0.4691343307495117, + "median": 0.47219666838645935, + "p05": 0.3906501829624176, + "p95": 0.5421361327171326 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 0.4133813679218292, + "group": 1, + "latest_source_mass": 0.15208598971366882, + "layer": 4, + "max_source_mass": 0.8479140400886536, + "mixer_index": 7, + "normalized_entropy": 0.5963832494967191, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.8479140400886536, + "median": 0.8514594435691833, + "p05": 0.7530128359794617, + "p95": 0.9402707815170288 + }, + { + "label": "current_group_partial", + "mean": 0.15208598971366882, + "median": 0.14854055643081665, + "p05": 0.05972917377948761, + "p95": 0.24698717892169952 + } + ], + "sources": 2 + }, + { + "branch": "attention", + "entropy_mean": 0.6850745677947998, + "group": 2, + "latest_source_mass": 0.4601486325263977, + "layer": 5, + "max_source_mass": 0.5398513674736023, + "mixer_index": 8, + "normalized_entropy": 0.9883536815967077, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.5398513674736023, + "median": 0.5387102365493774, + "p05": 0.46027374267578125, + "p95": 0.623892605304718 + }, + { + "label": "completed_group_1", + "mean": 0.4601486325263977, + "median": 0.46128979325294495, + "p05": 0.376107394695282, + "p95": 0.5397262573242188 + } + ], + "sources": 2 + }, + { + "branch": "mlp", + "entropy_mean": 1.0074119567871094, + "group": 2, + "latest_source_mass": 0.4627366364002228, + "layer": 5, + "max_source_mass": 0.4627366364002228, + "mixer_index": 9, + "normalized_entropy": 0.9169858804405273, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.32744842767715454, + "median": 0.30657994747161865, + "p05": 0.16577640175819397, + "p95": 0.5453717708587646 + }, + { + "label": "completed_group_1", + "mean": 0.20981493592262268, + "median": 0.2177819311618805, + "p05": 0.14342713356018066, + "p95": 0.2433384209871292 + }, + { + "label": "current_group_partial", + "mean": 0.4627366364002228, + "median": 0.4847465753555298, + "p05": 0.27109938859939575, + "p95": 0.6081984639167786 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0489801168441772, + "group": 2, + "latest_source_mass": 0.4763939380645752, + "layer": 6, + "max_source_mass": 0.4763939380645752, + "mixer_index": 10, + "normalized_entropy": 0.9548228503031734, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28095996379852295, + "median": 0.2765299677848816, + "p05": 0.23549969494342804, + "p95": 0.3424558639526367 + }, + { + "label": "completed_group_1", + "mean": 0.24264612793922424, + "median": 0.23937124013900757, + "p05": 0.21875721216201782, + "p95": 0.2814398407936096 + }, + { + "label": "current_group_partial", + "mean": 0.4763939380645752, + "median": 0.48882943391799927, + "p05": 0.38478636741638184, + "p95": 0.528400719165802 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.8527575731277466, + "group": 2, + "latest_source_mass": 0.6454645395278931, + "layer": 6, + "max_source_mass": 0.6454645395278931, + "mixer_index": 11, + "normalized_entropy": 0.7762133938639787, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20522089302539825, + "median": 0.1591135561466217, + "p05": 0.11052663624286652, + "p95": 0.47355735301971436 + }, + { + "label": "completed_group_1", + "mean": 0.1493145078420639, + "median": 0.15203402936458588, + "p05": 0.10004506260156631, + "p95": 0.18058395385742188 + }, + { + "label": "current_group_partial", + "mean": 0.6454645395278931, + "median": 0.68653404712677, + "p05": 0.4207671880722046, + "p95": 0.7333818674087524 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.049942135810852, + "group": 2, + "latest_source_mass": 0.4395729899406433, + "layer": 7, + "max_source_mass": 0.4395729899406433, + "mixer_index": 12, + "normalized_entropy": 0.9556985177033998, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35788866877555847, + "median": 0.36233147978782654, + "p05": 0.30507102608680725, + "p95": 0.3985886573791504 + }, + { + "label": "completed_group_1", + "mean": 0.20253834128379822, + "median": 0.19928529858589172, + "p05": 0.180739626288414, + "p95": 0.23214691877365112 + }, + { + "label": "current_group_partial", + "mean": 0.4395729899406433, + "median": 0.43975552916526794, + "p05": 0.3959323763847351, + "p95": 0.4790603220462799 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9791408181190491, + "group": 2, + "latest_source_mass": 0.5214842557907104, + "layer": 7, + "max_source_mass": 0.5214842557907104, + "mixer_index": 13, + "normalized_entropy": 0.8912523810434521, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.31465935707092285, + "median": 0.28992873430252075, + "p05": 0.20534247159957886, + "p95": 0.507034420967102 + }, + { + "label": "completed_group_1", + "mean": 0.1638563871383667, + "median": 0.1667816936969757, + "p05": 0.11925681680440903, + "p95": 0.19515612721443176 + }, + { + "label": "current_group_partial", + "mean": 0.5214842557907104, + "median": 0.5455081462860107, + "p05": 0.35950881242752075, + "p95": 0.6097541451454163 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0485806465148926, + "group": 2, + "latest_source_mass": 0.4457142949104309, + "layer": 8, + "max_source_mass": 0.4457142949104309, + "mixer_index": 14, + "normalized_entropy": 0.9544592367395849, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.346283882856369, + "median": 0.353412926197052, + "p05": 0.26436588168144226, + "p95": 0.4084087312221527 + }, + { + "label": "completed_group_1", + "mean": 0.20800183713436127, + "median": 0.20826736092567444, + "p05": 0.18089595437049866, + "p95": 0.23509056866168976 + }, + { + "label": "current_group_partial", + "mean": 0.4457142949104309, + "median": 0.4416901469230652, + "p05": 0.3883572816848755, + "p95": 0.5136083364486694 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 0.9801348447799683, + "group": 2, + "latest_source_mass": 0.543371319770813, + "layer": 8, + "max_source_mass": 0.543371319770813, + "mixer_index": 15, + "normalized_entropy": 0.8921571831025336, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2748759388923645, + "median": 0.2577607035636902, + "p05": 0.17991375923156738, + "p95": 0.41774845123291016 + }, + { + "label": "completed_group_1", + "mean": 0.1817527413368225, + "median": 0.17968356609344482, + "p05": 0.130966454744339, + "p95": 0.23766928911209106 + }, + { + "label": "current_group_partial", + "mean": 0.543371319770813, + "median": 0.5535831451416016, + "p05": 0.4452384412288666, + "p95": 0.6142411828041077 + } + ], + "sources": 3 + }, + { + "branch": "attention", + "entropy_mean": 1.0688585042953491, + "group": 3, + "latest_source_mass": 0.36256325244903564, + "layer": 9, + "max_source_mass": 0.4051227569580078, + "mixer_index": 16, + "normalized_entropy": 0.9729169383233166, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.4051227569580078, + "median": 0.4008670449256897, + "p05": 0.3286888897418976, + "p95": 0.48389261960983276 + }, + { + "label": "completed_group_1", + "mean": 0.23231396079063416, + "median": 0.23353952169418335, + "p05": 0.2008797824382782, + "p95": 0.2593720257282257 + }, + { + "label": "completed_group_2", + "mean": 0.36256325244903564, + "median": 0.3638863265514374, + "p05": 0.3081618845462799, + "p95": 0.4163197875022888 + } + ], + "sources": 3 + }, + { + "branch": "mlp", + "entropy_mean": 1.2915732860565186, + "group": 3, + "latest_source_mass": 0.36661261320114136, + "layer": 9, + "max_source_mass": 0.36661261320114136, + "mixer_index": 17, + "normalized_entropy": 0.931673187369201, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.26621371507644653, + "median": 0.23494771122932434, + "p05": 0.16815711557865143, + "p95": 0.4694361090660095 + }, + { + "label": "completed_group_1", + "mean": 0.1108347550034523, + "median": 0.11330658197402954, + "p05": 0.06620892137289047, + "p95": 0.13892118632793427 + }, + { + "label": "completed_group_2", + "mean": 0.2563389539718628, + "median": 0.26728466153144836, + "p05": 0.16010139882564545, + "p95": 0.3091832995414734 + }, + { + "label": "current_group_partial", + "mean": 0.36661261320114136, + "median": 0.36950358748435974, + "p05": 0.29210716485977173, + "p95": 0.42882072925567627 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3568191528320312, + "group": 3, + "latest_source_mass": 0.30908840894699097, + "layer": 10, + "max_source_mass": 0.30908840894699097, + "mixer_index": 18, + "normalized_entropy": 0.978738131586968, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.294538676738739, + "median": 0.2961598038673401, + "p05": 0.2467859834432602, + "p95": 0.34424129128456116 + }, + { + "label": "completed_group_1", + "mean": 0.1710844486951828, + "median": 0.17187124490737915, + "p05": 0.1508811116218567, + "p95": 0.1898101568222046 + }, + { + "label": "completed_group_2", + "mean": 0.2252884805202484, + "median": 0.22875282168388367, + "p05": 0.18443678319454193, + "p95": 0.25181543827056885 + }, + { + "label": "current_group_partial", + "mean": 0.30908840894699097, + "median": 0.3080518841743469, + "p05": 0.2719307243824005, + "p95": 0.3494151830673218 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.320021152496338, + "group": 3, + "latest_source_mass": 0.3312946557998657, + "layer": 10, + "max_source_mass": 0.3312946557998657, + "mixer_index": 19, + "normalized_entropy": 0.9521939852875004, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.30465462803840637, + "median": 0.311603307723999, + "p05": 0.21955373883247375, + "p95": 0.40013614296913147 + }, + { + "label": "completed_group_1", + "mean": 0.12911874055862427, + "median": 0.12981033325195312, + "p05": 0.09804825484752655, + "p95": 0.15542283654212952 + }, + { + "label": "completed_group_2", + "mean": 0.23493197560310364, + "median": 0.24103470146656036, + "p05": 0.16810482740402222, + "p95": 0.28502923250198364 + }, + { + "label": "current_group_partial", + "mean": 0.3312946557998657, + "median": 0.325168639421463, + "p05": 0.2647673785686493, + "p95": 0.41786038875579834 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3578683137893677, + "group": 3, + "latest_source_mass": 0.30723780393600464, + "layer": 11, + "max_source_mass": 0.30723780393600464, + "mixer_index": 20, + "normalized_entropy": 0.9794949412420898, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.29558902978897095, + "median": 0.2938143014907837, + "p05": 0.24335844814777374, + "p95": 0.3542291522026062 + }, + { + "label": "completed_group_1", + "mean": 0.17812997102737427, + "median": 0.17726755142211914, + "p05": 0.15839283168315887, + "p95": 0.20142368972301483 + }, + { + "label": "completed_group_2", + "mean": 0.21904319524765015, + "median": 0.2218058705329895, + "p05": 0.18076662719249725, + "p95": 0.24698719382286072 + }, + { + "label": "current_group_partial", + "mean": 0.30723780393600464, + "median": 0.30536961555480957, + "p05": 0.26194074749946594, + "p95": 0.35952210426330566 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.258887767791748, + "group": 3, + "latest_source_mass": 0.34303945302963257, + "layer": 11, + "max_source_mass": 0.35744374990463257, + "mixer_index": 21, + "normalized_entropy": 0.9080955698144659, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.35744374990463257, + "median": 0.359974205493927, + "p05": 0.17910680174827576, + "p95": 0.5207242965698242 + }, + { + "label": "completed_group_1", + "mean": 0.12067858874797821, + "median": 0.11769672483205795, + "p05": 0.08151324093341827, + "p95": 0.16476252675056458 + }, + { + "label": "completed_group_2", + "mean": 0.17883820831775665, + "median": 0.17648206651210785, + "p05": 0.0995911955833435, + "p95": 0.2739725708961487 + }, + { + "label": "current_group_partial", + "mean": 0.34303945302963257, + "median": 0.33267199993133545, + "p05": 0.22800606489181519, + "p95": 0.48691368103027344 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3539760112762451, + "group": 3, + "latest_source_mass": 0.3175881803035736, + "layer": 12, + "max_source_mass": 0.3175881803035736, + "mixer_index": 22, + "normalized_entropy": 0.976687238475429, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2736307978630066, + "median": 0.27999642491340637, + "p05": 0.18969643115997314, + "p95": 0.3464604914188385 + }, + { + "label": "completed_group_1", + "mean": 0.1722733974456787, + "median": 0.17083553969860077, + "p05": 0.1506597101688385, + "p95": 0.1988438367843628 + }, + { + "label": "completed_group_2", + "mean": 0.2365075945854187, + "median": 0.24049004912376404, + "p05": 0.19099700450897217, + "p95": 0.26942259073257446 + }, + { + "label": "current_group_partial", + "mean": 0.3175881803035736, + "median": 0.3092706799507141, + "p05": 0.268545538187027, + "p95": 0.3924510180950165 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.3203014135360718, + "group": 3, + "latest_source_mass": 0.3395293354988098, + "layer": 12, + "max_source_mass": 0.3395293354988098, + "mixer_index": 23, + "normalized_entropy": 0.9523961508935896, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.28207293152809143, + "median": 0.29242122173309326, + "p05": 0.1354818046092987, + "p95": 0.4195851981639862 + }, + { + "label": "completed_group_1", + "mean": 0.15420743823051453, + "median": 0.15396761894226074, + "p05": 0.11267110705375671, + "p95": 0.1951446533203125 + }, + { + "label": "completed_group_2", + "mean": 0.22419027984142303, + "median": 0.22542357444763184, + "p05": 0.1655150204896927, + "p95": 0.2747117578983307 + }, + { + "label": "current_group_partial", + "mean": 0.3395293354988098, + "median": 0.32681703567504883, + "p05": 0.2524653375148773, + "p95": 0.46652013063430786 + } + ], + "sources": 4 + }, + { + "branch": "attention", + "entropy_mean": 1.3734662532806396, + "group": 4, + "latest_source_mass": 0.283242404460907, + "layer": 13, + "max_source_mass": 0.283242404460907, + "mixer_index": 24, + "normalized_entropy": 0.9907464762181619, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2766023874282837, + "median": 0.282479852437973, + "p05": 0.22528076171875, + "p95": 0.3097468614578247 + }, + { + "label": "completed_group_1", + "mean": 0.21176087856292725, + "median": 0.2058793306350708, + "p05": 0.18811854720115662, + "p95": 0.2617422640323639 + }, + { + "label": "completed_group_2", + "mean": 0.22839435935020447, + "median": 0.22951346635818481, + "p05": 0.20004916191101074, + "p95": 0.2551718056201935 + }, + { + "label": "completed_group_3", + "mean": 0.283242404460907, + "median": 0.2849352955818176, + "p05": 0.2205706238746643, + "p95": 0.34444743394851685 + } + ], + "sources": 4 + }, + { + "branch": "mlp", + "entropy_mean": 1.5325813293457031, + "group": 4, + "latest_source_mass": 0.27591708302497864, + "layer": 13, + "max_source_mass": 0.27591708302497864, + "mixer_index": 25, + "normalized_entropy": 0.9522463199762954, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.17451074719429016, + "median": 0.16826045513153076, + "p05": 0.08149556815624237, + "p95": 0.3126368522644043 + }, + { + "label": "completed_group_1", + "mean": 0.10643941164016724, + "median": 0.10269765555858612, + "p05": 0.08178127557039261, + "p95": 0.13947664201259613 + }, + { + "label": "completed_group_2", + "mean": 0.1691833734512329, + "median": 0.17118802666664124, + "p05": 0.12382284551858902, + "p95": 0.2069271057844162 + }, + { + "label": "completed_group_3", + "mean": 0.27394935488700867, + "median": 0.27278125286102295, + "p05": 0.2122008204460144, + "p95": 0.33945947885513306 + }, + { + "label": "current_group_partial", + "mean": 0.27591708302497864, + "median": 0.27714359760284424, + "p05": 0.21523593366146088, + "p95": 0.33170604705810547 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.560360074043274, + "group": 4, + "latest_source_mass": 0.3255118429660797, + "layer": 14, + "max_source_mass": 0.3255118429660797, + "mixer_index": 26, + "normalized_entropy": 0.9695062244951087, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.19377154111862183, + "median": 0.19472965598106384, + "p05": 0.17005598545074463, + "p95": 0.22297024726867676 + }, + { + "label": "completed_group_1", + "mean": 0.17211595177650452, + "median": 0.17136064171791077, + "p05": 0.1543424427509308, + "p95": 0.1928609162569046 + }, + { + "label": "completed_group_2", + "mean": 0.14605136215686798, + "median": 0.14629822969436646, + "p05": 0.13246256113052368, + "p95": 0.15914371609687805 + }, + { + "label": "completed_group_3", + "mean": 0.16254930198192596, + "median": 0.16272446513175964, + "p05": 0.14075018465518951, + "p95": 0.18463709950447083 + }, + { + "label": "current_group_partial", + "mean": 0.3255118429660797, + "median": 0.3261105418205261, + "p05": 0.3035835921764374, + "p95": 0.3457673192024231 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.540909767150879, + "group": 4, + "latest_source_mass": 0.2995486855506897, + "layer": 14, + "max_source_mass": 0.2995486855506897, + "mixer_index": 27, + "normalized_entropy": 0.9574210693349581, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21921303868293762, + "median": 0.21722173690795898, + "p05": 0.1413341462612152, + "p95": 0.2906968593597412 + }, + { + "label": "completed_group_1", + "mean": 0.10205383598804474, + "median": 0.09984464198350906, + "p05": 0.07455356419086456, + "p95": 0.13362759351730347 + }, + { + "label": "completed_group_2", + "mean": 0.16381604969501495, + "median": 0.16600549221038818, + "p05": 0.11865930259227753, + "p95": 0.2019941210746765 + }, + { + "label": "completed_group_3", + "mean": 0.215368390083313, + "median": 0.2136734575033188, + "p05": 0.1728287935256958, + "p95": 0.2628125846385956 + }, + { + "label": "current_group_partial", + "mean": 0.2995486855506897, + "median": 0.2982075810432434, + "p05": 0.23863884806632996, + "p95": 0.3662579655647278 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5142419338226318, + "group": 4, + "latest_source_mass": 0.369165301322937, + "layer": 15, + "max_source_mass": 0.369165301322937, + "mixer_index": 28, + "normalized_entropy": 0.940851412859105, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2000826597213745, + "median": 0.20369741320610046, + "p05": 0.15802279114723206, + "p95": 0.24072687327861786 + }, + { + "label": "completed_group_1", + "mean": 0.15480534732341766, + "median": 0.15228059887886047, + "p05": 0.13543802499771118, + "p95": 0.1815909892320633 + }, + { + "label": "completed_group_2", + "mean": 0.10827909409999847, + "median": 0.10761409997940063, + "p05": 0.09670238941907883, + "p95": 0.12229038774967194 + }, + { + "label": "completed_group_3", + "mean": 0.16766759753227234, + "median": 0.16637766361236572, + "p05": 0.1398160457611084, + "p95": 0.20042110979557037 + }, + { + "label": "current_group_partial", + "mean": 0.369165301322937, + "median": 0.3697334825992584, + "p05": 0.3279486298561096, + "p95": 0.40613043308258057 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5333527326583862, + "group": 4, + "latest_source_mass": 0.29974091053009033, + "layer": 15, + "max_source_mass": 0.29974091053009033, + "mixer_index": 29, + "normalized_entropy": 0.9527256198031004, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21770668029785156, + "median": 0.19699469208717346, + "p05": 0.1311802864074707, + "p95": 0.3720983564853668 + }, + { + "label": "completed_group_1", + "mean": 0.10525081306695938, + "median": 0.10493603348731995, + "p05": 0.08330865204334259, + "p95": 0.12839283049106598 + }, + { + "label": "completed_group_2", + "mean": 0.15970060229301453, + "median": 0.16236470639705658, + "p05": 0.11309029906988144, + "p95": 0.19861777126789093 + }, + { + "label": "completed_group_3", + "mean": 0.217601016163826, + "median": 0.21904586255550385, + "p05": 0.16268108785152435, + "p95": 0.2699928283691406 + }, + { + "label": "current_group_partial", + "mean": 0.29974091053009033, + "median": 0.30079299211502075, + "p05": 0.22156940400600433, + "p95": 0.37281978130340576 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.512608289718628, + "group": 4, + "latest_source_mass": 0.35851576924324036, + "layer": 16, + "max_source_mass": 0.35851576924324036, + "mixer_index": 30, + "normalized_entropy": 0.9398363727066501, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.21392285823822021, + "median": 0.22146853804588318, + "p05": 0.1518818885087967, + "p95": 0.25874194502830505 + }, + { + "label": "completed_group_1", + "mean": 0.1860550194978714, + "median": 0.1838144212961197, + "p05": 0.1660098135471344, + "p95": 0.21311265230178833 + }, + { + "label": "completed_group_2", + "mean": 0.10291537642478943, + "median": 0.1024642288684845, + "p05": 0.09207107126712799, + "p95": 0.11504144221544266 + }, + { + "label": "completed_group_3", + "mean": 0.1385909616947174, + "median": 0.13646918535232544, + "p05": 0.11220520734786987, + "p95": 0.1728362739086151 + }, + { + "label": "current_group_partial", + "mean": 0.35851576924324036, + "median": 0.3558155298233032, + "p05": 0.31809645891189575, + "p95": 0.40622422099113464 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.5262645483016968, + "group": 4, + "latest_source_mass": 0.3142552077770233, + "layer": 16, + "max_source_mass": 0.3142552077770233, + "mixer_index": 31, + "normalized_entropy": 0.9483214832396903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.20246261358261108, + "median": 0.18459102511405945, + "p05": 0.12276223301887512, + "p95": 0.38892465829849243 + }, + { + "label": "completed_group_1", + "mean": 0.10530521720647812, + "median": 0.10532709956169128, + "p05": 0.0785030871629715, + "p95": 0.1342671513557434 + }, + { + "label": "completed_group_2", + "mean": 0.16650916635990143, + "median": 0.17111444473266602, + "p05": 0.1085265576839447, + "p95": 0.2094469517469406 + }, + { + "label": "completed_group_3", + "mean": 0.21146778762340546, + "median": 0.2099919468164444, + "p05": 0.15801574289798737, + "p95": 0.26882117986679077 + }, + { + "label": "current_group_partial", + "mean": 0.3142552077770233, + "median": 0.31609731912612915, + "p05": 0.22627028822898865, + "p95": 0.397154837846756 + } + ], + "sources": 5 + }, + { + "branch": "attention", + "entropy_mean": 1.5794764757156372, + "group": 5, + "latest_source_mass": 0.2760102152824402, + "layer": 17, + "max_source_mass": 0.2760102152824402, + "mixer_index": 32, + "normalized_entropy": 0.9813839126772218, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.2210189402103424, + "median": 0.22435861825942993, + "p05": 0.17291823029518127, + "p95": 0.262204647064209 + }, + { + "label": "completed_group_1", + "mean": 0.18839938938617706, + "median": 0.18652752041816711, + "p05": 0.16997642815113068, + "p95": 0.2126670628786087 + }, + { + "label": "completed_group_2", + "mean": 0.14016608893871307, + "median": 0.14067986607551575, + "p05": 0.1265786588191986, + "p95": 0.15091627836227417 + }, + { + "label": "completed_group_3", + "mean": 0.17440536618232727, + "median": 0.17192018032073975, + "p05": 0.15062645077705383, + "p95": 0.20691406726837158 + }, + { + "label": "completed_group_4", + "mean": 0.2760102152824402, + "median": 0.27511972188949585, + "p05": 0.24412119388580322, + "p95": 0.30901142954826355 + } + ], + "sources": 5 + }, + { + "branch": "mlp", + "entropy_mean": 1.6602650880813599, + "group": 5, + "latest_source_mass": 0.3575916588306427, + "layer": 17, + "max_source_mass": 0.3575916588306427, + "mixer_index": 33, + "normalized_entropy": 0.9266115885502495, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12695768475532532, + "median": 0.11564654856920242, + "p05": 0.07159632444381714, + "p95": 0.21949690580368042 + }, + { + "label": "completed_group_1", + "mean": 0.08036886155605316, + "median": 0.07964898645877838, + "p05": 0.06650256365537643, + "p95": 0.09638062119483948 + }, + { + "label": "completed_group_2", + "mean": 0.12672865390777588, + "median": 0.1287270039319992, + "p05": 0.09939455986022949, + "p95": 0.14712141454219818 + }, + { + "label": "completed_group_3", + "mean": 0.15391482412815094, + "median": 0.15372109413146973, + "p05": 0.13333335518836975, + "p95": 0.17581847310066223 + }, + { + "label": "completed_group_4", + "mean": 0.154438316822052, + "median": 0.15402798354625702, + "p05": 0.12481103092432022, + "p95": 0.18498417735099792 + }, + { + "label": "current_group_partial", + "mean": 0.3575916588306427, + "median": 0.36001941561698914, + "p05": 0.3110165596008301, + "p95": 0.39699214696884155 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.5925545692443848, + "group": 5, + "latest_source_mass": 0.41908538341522217, + "layer": 18, + "max_source_mass": 0.41908538341522217, + "mixer_index": 34, + "normalized_entropy": 0.8888216284580353, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14075040817260742, + "median": 0.1455841362476349, + "p05": 0.09744156897068024, + "p95": 0.1818566620349884 + }, + { + "label": "completed_group_1", + "mean": 0.11028306186199188, + "median": 0.10887216031551361, + "p05": 0.09629634767770767, + "p95": 0.12874113023281097 + }, + { + "label": "completed_group_2", + "mean": 0.07537610828876495, + "median": 0.07517752051353455, + "p05": 0.0666593387722969, + "p95": 0.08491044491529465 + }, + { + "label": "completed_group_3", + "mean": 0.10244421660900116, + "median": 0.10141433775424957, + "p05": 0.08653581142425537, + "p95": 0.12175601720809937 + }, + { + "label": "completed_group_4", + "mean": 0.1520608365535736, + "median": 0.1513972282409668, + "p05": 0.1301831305027008, + "p95": 0.17589205503463745 + }, + { + "label": "current_group_partial", + "mean": 0.41908538341522217, + "median": 0.4191858768463135, + "p05": 0.3649080991744995, + "p95": 0.4721684455871582 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5918848514556885, + "group": 5, + "latest_source_mass": 0.4151553511619568, + "layer": 18, + "max_source_mass": 0.4151553511619568, + "mixer_index": 35, + "normalized_entropy": 0.8884478518433735, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11773698031902313, + "median": 0.10857899487018585, + "p05": 0.06324320286512375, + "p95": 0.22339528799057007 + }, + { + "label": "completed_group_1", + "mean": 0.07344591617584229, + "median": 0.07228962332010269, + "p05": 0.05982384458184242, + "p95": 0.09060496836900711 + }, + { + "label": "completed_group_2", + "mean": 0.11976176500320435, + "median": 0.12151201069355011, + "p05": 0.08953659236431122, + "p95": 0.14441059529781342 + }, + { + "label": "completed_group_3", + "mean": 0.14735516905784607, + "median": 0.14620742201805115, + "p05": 0.11767779290676117, + "p95": 0.17902395129203796 + }, + { + "label": "completed_group_4", + "mean": 0.126544788479805, + "median": 0.12564796209335327, + "p05": 0.09858805686235428, + "p95": 0.1567486822605133 + }, + { + "label": "current_group_partial", + "mean": 0.4151553511619568, + "median": 0.4184337854385376, + "p05": 0.3375611901283264, + "p95": 0.4846380352973938 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.6208878755569458, + "group": 5, + "latest_source_mass": 0.3991435170173645, + "layer": 19, + "max_source_mass": 0.3991435170173645, + "mixer_index": 36, + "normalized_entropy": 0.9046347477964072, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14785674214363098, + "median": 0.15170978009700775, + "p05": 0.09146977216005325, + "p95": 0.20172694325447083 + }, + { + "label": "completed_group_1", + "mean": 0.12492769956588745, + "median": 0.12345598638057709, + "p05": 0.10790181905031204, + "p95": 0.14623555541038513 + }, + { + "label": "completed_group_2", + "mean": 0.08781818300485611, + "median": 0.08839264512062073, + "p05": 0.07453933358192444, + "p95": 0.09894395619630814 + }, + { + "label": "completed_group_3", + "mean": 0.10019385814666748, + "median": 0.09901996701955795, + "p05": 0.08430047333240509, + "p95": 0.12013883143663406 + }, + { + "label": "completed_group_4", + "mean": 0.14006000757217407, + "median": 0.13918349146842957, + "p05": 0.11816112697124481, + "p95": 0.16475799679756165 + }, + { + "label": "current_group_partial", + "mean": 0.3991435170173645, + "median": 0.39726996421813965, + "p05": 0.34116071462631226, + "p95": 0.4634247422218323 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5470075607299805, + "group": 5, + "latest_source_mass": 0.44330859184265137, + "layer": 19, + "max_source_mass": 0.44330859184265137, + "mixer_index": 37, + "normalized_entropy": 0.8634013589985261, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12043030560016632, + "median": 0.1012125313282013, + "p05": 0.061379045248031616, + "p95": 0.25343188643455505 + }, + { + "label": "completed_group_1", + "mean": 0.06855648756027222, + "median": 0.06761305779218674, + "p05": 0.05337192490696907, + "p95": 0.08790726959705353 + }, + { + "label": "completed_group_2", + "mean": 0.11753211170434952, + "median": 0.1186976358294487, + "p05": 0.08412563800811768, + "p95": 0.14895018935203552 + }, + { + "label": "completed_group_3", + "mean": 0.1339646875858307, + "median": 0.13253431022167206, + "p05": 0.09857159107923508, + "p95": 0.1730121374130249 + }, + { + "label": "completed_group_4", + "mean": 0.1162077933549881, + "median": 0.11592038720846176, + "p05": 0.08609852194786072, + "p95": 0.14767511188983917 + }, + { + "label": "current_group_partial", + "mean": 0.44330859184265137, + "median": 0.4434061646461487, + "p05": 0.3389148414134979, + "p95": 0.5490255355834961 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.683492660522461, + "group": 5, + "latest_source_mass": 0.3365415334701538, + "layer": 20, + "max_source_mass": 0.3365415334701538, + "mixer_index": 38, + "normalized_entropy": 0.9395751435586169, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1658952683210373, + "median": 0.1726911962032318, + "p05": 0.10183650255203247, + "p95": 0.22522984445095062 + }, + { + "label": "completed_group_1", + "mean": 0.1435280442237854, + "median": 0.14210543036460876, + "p05": 0.12450659275054932, + "p95": 0.16782894730567932 + }, + { + "label": "completed_group_2", + "mean": 0.09461991488933563, + "median": 0.0948714017868042, + "p05": 0.08168491721153259, + "p95": 0.10723742842674255 + }, + { + "label": "completed_group_3", + "mean": 0.10784181207418442, + "median": 0.10693192481994629, + "p05": 0.08930590003728867, + "p95": 0.12958656251430511 + }, + { + "label": "completed_group_4", + "mean": 0.15157344937324524, + "median": 0.15053030848503113, + "p05": 0.128616601228714, + "p95": 0.17799487709999084 + }, + { + "label": "current_group_partial", + "mean": 0.3365415334701538, + "median": 0.33349889516830444, + "p05": 0.2778944969177246, + "p95": 0.40724658966064453 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.5540567636489868, + "group": 5, + "latest_source_mass": 0.42266085743904114, + "layer": 20, + "max_source_mass": 0.42266085743904114, + "mixer_index": 39, + "normalized_entropy": 0.8673355940563396, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.12987135350704193, + "median": 0.1083056628704071, + "p05": 0.03783264383673668, + "p95": 0.3301126956939697 + }, + { + "label": "completed_group_1", + "mean": 0.07178181409835815, + "median": 0.07081792503595352, + "p05": 0.05334944650530815, + "p95": 0.09311570972204208 + }, + { + "label": "completed_group_2", + "mean": 0.12132491171360016, + "median": 0.12282539904117584, + "p05": 0.07926258444786072, + "p95": 0.15763431787490845 + }, + { + "label": "completed_group_3", + "mean": 0.1441602110862732, + "median": 0.14273977279663086, + "p05": 0.09847753494977951, + "p95": 0.19304539263248444 + }, + { + "label": "completed_group_4", + "mean": 0.11020088195800781, + "median": 0.10934801399707794, + "p05": 0.07139573246240616, + "p95": 0.1517774760723114 + }, + { + "label": "current_group_partial", + "mean": 0.42266085743904114, + "median": 0.42266106605529785, + "p05": 0.28481048345565796, + "p95": 0.5570361018180847 + } + ], + "sources": 6 + }, + { + "branch": "attention", + "entropy_mean": 1.7208337783813477, + "group": 6, + "latest_source_mass": 0.27070939540863037, + "layer": 21, + "max_source_mass": 0.27070939540863037, + "mixer_index": 40, + "normalized_entropy": 0.9604156182429642, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.18048501014709473, + "median": 0.18919861316680908, + "p05": 0.11199048161506653, + "p95": 0.23807701468467712 + }, + { + "label": "completed_group_1", + "mean": 0.1685236394405365, + "median": 0.1669977903366089, + "p05": 0.14678362011909485, + "p95": 0.19381441175937653 + }, + { + "label": "completed_group_2", + "mean": 0.08726631104946136, + "median": 0.08729851245880127, + "p05": 0.07629868388175964, + "p95": 0.0980045348405838 + }, + { + "label": "completed_group_3", + "mean": 0.11160479485988617, + "median": 0.1099197044968605, + "p05": 0.09190923720598221, + "p95": 0.13723208010196686 + }, + { + "label": "completed_group_4", + "mean": 0.18141084909439087, + "median": 0.18020854890346527, + "p05": 0.15656134486198425, + "p95": 0.2108723670244217 + }, + { + "label": "completed_group_5", + "mean": 0.27070939540863037, + "median": 0.2684924602508545, + "p05": 0.2159801721572876, + "p95": 0.33192309737205505 + } + ], + "sources": 6 + }, + { + "branch": "mlp", + "entropy_mean": 1.6729483604431152, + "group": 6, + "latest_source_mass": 0.3803577423095703, + "layer": 21, + "max_source_mass": 0.3803577423095703, + "mixer_index": 41, + "normalized_entropy": 0.8597253893019091, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09229261428117752, + "median": 0.07798201590776443, + "p05": 0.04784023016691208, + "p95": 0.20285779237747192 + }, + { + "label": "completed_group_1", + "mean": 0.051420148462057114, + "median": 0.05041736364364624, + "p05": 0.040625106543302536, + "p95": 0.06547587364912033 + }, + { + "label": "completed_group_2", + "mean": 0.08079522103071213, + "median": 0.08136115223169327, + "p05": 0.05947277322411537, + "p95": 0.10017222166061401 + }, + { + "label": "completed_group_3", + "mean": 0.09550285339355469, + "median": 0.09445887804031372, + "p05": 0.07334083318710327, + "p95": 0.12144046276807785 + }, + { + "label": "completed_group_4", + "mean": 0.07590493559837341, + "median": 0.07492390275001526, + "p05": 0.055510398000478745, + "p95": 0.10084333270788193 + }, + { + "label": "completed_group_5", + "mean": 0.22372648119926453, + "median": 0.22069036960601807, + "p05": 0.1407623589038849, + "p95": 0.32290777564048767 + }, + { + "label": "current_group_partial", + "mean": 0.3803577423095703, + "median": 0.37903881072998047, + "p05": 0.2855539321899414, + "p95": 0.476589173078537 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.710235595703125, + "group": 6, + "latest_source_mass": 0.4011716842651367, + "layer": 22, + "max_source_mass": 0.4011716842651367, + "mixer_index": 42, + "normalized_entropy": 0.8788872376935791, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.11657392978668213, + "median": 0.11877645552158356, + "p05": 0.06819256395101547, + "p95": 0.1684979349374771 + }, + { + "label": "completed_group_1", + "mean": 0.10617930442094803, + "median": 0.10509790480136871, + "p05": 0.090352863073349, + "p95": 0.12587188184261322 + }, + { + "label": "completed_group_2", + "mean": 0.061112720519304276, + "median": 0.061012350022792816, + "p05": 0.05164862796664238, + "p95": 0.07133162766695023 + }, + { + "label": "completed_group_3", + "mean": 0.06796726584434509, + "median": 0.06712733209133148, + "p05": 0.05444999784231186, + "p95": 0.08481802046298981 + }, + { + "label": "completed_group_4", + "mean": 0.103337861597538, + "median": 0.10192783921957016, + "p05": 0.0832459107041359, + "p95": 0.12805812060832977 + }, + { + "label": "completed_group_5", + "mean": 0.14365722239017487, + "median": 0.14159655570983887, + "p05": 0.10498540848493576, + "p95": 0.19020715355873108 + }, + { + "label": "current_group_partial", + "mean": 0.4011716842651367, + "median": 0.40084803104400635, + "p05": 0.32083970308303833, + "p95": 0.47994357347488403 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.5640671253204346, + "group": 6, + "latest_source_mass": 0.4689176678657532, + "layer": 22, + "max_source_mass": 0.4689176678657532, + "mixer_index": 43, + "normalized_entropy": 0.8037715030571925, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.090422622859478, + "median": 0.0774901732802391, + "p05": 0.04492149129509926, + "p95": 0.1988787055015564 + }, + { + "label": "completed_group_1", + "mean": 0.04561382159590721, + "median": 0.044634923338890076, + "p05": 0.03501313924789429, + "p95": 0.05900419130921364 + }, + { + "label": "completed_group_2", + "mean": 0.07565715163946152, + "median": 0.07550699263811111, + "p05": 0.053458306938409805, + "p95": 0.09821777045726776 + }, + { + "label": "completed_group_3", + "mean": 0.08878682553768158, + "median": 0.08692863583564758, + "p05": 0.06425619125366211, + "p95": 0.11807316541671753 + }, + { + "label": "completed_group_4", + "mean": 0.06555216014385223, + "median": 0.06414324045181274, + "p05": 0.04460844025015831, + "p95": 0.09090004861354828 + }, + { + "label": "completed_group_5", + "mean": 0.165049746632576, + "median": 0.15503892302513123, + "p05": 0.08368627727031708, + "p95": 0.2792147397994995 + }, + { + "label": "current_group_partial", + "mean": 0.4689176678657532, + "median": 0.46879667043685913, + "p05": 0.3151778280735016, + "p95": 0.6170428991317749 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7556228637695312, + "group": 6, + "latest_source_mass": 0.3706088662147522, + "layer": 23, + "max_source_mass": 0.3706088662147522, + "mixer_index": 44, + "normalized_entropy": 0.9022116795175967, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13281464576721191, + "median": 0.13658174872398376, + "p05": 0.08060654997825623, + "p95": 0.17402096092700958 + }, + { + "label": "completed_group_1", + "mean": 0.12175846099853516, + "median": 0.12066285312175751, + "p05": 0.10334118455648422, + "p95": 0.14384035766124725 + }, + { + "label": "completed_group_2", + "mean": 0.07235272228717804, + "median": 0.07218228280544281, + "p05": 0.059715576469898224, + "p95": 0.08584222197532654 + }, + { + "label": "completed_group_3", + "mean": 0.0707377940416336, + "median": 0.06942892074584961, + "p05": 0.05500924959778786, + "p95": 0.09052760899066925 + }, + { + "label": "completed_group_4", + "mean": 0.10551950335502625, + "median": 0.103996641933918, + "p05": 0.08408617228269577, + "p95": 0.13205930590629578 + }, + { + "label": "completed_group_5", + "mean": 0.12620802223682404, + "median": 0.12379512935876846, + "p05": 0.08937962353229523, + "p95": 0.17139491438865662 + }, + { + "label": "current_group_partial", + "mean": 0.3706088662147522, + "median": 0.3705657124519348, + "p05": 0.282781720161438, + "p95": 0.4561466872692108 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.594892978668213, + "group": 6, + "latest_source_mass": 0.4636557698249817, + "layer": 23, + "max_source_mass": 0.4636557698249817, + "mixer_index": 45, + "normalized_entropy": 0.8196128579947488, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10721131414175034, + "median": 0.10240983963012695, + "p05": 0.05563424155116081, + "p95": 0.18401460349559784 + }, + { + "label": "completed_group_1", + "mean": 0.053925663232803345, + "median": 0.0527028888463974, + "p05": 0.04038037359714508, + "p95": 0.07054222375154495 + }, + { + "label": "completed_group_2", + "mean": 0.07673212885856628, + "median": 0.07705660909414291, + "p05": 0.05211106687784195, + "p95": 0.10031480342149734 + }, + { + "label": "completed_group_3", + "mean": 0.08865120261907578, + "median": 0.08699032664299011, + "p05": 0.06242028623819351, + "p95": 0.11899784207344055 + }, + { + "label": "completed_group_4", + "mean": 0.06888366490602493, + "median": 0.06751501560211182, + "p05": 0.04784276708960533, + "p95": 0.09609999507665634 + }, + { + "label": "completed_group_5", + "mean": 0.14094024896621704, + "median": 0.13063763082027435, + "p05": 0.07297889143228531, + "p95": 0.24012833833694458 + }, + { + "label": "current_group_partial", + "mean": 0.4636557698249817, + "median": 0.46312135457992554, + "p05": 0.31259939074516296, + "p95": 0.6109440326690674 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.7775511741638184, + "group": 6, + "latest_source_mass": 0.3569435477256775, + "layer": 24, + "max_source_mass": 0.3569435477256775, + "mixer_index": 46, + "normalized_entropy": 0.9134806018801903, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.13637229800224304, + "median": 0.13456633687019348, + "p05": 0.09586701542139053, + "p95": 0.18876218795776367 + }, + { + "label": "completed_group_1", + "mean": 0.10478455573320389, + "median": 0.10326151549816132, + "p05": 0.0880400538444519, + "p95": 0.12566551566123962 + }, + { + "label": "completed_group_2", + "mean": 0.07710614800453186, + "median": 0.07678486406803131, + "p05": 0.06292064487934113, + "p95": 0.09218747913837433 + }, + { + "label": "completed_group_3", + "mean": 0.08548048138618469, + "median": 0.08430111408233643, + "p05": 0.0665682703256607, + "p95": 0.10857507586479187 + }, + { + "label": "completed_group_4", + "mean": 0.11000140011310577, + "median": 0.10904969274997711, + "p05": 0.0881001353263855, + "p95": 0.13561299443244934 + }, + { + "label": "completed_group_5", + "mean": 0.12931162118911743, + "median": 0.12692990899085999, + "p05": 0.08648453652858734, + "p95": 0.1814313679933548 + }, + { + "label": "current_group_partial", + "mean": 0.3569435477256775, + "median": 0.3568228483200073, + "p05": 0.26841166615486145, + "p95": 0.4493965804576874 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.654036045074463, + "group": 6, + "latest_source_mass": 0.4265251159667969, + "layer": 24, + "max_source_mass": 0.4265251159667969, + "mixer_index": 47, + "normalized_entropy": 0.8500063817835848, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10232305526733398, + "median": 0.08623291552066803, + "p05": 0.04679499566555023, + "p95": 0.23077142238616943 + }, + { + "label": "completed_group_1", + "mean": 0.061968062072992325, + "median": 0.060845747590065, + "p05": 0.0479469932615757, + "p95": 0.07941868156194687 + }, + { + "label": "completed_group_2", + "mean": 0.08900615572929382, + "median": 0.08864577114582062, + "p05": 0.0641680359840393, + "p95": 0.11413820832967758 + }, + { + "label": "completed_group_3", + "mean": 0.1005433052778244, + "median": 0.09813467413187027, + "p05": 0.0731433779001236, + "p95": 0.1343638002872467 + }, + { + "label": "completed_group_4", + "mean": 0.07864196598529816, + "median": 0.07639941573143005, + "p05": 0.05487418174743652, + "p95": 0.10941869765520096 + }, + { + "label": "completed_group_5", + "mean": 0.14099235832691193, + "median": 0.12885531783103943, + "p05": 0.07315157353878021, + "p95": 0.24481485784053802 + }, + { + "label": "current_group_partial", + "mean": 0.4265251159667969, + "median": 0.42903727293014526, + "p05": 0.2803359627723694, + "p95": 0.5635722875595093 + } + ], + "sources": 7 + }, + { + "branch": "attention", + "entropy_mean": 1.84956693649292, + "group": 7, + "latest_source_mass": 0.28775152564048767, + "layer": 25, + "max_source_mass": 0.28775152564048767, + "mixer_index": 48, + "normalized_entropy": 0.9504893827656096, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.14851760864257812, + "median": 0.14813844859600067, + "p05": 0.10312555730342865, + "p95": 0.20065288245677948 + }, + { + "label": "completed_group_1", + "mean": 0.11630287766456604, + "median": 0.11500488221645355, + "p05": 0.09825178980827332, + "p95": 0.13750022649765015 + }, + { + "label": "completed_group_2", + "mean": 0.08512166142463684, + "median": 0.08458003401756287, + "p05": 0.0710662305355072, + "p95": 0.10092862695455551 + }, + { + "label": "completed_group_3", + "mean": 0.09215595573186874, + "median": 0.09079311043024063, + "p05": 0.07304700464010239, + "p95": 0.11515989899635315 + }, + { + "label": "completed_group_4", + "mean": 0.1263950914144516, + "median": 0.12563186883926392, + "p05": 0.10441911220550537, + "p95": 0.1519687920808792 + }, + { + "label": "completed_group_5", + "mean": 0.14375528693199158, + "median": 0.1422564834356308, + "p05": 0.10313796997070312, + "p95": 0.1929876059293747 + }, + { + "label": "completed_group_6", + "mean": 0.28775152564048767, + "median": 0.28676605224609375, + "p05": 0.21690022945404053, + "p95": 0.3624127507209778 + } + ], + "sources": 7 + }, + { + "branch": "mlp", + "entropy_mean": 1.7161309719085693, + "group": 7, + "latest_source_mass": 0.32536065578460693, + "layer": 25, + "max_source_mass": 0.32536065578460693, + "mixer_index": 49, + "normalized_entropy": 0.8252845475628168, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0837959572672844, + "median": 0.0849243700504303, + "p05": 0.0271806251257658, + "p95": 0.15180703997612 + }, + { + "label": "completed_group_1", + "mean": 0.04434927552938461, + "median": 0.04302988201379776, + "p05": 0.03217232972383499, + "p95": 0.06002792343497276 + }, + { + "label": "completed_group_2", + "mean": 0.056708935648202896, + "median": 0.05563671886920929, + "p05": 0.03992946073412895, + "p95": 0.07731831818819046 + }, + { + "label": "completed_group_3", + "mean": 0.06495873630046844, + "median": 0.06223570555448532, + "p05": 0.04220382124185562, + "p95": 0.0949486792087555 + }, + { + "label": "completed_group_4", + "mean": 0.04885463789105415, + "median": 0.046898528933525085, + "p05": 0.030923232436180115, + "p95": 0.0743112787604332 + }, + { + "label": "completed_group_5", + "mean": 0.09997115284204483, + "median": 0.08650931715965271, + "p05": 0.04111029952764511, + "p95": 0.20292627811431885 + }, + { + "label": "completed_group_6", + "mean": 0.2760006785392761, + "median": 0.27142390608787537, + "p05": 0.13761089742183685, + "p95": 0.42605364322662354 + }, + { + "label": "current_group_partial", + "mean": 0.32536065578460693, + "median": 0.3348004221916199, + "p05": 0.10654139518737793, + "p95": 0.5236068964004517 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9285151958465576, + "group": 7, + "latest_source_mass": 0.2804902195930481, + "layer": 26, + "max_source_mass": 0.2804902195930481, + "mixer_index": 50, + "normalized_entropy": 0.9274197697756124, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10564712435007095, + "median": 0.10497278720140457, + "p05": 0.04608207941055298, + "p95": 0.16350959241390228 + }, + { + "label": "completed_group_1", + "mean": 0.09173665940761566, + "median": 0.09041133522987366, + "p05": 0.07509034872055054, + "p95": 0.11238400638103485 + }, + { + "label": "completed_group_2", + "mean": 0.07135222852230072, + "median": 0.06981296837329865, + "p05": 0.0522611029446125, + "p95": 0.09555281698703766 + }, + { + "label": "completed_group_3", + "mean": 0.07045470923185349, + "median": 0.06927657127380371, + "p05": 0.055566973984241486, + "p95": 0.08908769488334656 + }, + { + "label": "completed_group_4", + "mean": 0.09785081446170807, + "median": 0.0971219539642334, + "p05": 0.07590530812740326, + "p95": 0.12285276502370834 + }, + { + "label": "completed_group_5", + "mean": 0.10425601899623871, + "median": 0.10151226818561554, + "p05": 0.06786561012268066, + "p95": 0.15156632661819458 + }, + { + "label": "completed_group_6", + "mean": 0.1782122254371643, + "median": 0.1762663722038269, + "p05": 0.11771149188280106, + "p95": 0.24932436645030975 + }, + { + "label": "current_group_partial", + "mean": 0.2804902195930481, + "median": 0.2783043086528778, + "p05": 0.14811429381370544, + "p95": 0.4168073534965515 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7219383716583252, + "group": 7, + "latest_source_mass": 0.3661728501319885, + "layer": 26, + "max_source_mass": 0.3661728501319885, + "mixer_index": 51, + "normalized_entropy": 0.8280773165026276, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0923302173614502, + "median": 0.08522969484329224, + "p05": 0.056062981486320496, + "p95": 0.1509525179862976 + }, + { + "label": "completed_group_1", + "mean": 0.04646994173526764, + "median": 0.04543843865394592, + "p05": 0.032575178891420364, + "p95": 0.06306323409080505 + }, + { + "label": "completed_group_2", + "mean": 0.05750556290149689, + "median": 0.05649096518754959, + "p05": 0.03895717114210129, + "p95": 0.07940060645341873 + }, + { + "label": "completed_group_3", + "mean": 0.06549686938524246, + "median": 0.062135741114616394, + "p05": 0.04169235751032829, + "p95": 0.09948549419641495 + }, + { + "label": "completed_group_4", + "mean": 0.054404959082603455, + "median": 0.05241036415100098, + "p05": 0.03378590941429138, + "p95": 0.08157843351364136 + }, + { + "label": "completed_group_5", + "mean": 0.08427833020687103, + "median": 0.07576382160186768, + "p05": 0.041103728115558624, + "p95": 0.15204203128814697 + }, + { + "label": "completed_group_6", + "mean": 0.2333412617444992, + "median": 0.22215056419372559, + "p05": 0.10060560703277588, + "p95": 0.3983197808265686 + }, + { + "label": "current_group_partial", + "mean": 0.3661728501319885, + "median": 0.3673480749130249, + "p05": 0.16410952806472778, + "p95": 0.5635213255882263 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.9470824003219604, + "group": 7, + "latest_source_mass": 0.26050227880477905, + "layer": 27, + "max_source_mass": 0.26050227880477905, + "mixer_index": 52, + "normalized_entropy": 0.9363487077155573, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10804469138383865, + "median": 0.10379163920879364, + "p05": 0.05421917140483856, + "p95": 0.17028658092021942 + }, + { + "label": "completed_group_1", + "mean": 0.07850409299135208, + "median": 0.0776081532239914, + "p05": 0.063654825091362, + "p95": 0.09574449062347412 + }, + { + "label": "completed_group_2", + "mean": 0.07298290729522705, + "median": 0.07200999557971954, + "p05": 0.05533483251929283, + "p95": 0.09299935400485992 + }, + { + "label": "completed_group_3", + "mean": 0.08511961251497269, + "median": 0.08372659981250763, + "p05": 0.06449062377214432, + "p95": 0.11086944490671158 + }, + { + "label": "completed_group_4", + "mean": 0.09968686103820801, + "median": 0.09877584874629974, + "p05": 0.07606595754623413, + "p95": 0.1266063004732132 + }, + { + "label": "completed_group_5", + "mean": 0.11502514779567719, + "median": 0.11171717941761017, + "p05": 0.07302433997392654, + "p95": 0.1681303083896637 + }, + { + "label": "completed_group_6", + "mean": 0.18013443052768707, + "median": 0.1765173077583313, + "p05": 0.11514104157686234, + "p95": 0.2599799633026123 + }, + { + "label": "current_group_partial", + "mean": 0.26050227880477905, + "median": 0.2575443983078003, + "p05": 0.13687226176261902, + "p95": 0.39756521582603455 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.7855088710784912, + "group": 7, + "latest_source_mass": 0.28804832696914673, + "layer": 27, + "max_source_mass": 0.28804832696914673, + "mixer_index": 53, + "normalized_entropy": 0.858648264589397, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08493785560131073, + "median": 0.05951457470655441, + "p05": 0.02441210113465786, + "p95": 0.22703105211257935 + }, + { + "label": "completed_group_1", + "mean": 0.05972691997885704, + "median": 0.058127306401729584, + "p05": 0.03591413050889969, + "p95": 0.08903861790895462 + }, + { + "label": "completed_group_2", + "mean": 0.07017223536968231, + "median": 0.06906847655773163, + "p05": 0.04151676222681999, + "p95": 0.10273467004299164 + }, + { + "label": "completed_group_3", + "mean": 0.07963316142559052, + "median": 0.07547932863235474, + "p05": 0.042773112654685974, + "p95": 0.12874338030815125 + }, + { + "label": "completed_group_4", + "mean": 0.0645163506269455, + "median": 0.061876002699136734, + "p05": 0.0366181954741478, + "p95": 0.10098200291395187 + }, + { + "label": "completed_group_5", + "mean": 0.11586236953735352, + "median": 0.10182450711727142, + "p05": 0.048887670040130615, + "p95": 0.2313569337129593 + }, + { + "label": "completed_group_6", + "mean": 0.23710274696350098, + "median": 0.22610372304916382, + "p05": 0.10011275857686996, + "p95": 0.4084784686565399 + }, + { + "label": "current_group_partial", + "mean": 0.28804832696914673, + "median": 0.27438053488731384, + "p05": 0.07688213139772415, + "p95": 0.5529112815856934 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.973182201385498, + "group": 7, + "latest_source_mass": 0.23531918227672577, + "layer": 28, + "max_source_mass": 0.23531918227672577, + "mixer_index": 54, + "normalized_entropy": 0.9489000589030754, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.104981929063797, + "median": 0.09386023879051208, + "p05": 0.04212673753499985, + "p95": 0.18614700436592102 + }, + { + "label": "completed_group_1", + "mean": 0.0840599462389946, + "median": 0.08250434696674347, + "p05": 0.05170809105038643, + "p95": 0.12236068397760391 + }, + { + "label": "completed_group_2", + "mean": 0.09310521930456161, + "median": 0.0931512713432312, + "p05": 0.06374019384384155, + "p95": 0.12329407781362534 + }, + { + "label": "completed_group_3", + "mean": 0.11566174775362015, + "median": 0.11439734697341919, + "p05": 0.07886623591184616, + "p95": 0.1571493148803711 + }, + { + "label": "completed_group_4", + "mean": 0.0963074117898941, + "median": 0.09477876126766205, + "p05": 0.06686149537563324, + "p95": 0.13022682070732117 + }, + { + "label": "completed_group_5", + "mean": 0.1096896305680275, + "median": 0.10702179372310638, + "p05": 0.06579374521970749, + "p95": 0.1643676459789276 + }, + { + "label": "completed_group_6", + "mean": 0.16087493300437927, + "median": 0.15455034375190735, + "p05": 0.09876372665166855, + "p95": 0.24324838817119598 + }, + { + "label": "current_group_partial", + "mean": 0.23531918227672577, + "median": 0.22382870316505432, + "p05": 0.11656329780817032, + "p95": 0.38286590576171875 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.802741289138794, + "group": 7, + "latest_source_mass": 0.2497689425945282, + "layer": 28, + "max_source_mass": 0.2913346588611603, + "mixer_index": 55, + "normalized_entropy": 0.8669353059487718, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08190082013607025, + "median": 0.06415966153144836, + "p05": 0.039168842136859894, + "p95": 0.23471535742282867 + }, + { + "label": "completed_group_1", + "mean": 0.05632811039686203, + "median": 0.055233441293239594, + "p05": 0.03936203569173813, + "p95": 0.076178178191185 + }, + { + "label": "completed_group_2", + "mean": 0.07102417945861816, + "median": 0.06932097673416138, + "p05": 0.04159427434206009, + "p95": 0.1046731173992157 + }, + { + "label": "completed_group_3", + "mean": 0.08421063423156738, + "median": 0.0812690332531929, + "p05": 0.0486646331846714, + "p95": 0.12889066338539124 + }, + { + "label": "completed_group_4", + "mean": 0.06341387331485748, + "median": 0.061601825058460236, + "p05": 0.03807957097887993, + "p95": 0.09577237069606781 + }, + { + "label": "completed_group_5", + "mean": 0.10201875865459442, + "median": 0.09330011904239655, + "p05": 0.05116701498627663, + "p95": 0.18108239769935608 + }, + { + "label": "completed_group_6", + "mean": 0.2913346588611603, + "median": 0.28823426365852356, + "p05": 0.13295966386795044, + "p95": 0.4586564898490906 + }, + { + "label": "current_group_partial", + "mean": 0.2497689425945282, + "median": 0.234481081366539, + "p05": 0.09318265318870544, + "p95": 0.4530518054962158 + } + ], + "sources": 8 + }, + { + "branch": "attention", + "entropy_mean": 1.945815086364746, + "group": 8, + "latest_source_mass": 0.153213769197464, + "layer": 29, + "max_source_mass": 0.2451319396495819, + "mixer_index": 56, + "normalized_entropy": 0.9357392585284499, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.1064736545085907, + "median": 0.0881282389163971, + "p05": 0.029344500973820686, + "p95": 0.23970629274845123 + }, + { + "label": "completed_group_1", + "mean": 0.07925166189670563, + "median": 0.07691921293735504, + "p05": 0.062442928552627563, + "p95": 0.1030145138502121 + }, + { + "label": "completed_group_2", + "mean": 0.07823626697063446, + "median": 0.07715678215026855, + "p05": 0.059018202126026154, + "p95": 0.10055352747440338 + }, + { + "label": "completed_group_3", + "mean": 0.09447507560253143, + "median": 0.09318602085113525, + "p05": 0.0684189647436142, + "p95": 0.12381350249052048 + }, + { + "label": "completed_group_4", + "mean": 0.09724453091621399, + "median": 0.09628183394670486, + "p05": 0.0699525699019432, + "p95": 0.12776269018650055 + }, + { + "label": "completed_group_5", + "mean": 0.1459731012582779, + "median": 0.14035329222679138, + "p05": 0.08667999505996704, + "p95": 0.2258891463279724 + }, + { + "label": "completed_group_6", + "mean": 0.2451319396495819, + "median": 0.24505369365215302, + "p05": 0.16207969188690186, + "p95": 0.3307104706764221 + }, + { + "label": "completed_group_7", + "mean": 0.153213769197464, + "median": 0.13417893648147583, + "p05": 0.05588865652680397, + "p95": 0.30438560247421265 + } + ], + "sources": 8 + }, + { + "branch": "mlp", + "entropy_mean": 1.8717057704925537, + "group": 8, + "latest_source_mass": 0.3048541843891144, + "layer": 29, + "max_source_mass": 0.3048541843891144, + "mixer_index": 57, + "normalized_entropy": 0.8518500065030654, + "offset": 1, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.07336360216140747, + "median": 0.0611552819609642, + "p05": 0.024239415302872658, + "p95": 0.16916418075561523 + }, + { + "label": "completed_group_1", + "mean": 0.05720400810241699, + "median": 0.05402737483382225, + "p05": 0.0314149409532547, + "p95": 0.09488057345151901 + }, + { + "label": "completed_group_2", + "mean": 0.057025715708732605, + "median": 0.05547948181629181, + "p05": 0.03480810672044754, + "p95": 0.08415934443473816 + }, + { + "label": "completed_group_3", + "mean": 0.06655597686767578, + "median": 0.06265220046043396, + "p05": 0.035200610756874084, + "p95": 0.10949824750423431 + }, + { + "label": "completed_group_4", + "mean": 0.05032068490982056, + "median": 0.047904569655656815, + "p05": 0.02614513412117958, + "p95": 0.08244554698467255 + }, + { + "label": "completed_group_5", + "mean": 0.08329877257347107, + "median": 0.0731179416179657, + "p05": 0.02939312346279621, + "p95": 0.17248544096946716 + }, + { + "label": "completed_group_6", + "mean": 0.1901058554649353, + "median": 0.18234091997146606, + "p05": 0.08719424903392792, + "p95": 0.317789226770401 + }, + { + "label": "completed_group_7", + "mean": 0.11727118492126465, + "median": 0.09272899478673935, + "p05": 0.031939513981342316, + "p95": 0.2891285717487335 + }, + { + "label": "current_group_partial", + "mean": 0.3048541843891144, + "median": 0.29432767629623413, + "p05": 0.07913826406002045, + "p95": 0.5632759928703308 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.09096622467041, + "group": 8, + "latest_source_mass": 0.13795366883277893, + "layer": 30, + "max_source_mass": 0.17320571839809418, + "mixer_index": 58, + "normalized_entropy": 0.951639739623416, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10326676070690155, + "median": 0.09009090811014175, + "p05": 0.01757553033530712, + "p95": 0.2674753665924072 + }, + { + "label": "completed_group_1", + "mean": 0.08232800662517548, + "median": 0.08093428611755371, + "p05": 0.06430500745773315, + "p95": 0.10457758605480194 + }, + { + "label": "completed_group_2", + "mean": 0.07164021581411362, + "median": 0.0711187869310379, + "p05": 0.05190654471516609, + "p95": 0.09468771517276764 + }, + { + "label": "completed_group_3", + "mean": 0.08822821825742722, + "median": 0.08667590469121933, + "p05": 0.06148773431777954, + "p95": 0.12025904655456543 + }, + { + "label": "completed_group_4", + "mean": 0.10803373157978058, + "median": 0.10627314448356628, + "p05": 0.07042569667100906, + "p95": 0.1514367014169693 + }, + { + "label": "completed_group_5", + "mean": 0.13578416407108307, + "median": 0.12929600477218628, + "p05": 0.08227657526731491, + "p95": 0.21117588877677917 + }, + { + "label": "completed_group_6", + "mean": 0.17320571839809418, + "median": 0.1694163680076599, + "p05": 0.11322733014822006, + "p95": 0.24482715129852295 + }, + { + "label": "completed_group_7", + "mean": 0.09955951571464539, + "median": 0.08973086625337601, + "p05": 0.037010837346315384, + "p95": 0.1912725567817688 + }, + { + "label": "current_group_partial", + "mean": 0.13795366883277893, + "median": 0.1256815791130066, + "p05": 0.053015124052762985, + "p95": 0.2601332366466522 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.96881103515625, + "group": 8, + "latest_source_mass": 0.21570813655853271, + "layer": 30, + "max_source_mass": 0.21570813655853271, + "mixer_index": 59, + "normalized_entropy": 0.896044517007504, + "offset": 2, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08843819797039032, + "median": 0.08496083319187164, + "p05": 0.03690534085035324, + "p95": 0.1522325873374939 + }, + { + "label": "completed_group_1", + "mean": 0.06052382290363312, + "median": 0.058369796723127365, + "p05": 0.03549504280090332, + "p95": 0.09154202789068222 + }, + { + "label": "completed_group_2", + "mean": 0.0642135739326477, + "median": 0.06355445086956024, + "p05": 0.03948400169610977, + "p95": 0.09175601601600647 + }, + { + "label": "completed_group_3", + "mean": 0.07744920998811722, + "median": 0.07566337287425995, + "p05": 0.04564609378576279, + "p95": 0.11733324825763702 + }, + { + "label": "completed_group_4", + "mean": 0.051616813987493515, + "median": 0.049985647201538086, + "p05": 0.03238597884774208, + "p95": 0.07699409872293472 + }, + { + "label": "completed_group_5", + "mean": 0.06631231307983398, + "median": 0.061356961727142334, + "p05": 0.030705859884619713, + "p95": 0.1188126802444458 + }, + { + "label": "completed_group_6", + "mean": 0.16417551040649414, + "median": 0.15454982221126556, + "p05": 0.07227650284767151, + "p95": 0.28998225927352905 + }, + { + "label": "completed_group_7", + "mean": 0.2115624099969864, + "median": 0.19997704029083252, + "p05": 0.09131578356027603, + "p95": 0.3709705173969269 + }, + { + "label": "current_group_partial", + "mean": 0.21570813655853271, + "median": 0.2005728930234909, + "p05": 0.06839637458324432, + "p95": 0.4193759858608246 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.124063014984131, + "group": 8, + "latest_source_mass": 0.1258423924446106, + "layer": 31, + "max_source_mass": 0.17017140984535217, + "mixer_index": 60, + "normalized_entropy": 0.9667027380329118, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09220977127552032, + "median": 0.09152225404977798, + "p05": 0.03251190483570099, + "p95": 0.17071731388568878 + }, + { + "label": "completed_group_1", + "mean": 0.06839152425527573, + "median": 0.06702418625354767, + "p05": 0.04446800425648689, + "p95": 0.0977444052696228 + }, + { + "label": "completed_group_2", + "mean": 0.10039852559566498, + "median": 0.09736946225166321, + "p05": 0.06476961821317673, + "p95": 0.14545989036560059 + }, + { + "label": "completed_group_3", + "mean": 0.10902463644742966, + "median": 0.10749617218971252, + "p05": 0.07769118249416351, + "p95": 0.14509552717208862 + }, + { + "label": "completed_group_4", + "mean": 0.0849665105342865, + "median": 0.08355648815631866, + "p05": 0.059977494180202484, + "p95": 0.11480176448822021 + }, + { + "label": "completed_group_5", + "mean": 0.12137698382139206, + "median": 0.11852307617664337, + "p05": 0.07902064919471741, + "p95": 0.17207489907741547 + }, + { + "label": "completed_group_6", + "mean": 0.17017140984535217, + "median": 0.16817378997802734, + "p05": 0.11521546542644501, + "p95": 0.23236754536628723 + }, + { + "label": "completed_group_7", + "mean": 0.12761825323104858, + "median": 0.1237221211194992, + "p05": 0.062459878623485565, + "p95": 0.2092883139848709 + }, + { + "label": "current_group_partial", + "mean": 0.1258423924446106, + "median": 0.11723431199789047, + "p05": 0.06504593789577484, + "p95": 0.21421310305595398 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 1.9633058309555054, + "group": 8, + "latest_source_mass": 0.15763342380523682, + "layer": 31, + "max_source_mass": 0.23511825501918793, + "mixer_index": 61, + "normalized_entropy": 0.8935389906004497, + "offset": 3, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.0846441388130188, + "median": 0.06999644637107849, + "p05": 0.04030589759349823, + "p95": 0.20743247866630554 + }, + { + "label": "completed_group_1", + "mean": 0.05872901901602745, + "median": 0.05653517693281174, + "p05": 0.03710630163550377, + "p95": 0.08772030472755432 + }, + { + "label": "completed_group_2", + "mean": 0.07210008800029755, + "median": 0.07183568179607391, + "p05": 0.03840115666389465, + "p95": 0.10702595114707947 + }, + { + "label": "completed_group_3", + "mean": 0.07792744785547256, + "median": 0.0762966051697731, + "p05": 0.04050692170858383, + "p95": 0.122304767370224 + }, + { + "label": "completed_group_4", + "mean": 0.059186771512031555, + "median": 0.057271093130111694, + "p05": 0.03537197411060333, + "p95": 0.08958204090595245 + }, + { + "label": "completed_group_5", + "mean": 0.07445863634347916, + "median": 0.06928195059299469, + "p05": 0.03154807910323143, + "p95": 0.1351260244846344 + }, + { + "label": "completed_group_6", + "mean": 0.23511825501918793, + "median": 0.22703276574611664, + "p05": 0.0953572615981102, + "p95": 0.3982555568218231 + }, + { + "label": "completed_group_7", + "mean": 0.18020221590995789, + "median": 0.16846294701099396, + "p05": 0.07464539259672165, + "p95": 0.324230432510376 + }, + { + "label": "current_group_partial", + "mean": 0.15763342380523682, + "median": 0.12027859687805176, + "p05": 0.036883872002363205, + "p95": 0.4093724191188812 + } + ], + "sources": 9 + }, + { + "branch": "attention", + "entropy_mean": 2.1238560676574707, + "group": 8, + "latest_source_mass": 0.11516246944665909, + "layer": 32, + "max_source_mass": 0.1766587197780609, + "mixer_index": 62, + "normalized_entropy": 0.966608552245626, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.10231245309114456, + "median": 0.10458895564079285, + "p05": 0.025718798860907555, + "p95": 0.1875312179327011 + }, + { + "label": "completed_group_1", + "mean": 0.07575537264347076, + "median": 0.07447351515293121, + "p05": 0.05878028646111488, + "p95": 0.09651422500610352 + }, + { + "label": "completed_group_2", + "mean": 0.08897508680820465, + "median": 0.08751188218593597, + "p05": 0.06392349302768707, + "p95": 0.11800213158130646 + }, + { + "label": "completed_group_3", + "mean": 0.0967075377702713, + "median": 0.09579810500144958, + "p05": 0.06915527582168579, + "p95": 0.12801720201969147 + }, + { + "label": "completed_group_4", + "mean": 0.09801244735717773, + "median": 0.09654392302036285, + "p05": 0.07431379705667496, + "p95": 0.12754815816879272 + }, + { + "label": "completed_group_5", + "mean": 0.12438583374023438, + "median": 0.12010829150676727, + "p05": 0.08193086087703705, + "p95": 0.1824759691953659 + }, + { + "label": "completed_group_6", + "mean": 0.1766587197780609, + "median": 0.17520052194595337, + "p05": 0.12216726690530777, + "p95": 0.2341049611568451 + }, + { + "label": "completed_group_7", + "mean": 0.12203006446361542, + "median": 0.1200856938958168, + "p05": 0.05817953124642372, + "p95": 0.19647708535194397 + }, + { + "label": "current_group_partial", + "mean": 0.11516246944665909, + "median": 0.10430245846509933, + "p05": 0.05224763602018356, + "p95": 0.2294922173023224 + } + ], + "sources": 9 + }, + { + "branch": "mlp", + "entropy_mean": 2.0835556983947754, + "group": 8, + "latest_source_mass": 0.09660109132528305, + "layer": 32, + "max_source_mass": 0.22511571645736694, + "mixer_index": 63, + "normalized_entropy": 0.9482670637704002, + "offset": 4, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.09133295714855194, + "median": 0.08019149303436279, + "p05": 0.04441600665450096, + "p95": 0.19931957125663757 + }, + { + "label": "completed_group_1", + "mean": 0.07054752856492996, + "median": 0.06906121969223022, + "p05": 0.05099350959062576, + "p95": 0.09504306316375732 + }, + { + "label": "completed_group_2", + "mean": 0.09000501036643982, + "median": 0.0901019275188446, + "p05": 0.05998067185282707, + "p95": 0.11849517375230789 + }, + { + "label": "completed_group_3", + "mean": 0.10956105589866638, + "median": 0.1064089685678482, + "p05": 0.0759081020951271, + "p95": 0.1509535312652588 + }, + { + "label": "completed_group_4", + "mean": 0.07114207744598389, + "median": 0.06995616853237152, + "p05": 0.04991999268531799, + "p95": 0.09545011073350906 + }, + { + "label": "completed_group_5", + "mean": 0.11565887182950974, + "median": 0.11069735884666443, + "p05": 0.06978912651538849, + "p95": 0.17647549510002136 + }, + { + "label": "completed_group_6", + "mean": 0.22511571645736694, + "median": 0.22372552752494812, + "p05": 0.11895382404327393, + "p95": 0.33412501215934753 + }, + { + "label": "completed_group_7", + "mean": 0.13003569841384888, + "median": 0.12226496636867523, + "p05": 0.06884703785181046, + "p95": 0.21555939316749573 + }, + { + "label": "current_group_partial", + "mean": 0.09660109132528305, + "median": 0.09000182151794434, + "p05": 0.051288168877363205, + "p95": 0.16580010950565338 + } + ], + "sources": 9 + }, + { + "branch": "output", + "entropy_mean": 1.679484486579895, + "group": null, + "latest_source_mass": 0.2827349305152893, + "layer": null, + "max_source_mass": 0.32717907428741455, + "mixer_index": null, + "normalized_entropy": 0.7643663300981273, + "offset": null, + "source_summaries": [ + { + "label": "embedding", + "mean": 0.08993907272815704, + "median": 0.09290459752082825, + "p05": 0.022419869899749756, + "p95": 0.15114322304725647 + }, + { + "label": "completed_group_1", + "mean": 0.043493133038282394, + "median": 0.042318835854530334, + "p05": 0.026918282732367516, + "p95": 0.06356139481067657 + }, + { + "label": "completed_group_2", + "mean": 0.04922422394156456, + "median": 0.047891765832901, + "p05": 0.03095712885260582, + "p95": 0.07266643643379211 + }, + { + "label": "completed_group_3", + "mean": 0.037924449890851974, + "median": 0.03562554344534874, + "p05": 0.020909149199724197, + "p95": 0.06224166974425316 + }, + { + "label": "completed_group_4", + "mean": 0.03939433768391609, + "median": 0.036868683993816376, + "p05": 0.020651251077651978, + "p95": 0.06597590446472168 + }, + { + "label": "completed_group_5", + "mean": 0.03898489475250244, + "median": 0.03362073004245758, + "p05": 0.014276490546762943, + "p95": 0.08282073587179184 + }, + { + "label": "completed_group_6", + "mean": 0.09112587571144104, + "median": 0.07136045396327972, + "p05": 0.02230823040008545, + "p95": 0.22286513447761536 + }, + { + "label": "completed_group_7", + "mean": 0.32717907428741455, + "median": 0.3213679790496826, + "p05": 0.08271751552820206, + "p95": 0.6128557920455933 + }, + { + "label": "completed_group_8", + "mean": 0.2827349305152893, + "median": 0.2627606987953186, + "p05": 0.05262754485011101, + "p95": 0.571351170539856 + } + ], + "sources": 9 + } + ], + "interventions": { + "detached_learned": { + "statistics": { + "first_quartile_mean": 7.388767562588328e-05, + "first_to_last_ratio": 1.316792081076884, + "imbalance_abs_log_ratio": 0.2751985371595874, + "last_quartile_mean": 5.611187725662603e-05, + "mean": 8.168265617314319e-05, + "normalized": [ + 0.7533431897136043, + 0.7827542764436777, + 0.7759396123840813, + 0.7316445186865006, + 1.1365162577009855, + 1.1353975532420753, + 1.004206629730762, + 0.9167575048500237, + 1.0078390566063014, + 0.9220130731102486, + 0.8550917608474935, + 0.8013922543773514, + 0.7721726808717908, + 0.7274660564313035, + 0.7065479131389877, + 0.6779112969812525, + 1.6398021444587536, + 1.5049103837665865, + 1.3267399301434915, + 1.1597912607176666, + 2.3220162536772593, + 1.9302044249737353, + 1.584049170842242, + 1.3298950882483613, + 1.2567261692224045, + 0.9579394397382498, + 0.801457903329196, + 0.6752085554658249, + 0.5811903488324587, + 0.45264756329860717, + 0.3896296913930569, + 0.38079803677566654 + ], + "peak_layer": 21, + "peak_normalized": 2.3220162536772593, + "population_cv": 0.4315618932878077, + "rest_mean": 7.132743997283763e-05, + "spike_contrast": 1.9291428895694742, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001376008236547932, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.153507274575531e-05, + 6.39374484308064e-05, + 6.338080856949091e-05, + 5.976266766083427e-05, + 9.283366671297699e-05, + 9.274228796130046e-05, + 8.202626486308873e-05, + 7.488318806281313e-05, + 8.232297113863751e-05, + 7.531247683800757e-05, + 6.984616629779339e-05, + 6.54598479741253e-05, + 6.30731155979447e-05, + 5.942135976511054e-05, + 5.771271025878377e-05, + 5.537359538720921e-05, + 0.00013394339475780725, + 0.00012292507744859904, + 0.00010837164154509082, + 9.473483078181744e-05, + 0.0001896684552775696, + 0.00015766422438900918, + 0.0001293893437832594, + 0.00010862936323974282, + 0.00010265273158438504, + 7.824703789083287e-05, + 6.546521035488695e-05, + 5.515282828127965e-05, + 4.7473171434830874e-05, + 3.69734552805312e-05, + 3.1825988116906956e-05, + 3.1104595109354705e-05 + ] + }, + "learned": { + "statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "rest_mean": 7.697225831819836e-05, + "spike_contrast": 1.8814916302334765, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001448226597858593, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ] + }, + "uniform_value_backward": { + "statistics": { + "first_quartile_mean": 3.5575103993323864e-05, + "first_to_last_ratio": 2.443213468991071, + "imbalance_abs_log_ratio": 0.893314168315378, + "last_quartile_mean": 1.456078416595119e-05, + "mean": 2.3831100747884193e-05, + "normalized": [ + 1.614905261758675, + 1.6568148874352646, + 1.6283515751571318, + 1.4605379751384047, + 1.4717695441533916, + 1.4559185813526447, + 1.358376097224691, + 1.295738430337987, + 1.1776907075521434, + 1.1141057290944703, + 1.0640572841848701, + 1.0269673379730528, + 1.0041631647243947, + 0.9733861101490422, + 0.9597079874490758, + 0.9349675920893245, + 0.9158525986134215, + 0.9066797587654788, + 0.890357239209801, + 0.8734858422464332, + 0.8579185157197757, + 0.8414715045701692, + 0.8239471946120328, + 0.8048352542714806, + 0.7658980848440458, + 0.7173626211575344, + 0.6794957283273223, + 0.6347225670052176, + 0.5854538165358278, + 0.5328910383364602, + 0.49362355782826073, + 0.47854641218217414 + ], + "peak_layer": 2, + "peak_normalized": 1.6568148874352646, + "population_cv": 0.33383138748240165, + "rest_mean": 2.4630704299650257e-05, + "spike_contrast": 0.7922323832463255, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.951324156834744e-05, + "top_five_layers": [ + 2, + 3, + 1, + 5, + 4 + ], + "top_five_spike_overlap": 0 + }, + "values": [ + 3.848496999125928e-05, + 3.94837225030642e-05, + 3.880541044054553e-05, + 3.48062276316341e-05, + 3.507388828438707e-05, + 3.4696142392931506e-05, + 3.237159762647934e-05, + 3.087887307628989e-05, + 2.8065665901522152e-05, + 2.6550365873845294e-05, + 2.535765634092968e-05, + 2.4473762096022256e-05, + 2.393031354586128e-05, + 2.3196862457552925e-05, + 2.2870897737448104e-05, + 2.2281306883087382e-05, + 2.182577554776799e-05, + 2.1607176677207462e-05, + 2.1218193069216795e-05, + 2.081612910842523e-05, + 2.0445142581593245e-05, + 2.0053192201885395e-05, + 1.9635568605735898e-05, + 1.9180110029992647e-05, + 1.825219442253001e-05, + 1.7095540897571482e-05, + 1.6193131159525365e-05, + 1.5126137441257015e-05, + 1.3952008885098621e-05, + 1.2699380022240803e-05, + 1.176359273813432e-05, + 1.1404287761251908e-05 + ] + } + }, + "positions": { + "attention_branch_output": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0936665578119573e-05, + "first_to_last_ratio": 1.168787114339494, + "imbalance_abs_log_ratio": 0.1559665567011555, + "last_quartile_mean": 1.7913155716087203e-05, + "mean": 2.42622821247096e-05, + "normalized": [ + 0.7126129382274812, + 0.7437894782846245, + 0.7319401693963385, + 0.6938624658898646, + 1.0850314310673168, + 1.0981344950491732, + 0.9606310812212513, + 0.877442710157735, + 1.0089375010128527, + 0.8940805493086196, + 0.8303081011723588, + 0.7704924450335486, + 0.7457227037132441, + 0.6956545192296728, + 0.6629687198417477, + 0.6359294291467756, + 1.5989410181555679, + 1.4898579524033908, + 1.348196746912536, + 1.1824041858349024, + 2.2648751765532618, + 2.0078719511613548, + 1.669089976380829, + 1.384721107610681, + 1.3105185686643073, + 1.0955274221780484, + 0.8401582843314063, + 0.6990787108453305, + 0.7286997339090275, + 0.5032506445515866, + 0.392614310962981, + 0.33665547179218436 + ], + "peak_layer": 21, + "peak_normalized": 2.2648751765532618, + "population_cv": 0.43871254744666655, + "rest_mean": 2.099399386713264e-05, + "spike_contrast": 1.9963347127217863, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.191103871562518e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7289616152993403e-05, + 1.8046030163532123e-05, + 1.77585388883017e-05, + 1.6834686903166585e-05, + 2.6325338694732636e-05, + 2.6643248929758556e-05, + 2.330710231035482e-05, + 2.128876258211676e-05, + 2.4479126295773312e-05, + 2.169243452954106e-05, + 2.014516940107569e-05, + 1.869390507636126e-05, + 1.8092934624291956e-05, + 1.687816620687954e-05, + 1.608513412065804e-05, + 1.5429099221364595e-05, + 3.8793958083260804e-05, + 3.614735396695323e-05, + 3.271032983320765e-05, + 2.868782394216396e-05, + 5.49510405107867e-05, + 4.871555574936792e-05, + 4.0495931898476556e-05, + 3.35964941768907e-05, + 3.179617124260403e-05, + 2.6579995392239653e-05, + 2.0384157323860563e-05, + 1.696124490990769e-05, + 1.767991852830164e-05, + 1.2210009117552545e-05, + 9.52571917878231e-06, + 8.168030035449192e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.209008865378564e-05, + "first_to_last_ratio": 1.1473326374136463, + "imbalance_abs_log_ratio": 0.13743980256699795, + "last_quartile_mean": 7.154863897085306e-05, + "mean": 9.638685185109352e-05, + "normalized": [ + 0.7014672692081079, + 0.730745142012726, + 0.7197152784749434, + 0.6836365522681692, + 1.073826467208612, + 1.0803379033203697, + 0.9514146538604249, + 0.8722415119444402, + 1.0096074578076277, + 0.8908657485913869, + 0.8272757726491637, + 0.7688961382914924, + 0.7419179025068791, + 0.6953651992846211, + 0.6657440869339828, + 0.6349379036282153, + 1.6060213837194506, + 1.5018800747743897, + 1.362846846779478, + 1.1942969804897212, + 2.2656295070514414, + 2.0104706485090547, + 1.6756941049043201, + 1.3967091226783026, + 1.3149112210845808, + 1.1000992011684827, + 0.8430071951587476, + 0.7025118587919716, + 0.7359466516142374, + 0.5075845580254474, + 0.3944123577040649, + 0.33998329955514606 + ], + "peak_layer": 21, + "peak_normalized": 2.2656295070514414, + "population_cv": 0.4413317448728873, + "rest_mean": 8.330888886121102e-05, + "spike_contrast": 2.004682264754327, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016700785199645906, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761222175555304e-05, + 7.043422374408692e-05, + 6.93710899213329e-05, + 6.589357508346438e-05, + 0.00010350275260861963, + 0.00010413036943646148, + 9.17038632906042e-05, + 8.407261339016259e-05, + 9.731288446346298e-05, + 8.586774492869154e-05, + 7.973850733833387e-05, + 7.411147817038e-05, + 7.15111309546046e-05, + 6.70240624458529e-05, + 6.416897667804733e-05, + 6.119966565165669e-05, + 0.0001547993451822549, + 0.00014476149226538837, + 0.0001313605171162635, + 0.0001151145261246711, + 0.00021837689564563334, + 0.00019378293654881418, + 0.00016151487943716347, + 0.00013462439528666437, + 0.00012674015306401998, + 0.00010603509872453287, + 8.125480962917209e-05, + 6.771290645701811e-05, + 7.093558087944984e-05, + 4.8924477596301585e-05, + 3.801616549026221e-05, + 3.2769919926067814e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07279830891638994, + "first_to_last_ratio": 1.1473325978581586, + "imbalance_abs_log_ratio": 0.13743976809095573, + "last_quartile_mean": 0.06345004844479263, + "mean": 0.08547682355856523, + "normalized": [ + 0.701467243783159, + 0.7307451127750545, + 0.7197152636360212, + 0.6836365259909346, + 1.0738264172711294, + 1.080337898385664, + 0.951414616770227, + 0.8722414984608238, + 1.0096074000217996, + 0.8908656785314788, + 0.8272757305190417, + 0.7688961528411616, + 0.7419179123849904, + 0.6953651763038863, + 0.6657440954344426, + 0.6349378767127872, + 1.6060213451352348, + 1.5018801837842675, + 1.3628468172791657, + 1.19429705497508, + 2.2656295095184866, + 2.010470763800267, + 1.6756941843442625, + 1.3967091688883524, + 1.314911253304382, + 1.1000991448544435, + 0.8430072555120299, + 0.7025118281998033, + 0.7359466794525621, + 0.5075845660770223, + 0.3944123343969408, + 0.3399833106550978 + ], + "peak_layer": 21, + "peak_normalized": 2.2656295095184866, + "population_cv": 0.4413317644266601, + "rest_mean": 0.07387915439903736, + "spike_contrast": 2.0046823522109167, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1481042370200157, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05995919182896614, + 0.062461771070957184, + 0.06151897460222244, + 0.05843507871031761, + 0.09178727120161057, + 0.09234385192394257, + 0.08132389932870865, + 0.07455643266439438, + 0.08629803359508514, + 0.0761483684182167, + 0.07071290165185928, + 0.06572280079126358, + 0.06341678649187088, + 0.05943760648369789, + 0.05690569058060646, + 0.054272472858428955, + 0.13727760314941406, + 0.12837594747543335, + 0.11649181693792343, + 0.10208471864461899, + 0.19365881383419037, + 0.17184865474700928, + 0.1432330161333084, + 0.11938626319169998, + 0.11239443719387054, + 0.09403298050165176, + 0.0720575824379921, + 0.060048479586839676, + 0.06290638446807861, + 0.04338671639561653, + 0.03371311351656914, + 0.029060693457722664 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.120508836218505e-06, + "first_to_last_ratio": 0.6479745229308365, + "imbalance_abs_log_ratio": 0.43390389986753336, + "last_quartile_mean": 4.815789395706815e-06, + "mean": 5.18821034489747e-06, + "normalized": [ + 0.6392329797809361, + 0.5994300881226403, + 0.5850824179653978, + 0.6009191324861043, + 0.6062512409598932, + 0.6024392926484339, + 0.5897072336534216, + 0.5886297066862235, + 0.7270901470558813, + 0.6920316696984324, + 0.6762839200674405, + 0.6775578709931118, + 0.6128406906582687, + 0.6178016007991107, + 0.624771802695437, + 0.6621615984140956, + 1.5142793639941121, + 1.477704889855872, + 1.4442885378272925, + 1.3463135645628785, + 2.4904030104436794, + 2.3638034305258637, + 2.088000001391446, + 1.747233095035389, + 1.6663782972572876, + 1.4047981189097183, + 1.0886105900542489, + 0.8884508074141, + 0.8978880977348557, + 0.6001555244868991, + 0.4703385709663715, + 0.40912270685515684 + ], + "peak_layer": 21, + "peak_normalized": 2.4904030104436794, + "population_cv": 0.5663270603947061, + "rest_mean": 4.159058141373896e-06, + "spike_contrast": 2.5836696383318847, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0745632243924774e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.316475158499088e-06, + 3.109969384240685e-06, + 3.0355306535057025e-06, + 3.1176948596112197e-06, + 3.145358959955047e-06, + 3.125581770291319e-06, + 3.0595251701015513e-06, + 3.0539347335434286e-06, + 3.772296622628346e-06, + 3.5904058677260764e-06, + 3.508703230181709e-06, + 3.5153127555531682e-06, + 3.1795464110473404e-06, + 3.2052846563601634e-06, + 3.2414475299447076e-06, + 3.4354336548858555e-06, + 7.856399861339014e-06, + 7.666643796255812e-06, + 7.4932727329724e-06, + 6.984957963140914e-06, + 1.2920734661747701e-05, + 1.2263909411558416e-05, + 1.0832983207365032e-05, + 9.06501281860983e-06, + 8.645521120342892e-06, + 7.288388133019907e-06, + 5.647940724884393e-06, + 4.609469669958344e-06, + 4.658432317228289e-06, + 3.113733100690297e-06, + 2.4402154394920217e-06, + 2.12261466003838e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.9889651311095804e-05, + "first_to_last_ratio": 1.3326615656169287, + "imbalance_abs_log_ratio": 0.28717811970161894, + "last_quartile_mean": 4.493988035392249e-05, + "mean": 6.640704737037595e-05, + "normalized": [ + 0.7737542671823434, + 0.7960594489342157, + 0.7852142182091284, + 0.752976874377423, + 1.1002658507118863, + 1.0894970423039605, + 0.9958150050309817, + 0.921272183824516, + 1.0695745385738102, + 0.9470829860046872, + 0.8771170672423568, + 0.8087454440169668, + 0.8138319929245809, + 0.7582700648479997, + 0.7300932386851152, + 0.6932937290424763, + 1.5635426486844544, + 1.4635271461161115, + 1.3107699454980482, + 1.1481322979739153, + 2.32725688601457, + 1.991760393656834, + 1.584233429529925, + 1.2840444859649542, + 1.264803263909513, + 1.026616211913306, + 0.7846647444878888, + 0.6569396053772728, + 0.6050973350000657, + 0.42438660382108917, + 0.34340864002846966, + 0.3079524101111341 + ], + "peak_layer": 21, + "peak_normalized": 2.32725688601457, + "population_cv": 0.43279320435052254, + "rest_mean": 5.791654122596675e-05, + "spike_contrast": 1.9382335024498314, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011225578055018559, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.1382736273808405e-05, + 5.286395753500983e-05, + 5.2143757784506306e-05, + 5.000297096557915e-05, + 7.306540646823123e-05, + 7.23502816981636e-05, + 6.612913421122357e-05, + 6.117896555224434e-05, + 7.102728704921901e-05, + 6.289298471529037e-05, + 5.8246754633728415e-05, + 5.3706397011410445e-05, + 5.4044179705670103e-05, + 5.035447611589916e-05, + 4.8483336286153644e-05, + 4.6039589506108314e-05, + 0.00010383025073679164, + 9.718851651996374e-05, + 8.704436186235398e-05, + 7.624407589901239e-05, + 0.00015454625827260315, + 0.00013226692681200802, + 0.00010520426440052688, + 8.526960300514475e-05, + 8.399185026064515e-05, + 6.817455141572282e-05, + 5.2107268857071176e-05, + 4.362541949376464e-05, + 4.018272738903761e-05, + 2.8182261303300038e-05, + 2.2804753825766966e-05, + 2.0450210286071524e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.414140539665823e-05, + "first_to_last_ratio": 1.6936904784541074, + "imbalance_abs_log_ratio": 0.5269098631562702, + "last_quartile_mean": 2.606226223633712e-05, + "mean": 4.478222555803768e-05, + "normalized": [ + 0.874026545726401, + 0.9009891441910284, + 0.8916727093699365, + 0.8527000158254343, + 1.172545810112114, + 1.156435027941858, + 1.0621486326987206, + 0.9750045233682423, + 1.1460449634090804, + 1.011287374798366, + 0.9322859052145132, + 0.8551582509249812, + 0.907118809465358, + 0.8439303067882139, + 0.8153240377760277, + 0.7685666352135386, + 1.5553576798550997, + 1.437596357968014, + 1.2796321689041579, + 1.1209246602381584, + 2.385838207411615, + 1.9252114173862964, + 1.4016273255130902, + 1.0727508885758477, + 1.1785651553842893, + 0.9257194279728913, + 0.7253410162251672, + 0.6191416448731234, + 0.43450056827805394, + 0.29243769148345633, + 0.24680184853049614, + 0.23331524857642952 + ], + "peak_layer": 21, + "peak_normalized": 2.385838207411615, + "population_cv": 0.43498704004207234, + "rest_mean": 3.986614397204162e-05, + "spike_contrast": 1.7892140803093455, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.13290661224164e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.914085391443223e-05, + 4.034829908050597e-05, + 3.9931088394951075e-05, + 3.81858044420369e-05, + 5.2509210945572704e-05, + 5.178773426450789e-05, + 4.756537964567542e-05, + 4.366287248558365e-05, + 5.1322444051038474e-05, + 4.528769932221621e-05, + 4.1749837691895664e-05, + 3.829588968073949e-05, + 4.0622799133416265e-05, + 3.779307735385373e-05, + 3.6512024962576106e-05, + 3.441812441451475e-05, + 6.965237844269723e-05, + 6.437876436393708e-05, + 5.7304776419186965e-05, + 5.019750096835196e-05, + 0.00010684314474929124, + 8.621525194030255e-05, + 6.276799103943631e-05, + 4.804017225978896e-05, + 5.277877062326297e-05, + 4.1455776226939633e-05, + 3.2482384995091707e-05, + 2.7726540793082677e-05, + 1.9457902453723364e-05, + 1.3096010661683977e-05, + 1.1052336049033329e-05, + 1.0448376087879296e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015771267680975143, + "first_to_last_ratio": 1.0243014562120425, + "imbalance_abs_log_ratio": 0.024010874124910462, + "last_quartile_mean": 0.0001539709583084914, + "mean": 0.00019892108525709773, + "normalized": [ + 0.6590708750981731, + 0.6828508221536124, + 0.6699110125304764, + 0.6473590320765046, + 0.9801195462127644, + 0.9691430493294225, + 0.887896737937672, + 0.8463722857737056, + 0.988608068348346, + 0.8930841762598842, + 0.8341812095096794, + 0.7796271143503263, + 0.7202755659725629, + 0.6755421311480478, + 0.6527815148816499, + 0.632514433083119, + 1.5594662296068276, + 1.4738278124849744, + 1.3649033855466377, + 1.1834547507508266, + 2.341778688768703, + 2.0989567426400018, + 1.7777537161382369, + 1.4882782565844777, + 1.4127403878123095, + 1.1574678874298643, + 0.8851612824909503, + 0.7258447229726291, + 0.7423942540292697, + 0.5132421599816662, + 0.4020767269150468, + 0.35331542118163173 + ], + "peak_layer": 21, + "peak_normalized": 2.341778688768703, + "population_cv": 0.465450630495743, + "rest_mean": 0.00016857082745344896, + "spike_contrast": 2.1522850832359604, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00036281247739680114, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.0001311030937358737, + 0.000135833426611498, + 0.00013325942563824356, + 0.00012877336121164262, + 0.00019496644381433725, + 0.00019278298714198172, + 0.0001766213827067986, + 0.00016836129361763597, + 0.00019665498984977603, + 0.00017765327356755733, + 0.00016593623149674386, + 0.00015508427168242633, + 0.0001432779972674325, + 0.00013437957386486232, + 0.0001298520073760301, + 0.00012582045746967196, + 0.0003102107148151845, + 0.00029317542794160545, + 0.00027150806272402406, + 0.0002354141033720225, + 0.0004658291582018137, + 0.00041752675315365195, + 0.00035363269853405654, + 0.00029604992596432567, + 0.0002810238511301577, + 0.00023024476831778884, + 0.0001760772429406643, + 0.00014438582002185285, + 0.00014767787070013583, + 0.00010209468746325001, + 7.998153887456283e-05, + 7.028188701951876e-05 + ] + } + }, + "mlp_branch_output": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0824043303946382e-05, + "first_to_last_ratio": 1.424267056786447, + "imbalance_abs_log_ratio": 0.3536573352845024, + "last_quartile_mean": 1.4620883916904859e-05, + "mean": 2.2054508917790372e-05, + "normalized": [ + 0.7842104740080174, + 0.8183716989463029, + 0.8099804080045728, + 0.7638102866038118, + 1.1922163664937413, + 1.1967612607580402, + 1.039219827104321, + 0.9490935140140151, + 1.019003480899918, + 0.9331398258308763, + 0.8658119659177055, + 0.813018282797004, + 0.7774015040255573, + 0.7266621576498908, + 0.7030285425215664, + 0.6829301796443584, + 1.632275124747471, + 1.4994012377073724, + 1.316962971421436, + 1.1572238560225663, + 2.2864000936278734, + 1.8918326711874776, + 1.5430845130531994, + 1.2946151724909234, + 1.217184871626016, + 0.9287724220385267, + 0.7771760119482759, + 0.6578637330922696, + 0.5567850124802112, + 0.4325353313877837, + 0.37087941374039357, + 0.3623477882085046 + ], + "peak_layer": 21, + "peak_normalized": 2.2864000936278734, + "population_cv": 0.42456281874262314, + "rest_mean": 1.9413589850753648e-05, + "spike_contrast": 1.870621155539602, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6315471879788676e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7295376892434433e-05, + 1.8048785932478495e-05, + 1.7863720131572336e-05, + 1.6845460777403787e-05, + 2.6293746486771852e-05, + 2.6393981897854246e-05, + 2.2919482944416814e-05, + 2.0931791368639097e-05, + 2.247362135676667e-05, + 2.0579940610332415e-05, + 1.909505772346165e-05, + 1.793071896827314e-05, + 1.71452084032353e-05, + 1.6026177036110312e-05, + 1.5504949260503054e-05, + 1.5061689737194683e-05, + 3.599902629503049e-05, + 3.3068557968363166e-05, + 2.9044971597613767e-05, + 2.552200385252945e-05, + 5.042543125455268e-05, + 4.1723440517671406e-05, + 3.4031971154036e-05, + 2.855210186680779e-05, + 2.68444146058755e-05, + 2.048361966444645e-05, + 1.7140235286206007e-05, + 1.4508861568174325e-05, + 1.2279620023036841e-05, + 9.53935432335129e-06, + 8.179563337762374e-06, + 7.991402526386082e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.156287549354602e-05, + "first_to_last_ratio": 1.3967979973430424, + "imbalance_abs_log_ratio": 0.3341824723557829, + "last_quartile_mean": 5.839274945174111e-05, + "mean": 8.75740092851629e-05, + "normalized": [ + 0.7720751985061499, + 0.8035956764433113, + 0.7960739585260428, + 0.7524919125585545, + 1.1804156983562801, + 1.1778760010857563, + 1.0298326204965722, + 0.9385140198493713, + 1.0207128740101707, + 0.929812017339553, + 0.8640701821512353, + 0.8130222503970004, + 0.7736044339144612, + 0.7280851293348032, + 0.7061941984091654, + 0.6826155087046274, + 1.642189790371069, + 1.511740343745529, + 1.3334033796706957, + 1.1637276240936476, + 2.2882576030527964, + 1.895449319429852, + 1.5552551751392194, + 1.3067312409844312, + 1.222859749246748, + 0.9319757616478574, + 0.7801162715542103, + 0.6613743325062905, + 0.561952429248885, + 0.43456344744462927, + 0.3748431739999157, + 0.36656867778116903 + ], + "peak_layer": 21, + "peak_normalized": 2.2882576030527964, + "population_cv": 0.4268460433011898, + "rest_mean": 7.697251674943362e-05, + "spike_contrast": 1.8814776376552176, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00014482206897810103, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761372060282156e-05, + 7.037409523036331e-05, + 6.971538823563606e-05, + 6.589873373741284e-05, + 0.00010337373532820493, + 0.00010315132385585457, + 9.018657146953046e-05, + 8.21894354885444e-05, + 8.9387918706052e-05, + 8.142736623995006e-05, + 7.567009015474468e-05, + 7.119961810531095e-05, + 6.774764187866822e-05, + 6.376133387675509e-05, + 6.184425728861243e-05, + 5.977937689749524e-05, + 0.0001438131439499557, + 0.0001323891628999263, + 0.00011677147995214909, + 0.00010191229375777766, + 0.00020039189257659018, + 0.00016599209629930556, + 0.00013619993114843965, + 0.00011443569383118302, + 0.00010709073103498667, + 8.161685400409624e-05, + 6.831790960859507e-05, + 5.79192019358743e-05, + 4.92124272568617e-05, + 3.8056463381508365e-05, + 3.282651960034855e-05, + 3.2101888791657984e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07233077567070723, + "first_to_last_ratio": 1.3967980239296907, + "imbalance_abs_log_ratio": 0.33418249138977923, + "last_quartile_mean": 0.051783274626359344, + "mean": 0.07766150805400684, + "normalized": [ + 0.7720752144189068, + 0.8035957101529103, + 0.7960739453756324, + 0.7524919172501093, + 1.18041575335872, + 1.177876023895764, + 1.029832619521225, + 0.9385140290506606, + 1.020712886859698, + 0.9298120043210532, + 0.8640702058481301, + 0.8130222480432568, + 0.7736043957649873, + 0.7280851679574961, + 0.7061942119475418, + 0.6826155393692982, + 1.6421898574274953, + 1.51174026509585, + 1.333403346061099, + 1.1637275827667504, + 2.288257583675922, + 1.8954492273341264, + 1.5552551855736059, + 1.3067312459654492, + 1.2228597336334397, + 0.9319757582706437, + 0.7801162839815503, + 0.6613743139447658, + 0.5619524390301038, + 0.4345634639005874, + 0.3748431737420666, + 0.366568666461154 + ], + "peak_layer": 21, + "peak_normalized": 2.288257583675922, + "population_cv": 0.4268460335100709, + "rest_mean": 0.06825999867309022, + "spike_contrast": 1.8814776033915557, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12842965871095657, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05996052548289299, + 0.0624084547162056, + 0.06182430312037468, + 0.05843965709209442, + 0.0916728675365448, + 0.09147562831640244, + 0.07997835427522659, + 0.07288641482591629, + 0.07927010208368301, + 0.07221060246229172, + 0.0671049952507019, + 0.06314053386449814, + 0.06007928401231766, + 0.056544192135334015, + 0.05484410747885704, + 0.05301295220851898, + 0.12753494083881378, + 0.1174040287733078, + 0.10355411469936371, + 0.0903768390417099, + 0.17770953476428986, + 0.1472034454345703, + 0.12078346312046051, + 0.10148271918296814, + 0.09496913105249405, + 0.07237864285707474, + 0.060585007071495056, + 0.051363326609134674, + 0.0436420738697052, + 0.03374885395169258, + 0.02911088615655899, + 0.02846827544271946 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.100046171766735e-06, + "first_to_last_ratio": 0.7824902199643116, + "imbalance_abs_log_ratio": 0.24527385509409916, + "last_quartile_mean": 3.961769863281006e-06, + "mean": 4.656072917441634e-06, + "normalized": [ + 0.7040746582663436, + 0.6663619356936908, + 0.6544453195323515, + 0.6681125781418429, + 0.6743036280537461, + 0.6669931612309391, + 0.6466275673249758, + 0.6455375483541481, + 0.766065733134754, + 0.7325311317531802, + 0.713184552491017, + 0.7279868059896252, + 0.6487420214900163, + 0.6567739586893749, + 0.6698952555545961, + 0.731415182041189, + 1.5336327145648883, + 1.46016794598778, + 1.3941212250673451, + 1.3066200632810194, + 2.5346433175887912, + 2.2579967860286922, + 1.9597071198477958, + 1.7730016814327627, + 1.6211494054956397, + 1.2094753028064131, + 1.0219643022327503, + 0.8604311185936954, + 0.6846742153365659, + 0.5134546057775913, + 0.45040862343156707, + 0.44550053478491114 + ], + "peak_layer": 21, + "peak_normalized": 2.5346433175887912, + "population_cv": 0.5361380970640629, + "rest_mean": 3.768573976601517e-06, + "spike_contrast": 2.50719960829824, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.448567197978264e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.278222948210896e-06, + 3.1026297619973775e-06, + 3.0471451282210182e-06, + 3.110780880888342e-06, + 3.139606860713684e-06, + 3.1055687941261567e-06, + 3.0107451038929867e-06, + 3.0056698960834183e-06, + 3.566857913028798e-06, + 3.4107183637388516e-06, + 3.3206392799911555e-06, + 3.3895596516231308e-06, + 3.020590156666003e-06, + 3.0579874419345288e-06, + 3.119081156910397e-06, + 3.405522420507623e-06, + 7.140705747588072e-06, + 6.798648428230081e-06, + 6.491130079666618e-06, + 6.083718290028628e-06, + 1.1801484106399585e-05, + 1.0513397683098447e-05, + 9.124539246840868e-06, + 8.255225111497566e-06, + 7.548189842054853e-06, + 5.631405201711459e-06, + 4.7583403102180455e-06, + 4.006230028608115e-06, + 3.187893071299186e-06, + 2.3906820842967136e-06, + 2.097135393341887e-06, + 2.0742829747177893e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.958586689303047e-05, + "first_to_last_ratio": 1.5974368171613162, + "imbalance_abs_log_ratio": 0.46840035541466, + "last_quartile_mean": 3.730092248588335e-05, + "mean": 6.105391446453723e-05, + "normalized": [ + 0.841921083374255, + 0.8656847679595863, + 0.8583386074530126, + 0.8192653470430583, + 1.1963759198218682, + 1.175200963423096, + 1.067618556331515, + 0.9832339424318312, + 1.0816814073351606, + 0.984735398821402, + 0.9088666847085389, + 0.8483372799998844, + 0.8441732679586115, + 0.788240769992009, + 0.7677328825781261, + 0.7384848720107285, + 1.592019271873236, + 1.469773741071107, + 1.2839883253170383, + 1.1290190536252611, + 2.340759428283372, + 1.8604406810329672, + 1.4645591028269243, + 1.2019442625318217, + 1.1579689537744897, + 0.8703227892706042, + 0.7241054984123771, + 0.6218678661137057, + 0.4685539623066287, + 0.375211854314897, + 0.3379493629430677, + 0.3316240950598181 + ], + "peak_layer": 21, + "peak_normalized": 2.340759428283372, + "population_cv": 0.4213561912347865, + "rest_mean": 5.421209425549023e-05, + "spike_contrast": 1.8077099757766004, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.799974359339103e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.140257781022228e-05, + 5.2853443776257336e-05, + 5.240493192104623e-05, + 5.001935642212629e-05, + 7.30434330762364e-05, + 7.175061909947544e-05, + 6.518229201901704e-05, + 6.003028101986274e-05, + 6.604088412132114e-05, + 6.012195080984384e-05, + 5.5489868827862665e-05, + 5.179431173019111e-05, + 5.154008249519393e-05, + 4.812518454855308e-05, + 4.687309774453752e-05, + 4.508739220909774e-05, + 9.71990084508434e-05, + 8.973544026957825e-05, + 7.839251338737085e-05, + 6.893103272886947e-05, + 0.00014291252591647208, + 0.00011358718620613217, + 8.941706619225442e-05, + 7.338340219575912e-05, + 7.069853745633736e-05, + 5.3136613132664934e-05, + 4.420947516337037e-05, + 3.796746750595048e-05, + 2.860705353668891e-05, + 2.2908152459422126e-05, + 2.0633131498470902e-05, + 2.0246949134161696e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.3941625335719436e-05, + "first_to_last_ratio": 1.9900849942152503, + "imbalance_abs_log_ratio": 0.6881773484852566, + "last_quartile_mean": 2.2080275698499463e-05, + "mean": 4.1715027776945135e-05, + "normalized": [ + 0.9392326235815728, + 0.967237934106481, + 0.9603764045369728, + 0.9164866155582375, + 1.2577449343388738, + 1.232917566027963, + 1.1238782174568975, + 1.0291368181202014, + 1.1532132229581178, + 1.0411575328021039, + 0.9502116988067248, + 0.8875233874351255, + 0.938078657297939, + 0.8686978298587725, + 0.846151620562448, + 0.8110684117898503, + 1.5702254340289712, + 1.447397095355284, + 1.2505015110000162, + 1.0952804003622336, + 2.3931370003923895, + 1.7749904370362708, + 1.2920876407146809, + 1.0187689124624395, + 1.0659664997461884, + 0.8050885780431608, + 0.6754901061622123, + 0.6027145753846552, + 0.3199047099172539, + 0.266999929991275, + 0.2517914183349422, + 0.24654227582974492 + ], + "peak_layer": 21, + "peak_normalized": 2.3931370003923895, + "population_cv": 0.43040686028555775, + "rest_mean": 3.7783076651168005e-05, + "spike_contrast": 1.666025359377284, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.294756385614165e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.918011498171836e-05, + 4.034835728816688e-05, + 4.006212839158252e-05, + 3.8231264625210315e-05, + 5.2466864872258157e-05, + 5.143119051354006e-05, + 4.688261105911806e-05, + 4.293047095416114e-05, + 4.810632162843831e-05, + 4.343191540101543e-05, + 3.963810740970075e-05, + 3.70230627595447e-05, + 3.9131977246142924e-05, + 3.6237754102330655e-05, + 3.5297238355269656e-05, + 3.383374132681638e-05, + 6.550199759658426e-05, + 6.037821003701538e-05, + 5.216470526647754e-05, + 4.568965232465416e-05, + 9.98297764454037e-05, + 7.404377538478002e-05, + 5.3899471822660416e-05, + 4.2497973481658846e-05, + 4.446682214620523e-05, + 3.3584292395971715e-05, + 2.8178088541608304e-05, + 2.5142255253740586e-05, + 1.3344833860173821e-05, + 1.1137909496028442e-05, + 1.0503486009838525e-05, + 1.0284517884429079e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015718752911197953, + "first_to_last_ratio": 1.257989643214417, + "imbalance_abs_log_ratio": 0.22951492550542246, + "last_quartile_mean": 0.0001249513697985094, + "mean": 0.0001807773865039053, + "normalized": [ + 0.7253332250389443, + 0.7487212712295568, + 0.7410605164905312, + 0.7105636788603447, + 1.0746269026158506, + 1.051977329709352, + 0.9748117817952817, + 0.9289757254955786, + 1.003966745690328, + 0.9387623076773945, + 0.8790880476775305, + 0.8198592543393499, + 0.76275878383003, + 0.7097319103357375, + 0.6897181090899124, + 0.6787200572192026, + 1.6053206581539505, + 1.5070819773591158, + 1.3319581978459651, + 1.179141640369754, + 2.363943782302499, + 1.985198270384524, + 1.6420557999668541, + 1.4171107521036512, + 1.2788263571440766, + 0.9695939295226049, + 0.8072936975349638, + 0.6913356827376413, + 0.5679249559528513, + 0.4460061883008719, + 0.39061284360066933, + 0.37791961962508147 + ], + "peak_layer": 21, + "peak_normalized": 2.363943782302499, + "population_cv": 0.44512103777248124, + "rest_mean": 0.00015609032642613683, + "spike_contrast": 2.012216375705279, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.000314087510923855, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00013112384476698935, + 0.00013535187463276088, + 0.00013396698341239244, + 0.00012845384480897337, + 0.0001942682429216802, + 0.00019017371232621372, + 0.00017622392624616623, + 0.00016793780378066003, + 0.00018149448442272842, + 0.00016970699653029442, + 0.00015891923976596445, + 0.00014821201330050826, + 0.0001378895394736901, + 0.00012830347986891866, + 0.0001246854371856898, + 0.0001226972381118685, + 0.00029020567308180034, + 0.0002724463411141187, + 0.0002407879219390452, + 0.00021316214406397194, + 0.00042734757880680263, + 0.00035887895501218736, + 0.0002968465560115874, + 0.0002561815781518817, + 0.00023118288663681597, + 0.00017528065654914826, + 0.00014594044478144497, + 0.00012497785792220384, + 0.00010266798926750198, + 8.062783308560029e-05, + 7.061396900098771e-05, + 6.831932114437222e-05 + ] + } + }, + "post_attention_state": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0937054614478257e-05, + "first_to_last_ratio": 1.1688073264187404, + "imbalance_abs_log_ratio": 0.15598384976011784, + "last_quartile_mean": 1.791317879451526e-05, + "mean": 2.4262353377935142e-05, + "normalized": [ + 0.7126199170124646, + 0.7437761981288968, + 0.7319360705844501, + 0.6938843441336064, + 1.08510119201235, + 1.0981398918115777, + 0.9606406303851301, + 0.877454527863037, + 1.0089327386637434, + 0.8940806225700821, + 0.830291493091461, + 0.7704822352676253, + 0.7457403811840673, + 0.695661023016061, + 0.6629855157677164, + 0.6359363332477325, + 1.598966161144301, + 1.4898256875513118, + 1.3481551517699815, + 1.1823992139387995, + 2.2648560798112616, + 2.007845512240135, + 1.6690690306890001, + 1.384726637362728, + 1.3105208676356837, + 1.0955359004357528, + 0.840135724559922, + 0.6990787570111011, + 0.728702616979868, + 0.5032438811110385, + 0.39261683155295724, + 0.3366588314661564 + ], + "peak_layer": 21, + "peak_normalized": 2.2648560798112616, + "population_cv": 0.43870656486100834, + "rest_mean": 2.0994108227670142e-05, + "spike_contrast": 1.9963161442660275, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.1910877189366144e-05, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7289836250711232e-05, + 1.8045760953100398e-05, + 1.7758491594577208e-05, + 1.6835267160786316e-05, + 2.6327108571422286e-05, + 2.6643458113539964e-05, + 2.3307402443606406e-05, + 2.128911182808224e-05, + 2.4479082640027627e-05, + 2.1692500013159588e-05, + 2.014482561207842e-05, + 1.8693712263484485e-05, + 1.8093416656483896e-05, + 1.6878373571671546e-05, + 1.6085588868008927e-05, + 1.542931204312481e-05, + 3.8794682041043416e-05, + 3.6146677302895114e-05, + 3.270941670052707e-05, + 2.8687787562375888e-05, + 5.495073855854571e-05, + 4.871505734627135e-05, + 4.04955426347442e-05, + 3.3596727007534355e-05, + 3.179632039973512e-05, + 2.6580279154586606e-05, + 2.0383669834700413e-05, + 1.696129584161099e-05, + 1.768004040059168e-05, + 1.2209880878799595e-05, + 9.525808309263084e-06, + 8.168135536834598e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.209185853047529e-05, + "first_to_last_ratio": 1.1473585117140146, + "imbalance_abs_log_ratio": 0.13746235401177884, + "last_quartile_mean": 7.154856803026632e-05, + "mean": 9.638741801154538e-05, + "normalized": [ + 0.701459601059617, + 0.7307291493400122, + 0.7197289413331719, + 0.6836538994288254, + 1.0739127818136336, + 1.080318121017968, + 0.9514366180464413, + 0.8722525427024255, + 1.0096073400424832, + 0.8908667057242938, + 0.8272733289673383, + 0.7688963021177191, + 0.7419276606217693, + 0.6953759857067083, + 0.6657591236204922, + 0.6349357593481342, + 1.6060262927227642, + 1.5018600810151583, + 1.362811062623623, + 1.194289437021534, + 2.265619973525555, + 2.010448271283051, + 1.6756913579512287, + 1.3967040891245648, + 1.314898968363092, + 1.100096740194715, + 0.8429885804297745, + 0.7025081852971004, + 0.7359520665807775, + 0.5075723672109625, + 0.3944131736981434, + 0.33998549206692247 + ], + "peak_layer": 21, + "peak_normalized": 2.265619973525555, + "population_cv": 0.44132695283622186, + "rest_mean": 8.330956363890975e-05, + "spike_contrast": 2.0046657830023333, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00016700783162377776, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761187978554517e-05, + 7.043309597065672e-05, + 6.937281432328746e-05, + 6.58956341794692e-05, + 0.00010351168020861223, + 0.00010412907431600615, + 9.170651901513338e-05, + 8.407417044509202e-05, + 9.731344471219927e-05, + 8.58683415572159e-05, + 7.973874016897753e-05, + 7.411192927975208e-05, + 7.151249155867845e-05, + 6.70254958095029e-05, + 6.417080294340849e-05, + 6.119981844676659e-05, + 0.0001548007276142016, + 0.00014476041542366147, + 0.0001313578395638615, + 0.0001151144751929678, + 0.00021837725944351405, + 0.0001937819179147482, + 0.0001615155633771792, + 0.00013462470087688416, + 0.00012673971650656313, + 0.00010603548435028642, + 8.125349268084392e-05, + 6.771295011276379e-05, + 7.093651947798207e-05, + 4.8923589929472655e-05, + 3.80164674425032e-05, + 3.27703237417154e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07279988285154104, + "first_to_last_ratio": 1.147358527887077, + "imbalance_abs_log_ratio": 0.13746236810768883, + "last_quartile_mean": 0.06344998627901077, + "mean": 0.08547732653096318, + "normalized": [ + 0.7014596295781764, + 0.7307291764187523, + 0.7197289409168642, + 0.6836539457276586, + 1.0739127543434965, + 1.0803181180576646, + 0.9514366494990735, + 0.8722525785288534, + 1.0096072992260057, + 0.8908667122670901, + 0.8272733032016506, + 0.7688963353200844, + 0.7419276673674994, + 0.6953759896953364, + 0.6657591362695728, + 0.6349357531030216, + 1.6060263641500938, + 1.5018600149298467, + 1.3628109924626157, + 1.1942894172476932, + 2.2656200131573585, + 2.010448125243662, + 1.6756912972456086, + 1.3967041753456768, + 1.3148989834510798, + 1.10009676829185, + 0.8429886102104137, + 0.7025081738287764, + 0.7359520241866415, + 0.5075723834629702, + 0.3944131950658124, + 0.3399854721990999 + ], + "peak_layer": 21, + "peak_normalized": 2.2656200131573585, + "population_cv": 0.44132694138829454, + "rest_mean": 0.07387975465368342, + "spike_contrast": 2.0046657621228294, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.14810421466827392, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.059958893805742264, + 0.062460776418447495, + 0.06152050569653511, + 0.058436911553144455, + 0.0917951911687851, + 0.09234270453453064, + 0.08132626116275787, + 0.0745578184723854, + 0.08629853278398514, + 0.07614890486001968, + 0.070713110268116, + 0.06572320312261581, + 0.06341799348592758, + 0.059438880532979965, + 0.05690731108188629, + 0.05427261069417, + 0.13727883994579315, + 0.12837497889995575, + 0.11648944020271301, + 0.10208466649055481, + 0.19365914165973663, + 0.17184773087501526, + 0.14323361217975616, + 0.11938653886318207, + 0.1123940497636795, + 0.09403333067893982, + 0.07205641269683838, + 0.06004852056503296, + 0.06290721148252487, + 0.043385930359363556, + 0.033713385462760925, + 0.029061049222946167 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.1205255481836502e-06, + "first_to_last_ratio": 0.6479717368101099, + "imbalance_abs_log_ratio": 0.4339081996148503, + "last_quartile_mean": 4.8158358936234436e-06, + "mean": 5.188215141060937e-06, + "normalized": [ + 0.6392337912530196, + 0.5994569684566948, + 0.58506750248592, + 0.6009069195183814, + 0.6062570789746512, + 0.60249307876806, + 0.5896500227462853, + 0.5886480511239991, + 0.7270810166784253, + 0.6920427312432549, + 0.6762719442051294, + 0.6775019374497203, + 0.6128892519841179, + 0.6178328466527419, + 0.6247970380766965, + 0.662130571719626, + 1.514294792953365, + 1.477854106612226, + 1.4442983342347777, + 1.3462892680196958, + 2.490378094515379, + 2.363646805945382, + 2.0879100705160045, + 1.7472602290519488, + 1.6663413461773389, + 1.404838629343315, + 1.0886161574596454, + 0.888478735318369, + 0.8979335469262955, + 0.6001271407896346, + 0.4703453234739583, + 0.40912666732594116 + ], + "peak_layer": 21, + "peak_normalized": 2.490378094515379, + "population_cv": 0.5663116287539589, + "rest_mean": 4.159116037448661e-06, + "spike_contrast": 2.583565883667621, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.0745350300567224e-05, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.3164824344567023e-06, + 3.1101117201615125e-06, + 3.035456074940157e-06, + 3.117634378213552e-06, + 3.1453921565116616e-06, + 3.1258637136488687e-06, + 3.0592311759392032e-06, + 3.0540327315975446e-06, + 3.7722527395089855e-06, + 3.5904665764974197e-06, + 3.508644340399769e-06, + 3.515025809974759e-06, + 3.1798012969375122e-06, + 3.205449729648535e-06, + 3.2415814530395437e-06, + 3.435275857555098e-06, + 7.856487172830384e-06, + 7.667425052204635e-06, + 7.493330485885963e-06, + 6.984838364587631e-06, + 1.2920617336931173e-05, + 1.2263108146726154e-05, + 1.0832526641024742e-05, + 9.06516197574092e-06, + 8.645337402413134e-06, + 7.28860504750628e-06, + 5.647974830935709e-06, + 4.6096188270894345e-06, + 4.658672423829557e-06, + 3.1135887184063904e-06, + 2.4402527287747944e-06, + 2.122637170032249e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.989002966089174e-05, + "first_to_last_ratio": 1.332672418740526, + "imbalance_abs_log_ratio": 0.2871862636142851, + "last_quartile_mean": 4.4939798272025655e-05, + "mean": 6.640715508865469e-05, + "normalized": [ + 0.7737597503853951, + 0.7960652246538951, + 0.7852109175536554, + 0.752979652136534, + 1.1002854313277015, + 1.0894969185314263, + 0.9958159097459445, + 0.9212749625084216, + 1.0695780627909957, + 0.9470852845577153, + 0.8771168497054519, + 0.8087458852142236, + 0.8138331380491499, + 0.758274367943245, + 0.7300990118419535, + 0.6932937549003495, + 1.563540331611732, + 1.463522142564802, + 1.3107588349120618, + 1.1481313121287318, + 2.3272535492615467, + 1.9917532184740583, + 1.5842339276075041, + 1.284041417038419, + 1.2647947479005226, + 1.026614217953409, + 0.784658048181791, + 0.6569384849801976, + 0.6050998048035263, + 0.4243821080145578, + 0.34340906908209917, + 0.3079536636389831 + ], + "peak_layer": 21, + "peak_normalized": 2.3272535492615467, + "population_cv": 0.432791558992841, + "rest_mean": 5.791668829462853e-05, + "spike_contrast": 1.938226771623043, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011225567577639594, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.138318374520168e-05, + 5.2864426834275946e-05, + 5.2143623179290444e-05, + 5.000323653803207e-05, + 7.3066825279966e-05, + 7.235039083752781e-05, + 6.61293015582487e-05, + 6.117924931459129e-05, + 7.102763629518449e-05, + 6.289323937380686e-05, + 5.824683466926217e-05, + 5.370651342673227e-05, + 5.404434341471642e-05, + 5.035484355175868e-05, + 4.8483798309462145e-05, + 4.603966590366326e-05, + 0.00010383026528870687, + 9.7188341896981e-05, + 8.704376523382962e-05, + 7.62441341066733e-05, + 0.0001545462873764336, + 0.0001322666648775339, + 0.00010520446812734008, + 8.526953752152622e-05, + 8.399142097914591e-05, + 6.817452958784997e-05, + 5.2106908697169274e-05, + 4.362541585578583e-05, + 4.0182956581702456e-05, + 2.8182008463772945e-05, + 2.2804819309385493e-05, + 2.045032670139335e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.413846727402415e-05, + "first_to_last_ratio": 1.6934944304798134, + "imbalance_abs_log_ratio": 0.5267941045072542, + "last_quartile_mean": 2.6063544396492944e-05, + "mean": 4.4781047250808115e-05, + "normalized": [ + 0.8740739154412697, + 0.9009770250766957, + 0.891219784649098, + 0.852663960345562, + 1.1726907227894316, + 1.1562245824076949, + 1.0625075493400142, + 0.9748474712247537, + 1.1459503353768645, + 1.011393273941681, + 0.9321659114804322, + 0.8550679923070413, + 0.9071127008648373, + 0.8438105066025905, + 0.8153663695855309, + 0.7686703721764182, + 1.5553617227891403, + 1.437584791446802, + 1.2800134621565402, + 1.1209839695222465, + 2.385210126491363, + 1.925331940523583, + 1.4016057137556681, + 1.0729916373601829, + 1.178493073924358, + 0.9257430549345239, + 0.725497883628933, + 0.6192210996384143, + 0.4345528238561406, + 0.29242012088135483, + 0.2468894193108289, + 0.23335668617000463 + ], + "peak_layer": 21, + "peak_normalized": 2.385210126491363, + "population_cv": 0.4349311871982457, + "rest_mean": 3.986569293003952e-05, + "spike_contrast": 1.7891062550480503, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.132396058295853e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.9141945308074355e-05, + 4.034669473185204e-05, + 3.990975528722629e-05, + 3.818318509729579e-05, + 5.251431866781786e-05, + 5.177694765734486e-05, + 4.758020077133551e-05, + 4.36546906712465e-05, + 5.131685611559078e-05, + 4.5291249989531934e-05, + 4.174336572759785e-05, + 3.829084016615525e-05, + 4.062145671923645e-05, + 3.7786718166898936e-05, + 3.651295992312953e-05, + 3.442186425672844e-05, + 6.965072680031881e-05, + 6.437655247282237e-05, + 5.732034333050251e-05, + 5.019883610657416e-05, + 0.00010681220737751573, + 8.621838060207665e-05, + 6.27653716946952e-05, + 4.804968921234831e-05, + 5.277415402815677e-05, + 4.145574348513037e-05, + 3.2488555007148534e-05, + 2.772936932160519e-05, + 1.9459730538073927e-05, + 1.3094879250274971e-05, + 1.1055966751882806e-05, + 1.0449956789670978e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015772300503158476, + "first_to_last_ratio": 1.024373509074145, + "imbalance_abs_log_ratio": 0.02408121506561912, + "last_quartile_mean": 0.00015397021070384653, + "mean": 0.0001989226409477851, + "normalized": [ + 0.6591656486528042, + 0.6829581384685631, + 0.6699127961863572, + 0.6472837418583299, + 0.9802229283334125, + 0.9691875554589603, + 0.8879032543387678, + 0.8464550603982911, + 0.9884366921662843, + 0.8930886037902819, + 0.8342806121981681, + 0.779517431659101, + 0.7202640807077498, + 0.675490688069157, + 0.6529307639307617, + 0.6324384542682998, + 1.559547816640635, + 1.474378252555848, + 1.3651032473841427, + 1.1832746816977402, + 2.3416687863410064, + 2.09906600553135, + 1.7771997929015884, + 1.4880606167336252, + 1.412667012460112, + 1.1576278202661294, + 0.8849667940826665, + 0.7258688199763643, + 0.7424188068351703, + 0.5131968143231243, + 0.4021086961884269, + 0.3533095855967808 + ], + "peak_layer": 21, + "peak_normalized": 2.3416687863410064, + "population_cv": 0.4654332615698335, + "rest_mean": 0.0001685783755857739, + "spike_contrast": 2.152005989155233, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0003627816739026457, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 24 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 0.00013112297165207565, + 0.00013585583656094968, + 0.00013326082262210548, + 0.0001287593913730234, + 0.00019498853362165391, + 0.00019279334810562432, + 0.00017662406025920063, + 0.00016837907605804503, + 0.00019662243721541017, + 0.00017765554366633296, + 0.00016595730266999453, + 0.00015506366617046297, + 0.00014327683311421424, + 0.00013437039160635322, + 0.00012988271191716194, + 0.00012580632755998522, + 0.00031022937037050724, + 0.00029328721575438976, + 0.0002715499431360513, + 0.00023538012464996427, + 0.0004658109392039478, + 0.0004175517533440143, + 0.00035352527629584074, + 0.0002960089477710426, + 0.00028101145289838314, + 0.00023027838324196637, + 0.00017603993183001876, + 0.0001443917426513508, + 0.0001476839097449556, + 0.00010208646563114598, + 7.998852379387245e-05, + 7.028127583907917e-05 + ] + } + }, + "post_mlp_state": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 2.0823702925554244e-05, + "first_to_last_ratio": 1.4242510302785252, + "imbalance_abs_log_ratio": 0.35364608276130804, + "last_quartile_mean": 1.4620809452026151e-05, + "mean": 2.2054440051988422e-05, + "normalized": [ + 0.7841928807641645, + 0.8183408510588258, + 0.809986731151413, + 0.7637938668182616, + 1.192276091281044, + 1.1966830153099992, + 1.039215154291241, + 0.9490753634193776, + 1.0189872806238243, + 0.9331490903420598, + 0.8658120301757168, + 0.81301306861575, + 0.7774052511262896, + 0.7266654164061733, + 0.7030327996851689, + 0.6829495498627286, + 1.6322510246428148, + 1.499422250137046, + 1.3169815172043178, + 1.1572342326306069, + 2.2864238933994527, + 1.8918479809103845, + 1.543069371898747, + 1.294657154506133, + 1.2171810844243176, + 0.9287685590307506, + 0.7771677166674748, + 0.6578712720326597, + 0.5567773486562734, + 0.43254113576640124, + 0.3708732313504376, + 0.3623537858101448 + ], + "peak_layer": 21, + "peak_normalized": 2.2864238933994527, + "population_cv": 0.4245664907667088, + "rest_mean": 1.9413478454495175e-05, + "spike_contrast": 1.8706401721656991, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 3.6315632678451947e-05, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.729493487800937e-05, + 1.804804924177006e-05, + 1.78638038050849e-05, + 1.684504604781978e-05, + 2.629498158057686e-05, + 2.639217382238712e-05, + 2.2919308321434073e-05, + 2.093132570735179e-05, + 2.2473193894256838e-05, + 2.0580080672516488e-05, + 1.9094999515800737e-05, + 1.7930547983269207e-05, + 1.7145237507065758e-05, + 1.6026198863983154e-05, + 1.5504994735238142e-05, + 1.5062069905980024e-05, + 3.5998382372781634e-05, + 3.306891812826507e-05, + 2.9045289920759387e-05, + 2.5522153009660542e-05, + 5.0425798690412194e-05, + 4.172364788246341e-05, + 3.403153095860034e-05, + 2.855293860193342e-05, + 2.6844247258850373e-05, + 2.048347050731536e-05, + 1.7139998817583546e-05, + 1.450898253096966e-05, + 1.2279412658244837e-05, + 9.539452548779082e-06, + 8.179401447705459e-06, + 7.991509846760891e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 8.156212697940646e-05, + "first_to_last_ratio": 1.3967870279562358, + "imbalance_abs_log_ratio": 0.33417461908716395, + "last_quartile_mean": 5.839267214469146e-05, + "mean": 8.757388354752038e-05, + "normalized": [ + 0.7720675832605176, + 0.803585946279625, + 0.7960780094481321, + 0.752473219070419, + 1.180467409539882, + 1.1778079020041263, + 1.029829529518624, + 0.9385078067458692, + 1.0206984705622164, + 0.9298229900567075, + 0.8640759923745102, + 0.8130195958778703, + 0.7736070401530517, + 0.7280910766467854, + 0.7061969571118035, + 0.6826252956632183, + 1.642168884788434, + 1.5117445082936212, + 1.3334176736203447, + 1.1637251407789435, + 2.2882673690329693, + 1.8954517085610634, + 1.5552436162725376, + 1.306771834154562, + 1.2228641636923565, + 0.9319785952743014, + 0.7801153976292595, + 0.661371584878177, + 0.561946672486053, + 0.4345672285647456, + 0.37483694087850516, + 0.36657385678076787 + ], + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "rest_mean": 7.697225831819836e-05, + "spike_contrast": 1.8814916302334765, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001448226597858593, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 6.761295662727207e-05, + 7.037314207991585e-05, + 6.971564289415255e-05, + 6.589700205950066e-05, + 0.00010337811545468867, + 0.00010314521205145866, + 9.018617129186168e-05, + 8.218877337640151e-05, + 8.93865289981477e-05, + 8.14282102510333e-05, + 7.567049033241346e-05, + 7.11992834112607e-05, + 6.774777284590527e-05, + 6.376176315825433e-05, + 6.184441008372232e-05, + 5.9780148149002343e-05, + 0.00014381110668182373, + 0.00013238933752290905, + 0.0001167725640698336, + 0.00010191192995989695, + 0.0002003924601012841, + 0.0001659920671954751, + 0.0001361987233394757, + 0.00011443908442743123, + 0.00010709096386563033, + 8.16169849713333e-05, + 6.831773498561233e-05, + 5.791887815576047e-05, + 4.921185245621018e-05, + 3.8056739867897704e-05, + 3.282592660980299e-05, + 3.210229624528438e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.07233011117205024, + "first_to_last_ratio": 1.3967870631582553, + "imbalance_abs_log_ratio": 0.3341746442893017, + "last_quartile_mean": 0.051783205242827535, + "mean": 0.07766139676095918, + "normalized": [ + 0.7720675426341356, + 0.8035859729306402, + 0.7960780122636479, + 0.7524732326455996, + 1.1804674279966365, + 1.1778079658527474, + 1.0298294903648106, + 0.9385077949910019, + 1.020698424099846, + 0.9298230264026498, + 0.8640760490713295, + 0.8130195756795249, + 0.773607039370292, + 0.7280910561484961, + 0.7061969508238138, + 0.6826252958058289, + 1.64216899408666, + 1.511744542110794, + 1.3334176327397504, + 1.1637251251728324, + 2.288267386572505, + 1.8954517517435265, + 1.5552435035136007, + 1.3067718770227776, + 1.2228641722871045, + 0.9319785328940787, + 0.7801153392913037, + 0.6613715681663117, + 0.5619466726726858, + 0.43456725256605405, + 0.37483694737332934, + 0.36657384470568466 + ], + "peak_layer": 21, + "peak_normalized": 2.288267386572505, + "population_cv": 0.4268470914878033, + "rest_mean": 0.0682597693607763, + "spike_contrast": 1.8814916300573643, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.12843018472194673, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.05995984375476837, + 0.062407609075307846, + 0.061824530363082886, + 0.058438122272491455, + 0.0916767492890358, + 0.09147021174430847, + 0.07997799664735794, + 0.07288582623004913, + 0.07926886528730392, + 0.072211354970932, + 0.06710535287857056, + 0.06314023584127426, + 0.06007940322160721, + 0.05654456838965416, + 0.054844241589307785, + 0.0530136339366436, + 0.12753313779830933, + 0.11740419268608093, + 0.10355507582426071, + 0.09037651866674423, + 0.17771004140377045, + 0.14720343053340912, + 0.12078238278627396, + 0.1014857292175293, + 0.09496933966875076, + 0.07237875461578369, + 0.06058484688401222, + 0.05136303976178169, + 0.04364156350493431, + 0.03374909982085228, + 0.029110360890626907, + 0.02846863679587841 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 3.1000530213987076e-06, + "first_to_last_ratio": 0.7825005490137864, + "imbalance_abs_log_ratio": 0.2452606549530124, + "last_quartile_mean": 3.961726321222159e-06, + "mean": 4.656059921614997e-06, + "normalized": [ + 0.7040313055728622, + 0.6663442620495864, + 0.6544029514954961, + 0.6681637652166335, + 0.6743674803950376, + 0.6670078174062475, + 0.6466284443239834, + 0.6455370061269436, + 0.7660325155889827, + 0.7325145706427032, + 0.7131598309497137, + 0.7280186266136806, + 0.64870437442493, + 0.656793469733594, + 0.6699169519151534, + 0.7314456448845376, + 1.53365232903732, + 1.4601427212016398, + 1.3940973786195396, + 1.306558663485338, + 2.534675981169938, + 2.2580403975857246, + 1.9597194264610087, + 1.773041790596292, + 1.6211455309545355, + 1.2094494759638503, + 1.0219317012739941, + 0.8604275624593173, + 0.6846808632651675, + 0.5134649266877133, + 0.4503974279453127, + 0.4455048059532228 + ], + "peak_layer": 21, + "peak_normalized": 2.534675981169938, + "population_cv": 0.5361438958058841, + "rest_mean": 3.768541933755774e-06, + "spike_contrast": 2.5072447705625382, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.4486570560548e-06, + "top_five_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.2780119454400847e-06, + 3.1025388125272e-06, + 3.046939355044742e-06, + 3.1110105283005396e-06, + 3.1398953979078215e-06, + 3.105628366029123e-06, + 3.0107407837931532e-06, + 3.005658982146997e-06, + 3.566693294487777e-06, + 3.4106317343685078e-06, + 3.320514906590688e-06, + 3.389698349565151e-06, + 3.020406438736245e-06, + 3.058069751205039e-06, + 3.119173470622627e-06, + 3.405654751986731e-06, + 7.140777142922161e-06, + 6.798512004024815e-06, + 6.4910009314189665e-06, + 6.083415428292938e-06, + 1.1801603250205517e-05, + 1.0513571396586485e-05, + 9.12457107915543e-06, + 8.255388820543885e-06, + 7.548150733782677e-06, + 5.631269232253544e-06, + 4.758175236929674e-06, + 4.006202289019711e-06, + 3.187915126545704e-06, + 2.3907234663056443e-06, + 2.0970774130546488e-06, + 2.074297071885667e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 5.95857286498358e-05, + "first_to_last_ratio": 1.5974327409797537, + "imbalance_abs_log_ratio": 0.4683978037101298, + "last_quartile_mean": 3.7300931126083015e-05, + "mean": 6.105390025368251e-05, + "normalized": [ + 0.8419145460820371, + 0.8656854461464137, + 0.8583401181387939, + 0.8192663123570689, + 1.1963719080724353, + 1.1751919414919585, + 1.0676149913033914, + 0.9832376272965914, + 1.0816749854357006, + 0.984736879340849, + 0.9088680283961561, + 0.8483339022774173, + 0.8441772779737997, + 0.7882426814662962, + 0.7677338358974833, + 0.7384876656990601, + 1.5920129687595903, + 1.4697744406924866, + 1.283993033566668, + 1.1290203889690138, + 2.340760926498109, + 1.860435632123778, + 1.4645539617726704, + 1.2019638482713701, + 1.15797649283655, + 0.8703237664687024, + 0.7241030451553558, + 0.6218659849233957, + 0.4685510324632258, + 0.37521378882561096, + 0.3379462835275562, + 0.3316262577704645 + ], + "peak_layer": 21, + "peak_normalized": 2.340760926498109, + "population_cv": 0.4213559298637173, + "rest_mean": 5.4212039955288036e-05, + "spike_contrast": 1.8077155175462722, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.79999458650127e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 5.140216671861708e-05, + 5.285347288008779e-05, + 5.240501195657998e-05, + 5.0019403715850785e-05, + 7.304317114176229e-05, + 7.175005157478154e-05, + 6.518205918837339e-05, + 6.003049202263355e-05, + 6.604047666769475e-05, + 6.012202720739879e-05, + 5.548993794946e-05, + 5.1794093451462686e-05, + 5.154031532583758e-05, + 4.8125290049938485e-05, + 4.687314503826201e-05, + 4.5087552280165255e-05, + 9.7198600997217e-05, + 8.973546209745109e-05, + 7.839278259780258e-05, + 6.8931098212488e-05, + 0.000142912584124133, + 0.00011358685151208192, + 8.941673149820417e-05, + 7.338458090089262e-05, + 7.069898128975183e-05, + 5.3136660426389426e-05, + 4.420931509230286e-05, + 3.796734381467104e-05, + 2.8606867999769747e-05, + 2.2908265236765146e-05, + 2.0632938685594127e-05, + 2.0247076463419944e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 4.39446339441929e-05, + "first_to_last_ratio": 1.9902838030955072, + "imbalance_abs_log_ratio": 0.6882772431887607, + "last_quartile_mean": 2.207958175404201e-05, + "mean": 4.171556903997953e-05, + "normalized": [ + 0.9393219483952423, + 0.9671725353750462, + 0.9606035942825071, + 0.9165296658237206, + 1.25754251069206, + 1.233040231347365, + 1.1241638961033311, + 1.029104366181827, + 1.1530467776467852, + 1.041181174787795, + 0.9503458810810488, + 0.8874956508445306, + 0.9380795670187313, + 0.868712023499227, + 0.8459898570744635, + 0.811097306638917, + 1.57041471098841, + 1.447423664007262, + 1.250399123009104, + 1.0954141829132344, + 2.392471764387743, + 1.7753028127291355, + 1.2921083756975622, + 1.0187283101804303, + 1.065894151409315, + 0.8051193818729687, + 0.6754751497722067, + 0.6026093861097862, + 0.3198852539229254, + 0.2669904264118248, + 0.2517857094641488, + 0.2465506103313451 + ], + "peak_layer": 21, + "peak_normalized": 2.392471764387743, + "population_cv": 0.4303681174959756, + "rest_mean": 3.778425454786177e-05, + "spike_contrast": 1.6658967617763285, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.294466729741543e-05, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 3.9184349589049816e-05, + 4.034615267300978e-05, + 4.007212555734441e-05, + 3.823355655185878e-05, + 5.245910142548382e-05, + 5.1436974899843335e-05, + 4.689513662015088e-05, + 4.2929674236802384e-05, + 4.810000245925039e-05, + 4.343346517998725e-05, + 3.9644219214096665e-05, + 3.702238609548658e-05, + 3.913252294296399e-05, + 3.6238816392142326e-05, + 3.5290948289912194e-05, + 3.3835385693237185e-05, + 6.551074329763651e-05, + 6.0380101785995066e-05, + 5.2161110943416134e-05, + 4.5695825974689797e-05, + 9.980332106351852e-05, + 7.40577670512721e-05, + 5.3901036153547466e-05, + 4.249683115631342e-05, + 4.446438106242567e-05, + 3.358601315994747e-05, + 2.8177830245113e-05, + 2.513819345040247e-05, + 1.3344195394893177e-05, + 1.113765756599605e-05, + 1.0503384146431927e-05, + 1.0284999007126316e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 0.00015719397015345749, + "first_to_last_ratio": 1.2580343238755638, + "imbalance_abs_log_ratio": 0.22955044238581457, + "last_quartile_mean": 0.00012495205191953573, + "mean": 0.00018078252287523355, + "normalized": [ + 0.7253227592072276, + 0.7485993006094415, + 0.7410050906139998, + 0.7104508417729404, + 1.0745241672934553, + 1.0524252536641314, + 0.9748384190355198, + 0.9289919933888637, + 1.0038768041568893, + 0.9389629508381758, + 0.8789427325469985, + 0.8198897306139241, + 0.7627520038160613, + 0.7096499260969147, + 0.6897553416925117, + 0.6787601781063402, + 1.6052869610938005, + 1.5071148227224969, + 1.3317635520239959, + 1.1792135859285877, + 2.364258320942869, + 1.9856700690456726, + 1.6414783683894763, + 1.4170804705838413, + 1.2788477375082987, + 0.9694880608375548, + 0.8072605380605845, + 0.691415531221962, + 0.5679201295495192, + 0.44599504581333255, + 0.3904354851204105, + 0.3780238277042025 + ], + "peak_layer": 21, + "peak_normalized": 2.364258320942869, + "population_cv": 0.4451626458968529, + "rest_mean": 0.0001560934222113617, + "spike_contrast": 2.0122799667677387, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00031410366646014156, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.00013112567830830812, + 0.0001353336701868102, + 0.00013396076974458992, + 0.00012843709555454552, + 0.00019425518985372037, + 0.0001902600924950093, + 0.00017623374878894538, + 0.00016794551629573107, + 0.00018148338131140918, + 0.00016974809113889933, + 0.00015889748465269804, + 0.0001482217339798808, + 0.00013789223157800734, + 0.00012829230399802327, + 0.00012469571083784103, + 0.00012270797742530704, + 0.00029020782676525414, + 0.00027246001991443336, + 0.00024075957480818033, + 0.0002131812070729211, + 0.00042741658398881555, + 0.0003589744446799159, + 0.00029675060068257153, + 0.00025618338258937, + 0.0002311933203600347, + 0.00017526649753563106, + 0.00014593859668821096, + 0.0001249958440894261, + 0.00010267003381159157, + 8.062810957198963e-05, + 7.058391202008352e-05, + 6.834010127931833e-05 + ] + } + }, + "pre_attention_input": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0717411839777924e-05, + "first_to_last_ratio": 1.3606399895734356, + "imbalance_abs_log_ratio": 0.3079551696283513, + "last_quartile_mean": 7.876743239876305e-06, + "mean": 1.4356073158694471e-05, + "normalized": [ + 0.6895052247736269, + 0.5576007830925034, + 0.38530829240192566, + 0.5472592297661263, + 0.37729668968423546, + 0.9131958145391527, + 1.3435179101111983, + 1.1586522070597358, + 1.5436961923733754, + 1.144684350345656, + 0.8911764746394834, + 0.9261744836030649, + 0.5468539631209283, + 0.8612404659079811, + 0.9670035934822814, + 0.889769793290417, + 0.9785434614239343, + 1.8290819610081128, + 1.5223690430889059, + 1.8851307751720068, + 1.8951492301881587, + 2.5144698996677146, + 2.058593895387357, + 1.1843681717440393, + 1.0244158825153251, + 0.9136429572594602, + 0.5473929037729057, + 0.6024761608101054, + 0.3781148252878133, + 0.4421926007844957, + 0.24629038211477544, + 0.23483238158319855 + ], + "peak_layer": 22, + "peak_normalized": 2.5144698996677146, + "population_cv": 0.5646113035804416, + "rest_mean": 1.2400990229892799e-05, + "spike_contrast": 2.008994484502458, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 2.4913520974223502e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 9.898587450152263e-06, + 8.004957635421306e-06, + 5.531514034373686e-06, + 7.856493539293297e-06, + 5.41649887964013e-06, + 1.3109905921737663e-05, + 1.9287641407572664e-05, + 1.6633695850032382e-05, + 2.216141547251027e-05, + 1.643317227717489e-05, + 1.279379466723185e-05, + 1.3296228644321673e-05, + 7.850675501686055e-06, + 1.2364031135803089e-05, + 1.3882374332752079e-05, + 1.2773600246873684e-05, + 1.4048041521164123e-05, + 2.6258434445480816e-05, + 2.185524135711603e-05, + 2.7063075322075747e-05, + 2.7206900995224714e-05, + 3.6097913834964857e-05, + 2.9553324566222727e-05, + 1.7002876120386645e-05, + 1.4706589354318567e-05, + 1.3116325135342777e-05, + 7.858412573114038e-06, + 8.649191840959247e-06, + 5.428244094218826e-06, + 6.348149327095598e-06, + 3.5357627439225325e-06, + 3.3712708500388544e-06 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 4.013745751763054e-05, + "first_to_last_ratio": 1.3002217470857882, + "imbalance_abs_log_ratio": 0.26253482460265976, + "last_quartile_mean": 3.0869701731717214e-05, + "mean": 5.617674656832605e-05, + "normalized": [ + 0.473654064493233, + 0.5256902444154287, + 0.3705068590661575, + 0.5387329076786941, + 0.37665460925127686, + 0.9021763170314637, + 1.3591261829588805, + 1.169340915014312, + 1.4504557484696843, + 1.2215794360380687, + 0.9103877586499164, + 0.9321134746924985, + 0.5414129154069731, + 0.8525397685215896, + 0.9808579608967221, + 0.9142526712062526, + 1.0619023556310665, + 1.8034129735144708, + 1.5888135435441055, + 1.899281403844036, + 1.9024910148779852, + 2.5497535502321647, + 2.089500431013523, + 1.18928036938163, + 1.0430326636758747, + 0.9143825787913804, + 0.5551399248346642, + 0.5581364123079678, + 0.38887965127025986, + 0.4518317596354874, + 0.24874412658654727, + 0.2359354070676856 + ], + "peak_layer": 22, + "peak_normalized": 2.5497535502321647, + "population_cv": 0.5766137085977164, + "rest_mean": 4.832436503353098e-05, + "spike_contrast": 2.0399565889343343, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 9.857960685621947e-05, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 2.6608344342093915e-05, + 2.9531567633966915e-05, + 2.0813869923586026e-05, + 3.0264262022683397e-05, + 2.1159230527700856e-05, + 5.0681330321822315e-05, + 7.635128713445738e-05, + 6.568976823473349e-05, + 8.148188499035314e-05, + 6.862435839138925e-05, + 5.114262239658274e-05, + 5.236310244072229e-05, + 3.041481613763608e-05, + 4.7892910515656695e-05, + 5.510140908882022e-05, + 5.135974060976878e-05, + 5.9654219512594864e-05, + 0.00010130987357115373, + 8.925437578000128e-05, + 0.00010669545008568093, + 0.00010687575559131801, + 0.00014323685900308192, + 0.00011738133616745472, + 6.680990190943703e-05, + 5.859418160980567e-05, + 5.1367038395255804e-05, + 3.1185954867396504e-05, + 3.1354287784779444e-05, + 2.18459936149884e-05, + 2.538243825256359e-05, + 1.397363575961208e-05, + 1.3254083569336217e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.03559429990127683, + "first_to_last_ratio": 1.3002217714306916, + "imbalance_abs_log_ratio": 0.26253484332631455, + "last_quartile_mean": 0.027375560603104532, + "mean": 0.04981810107710771, + "normalized": [ + 0.4736540568825577, + 0.5256902401367205, + 0.3705068792758801, + 0.5387329172865728, + 0.3766546277313775, + 0.9021763186028472, + 1.3591262384871188, + 1.1693409622603343, + 1.4504557122308583, + 1.2215794195883674, + 0.9103877486678197, + 0.9321134797583928, + 0.5414129179041209, + 0.8525397564320455, + 0.9808579422244733, + 0.9142526416467389, + 1.0619023214513024, + 1.8034129787908137, + 1.5888135716490905, + 1.8992814677966294, + 1.9024909329539326, + 2.549753462436128, + 2.0895004555311574, + 1.1892804001617727, + 1.0430326554170677, + 0.9143826055433697, + 0.5551398871242083, + 0.5581364223645632, + 0.38887964652236373, + 0.4518317875925886, + 0.24874413075622867, + 0.23593541479255786 + ], + "peak_layer": 22, + "peak_normalized": 2.549753462436128, + "population_cv": 0.576613701889011, + "rest_mean": 0.04285453077129744, + "spike_contrast": 2.0399565496359413, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0874213807284832, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 0.023596545681357384, + 0.026188889518380165, + 0.01845794916152954, + 0.026838650926947594, + 0.01876421831548214, + 0.04494471102952957, + 0.06770908832550049, + 0.05825434625148773, + 0.07225894927978516, + 0.06085676699876785, + 0.04535378888249397, + 0.0464361235499382, + 0.02697216346859932, + 0.04247191175818443, + 0.04886448010802269, + 0.04554633051156998, + 0.05290195718407631, + 0.08984261006116867, + 0.07915167510509491, + 0.09461859613656998, + 0.09477848559617996, + 0.1270238757133484, + 0.10409494489431381, + 0.0592476911842823, + 0.051961906254291534, + 0.045552805066108704, + 0.027656015008687973, + 0.027805296704173088, + 0.019373245537281036, + 0.02250940166413784, + 0.012391960248351097, + 0.011753854341804981 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 2.9162920753833532e-06, + "first_to_last_ratio": 0.5172826991978068, + "imbalance_abs_log_ratio": 0.6591657469591501, + "last_quartile_mean": 5.637714309614239e-06, + "mean": 5.31696678152116e-06, + "normalized": [ + 2.0689578843648566, + 0.7607510573443436, + 0.20852787293451516, + 0.2204829621221139, + 0.15259464703358938, + 0.24520248807427852, + 0.36182592024390203, + 0.3695605650129213, + 0.49013872876325654, + 0.6336599249096766, + 0.533666626864933, + 0.7065919516088963, + 0.4409106734215795, + 0.6204696462782906, + 0.6094215771745697, + 0.6245105255232631, + 0.9007337205411854, + 1.4762581910365227, + 1.5659025652557783, + 1.5082894705927152, + 1.6990202709747757, + 2.997351654903827, + 2.2171258799622118, + 2.105442927603072, + 1.6398303896765056, + 1.6495827585670528, + 1.159798242314629, + 1.3298749851714036, + 0.9845526396022054, + 0.7951372535426381, + 0.492335162782499, + 0.4314908357979929 + ], + "peak_layer": 22, + "peak_normalized": 2.997351654903827, + "population_cv": 0.6973153224686441, + "rest_mean": 4.202615000563297e-06, + "spike_contrast": 2.697003269910379, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.133446639869362e-05, + "top_five_layers": [ + 22, + 23, + 24, + 1, + 21 + ], + "top_five_spike_overlap": 4 + }, + "values": [ + 1.1000580343534239e-05, + 4.044888100906974e-06, + 1.1087357734140824e-06, + 1.1723005854946678e-06, + 8.113406693155412e-07, + 1.3037334838372772e-06, + 1.9238163986301515e-06, + 1.9649412479338935e-06, + 2.6060513391712448e-06, + 3.3691487715259427e-06, + 2.8374877274472965e-06, + 3.756925934794708e-06, + 2.344307404200663e-06, + 3.299016498203855e-06, + 3.2402742817794206e-06, + 3.3205017189175123e-06, + 4.789171271113446e-06, + 7.84921576268971e-06, + 8.325851922563743e-06, + 8.019525012059603e-06, + 9.033634341903962e-06, + 1.593681918166112e-05, + 1.1788384654209949e-05, + 1.1194570106454194e-05, + 8.71892370923888e-06, + 8.77077673067106e-06, + 6.166608727653511e-06, + 7.070901119732298e-06, + 5.2348336794239e-06, + 4.227718363836175e-06, + 2.61772970588936e-06, + 2.2942224404687295e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 2.6059751689899713e-05, + "first_to_last_ratio": 1.3136640344119184, + "imbalance_abs_log_ratio": 0.27282020579643, + "last_quartile_mean": 1.983745539746451e-05, + "mean": 3.78474055935385e-05, + "normalized": [ + 0.4589265841635195, + 0.5077348811615244, + 0.3979439201623806, + 0.5340807477186122, + 0.3758546364010748, + 0.8955899993665016, + 1.2433535731478862, + 1.0948988680288454, + 1.2525996711021592, + 1.2070776025128485, + 0.9384156458392604, + 0.9009485279466896, + 0.5787782313540217, + 0.9014501901107522, + 1.0394159362329198, + 0.9428128555264055, + 1.0906052781563889, + 1.901538177289176, + 1.645145075817088, + 1.8092152293363273, + 1.9503048533412066, + 2.7077203289997303, + 2.1756312348621445, + 1.2568133833623651, + 1.064835664650375, + 0.9479102198797764, + 0.5517473493893477, + 0.45064300663075274, + 0.3856334920760448, + 0.38842166290606656, + 0.2110006297428454, + 0.19295254278496235 + ], + "peak_layer": 22, + "peak_normalized": 2.7077203289997303, + "population_cv": 0.6018204393551218, + "rest_mean": 3.202268221179515e-05, + "spike_contrast": 2.164119526172185, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 6.930091185495258e-05, + "top_five_layers": [ + 22, + 23, + 21, + 18, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.7369180568493903e-05, + 1.9216447981307283e-05, + 1.5061144949868321e-05, + 2.021357067860663e-05, + 1.4225122868083417e-05, + 3.389575795154087e-05, + 4.7057706979103386e-05, + 4.143908154219389e-05, + 4.7407647798536345e-05, + 4.5684755605179816e-05, + 3.5516597563400865e-05, + 3.4098564356099814e-05, + 2.1905254470766522e-05, + 3.4117550967494026e-05, + 3.933919651899487e-05, + 3.568302054191008e-05, + 4.127658030483872e-05, + 7.196828664746135e-05, + 6.226447294466197e-05, + 6.847410259069875e-05, + 7.381397881545126e-05, + 0.00010248018952552229, + 8.23419977677986e-05, + 4.7567125875502825e-05, + 4.0301267290487885e-05, + 3.5875942558050156e-05, + 2.0882205717498437e-05, + 1.7055668649845757e-05, + 1.4595227185054682e-05, + 1.4700752217322588e-05, + 7.985826414369512e-06, + 7.30275314708706e-06 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 1.7762501443030487e-05, + "first_to_last_ratio": 1.3504079548741488, + "imbalance_abs_log_ratio": 0.3004067355961661, + "last_quartile_mean": 1.315343365604349e-05, + "mean": 2.547085748005884e-05, + "normalized": [ + 0.4678386796616822, + 0.5235377308817779, + 0.4345577931245205, + 0.5165838835160134, + 0.4023318777827698, + 0.9134377309740589, + 1.2276035467300503, + 1.0930340705226127, + 1.0712949866382682, + 1.183459795509594, + 0.9400000096186449, + 0.8234489915172668, + 0.6264659982400982, + 0.9214742940814803, + 1.1003532746415148, + 0.9455412775127405, + 1.1042840733917474, + 2.0049913068036904, + 1.659392669807193, + 1.6624482120354451, + 1.950545580645667, + 2.815676235928119, + 2.2178876005698376, + 1.2625215400320522, + 1.0704850029998623, + 1.0091870523571758, + 0.5486605781901173, + 0.4093310376700093, + 0.39616480433203954, + 0.33219377199499983, + 0.19307701107731026, + 0.17218958121164055 + ], + "peak_layer": 22, + "peak_normalized": 2.815676235928119, + "population_cv": 0.6142210141608739, + "rest_mean": 2.1398240985743338e-05, + "spike_contrast": 2.218078886997531, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.746298654936254e-05, + "top_five_layers": [ + 22, + 23, + 18, + 21, + 20 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 1.1916252333321609e-05, + 1.3334954928723164e-05, + 1.1068559615523554e-05, + 1.3157834473531693e-05, + 1.0247737918689381e-05, + 2.326604226254858e-05, + 3.126811498077586e-05, + 2.7840515031130053e-05, + 2.728680192376487e-05, + 3.014373578480445e-05, + 2.3942606276250444e-05, + 2.0973951905034482e-05, + 1.5956626157276332e-05, + 2.347074041608721e-05, + 2.8026941436110064e-05, + 2.4083747121039778e-05, + 2.8127062250860035e-05, + 5.1068847824353725e-05, + 4.226615419611335e-05, + 4.234398147673346e-05, + 4.96820684929844e-05, + 7.171768811531365e-05, + 5.6491498980904e-05, + 3.21575062116608e-05, + 2.7266170945949852e-05, + 2.5704859581310302e-05, + 1.3974855392007157e-05, + 1.0426012522657402e-05, + 1.0090657269756775e-05, + 8.461260222247802e-06, + 4.917837031825911e-06, + 4.385816282592714e-06 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 7.240166451083496e-05, + "first_to_last_ratio": 1.2489384660588578, + "imbalance_abs_log_ratio": 0.22229396336363683, + "last_quartile_mean": 5.797056178380444e-05, + "mean": 0.0001109465127342446, + "normalized": [ + 0.42709475759614274, + 0.4935186328821003, + 0.35950105255417747, + 0.5439303076283766, + 0.3371404368406644, + 0.8629008054703304, + 1.152554863343196, + 1.0440127885102133, + 1.3928110595765304, + 1.2466636602290324, + 0.9528986062756032, + 0.9925221789536534, + 0.5398804000340625, + 0.8579108303439223, + 1.0148735477787485, + 0.9293336653216302, + 1.0803774560382864, + 1.8519808466056424, + 1.6712809075368884, + 1.9834945531761723, + 1.984928279946317, + 2.6376973498062735, + 2.185821980542675, + 1.2767982858711757, + 1.0594658474140766, + 0.9033078060139075, + 0.5489550405623127, + 0.4646003337611295, + 0.361861238846865, + 0.4230227492827023, + 0.22047516727493316, + 0.19838456398225895 + ], + "peak_layer": 22, + "peak_normalized": 2.6376973498062735, + "population_cv": 0.610312229060459, + "rest_mean": 9.391535294761761e-05, + "spike_contrast": 2.1606134589645687, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0002029147755820304, + "top_five_layers": [ + 22, + 23, + 21, + 20, + 18 + ], + "top_five_spike_overlap": 3 + }, + "values": [ + 4.738467396236956e-05, + 5.475417128764093e-05, + 3.988538810517639e-05, + 6.034717080183327e-05, + 3.7404555769171566e-05, + 9.573583520250395e-05, + 0.00012787194282282144, + 0.00011582957813516259, + 0.00015452752995770425, + 0.00013831298565492034, + 0.00010572077735560015, + 0.0001101168745663017, + 5.989784767734818e-05, + 9.518221486359835e-05, + 0.00011259668099228293, + 0.00010310632933396846, + 0.00011986411118414253, + 0.00020547081658151, + 0.00018542278849054128, + 0.00022006180370226502, + 0.0002202208706876263, + 0.000292643322609365, + 0.00024250932619906962, + 0.00014165631728246808, + 0.00011754404113162309, + 0.00010021885100286454, + 6.090464739827439e-05, + 5.154578684596345e-05, + 4.0147242543753237e-05, + 4.693289884016849e-05, + 2.4460950953653082e-05, + 2.2010075554135256e-05 + ] + } + }, + "pre_mlp_input": { + "batch_mean_rms": { + "statistics": { + "first_quartile_mean": 5.254158395473496e-06, + "first_to_last_ratio": 0.18864041654826996, + "imbalance_abs_log_ratio": 1.6679126340303851, + "last_quartile_mean": 2.7852771381731145e-05, + "mean": 2.0484676667820167e-05, + "normalized": [ + 0.5568020220803858, + 0.21809281295181746, + 0.09241273564310358, + 0.28328072581730895, + 0.22196567434120804, + 0.13250710048221157, + 0.17764328005295377, + 0.3692327060422689, + 0.6392853696302555, + 0.4685551353916866, + 0.4862788629304224, + 0.46993740268393913, + 0.5775584528492825, + 0.5275503726871256, + 0.4833126690762866, + 0.4805557266835708, + 1.3911160583811022, + 1.414259456489031, + 1.4619454359332327, + 1.4084306067413008, + 2.477707616941058, + 2.525352305517634, + 2.2589272888438794, + 1.9997852708388022, + 2.0849570534601547, + 1.8699232722164576, + 1.4787615635622833, + 1.3487564232655025, + 1.428990706094454, + 1.0620321056863733, + 0.8416254501901257, + 0.762458336494781 + ], + "peak_layer": 22, + "peak_normalized": 2.525352305517634, + "population_cv": 0.7251468208972824, + "rest_mean": 1.566946546652751e-05, + "spike_contrast": 2.966713654279006, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 4.648681715480052e-05, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 1.1405909390305169e-05, + 4.467560756893363e-06, + 1.8930450096377172e-06, + 5.802914074592991e-06, + 4.546895070234314e-06, + 2.7143651095684618e-06, + 3.6389651540957857e-06, + 7.563612598460168e-06, + 1.3095554095343687e-05, + 9.598200449545402e-06, + 9.961265277524944e-06, + 9.626515748095699e-06, + 1.1831098163384013e-05, + 1.0806698810483795e-05, + 9.900503755488899e-06, + 9.844028681982309e-06, + 2.8496562663349323e-05, + 2.8970647690584883e-05, + 2.9947479561087675e-05, + 2.8851245588157326e-05, + 5.07550394104328e-05, + 5.1731025450862944e-05, + 4.6273395128082484e-05, + 4.096495467820205e-05, + 4.270967110642232e-05, + 3.830477362498641e-05, + 3.0291952498373576e-05, + 2.7628839234239422e-05, + 2.927241257566493e-05, + 2.1755384295829572e-05, + 1.7240425222553313e-05, + 1.561871249577962e-05 + ] + }, + "element_rms": { + "statistics": { + "first_quartile_mean": 2.0563243310789403e-05, + "first_to_last_ratio": 0.184584542859635, + "imbalance_abs_log_ratio": 1.6896476935739415, + "last_quartile_mean": 0.00011140284550492652, + "mean": 8.183052003118974e-05, + "normalized": [ + 0.5514618447323059, + 0.2169408826973503, + 0.08472485593965659, + 0.28295099514537675, + 0.21989392642956848, + 0.1303640614627248, + 0.17290402482659964, + 0.35108448581118906, + 0.6388397786640535, + 0.4693907558756991, + 0.4878178928069798, + 0.4684568375678178, + 0.5735758405807379, + 0.5222431877258841, + 0.48692296371012395, + 0.47894342377930604, + 1.3828038952387727, + 1.4137992065669855, + 1.4773604231197706, + 1.4101154596735193, + 2.482879371711871, + 2.526943137473127, + 2.259311590178407, + 2.019190865749238, + 2.085961378955964, + 1.8764944434843154, + 1.4765035496268348, + 1.350239590483524, + 1.4291199659133378, + 1.0658000032658883, + 0.838995628185017, + 0.7679657326180543 + ], + "peak_layer": 22, + "peak_normalized": 2.526943137473127, + "population_cv": 0.7282772544392188, + "rest_mean": 6.251158794367695e-05, + "spike_contrast": 2.977891930556664, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.0001861527533037588, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 4.51264095318038e-05, + 1.7752385247149505e-05, + 6.933079021109734e-06, + 2.3154027076088823e-05, + 1.7994034351431765e-05, + 1.0667758942872752e-05, + 1.4148826267046388e-05, + 2.8729426048812456e-05, + 5.2276591304689646e-05, + 3.841048965114169e-05, + 3.9918391848914325e-05, + 3.833406663034111e-05, + 4.6936009312048554e-05, + 4.2735431634355336e-05, + 3.984515933552757e-05, + 3.91921894333791e-05, + 0.00011315556184854358, + 0.00011569192429305986, + 0.00012089317169738933, + 0.00011539048136910424, + 0.00020317531016189605, + 0.00020678107102867216, + 0.00018488064233679324, + 0.00016523143858648837, + 0.00017069530440494418, + 0.0001535545161459595, + 0.00012082305329386145, + 0.00011049080785596743, + 0.00011694562999764457, + 8.721496851649135e-05, + 6.865544855827466e-05, + 6.2843035266269e-05 + ] + }, + "global_l2": { + "statistics": { + "first_quartile_mean": 0.018235690018627793, + "first_to_last_ratio": 0.1845845477639835, + "imbalance_abs_log_ratio": 1.6896476670042821, + "last_quartile_mean": 0.09879315597936511, + "mean": 0.07256812266132329, + "normalized": [ + 0.5514618483157399, + 0.21694088816023477, + 0.08472485792509094, + 0.28295099765712173, + 0.21989393857080072, + 0.13036405951384217, + 0.17290402763813106, + 0.3510845056878191, + 0.6388397993688198, + 0.4693907637084327, + 0.48781790094052196, + 0.46845687595408214, + 0.5735758195754074, + 0.5222432048140645, + 0.48692297651101485, + 0.47894340052024936, + 1.3828038812071648, + 1.413799283122749, + 1.4773604270157898, + 1.4101154778648226, + 2.482879404994159, + 2.526942972591032, + 2.259311440497952, + 2.0191909931789223, + 2.0859612981908873, + 1.8764944638324905, + 1.4765035418797068, + 1.350239576611916, + 1.4291200320637618, + 1.065800015673522, + 0.8389956311315239, + 0.7679656952822267 + ], + "peak_layer": 22, + "peak_normalized": 2.526942972591032, + "population_cv": 0.7282772403307106, + "rest_mean": 0.0554359013180214, + "spike_contrast": 2.977891835258891, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.1650821179151535, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 0.04001855105161667, + 0.015742992982268333, + 0.006148323882371187, + 0.02053322270512581, + 0.015957290306687355, + 0.009460275061428547, + 0.012547320686280727, + 0.025477543473243713, + 0.04635940492153168, + 0.03406280651688576, + 0.03540002927184105, + 0.03399503603577614, + 0.0416233204305172, + 0.03789820894598961, + 0.03533508628606796, + 0.034756023436784744, + 0.10034748166799545, + 0.10259675979614258, + 0.10720927268266678, + 0.10232943296432495, + 0.18017789721488953, + 0.1833755075931549, + 0.1639539897441864, + 0.14652889966964722, + 0.15137429535388947, + 0.13617368042469025, + 0.10714709013700485, + 0.09798435121774673, + 0.10370855778455734, + 0.07734310626983643, + 0.06088433787226677, + 0.05572982877492905 + ] + }, + "token_mean_rms": { + "statistics": { + "first_quartile_mean": 1.0810431092522776e-06, + "first_to_last_ratio": 0.14665609099981078, + "imbalance_abs_log_ratio": 1.9196649501572336, + "last_quartile_mean": 7.371279991730262e-06, + "mean": 5.266313571894443e-06, + "normalized": [ + 0.5313084313996962, + 0.2061221485022935, + 0.06868365811035125, + 0.2784811311506817, + 0.146950226496972, + 0.10534296524196422, + 0.12013039774147215, + 0.1851819388065788, + 0.48851680128093, + 0.4019870994251106, + 0.45293299696953215, + 0.4442480210738259, + 0.5787785597712432, + 0.5090619802943529, + 0.49010374567770754, + 0.468585442827116, + 1.3525598419019165, + 1.4672711047348668, + 1.4843408168011953, + 1.4896119798089702, + 2.508208189766812, + 2.524495568813508, + 2.388845240425297, + 2.1106199865726754, + 2.1011836342122523, + 1.9283517633380012, + 1.5593850323316485, + 1.452268444304737, + 1.4451925616379866, + 1.080259681295224, + 0.833132510373883, + 0.7978580989111985 + ], + "peak_layer": 22, + "peak_normalized": 2.524495568813508, + "population_cv": 0.7672111824742712, + "rest_mean": 3.972487093051313e-06, + "spike_contrast": 3.084459752954336, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 1.2252976557647343e-05, + "top_five_layers": [ + 22, + 21, + 23, + 24, + 25 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.7980368031421676e-06, + 1.0855038681256701e-06, + 3.6170968087390065e-07, + 1.4665689604953513e-06, + 7.738859721939662e-07, + 5.547690875573608e-07, + 6.326443440229923e-07, + 9.752261576068122e-07, + 2.5726826606842224e-06, + 2.1169901174289407e-06, + 2.385287189099472e-06, + 2.3395493826683378e-06, + 3.048029384444817e-06, + 2.6808800157596124e-06, + 2.581040007498814e-06, + 2.4677178771526087e-06, + 7.1230042522074655e-06, + 7.727109732513782e-06, + 7.817004188837018e-06, + 7.844763786124531e-06, + 1.3209010830905754e-05, + 1.3294785276229959e-05, + 1.2580408110807184e-05, + 1.1115186680399347e-05, + 1.1065491889894474e-05, + 1.0155305062653497e-05, + 8.212210559577215e-06, + 7.648101018276066e-06, + 7.610837201355025e-06, + 5.688986220775405e-06, + 4.387537046568468e-06, + 4.201770934741944e-06 + ] + }, + "token_rms_mean": { + "statistics": { + "first_quartile_mean": 1.3072703836769506e-05, + "first_to_last_ratio": 0.1965017087277629, + "imbalance_abs_log_ratio": 1.6270841518950423, + "last_quartile_mean": 6.65271763864439e-05, + "mean": 5.1720684069778144e-05, + "normalized": [ + 0.5903214985473635, + 0.21317222278028428, + 0.07861556264200854, + 0.2924267620633159, + 0.20614520303339837, + 0.10942131635502071, + 0.17754724840214117, + 0.3543967343826241, + 0.6616591248095567, + 0.5148181180855445, + 0.5447776590149115, + 0.5218723062533341, + 0.647120485093606, + 0.5989696729166056, + 0.5592579173992114, + 0.5462615999231498, + 1.3998993635309975, + 1.4086638785298187, + 1.4578445936683029, + 1.4102355321134408, + 2.6382627874666973, + 2.6317584037312556, + 2.2233659362975056, + 1.922962332424717, + 2.1034577149820564, + 1.8142894928770414, + 1.4101545016386676, + 1.2891070706275354, + 1.273679909247231, + 0.9355476159814242, + 0.756254888856092, + 0.7077325463251402 + ], + "peak_layer": 21, + "peak_normalized": 2.6382627874666973, + "population_cv": 0.7263037787473766, + "rest_mean": 3.923146602944514e-05, + "spike_contrast": 3.0374205592556516, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00011916246148757637, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 3.0531831725966185e-05, + 1.1025413186871447e-05, + 4.066050678375177e-06, + 1.5124512174224947e-05, + 1.0661970918590669e-05, + 5.659345333697274e-06, + 9.182865142065566e-06, + 1.8329641534364782e-05, + 3.4221462556160986e-05, + 2.6626745238900185e-05, + 2.8176273190183565e-05, + 2.6991592676495202e-05, + 3.346951416460797e-05, + 3.097912122029811e-05, + 2.8925202059326693e-05, + 2.825302362907678e-05, + 7.240375271067023e-05, + 7.285705942194909e-05, + 7.540071965195239e-05, + 7.293834642041475e-05, + 0.0001364527561236173, + 0.0001361163449473679, + 0.00011499400716274977, + 9.945692727342248e-05, + 0.00010879227193072438, + 9.383629367221147e-05, + 7.293415546882898e-05, + 6.667349953204393e-05, + 6.587559619219974e-05, + 4.838716267840937e-05, + 3.911402018275112e-05, + 3.66044114343822e-05 + ] + }, + "token_rms_median": { + "statistics": { + "first_quartile_mean": 8.603489931147124e-06, + "first_to_last_ratio": 0.24312184321824193, + "imbalance_abs_log_ratio": 1.4141925488860427, + "last_quartile_mean": 3.538756459420256e-05, + "mean": 3.111726851301455e-05, + "normalized": [ + 0.6644756490180851, + 0.23325159254807834, + 0.07539325824570257, + 0.31873771579525234, + 0.21217410245742602, + 0.11960013716276979, + 0.20155955466263742, + 0.38669620325415055, + 0.7284622302362078, + 0.5930713003867844, + 0.6319198952855785, + 0.6040604923468361, + 0.7580808064483824, + 0.7072262953008145, + 0.6749843915485383, + 0.6486223376959581, + 1.4735493318409316, + 1.4845067822216826, + 1.4896409680595757, + 1.476749210387741, + 2.817405493456774, + 2.7673279203027326, + 2.139795377674301, + 1.694850102870422, + 2.0495665188393017, + 1.6765832039465556, + 1.3012719394146397, + 1.1897383429944555, + 1.0088778638486162, + 0.720783924666597, + 0.5951684074906716, + 0.5558686495917995 + ], + "peak_layer": 21, + "peak_normalized": 2.817405493456774, + "population_cv": 0.7201026839240032, + "rest_mean": 2.366186438646562e-05, + "spike_contrast": 3.0165184632367974, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 7.137645079637877e-05, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 2.067666719085537e-05, + 7.258152436406817e-06, + 2.3460322609025752e-06, + 9.918247087625787e-06, + 6.602278517675586e-06, + 3.721629582287278e-06, + 6.271982783800922e-06, + 1.2032929589622654e-05, + 2.2667754819849506e-05, + 1.845475890149828e-05, + 1.9663621060317382e-05, + 1.879671253846027e-05, + 2.3589404008816928e-05, + 2.2006950530339964e-05, + 2.1003670553909615e-05, + 2.018335544562433e-05, + 4.5852830226067454e-05, + 4.619379615178332e-05, + 4.635355799109675e-05, + 4.5952401706017554e-05, + 8.76699632499367e-05, + 8.611168595962226e-05, + 6.658458732999861e-05, + 5.273910574032925e-05, + 6.377691170200706e-05, + 5.2170689741615206e-05, + 4.0492028347216547e-05, + 3.7021407479187474e-05, + 3.139352338621393e-05, + 2.242882692371495e-05, + 1.8520015146350488e-05, + 1.729711402731482e-05 + ] + }, + "token_rms_p95": { + "statistics": { + "first_quartile_mean": 3.8654464106002706e-05, + "first_to_last_ratio": 0.16200207727357052, + "imbalance_abs_log_ratio": 1.8201461211556016, + "last_quartile_mean": 0.0002386047435720684, + "mean": 0.00017063225249103198, + "normalized": [ + 0.5177972227971404, + 0.18422958638068, + 0.07755682312933251, + 0.27342225532372755, + 0.19007362166945208, + 0.0968768622809422, + 0.1517726472113855, + 0.32056442710365024, + 0.6168627493486994, + 0.4509605691569947, + 0.4954745315627688, + 0.46627480155634954, + 0.5531570230685467, + 0.5170724935984963, + 0.45563757978860675, + 0.4789901403808068, + 1.3287627572643317, + 1.3383475524992587, + 1.407284240753668, + 1.3213535994248635, + 2.4744625690346806, + 2.5664095372695273, + 2.4114518937040392, + 2.1183513814824377, + 2.201741632911359, + 1.9351067846360057, + 1.5669730065860399, + 1.4334150862915997, + 1.3851211553907088, + 1.0454346613928627, + 0.8612433454555685, + 0.7578174615454693 + ], + "peak_layer": 22, + "peak_normalized": 2.5664095372695273, + "population_cv": 0.7569752509959051, + "rest_mean": 0.00012783252026969826, + "spike_contrast": 3.14279031375294, + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "spike_mean": 0.00040175080648623406, + "top_five_layers": [ + 22, + 21, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + "values": [ + 8.83529064594768e-05, + 3.1435509299626574e-05, + 1.3233695426606573e-05, + 4.6654655307065696e-05, + 3.2432690204586834e-05, + 1.653031722526066e-05, + 2.5897308660205454e-05, + 5.469863026519306e-05, + 0.00010525668039917946, + 7.69484176998958e-05, + 8.454393537249416e-05, + 7.956151966936886e-05, + 9.438642882741988e-05, + 8.822924428386614e-05, + 7.774646655889228e-05, + 8.173116657417268e-05, + 0.00022672978229820728, + 0.0002283652574988082, + 0.00024012807989493012, + 0.00022546554100699723, + 0.00042222312185913324, + 0.0004379122401587665, + 0.00041147146839648485, + 0.0003614590677898377, + 0.000375688134226948, + 0.00033019162947312, + 0.00026737613370642066, + 0.00024458684492856264, + 0.00023634634271729738, + 0.00017838487110566348, + 0.00014695589197799563, + 0.00012930810044053942 + ] + } + } + }, + "seed": 2026073003 + } + ], + "gates": { + "forward_identity": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "initialization_negative_control": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "loss_scale": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "replay": { + "checks": { + "architecture": true, + "artifacts": true, + "batch_size": true, + "depth": true, + "diagnostics": true, + "environment": true, + "evaluations": true, + "hashes": true, + "manifest": true, + "model": true, + "optimizer": true, + "round05_equivalence": true, + "seed": true, + "steps": true, + "target_bytes_seen": true, + "training_history": true + }, + "frozen_compare_sha256": "05396eaf56429ca8d87815ecf00a5f78be5a9b2ec7cc96014856c09dcedcbb87", + "passed": true + }, + "round05_equivalence": [ + { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true, + "seed": 2026073001 + }, + { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true, + "seed": 2026073002 + }, + { + "checks": { + "evaluations": true, + "final_model_state": true, + "final_optimizer_state": true, + "post_mlp_element_rms_all_steps": true, + "training_history": true + }, + "passed": true, + "seed": 2026073003 + } + ] + }, + "mixer_associations": { + "all_layers": { + "attention_entropy": { + "pearson": -0.29619123279185056, + "spearman": -0.2883517139673049 + }, + "attention_latest": { + "pearson": 0.06600313295625082, + "spearman": 0.29780653565593507 + }, + "attention_max": { + "pearson": -0.024396635036849867, + "spearman": 0.20476403642736357 + }, + "mlp_entropy": { + "pearson": -0.2736346519450915, + "spearman": -0.41707813347802497 + }, + "mlp_latest": { + "pearson": 0.6507856318169407, + "spearman": 0.692444384156267 + }, + "mlp_max": { + "pearson": 0.23015916710007592, + "spearman": 0.4651112316874661 + }, + "points": 96 + }, + "identity": "observational association; not causal attribution", + "layers_19_28": { + "attention_entropy": { + "pearson": -0.05745838348637786, + "spearman": -0.11279199110122358 + }, + "attention_latest": { + "pearson": 0.15799271238887527, + "spearman": 0.3388209121245829 + }, + "attention_max": { + "pearson": 0.15799271238887527, + "spearman": 0.3388209121245829 + }, + "mlp_entropy": { + "pearson": -0.6355230712915908, + "spearman": -0.6400444938820912 + }, + "mlp_latest": { + "pearson": 0.6896916851820697, + "spearman": 0.692992213570634 + }, + "mlp_max": { + "pearson": 0.7022594570959072, + "spearman": 0.7045606229143493 + }, + "points": 30 + }, + "target_layers": [ + { + "attention_entropy": 0.9286576240857585, + "attention_latest": 0.34922462701797485, + "attention_max": 0.34922462701797485, + "group": 5, + "layer": 19, + "mlp_entropy": 0.8417470208868109, + "mlp_latest": 0.4519178469975789, + "mlp_max": 0.4519178469975789, + "normalized_gradient": 1.0628893270643442, + "offset": 3, + "per_seed_normalized_gradient": [ + 0.6978155985835179, + 1.1574347089891701, + 1.3334176736203447 + ] + }, + { + "attention_entropy": 0.9460862283961503, + "attention_latest": 0.3155268132686615, + "attention_max": 0.3155268132686615, + "group": 5, + "layer": 20, + "mlp_entropy": 0.862835637646716, + "mlp_latest": 0.4188626905282338, + "mlp_max": 0.4188626905282338, + "normalized_gradient": 1.0135704098952112, + "offset": 4, + "per_seed_normalized_gradient": [ + 0.8168473226649812, + 1.060138766241709, + 1.1637251407789435 + ] + }, + { + "attention_entropy": 0.9687891325525921, + "attention_latest": 0.2497197538614273, + "attention_max": 0.2497197538614273, + "group": 6, + "layer": 21, + "mlp_entropy": 0.7626460544111615, + "mlp_latest": 0.512785017490387, + "mlp_max": 0.512785017490387, + "normalized_gradient": 3.0420861976019666, + "offset": 1, + "per_seed_normalized_gradient": [ + 3.219066994145898, + 3.6189242296270336, + 2.2882673690329693 + ] + }, + { + "attention_entropy": 0.8507401102456327, + "attention_latest": 0.4364195068677266, + "attention_max": 0.4364195068677266, + "group": 6, + "layer": 22, + "mlp_entropy": 0.7208451982094757, + "mlp_latest": 0.563143769900004, + "mlp_max": 0.563143769900004, + "normalized_gradient": 2.405186320789127, + "offset": 2, + "per_seed_normalized_gradient": [ + 2.5077095541323673, + 2.81239769967395, + 1.8954517085610634 + ] + }, + { + "attention_entropy": 0.879444880309749, + "attention_latest": 0.40026357769966125, + "attention_max": 0.40026357769966125, + "group": 6, + "layer": 23, + "mlp_entropy": 0.7369153442658093, + "mlp_latest": 0.5514553984006246, + "mlp_max": 0.5514553984006246, + "normalized_gradient": 1.9103037349623444, + "offset": 3, + "per_seed_normalized_gradient": [ + 1.973834673592298, + 2.2018329150221976, + 1.5552436162725376 + ] + }, + { + "attention_entropy": 0.8894285781602397, + "attention_latest": 0.38436253865559894, + "attention_max": 0.38436253865559894, + "group": 6, + "layer": 24, + "mlp_entropy": 0.7713335619488723, + "mlp_latest": 0.5146652460098267, + "mlp_max": 0.5146652460098267, + "normalized_gradient": 1.548088097624808, + "offset": 4, + "per_seed_normalized_gradient": [ + 1.5759251613516239, + 1.7615672973682381, + 1.306771834154562 + ] + }, + { + "attention_entropy": 0.9336027031788025, + "attention_latest": 0.3171870708465576, + "attention_max": 0.3171870708465576, + "group": 7, + "layer": 25, + "mlp_entropy": 0.7657190825854533, + "mlp_latest": 0.3575698534647624, + "mlp_max": 0.3722212314605713, + "normalized_gradient": 1.7685028111508305, + "offset": 1, + "per_seed_normalized_gradient": [ + 2.2647260999477155, + 1.8179181698124194, + 1.2228641636923565 + ] + }, + { + "attention_entropy": 0.8756262007907392, + "attention_latest": 0.3432680070400238, + "attention_max": 0.3432680070400238, + "group": 7, + "layer": 26, + "mlp_entropy": 0.7635627832181188, + "mlp_latest": 0.4196455279986064, + "mlp_max": 0.4196455279986064, + "normalized_gradient": 1.3690565893386435, + "offset": 2, + "per_seed_normalized_gradient": [ + 1.6880724220748888, + 1.4871187506667403, + 0.9319785952743014 + ] + }, + { + "attention_entropy": 0.891179394493967, + "attention_latest": 0.32520759105682373, + "attention_max": 0.32520759105682373, + "group": 7, + "layer": 27, + "mlp_entropy": 0.7748494313187421, + "mlp_latest": 0.40247613191604614, + "mlp_max": 0.40247613191604614, + "normalized_gradient": 0.9864745886014125, + "offset": 3, + "per_seed_normalized_gradient": [ + 1.092454232265147, + 1.086854135909831, + 0.7801153976292595 + ] + }, + { + "attention_entropy": 0.9019494449297653, + "attention_latest": 0.2957511991262436, + "attention_max": 0.2957511991262436, + "group": 7, + "layer": 28, + "mlp_entropy": 0.821866239503451, + "mlp_latest": 0.30352112650871277, + "mlp_max": 0.3173763652642568, + "normalized_gradient": 0.8024500092961132, + "offset": 4, + "per_seed_normalized_gradient": [ + 0.8732751258075867, + 0.872703317202576, + 0.661371584878177 + ] + } + ] + }, + "protocol_id": "llm-atlas-k3-attnres-spike-path-v1", + "reproduction_canonical_sha256": "7c522314742920961be84b750d823ce41814f10fd6129f0531208116e2fe3515", + "runs": [ + { + "canonical_sha256": "76b0ccfb55c38baef50c395788ac4b351cbe0d58de70064b50702acb5c93f515", + "final_bpc": 1.7123525940760473, + "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", + "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", + "seed": 2026073001, + "timing": { + "mean_ms": 224.20719212223366, + "measured_steps": 7980, + "median_ms": 227.38306851533707, + "p95_ms": 230.37451171875, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + } + }, + { + "canonical_sha256": "5352c74eca853b375c0e85933dafd7c5916c39fc59052e742ca14ffd6d68bc78", + "final_bpc": 1.7093240655856785, + "final_model_state": "bd2556388aeaa211b798c283c7cbd8ccd29edf166a2922fa13d172e8dfdc38d1", + "final_optimizer_state": "0b101eab3bc7d8d654be2ea335c86fc25563ce19912d721844ee4e639c569e77", + "seed": 2026073002, + "timing": { + "mean_ms": 223.82424076982844, + "measured_steps": 7980, + "median_ms": 227.36876149429008, + "p95_ms": 229.48265075683594, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + } + }, + { + "canonical_sha256": "7e764c07e90b78c4cd0acc2e99600225f16428cbb25d5188a0a5a8fe797f8766", + "final_bpc": 1.7030966812870931, + "final_model_state": "638568aede21890773b6932a19ec4e112f5ac0a4770ba3402fcd82980a9ecf76", + "final_optimizer_state": "83947fd743ec8e3e31ca7788fd201981846f0f1c7e9e38c88afcef95cfc6ec4e", + "seed": 2026073003, + "timing": { + "mean_ms": 224.65581085475935, + "measured_steps": 7980, + "median_ms": 227.5267465010984, + "p95_ms": 230.52696228027344, + "peak_allocated_bytes": 12841009152, + "peak_reserved_bytes": 13922992128, + "warmup_steps_excluded": 20 + } + } + ], + "schema_version": 1, + "study": { + "architecture": "block", + "depth": 32, + "identity": "targeted follow-up informed by Round 05; not blind discovery", + "seeds": [ + 2026073001, + 2026073002, + 2026073003 + ], + "spike_layers": [ + 21, + 22, + 23, + 24, + 25 + ], + "steps": 8000 + }, + "trajectory": [ + { + "points": [ + { + "bits_per_byte": 7.9557801867581395, + "peak_layer": 1, + "peak_normalized": 1.7717451736709768, + "population_cv": 0.2714186609524326, + "spike_contrast": 0.7816308178118705, + "step": 0 + }, + { + "bits_per_byte": 5.165135820292823, + "peak_layer": 1, + "peak_normalized": 2.145023601546889, + "population_cv": 0.5072011380661324, + "spike_contrast": 0.5596113913680666, + "step": 100 + }, + { + "bits_per_byte": 3.109974443324717, + "peak_layer": 2, + "peak_normalized": 2.1343785185339303, + "population_cv": 0.5868596491376813, + "spike_contrast": 0.5221542367874475, + "step": 500 + }, + { + "bits_per_byte": 1.9592225965086283, + "peak_layer": 21, + "peak_normalized": 2.863607509132394, + "population_cv": 0.5384894543542103, + "spike_contrast": 2.4966108002124767, + "step": 2000 + }, + { + "bits_per_byte": 1.8027438806514628, + "peak_layer": 21, + "peak_normalized": 3.115049509935762, + "population_cv": 0.6027314599912318, + "spike_contrast": 2.8510104941924466, + "step": 4000 + }, + { + "bits_per_byte": 1.7123525940760473, + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "population_cv": 0.6420341358971389, + "spike_contrast": 3.04626897714722, + "step": 8000 + } + ], + "seed": 2026073001 + }, + { + "points": [ + { + "bits_per_byte": 8.034013546791773, + "peak_layer": 1, + "peak_normalized": 2.16920774571012, + "population_cv": 0.39438035124793563, + "spike_contrast": 0.6908033638721538, + "step": 0 + }, + { + "bits_per_byte": 5.132431212443787, + "peak_layer": 1, + "peak_normalized": 2.254829765809383, + "population_cv": 0.5330470009230729, + "spike_contrast": 0.5561530270868602, + "step": 100 + }, + { + "bits_per_byte": 3.0911456537212687, + "peak_layer": 2, + "peak_normalized": 2.173061151786652, + "population_cv": 0.5858213540607353, + "spike_contrast": 0.5734017265344913, + "step": 500 + }, + { + "bits_per_byte": 1.947918907793682, + "peak_layer": 21, + "peak_normalized": 3.189751492495938, + "population_cv": 0.6175097196118516, + "spike_contrast": 2.740373880872814, + "step": 2000 + }, + { + "bits_per_byte": 1.7911587423283797, + "peak_layer": 21, + "peak_normalized": 3.420348943656972, + "population_cv": 0.6827037752697015, + "spike_contrast": 3.1572107254761765, + "step": 4000 + }, + { + "bits_per_byte": 1.7093240655856785, + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "population_cv": 0.7299683279122143, + "spike_contrast": 3.3328477735441013, + "step": 8000 + } + ], + "seed": 2026073002 + }, + { + "points": [ + { + "bits_per_byte": 8.061194200882085, + "peak_layer": 1, + "peak_normalized": 1.7301666786506156, + "population_cv": 0.2498932502086342, + "spike_contrast": 0.7974048446155259, + "step": 0 + }, + { + "bits_per_byte": 5.191908918496252, + "peak_layer": 1, + "peak_normalized": 2.133789250541258, + "population_cv": 0.49221598973047537, + "spike_contrast": 0.5746026267549044, + "step": 100 + }, + { + "bits_per_byte": 3.0881952483500545, + "peak_layer": 2, + "peak_normalized": 2.195594858531836, + "population_cv": 0.606636113722199, + "spike_contrast": 0.47779529155407807, + "step": 500 + }, + { + "bits_per_byte": 1.948972795980338, + "peak_layer": 21, + "peak_normalized": 1.8057756999530474, + "population_cv": 0.34914284904214865, + "spike_contrast": 1.4606649819879878, + "step": 2000 + }, + { + "bits_per_byte": 1.7872753095784035, + "peak_layer": 21, + "peak_normalized": 2.1130451955668876, + "population_cv": 0.3859968358313239, + "spike_contrast": 1.7734534826874866, + "step": 4000 + }, + { + "bits_per_byte": 1.7030966812870931, + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "population_cv": 0.42684708327825066, + "spike_contrast": 1.8814916302334765, + "step": 8000 + } + ], + "seed": 2026073003 + } + ], + "verdicts": { + "interventions": { + "forward_identity": [ + { + "passed": true, + "seed": 2026073001 + }, + { + "passed": true, + "seed": 2026073002 + }, + { + "passed": true, + "seed": 2026073003 + } + ], + "scope": "all 64 depth mixers plus the output mixer", + "softmax_key_path": { + "label": "global removal of all mixer softmax/query/key source-gradient paths", + "mean_relative_drop_contrast": -0.0200106813233286, + "mean_relative_drop_peak": -0.008401901006156563, + "opposite_direction_3_of_3": true, + "passed_3_of_3": false, + "per_seed": [ + { + "passed": false, + "relative_drop_contrast": -0.015414390900329828, + "relative_drop_peak": -0.006668985338663224, + "seed": 2026073001, + "source_peak_normalized": 3.219066994145898, + "source_spike_contrast": 3.04626897714722, + "target_peak_normalized": 3.2405349047340315, + "target_spike_contrast": 3.093225357948515 + }, + { + "passed": false, + "relative_drop_contrast": -0.01929133294667749, + "relative_drop_peak": -0.003788054520504258, + "seed": 2026073002, + "source_peak_normalized": 3.6189242296270336, + "source_spike_contrast": 3.3328477735441013, + "target_peak_normalized": 3.6326329119144347, + "target_spike_contrast": 3.3971428496041334 + }, + { + "passed": false, + "relative_drop_contrast": -0.025326320122978477, + "relative_drop_peak": -0.014748663159302204, + "seed": 2026073003, + "source_peak_normalized": 2.2882673690329693, + "source_spike_contrast": 1.8814916302334765, + "target_peak_normalized": 2.3220162536772593, + "target_spike_contrast": 1.9291428895694742 + } + ], + "same_direction_3_of_3": false, + "source_mode": "learned", + "target_mode": "detached_learned", + "threshold": 0.2, + "verdict": "opposite direction in 3/3 seeds; no material reduction" + }, + "value_coefficients": { + "label": "global replacement of learned value-backward coefficients with 1/N", + "mean_relative_drop_contrast": 0.701998806119879, + "mean_relative_drop_peak": 0.3702754647109951, + "opposite_direction_3_of_3": false, + "passed_3_of_3": true, + "per_seed": [ + { + "passed": true, + "relative_drop_contrast": 0.7479318970064935, + "relative_drop_peak": 0.3925047809741125, + "seed": 2026073001, + "source_peak_normalized": 3.2405349047340315, + "source_spike_contrast": 3.093225357948515, + "target_peak_normalized": 1.968609461712434, + "target_spike_contrast": 0.7797034481094922 + }, + { + "passed": true, + "relative_drop_contrast": 0.7687299974719846, + "relative_drop_peak": 0.4318458552099138, + "seed": 2026073002, + "source_peak_normalized": 3.6326329119144347, + "source_spike_contrast": 3.3971428496041334, + "target_peak_normalized": 2.063895445405066, + "target_spike_contrast": 0.7856572354159773 + }, + { + "passed": true, + "relative_drop_contrast": 0.5893345238811586, + "relative_drop_peak": 0.2864757579489588, + "seed": 2026073003, + "source_peak_normalized": 2.3220162536772593, + "source_spike_contrast": 1.9291428895694742, + "target_peak_normalized": 1.6568148874352646, + "target_spike_contrast": 0.7922323832463255 + } + ], + "same_direction_3_of_3": true, + "source_mode": "detached_learned", + "target_mode": "uniform_value_backward", + "threshold": 0.2, + "verdict": "material sensitivity at the preregistered threshold" + } + }, + "reduction_robustness": { + "cells": [ + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "element_rms", + "seed": 2026073001, + "spearman_vs_element_rms": 1.0, + "spike_contrast": 3.04626897714722, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_mean", + "seed": 2026073001, + "spearman_vs_element_rms": 0.9578445747800587, + "spike_contrast": 2.869084843241085, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_median", + "seed": 2026073001, + "spearman_vs_element_rms": 0.9013929618768328, + "spike_contrast": 2.6456990159702416, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_p95", + "seed": 2026073001, + "spearman_vs_element_rms": 0.9967008797653959, + "spike_contrast": 3.143466990402101, + "top_five_layers": [ + 21, + 22, + 25, + 23, + 26 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "element_rms", + "seed": 2026073002, + "spearman_vs_element_rms": 1.0, + "spike_contrast": 3.3328477735441013, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_mean", + "seed": 2026073002, + "spearman_vs_element_rms": 0.9732404692082112, + "spike_contrast": 3.18249263251226, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_median", + "seed": 2026073002, + "spearman_vs_element_rms": 0.8951612903225806, + "spike_contrast": 2.9052606637917697, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 18 + ], + "top_five_spike_overlap": 4 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_p95", + "seed": 2026073002, + "spearman_vs_element_rms": 0.9970674486803519, + "spike_contrast": 3.5324549012779802, + "top_five_layers": [ + 21, + 22, + 23, + 25, + 24 + ], + "top_five_spike_overlap": 5 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "element_rms", + "seed": 2026073003, + "spearman_vs_element_rms": 1.0, + "spike_contrast": 1.8814916302334765, + "top_five_layers": [ + 21, + 22, + 17, + 23, + 18 + ], + "top_five_spike_overlap": 3 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_mean", + "seed": 2026073003, + "spearman_vs_element_rms": 0.9875366568914956, + "spike_contrast": 1.8077155175462722, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_median", + "seed": 2026073003, + "spearman_vs_element_rms": 0.9343841642228738, + "spike_contrast": 1.6658967617763285, + "top_five_layers": [ + 21, + 22, + 17, + 18, + 23 + ], + "top_five_spike_overlap": 3 + }, + { + "checks": { + "spearman": true, + "spike_contrast": true, + "top_five_overlap": true + }, + "passed": true, + "reduction": "token_rms_p95", + "seed": 2026073003, + "spearman_vs_element_rms": 0.9926686217008798, + "spike_contrast": 2.0122799667677387, + "top_five_layers": [ + 21, + 22, + 23, + 17, + 18 + ], + "top_five_spike_overlap": 3 + } + ], + "passed_cells": 12, + "total_cells": 12, + "verdict": "robust within the preregistered reduction family" + }, + "visible_positions": { + "criterion": "step 8000 element-RMS spike contrast >= 1.5 in 3/3 seeds", + "earliest_observed_tensor": "pre_attention_input", + "positions": [ + { + "mean_spike_contrast": 2.9842437043760683, + "per_seed": [ + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 4.232714993086349, + "seed": 2026073001, + "spike_contrast": 3.7241794809927478 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.51885122958336, + "seed": 2026073002, + "spike_contrast": 3.1885950432011234 + }, + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 2.5497535502321647, + "seed": 2026073003, + "spike_contrast": 2.0399565889343343 + } + ], + "position": "pre_attention_input", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.9553267731281916, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.138964807406113, + "seed": 2026073001, + "spike_contrast": 3.1996375366532934 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.7925106998539326, + "seed": 2026073002, + "spike_contrast": 3.6616605179769537 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.2656295070514414, + "seed": 2026073003, + "spike_contrast": 2.004682264754327 + } + ], + "position": "attention_branch_output", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.9553191807218897, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.1389231589965445, + "seed": 2026073001, + "spike_contrast": 3.1996384204494492 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.7925041485004933, + "seed": 2026073002, + "spike_contrast": 3.6616533387138865 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.265619973525555, + "seed": 2026073003, + "spike_contrast": 2.0046657830023333 + } + ], + "position": "post_attention_state", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 3.7778825845934967, + "per_seed": [ + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 2.987231504916195, + "seed": 2026073001, + "spike_contrast": 3.828767846228416 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.828274701557073, + "seed": 2026073002, + "spike_contrast": 4.52698797699541 + }, + { + "passed": true, + "peak_layer": 22, + "peak_normalized": 2.526943137473127, + "seed": 2026073003, + "spike_contrast": 2.977891930556664 + } + ], + "position": "pre_mlp_input", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.7535328779602537, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.2190719364475022, + "seed": 2026073001, + "spike_contrast": 3.0462648281411457 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.6189237818679025, + "seed": 2026073002, + "spike_contrast": 3.3328561680843984 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.2882576030527964, + "seed": 2026073003, + "spike_contrast": 1.8814776376552176 + } + ], + "position": "mlp_branch_output", + "visible_3_of_3": true + }, + { + "mean_spike_contrast": 2.7535361269749323, + "per_seed": [ + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.219066994145898, + "seed": 2026073001, + "spike_contrast": 3.04626897714722 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 3.6189242296270336, + "seed": 2026073002, + "spike_contrast": 3.3328477735441013 + }, + { + "passed": true, + "peak_layer": 21, + "peak_normalized": 2.2882673690329693, + "seed": 2026073003, + "spike_contrast": 1.8814916302334765 + } + ], + "position": "post_mlp_state", + "visible_3_of_3": true + } + ], + "verdict": "visible at one or more positions", + "visible_positions": [ + "pre_attention_input", + "attention_branch_output", + "post_attention_state", + "pre_mlp_input", + "mlp_branch_output", + "post_mlp_state" + ] + } + } +}